logos-storage/logos-storage-go-bindings
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-go-bindings.git synced 2026-01-04 06:23:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 7b5512e8482d579ddbe763c9708a1032874868da into 59437382121afb90f291e9c24d22c57c681456aa

Browse Source
This commit is contained in:
Arnaud 2025-12-24 13:12:36 +00:00 committed by GitHub
commit 45004f438a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 598 additions and 581 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
.github/workflows/artifacts.yml vendored
View File

@ -46,14 +46,14 @@ jobs:
toolchain: 1.85.0 toolchain: 1.85.0
- name: Record submodule commit - name: Record submodule commit
run: git -C vendor/nim-codex rev-parse HEAD > vendor/nim-codex/.codex-commit run: git -C vendor/logos-storage-nim rev-parse HEAD > vendor/logos-storage-nim/.storage-commit
- name: Cache libcodex build - name: Cache libstorage build
id: cache-libcodex id: cache-libstorage
uses: actions/cache@v4 uses: actions/cache@v4
with: with:
path: vendor/nim-codex/build path: vendor/logos-storage-nim/build
key: ${{ runner.os }}-${{ matrix.target.cpu }}-libcodex-${{ hashFiles('vendor/nim-codex/.codex-commit') }} key: ${{ runner.os }}-${{ matrix.target.cpu }}-libstorage-${{ hashFiles('vendor/logos-storage-nim/.storage-commit') }}
- name: MSYS2 (Windows amd64) - name: MSYS2 (Windows amd64)
if: matrix.target.os == 'windows-latest' && matrix.target.cpu == 'amd64' if: matrix.target.os == 'windows-latest' && matrix.target.cpu == 'amd64'
@ -69,49 +69,49 @@ jobs:
mingw-w64-ucrt-x86_64-ntldd-git mingw-w64-ucrt-x86_64-ntldd-git
mingw-w64-ucrt-x86_64-rust mingw-w64-ucrt-x86_64-rust
- name: Build libcodex (Linux) - name: Build libstorage (Linux)
if: matrix.target.lib_ext == 'so' && steps.cache-libcodex.outputs.cache-hit != 'true' if: matrix.target.lib_ext == 'so' && steps.cache-libstorage.outputs.cache-hit != 'true'
run: | run: |
make update make update
make libcodex make libstorage
- name: Build libcodex (MacOS) - name: Build libstorage (MacOS)
if: matrix.target.os == 'macos-latest' && steps.cache-libcodex.outputs.cache-hit != 'true' if: matrix.target.os == 'macos-latest' && steps.cache-libstorage.outputs.cache-hit != 'true'
run: | run: |
make update make update
CODEX_LIB_PARAMS="--passL:\"-Wl,-install_name,@rpath/libcodex.dylib\"" make libcodex STORAGE_LIB_PARAMS="--passL:\"-Wl,-install_name,@rpath/libstorage.dylib\"" make libstorage
- name: Build libcodex (Windows) - name: Build libstorage (Windows)
if: matrix.target.os == 'windows-latest' && steps.cache-libcodex.outputs.cache-hit != 'true' if: matrix.target.os == 'windows-latest' && steps.cache-libstorage.outputs.cache-hit != 'true'
shell: msys2 {0} shell: msys2 {0}
run: | run: |
pacman -Sy --noconfirm make pacman -Sy --noconfirm make
git config --global core.symlinks false git config --global core.symlinks false
make update make update
make libcodex make libstorage
- name: Package artifacts Linux - name: Package artifacts Linux
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm' if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
run: | run: |
sudo apt-get update && sudo apt-get install -y zip sudo apt-get update && sudo apt-get install -y zip
ZIPFILE=codex-linux-${{ matrix.target.cpu }}.zip ZIPFILE=storage-linux-${{ matrix.target.cpu }}.zip
zip -j $ZIPFILE vendor/nim-codex/build/libcodex.${{ matrix.target.lib_ext }} vendor/nim-codex/library/libcodex.h zip -j $ZIPFILE vendor/logos-storage-nim/build/libstorage.${{ matrix.target.lib_ext }} vendor/logos-storage-nim/library/libstorage.h
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Package artifacts MacOS - name: Package artifacts MacOS
if: matrix.target.os == 'macos-latest' if: matrix.target.os == 'macos-latest'
run: | run: |
ZIPFILE=codex-macos-${{ matrix.target.cpu }}.zip ZIPFILE=storage-macos-${{ matrix.target.cpu }}.zip
zip -j $ZIPFILE vendor/nim-codex/build/libcodex.${{ matrix.target.lib_ext }} vendor/nim-codex/library/libcodex.h zip -j $ZIPFILE vendor/logos-storage-nim/build/libstorage.${{ matrix.target.lib_ext }} vendor/logos-storage-nim/library/libstorage.h
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Package artifacts (Windows) - name: Package artifacts (Windows)
if: matrix.target.os == 'windows-latest' if: matrix.target.os == 'windows-latest'
shell: msys2 {0} shell: msys2 {0}
run: | run: |
ZIPFILE=codex-windows-${{ matrix.target.cpu }}.zip ZIPFILE=storage-windows-${{ matrix.target.cpu }}.zip
(cd vendor/nim-codex/build && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libcodex.dll) (cd vendor/logos-storage-nim/build && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libstorage.dll)
(cd vendor/nim-codex/library && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libcodex.h) (cd vendor/logos-storage-nim/library && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libstorage.h)
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Upload artifacts - name: Upload artifacts

18
.github/workflows/ci.yml vendored
View File

@ -28,22 +28,22 @@ jobs:
toolchain: 1.85.0 toolchain: 1.85.0
- name: Record submodule commit - name: Record submodule commit
run: git -C vendor/nim-codex rev-parse HEAD > vendor/nim-codex/.codex-commit run: git -C vendor/logos-storage-nim rev-parse HEAD > vendor/logos-storage-nim/.storage-commit
- name: Cache libcodex build - name: Cache libstorage build
id: cache-libcodex id: cache-libstorage
uses: actions/cache@v4 uses: actions/cache@v4
with: with:
path: vendor/nim-codex/build path: vendor/logos-storage-nim/build
key: ${{ runner.os }}-libcodex-${{ hashFiles('vendor/nim-codex/.codex-commit') }} key: ${{ runner.os }}-libstorage-${{ hashFiles('vendor/logos-storage-nim/.storage-commit') }}
- name: Build libcodex - name: Build libstorage
if: steps.cache-libcodex.outputs.cache-hit != 'true' if: steps.cache-libstorage.outputs.cache-hit != 'true'
run: | run: |
make update make update
CODEX_LIB_PARAMS="-d:codex_enable_api_debug_peers=true -d:LeopardCmakeFlags=\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON\"" make libcodex STORAGE_LIB_PARAMS="-d:storage_enable_api_debug_peers=true -d:LeopardCmakeFlags=\"-DCMAKE_POSITION_INDEPENDENT_CODE=ON\"" make libstorage
- name: Build codex go - name: Build Logos Storage go
run: make run: make
- name: Install gotestsum - name: Install gotestsum

6
.gitignore vendored
View File

@ -18,8 +18,8 @@
nimcache nimcache
# Test files # Test files
codex/testdata/hello.downloaded.txt storage/testdata/hello.downloaded.txt
codex/testdata/hello.downloaded.writer.txt storage/testdata/hello.downloaded.writer.txt
# Bin # Bin
codex-go storage-go

6
.gitmodules vendored
View File

@ -1,3 +1,3 @@
[submodule "vendor/nim-codex"] [submodule "vendor/logos-storage-nim"]
path = vendor/nim-codex path = vendor/logos-storage-nim
url = https://github.com/codex-storage/nim-codex.git url = https://github.com/logos-storage/logos-storage-nim.git

27
.vscode/launch.json vendored
View File

@ -10,7 +10,11 @@
"request": "launch", "request": "launch",
"mode": "test", "mode": "test",
"program": "${fileDirname}", "program": "${fileDirname}",
"args": ["-test.v", "-test.run", "^${selectedText}$"], "args": [
"-test.v",
"-test.run",
"^${selectedText}$"
],
"env": { "env": {
"CGO_ENABLED": "1" "CGO_ENABLED": "1"
} }
@ -31,7 +35,9 @@
"request": "launch", "request": "launch",
"mode": "test", "mode": "test",
"program": "${fileDirname}", "program": "${fileDirname}",
"args": ["-test.v"], "args": [
"-test.v"
],
"env": { "env": {
"CGO_ENABLED": "1" "CGO_ENABLED": "1"
} }
@ -42,18 +48,23 @@
"request": "launch", "request": "launch",
"mode": "test", "mode": "test",
"program": "${fileDirname}", "program": "${fileDirname}",
"args": ["-test.v", "-count=1"], "args": [
"-test.v",
"-count=1"
],
"env": { "env": {
"CGO_ENABLED": "1" "CGO_ENABLED": "1"
} }
}, },
{ {
"name": "Debug Codex Tests", "name": "Debug Logos Storage Tests",
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "test", "mode": "test",
"program": "${workspaceFolder}/codex", "program": "${workspaceFolder}/storage",
"args": ["-test.v"], "args": [
"-test.v"
],
"env": { "env": {
"CGO_ENABLED": "1" "CGO_ENABLED": "1"
} }
@ -63,7 +74,7 @@
"type": "go", "type": "go",
"request": "launch", "request": "launch",
"mode": "test", "mode": "test",
"program": "${workspaceFolder}/codex", "program": "${workspaceFolder}/storage",
"args": [ "args": [
"-test.v", "-test.v",
"-test.run", "-test.run",
@ -74,4 +85,4 @@
} }
} }
] ]
} }

6
.vscode/settings.json vendored
View File

@ -1,9 +1,9 @@
{ {
"go.toolsEnvVars": { "go.toolsEnvVars": {
"CGO_ENABLED": "1", "CGO_ENABLED": "1",
"CGO_CFLAGS": "-I${workspaceFolder}/vendor/nim-codex/library", "CGO_CFLAGS": "-I${workspaceFolder}/vendor/logos-storage-nim/library",
"CGO_LDFLAGS": "-L${workspaceFolder}/vendor/nim-codex/build -lcodex -Wl,-rpath,${workspaceFolder}/vendor/nim-codex/build", "CGO_LDFLAGS": "-L${workspaceFolder}/vendor/logos-storage-nim/build -lstorage -Wl,-rpath,${workspaceFolder}/vendor/logos-storage-nim/build",
"LD_LIBRARY_PATH": "${workspaceFolder}/vendor/nim-codex/build:${env:LD_LIBRARY_PATH}" "LD_LIBRARY_PATH": "${workspaceFolder}/vendor/logos-storage-nim/build:${env:LD_LIBRARY_PATH}"
}, },
"go.testTimeout": "2m" "go.testTimeout": "2m"
} }

38
Makefile
View File

@ -1,13 +1,13 @@
# Makefile for Codex Go Bindings # Makefile for Logos Storage Go Bindings
NIM_CODEX_DIR := vendor/nim-codex LOGOS_STORAGE_NIM_DIR := vendor/logos-storage-nim
NIM_CODEX_LIB_DIR := $(abspath $(NIM_CODEX_DIR)/library) LOGOS_STORAGE_NIM_LIB_DIR := $(abspath $(LOGOS_STORAGE_NIM_DIR)/library)
NIM_CODEX_BUILD_DIR := $(abspath $(NIM_CODEX_DIR)/build) LOGOS_STORAGE_NIM_BUILD_DIR := $(abspath $(LOGOS_STORAGE_NIM_DIR)/build)
CGO_CFLAGS := -I$(NIM_CODEX_LIB_DIR) CGO_CFLAGS := -I$(LOGOS_STORAGE_NIM_LIB_DIR)
CGO_LDFLAGS := -L$(NIM_CODEX_BUILD_DIR) -lcodex -Wl,-rpath,$(NIM_CODEX_BUILD_DIR) CGO_LDFLAGS := -L$(LOGOS_STORAGE_NIM_BUILD_DIR) -lstorage -Wl,-rpath,$(LOGOS_STORAGE_NIM_BUILD_DIR)
.PHONY: all clean update libcodex build test .PHONY: all clean update libstorage build test
all: build all: build
@ -16,20 +16,20 @@ submodules:
@git submodule update --init --recursive @git submodule update --init --recursive
update: | submodules update: | submodules
@echo "Updating nim-codex..." @echo "Updating logos-storage-nim..."
@$(MAKE) -C $(NIM_CODEX_DIR) update @$(MAKE) -C $(LOGOS_STORAGE_NIM_DIR) update
libcodex: libstorage:
@echo "Building libcodex..." @echo "Building libstorage..."
@$(MAKE) -C $(NIM_CODEX_DIR) libcodex @$(MAKE) -C $(LOGOS_STORAGE_NIM_DIR) libstorage
libcodex-with-debug-api: libstorage-with-debug-api:
@echo "Building libcodex..." @echo "Building libstorage..."
@$(MAKE) -C $(NIM_CODEX_DIR) libcodex CODEX_LIB_PARAMS="-d:codex_enable_api_debug_peers" @$(MAKE) -C $(LOGOS_STORAGE_NIM_DIR) libstorage STORAGE_LIB_PARAMS="-d:storage_enable_api_debug_peers"
build: build:
@echo "Building Codex Go Bindings..." @echo "Building Logos Storage Go Bindings..."
CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o codex-go ./codex CGO_ENABLED=1 CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" go build -o storage-go ./storage
test: test:
@echo "Running tests..." @echo "Running tests..."
@ -44,5 +44,5 @@ test-with-params:
clean: clean:
@echo "Cleaning up..." @echo "Cleaning up..."
@git submodule deinit -f $(NIM_CODEX_DIR) @git submodule deinit -f $(LOGOS_STORAGE_NIM_DIR)
@rm -f codex-go @rm -f storage-go

68
README.md
View File

@ -1,13 +1,13 @@
# Codex Go Bindings # Logos Storage Go Bindings
This repository provides Go bindings for the Codex library, enabling seamless integration with Go projects. This repository provides Go bindings for the Logos Stroage library, enabling seamless integration with Go projects.
## Usage ## Usage
Include in your Go project: Include in your Go project:
```sh ```sh
go get github.com/codex-storage/codex-go-bindings go get github.com/logos-storage/logos-storage-go-bindings
``` ```
Then the easiest way is to download our prebuilt artifacts and configure your project. Then the easiest way is to download our prebuilt artifacts and configure your project.
@ -17,13 +17,13 @@ You can use this `Makefile` (or integrates the commands in your build process):
# Path configuration # Path configuration
LIBS_DIR := $(abspath ./libs) LIBS_DIR := $(abspath ./libs)
CGO_CFLAGS := -I$(LIBS_DIR) CGO_CFLAGS := -I$(LIBS_DIR)
CGO_LDFLAGS := -L$(LIBS_DIR) -lcodex -Wl,-rpath,$(LIBS_DIR) CGO_LDFLAGS := -L$(LIBS_DIR) -lstorage -Wl,-rpath,$(LIBS_DIR)
# Fetch configuration # Fetch configuration
OS ?= "linux" OS ?= "linux"
ARCH ?= "amd64" ARCH ?= "amd64"
VERSION ?= "v0.0.21" VERSION ?= "v0.0.21"
DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(VERSION)/codex-${OS}-${ARCH}.zip" DOWNLOAD_URL := "https://github.com/logos-storage/logos-storage-go-bindings/releases/download/$(VERSION)/storage-${OS}-${ARCH}.zip"
# Edit your binary name here # Edit your binary name here
ifeq ($(OS),Windows_NT) ifeq ($(OS),Windows_NT)
@ -33,9 +33,9 @@ else
endif endif
fetch: fetch:
@echo "Fetching libcodex from GitHub Actions from: ${DOWNLOAD_URL}" @echo "Fetching libstorage from GitHub Actions from: ${DOWNLOAD_URL}"
@curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${OS}-${ARCH}.zip ${DOWNLOAD_URL} @curl -fSL --create-dirs -o $(LIBS_DIR)/storage-${OS}-${ARCH}.zip ${DOWNLOAD_URL}
@unzip -o -qq $(LIBS_DIR)/codex-${OS}-${ARCH}.zip -d $(LIBS_DIR) @unzip -o -qq $(LIBS_DIR)/storage-${OS}-${ARCH}.zip -d $(LIBS_DIR)
@rm -f $(LIBS_DIR)/*.zip @rm -f $(LIBS_DIR)/*.zip
build: build:
@ -60,7 +60,7 @@ make build
That's it! That's it!
For an example on how to use this package, please take a look at our [example-go-bindings](https://github.com/codex-storage/example-codex-go-bindings) repo. For an example on how to use this package, please take a look at our [example-go-bindings](https://github.com/logos-storage/example-logos-storage-go-bindings) repo.
If you want to build the library yourself, you need to clone this repo and follow the instructions If you want to build the library yourself, you need to clone this repo and follow the instructions
of the next step. of the next step.
@ -69,13 +69,13 @@ of the next step.
To build the required dependencies for this module, the `make` command needs to be executed. To build the required dependencies for this module, the `make` command needs to be executed.
If you are integrating this module into another project via `go get`, ensure that you navigate If you are integrating this module into another project via `go get`, ensure that you navigate
to the `codex-go-bindings` module directory and run the `make` commands. to the `logos-storage-go-bindings` module directory and run the `make` commands.
### Steps to install ### Steps to install
Follow these steps to install and set up the module: Follow these steps to install and set up the module:
1. Make sure your system has the [prerequisites](https://github.com/codex-storage/nim-codex) to run a local Codex node. 1. Make sure your system has the [prerequisites](https://github.com/logos-storage/logos-storage-nim) to run a local Logos Storage node.
2. Fetch the dependencies: 2. Fetch the dependencies:
```sh ```sh
@ -84,20 +84,20 @@ Follow these steps to install and set up the module:
3. Build the library: 3. Build the library:
```sh ```sh
make libcodex make libstorage
``` ```
You can pass flags to the Codex building step by using `CODEX_LIB_PARAMS`. For example, You can pass flags to the Logos Storage building step by using `STORAGE_LIB_PARAMS`. For example,
if you want to enable debug API for peers, you can build the library using: if you want to enable debug API for peers, you can build the library using:
```sh ```sh
CODEX_LIB_PARAMS="-d:codex_enable_api_debug_peers=true" make libcodex STORAGE_LIB_PARAMS="-d:storage_enable_api_debug_peers=true" make libstorage
``` ```
or you can use a convenience `libcodex-with-debug-api` make target: or you can use a convenience `libstorage-with-debug-api` make target:
```sh ```sh
make libcodex-with-debug-api make libstorage-with-debug-api
``` ```
To run the test, you have to make sure you have `gotestsum` installed on your system, e.g.: To run the test, you have to make sure you have `gotestsum` installed on your system, e.g.:
@ -143,11 +143,11 @@ The release process is defined [here](./RELEASE.md).
### Init ### Init
First you need to create a Codex node: First you need to create a Logos Storage node:
```go ```go
dataDir := "..." dataDir := "..."
node, err := CodexNew(CodexConfig{ node, err := StorageNew(StorageConfig{
DataDir: dataDir, DataDir: dataDir,
BlockRetries: 10, BlockRetries: 10,
}) })
@ -155,7 +155,7 @@ node, err := CodexNew(CodexConfig{
err := node.Destroy() err := node.Destroy()
``` ```
The `CodexConfig` object provides several options to configure your node. You should at least The `StorageConfig` object provides several options to configure your node. You should at least
adjust the `DataDir` folder and the `BlockRetries` setting to avoid long retrieval times when adjust the `DataDir` folder and the `BlockRetries` setting to avoid long retrieval times when
the data is unavailable. the data is unavailable.
@ -198,7 +198,7 @@ There are 3 strategies for uploading: `reader`, `file` or `chunks`. Each one req
The `reader` strategy is the easiest option when you already have a Go `Reader`. The `reader` strategy is the easiest option when you already have a Go `Reader`.
It handles creating the upload session and cancels it if an error occurs. It handles creating the upload session and cancels it if an error occurs.
The `filepath` should contain the data’s name with its extension, because Codex uses that to The `filepath` should contain the data’s name with its extension, because Logos Storage uses that to
infer the MIME type. infer the MIME type.
An `onProgress` callback is available to receive progress updates and notify the user. An `onProgress` callback is available to receive progress updates and notify the user.
@ -213,12 +213,12 @@ onProgress := func(read, total int, percent float64, err error) {
// Do something with the data // Do something with the data
} }
ctx := context.Background() ctx := context.Background()
cid, err := codex.UploadReader(ctx, UploadOptions{filepath: "hello.txt", onProgress: onProgress}, buf) cid, err := storage.UploadReader(ctx, UploadOptions{filepath: "hello.txt", onProgress: onProgress}, buf)
``` ```
#### file #### file
The `file` strategy allows you to upload a file on Codex using the path. The `file` strategy allows you to upload a file on Logos Storage using the path.
It handles creating the upload session and cancels it if an error occurs. It handles creating the upload session and cancels it if an error occurs.
The `onProgress` callback is the same as for `reader` strategy. The `onProgress` callback is the same as for `reader` strategy.
@ -230,7 +230,7 @@ onProgress := func(read, total int, percent float64, err error) {
// Do something with the data // Do something with the data
} }
ctx := context.Background() ctx := context.Background()
cid, err := codex.UploadFile(ctx, UploadOptions{filepath: "./testdata/hello.txt", onProgress: onProgress}) cid, err := storage.UploadFile(ctx, UploadOptions{filepath: "./testdata/hello.txt", onProgress: onProgress})
``` ```
#### chunks #### chunks
@ -240,13 +240,13 @@ but provides more flexibility. You have to create the upload session, send the c
and then finalize to get the cid. and then finalize to get the cid.
```go ```go
sessionId, err := codex.UploadInit(&UploadOptions{filepath: "hello.txt"}) sessionId, err := storage.UploadInit(&UploadOptions{filepath: "hello.txt"})
err = codex.UploadChunk(sessionId, []byte("Hello ")) err = storage.UploadChunk(sessionId, []byte("Hello "))
err = codex.UploadChunk(sessionId, []byte("World!")) err = storage.UploadChunk(sessionId, []byte("World!"))
cid, err := codex.UploadFinalize(sessionId) cid, err := storage.UploadFinalize(sessionId)
``` ```
Using this strategy, you can handle resumable uploads and cancel the upload Using this strategy, you can handle resumable uploads and cancel the upload
@ -257,7 +257,7 @@ whenever you want!
When you receive a cid, you can download the `Manifest` to get information about the data: When you receive a cid, you can download the `Manifest` to get information about the data:
```go ```go
manifest, err := codex.DownloadManifest(cid) manifest, err := storage.DownloadManifest(cid)
``` ```
It is not mandatory for downloading the data but it is really useful. It is not mandatory for downloading the data but it is really useful.
@ -286,7 +286,7 @@ opt := DownloadStreamOptions{
}, },
} }
ctx := context.Background() ctx := context.Background()
err := codex.DownloadStream(ctx, cid, opt) err := storage.DownloadStream(ctx, cid, opt)
``` ```
#### chunks #### chunks
@ -300,9 +300,9 @@ to terminate the download session.
```go ```go
cid := "..." cid := "..."
err := codex.DownloadInit(cid, DownloadInitOptions{}) err := storage.DownloadInit(cid, DownloadInitOptions{})
chunk, err := codex.DownloadChunk(cid) chunk, err := storage.DownloadChunk(cid)
err := codex.DownloadCancel(cid) err := storage.DownloadCancel(cid)
``` ```
Using this strategy, you can handle resumable downloads and cancel the download Using this strategy, you can handle resumable downloads and cancel the download
@ -345,10 +345,10 @@ info, err := node.Debug()
err := node.UpdateLogLevel("DEBUG") err := node.UpdateLogLevel("DEBUG")
peerId := "..." peerId := "..."
record, err := node.CodexPeerDebug(peerId) record, err := node.StoragePeerDebug(peerId)
``` ```
`CodexPeerDebug` is only available if you built with `-d:codex_enable_api_debug_peers=true` flag. `StoragePeerDebug` is only available if you built with `-d:STORAGE_enable_api_debug_peers=true` flag.
### Context and cancellation ### Context and cancellation

40
RELEASE.md
View File

@ -23,7 +23,7 @@ git push --tags
Once published, the artifacts can be downloaded using the `version`, example: Once published, the artifacts can be downloaded using the `version`, example:
`https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.16/codex-linux-amd64.zip` `https://github.com/logos-storage/logos-storage-go-bindings/releases/download/v0.0.16/storage-linux-amd64.zip`
It is not recommended to use the `latest` URL because you may face cache issues. It is not recommended to use the `latest` URL because you may face cache issues.
@ -32,33 +32,33 @@ It is not recommended to use the `latest` URL because you may face cache issues.
Once released, you can integrate it into your Go project using: Once released, you can integrate it into your Go project using:
```bash ```bash
go get github.com/codex-storage/codex-go-bindings@v0.0.26 go get github.com/logos-storage/logos-storage-go-bindings@v0.0.26
``` ```
Then you can use the following `Makefile` command to fetch the artifact: Then you can use the following `Makefile` command to fetch the artifact:
```bash ```bash
LIBS_DIR := $(abspath ./libs) LIBS_DIR := $(abspath ./libs)
CODEX_OS := linux STORAGE_OS := linux
CODEX_ARCH := amd64 STORAGE_ARCH := amd64
CODEX_VERSION := $(shell go list -m -f '{{.Version}}' github.com/codex-storage/codex-go-bindings 2>/dev/null) STORAGE_VERSION := $(shell go list -m -f '{{.Version}}' github.com/logos-storage/logos-storage-go-bindings 2>/dev/null)
CODEX_DOWNLOAD_URL := "https://github.com/codex-storage/codex-go-bindings/releases/download/$(CODEX_VERSION)/codex-${CODEX_OS}-${CODEX_ARCH}.zip" STORAGE_DOWNLOAD_URL := "https://github.com/logos-storage/logos-storage-go-bindings/releases/download/$(STORAGE_VERSION)/storage-${STORAGE_OS}-${STORAGE_ARCH}.zip"
fetch-libcodex: fetch-libstorage:
mkdir -p $(LIBS_DIR); \ mkdir -p $(LIBS_DIR); \
curl -fSL --create-dirs -o $(LIBS_DIR)/codex-${CODEX_OS}-${CODEX_ARCH}.zip ${CODEX_DOWNLOAD_URL}; \ curl -fSL --create-dirs -o $(LIBS_DIR)/storage-${STORAGE_OS}-${STORAGE_ARCH}.zip ${STORAGE_DOWNLOAD_URL}; \
unzip -o -qq $(LIBS_DIR)/codex-${CODEX_OS}-${CODEX_ARCH}.zip -d $(LIBS_DIR); \ unzip -o -qq $(LIBS_DIR)/storage-${STORAGE_OS}-${STORAGE_ARCH}.zip -d $(LIBS_DIR); \
rm -f $(LIBS_DIR)/codex*.zip; rm -f $(LIBS_DIR)/storage*.zip;
``` ```
`CODEX_VERSION` uses the same version as the Codex Go dependency declared in your project. `STORAGE_VERSION` uses the same version as the Logos Storage Go dependency declared in your project.
### Nix ### Nix
If you use Nix in a sandboxed environment, you cannot use curl to download the artifacts, so you have to prefetch them using the artifacts `SHA-256` hash. To generate the hash, you can use the following command: If you use Nix in a sandboxed environment, you cannot use curl to download the artifacts, so you have to prefetch them using the artifacts `SHA-256` hash. To generate the hash, you can use the following command:
```bash ```bash
nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-macos-arm64.zip | jq -r .hash nix store prefetch-file --json --unpack https://github.com/logos-storage/logos-storage-go-bindings/releases/download/v0.0.26/storage-macos-arm64.zip | jq -r .hash
# [10.4 MiB DL] sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE= # [10.4 MiB DL] sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE=
``` ```
@ -68,7 +68,7 @@ Then include this hash in your Nix configuration. For example:
```nix ```nix
let let
optionalString = pkgs.lib.optionalString; optionalString = pkgs.lib.optionalString;
codexVersion = "v0.0.26"; storageVersion = "v0.0.26";
arch = arch =
if stdenv.hostPlatform.isx86_64 then "amd64" if stdenv.hostPlatform.isx86_64 then "amd64"
else if stdenv.hostPlatform.isAarch64 then "arm64" else if stdenv.hostPlatform.isAarch64 then "arm64"
@ -76,21 +76,21 @@ let
os = if stdenv.isDarwin then "macos" else "Linux"; os = if stdenv.isDarwin then "macos" else "Linux";
hash = hash =
if stdenv.hostPlatform.isDarwin if stdenv.hostPlatform.isDarwin
# nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-macos-arm64.zip | jq -r .hash # nix store prefetch-file --json --unpack https://github.com/logos-storage/logos-storage-go-bindings/releases/download/v0.0.26/storage-macos-arm64.zip | jq -r .hash
then "sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE=" then "sha256-3CHIWoSjo0plsYqzXQWm1EtY1STcljV4yfXTPon90uE="
# nix store prefetch-file --json --unpack https://github.com/codex-storage/codex-go-bindings/releases/download/v0.0.26/codex-Linux-amd64.zip | jq -r .hash # nix store prefetch-file --json --unpack https://github.com/logos-storage/logos-storage-go-bindings/releases/download/v0.0.26/storage-Linux-amd64.zip | jq -r .hash
else "sha256-YxW2vFZlcLrOx1PYgWW4MIstH/oFBRF0ooS0sl3v6ig="; else "sha256-YxW2vFZlcLrOx1PYgWW4MIstH/oFBRF0ooS0sl3v6ig=";
# Pre-fetch libcodex to avoid network during build # Pre-fetch libstorage to avoid network during build
codexLib = pkgs.fetchzip { storageLib = pkgs.fetchzip {
url = "https://github.com/codex-storage/codex-go-bindings/releases/download/${codexVersion}/codex-${os}-${arch}.zip"; url = "https://github.com/logos-storage/logos-storage-go-bindings/releases/download/${storageVersion}/storage-${os}-${arch}.zip";
hash = hash; hash = hash;
stripRoot = false; stripRoot = false;
}; };
preBuild = '' preBuild = ''
export LIBS_DIR="${codexLib}" export LIBS_DIR="${storageLib}"
# Build something cool with Codex # Build something cool with Logos Storage
''; '';
``` ```

154
codex/codex_test.go
View File

@ -1,154 +0,0 @@
package codex
import (
"testing"
)
func TestCodexVersion(t *testing.T) {
config := defaultConfigHelper(t)
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Codex node: %v", err)
}
version, err := node.Version()
if err != nil {
t.Fatalf("Failed to get Codex version: %v", err)
}
if version == "" {
t.Fatal("Codex version is empty")
}
t.Logf("Codex version: %s", version)
}
func TestCodexRevision(t *testing.T) {
config := defaultConfigHelper(t)
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Codex node: %v", err)
}
revision, err := node.Revision()
if err != nil {
t.Fatalf("Failed to get Codex revision: %v", err)
}
if revision == "" {
t.Fatal("Codex revision is empty")
}
t.Logf("Codex revision: %s", revision)
}
func TestCodexRepo(t *testing.T) {
node := newCodexNode(t)
repo, err := node.Repo()
if err != nil {
t.Fatalf("Failed to get Codex repo: %v", err)
}
if repo == "" {
t.Fatal("Codex repo is empty")
}
t.Logf("Codex repo: %s", repo)
}
func TestSpr(t *testing.T) {
node := newCodexNode(t)
spr, err := node.Spr()
if err != nil {
t.Fatalf("Failed to get Codex SPR: %v", err)
}
if spr == "" {
t.Fatal("Codex SPR is empty")
}
t.Logf("Codex SPR: %s", spr)
}
func TestPeerId(t *testing.T) {
node := newCodexNode(t)
peerId, err := node.PeerId()
if err != nil {
t.Fatalf("Failed to get Codex PeerId: %v", err)
}
if peerId == "" {
t.Fatal("Codex PeerId is empty")
}
t.Logf("Codex PeerId: %s", peerId)
}
func TestStorageQuota(t *testing.T) {
node := newCodexNode(t, Config{
StorageQuota: 1024 * 1024 * 1024, // 1GB
})
if node == nil {
t.Fatal("expected codex node to be created")
}
}
func TestCreateAndDestroyMultipleInstancesWithSameDatadir(t *testing.T) {
datadir := t.TempDir()
config := Config{
DataDir: datadir,
LogFormat: LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
Nat: "none",
}
for range 2 {
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Codex node: %v", err)
}
if err := node.Start(); err != nil {
t.Fatalf("Failed to start Codex node: %v", err)
}
if err := node.Stop(); err != nil {
t.Fatalf("Failed to stop Codex node: %v", err)
}
if err := node.Destroy(); err != nil {
t.Fatalf("Failed to stop Codex node after restart: %v", err)
}
}
}
func TestNumThreads(t *testing.T) {
node := newCodexNode(t, Config{
NumThreads: 1,
})
if node == nil {
t.Fatal("expected codex node to be created")
}
}
func TestBlockTtl(t *testing.T) {
node := newCodexNode(t, Config{
BlockTtl: "10H",
})
if node == nil {
t.Fatal("expected codex node to be created")
}
}
func TestBlockMaintenanceInterval(t *testing.T) {
node := newCodexNode(t, Config{
BlockMaintenanceInterval: "10H",
})
if node == nil {
t.Fatal("expected codex node to be created")
}
}

2
go.mod
View File

@ -1,3 +1,3 @@
module github.com/codex-storage/codex-go-bindings module github.com/logos-storage/logos-go-bindings
go 1.24.0 go 1.24.0

4
codex/bridge.go → storage/bridge.go
View File

@ -1,9 +1,9 @@
package codex package storage
/* /*
#include <stdbool.h> #include <stdbool.h>
#include <stdlib.h> #include <stdlib.h>
#include "libcodex.h" #include "libstorage.h"
typedef struct { typedef struct {
int ret; int ret;

2
codex/bridge.h → storage/bridge.h
View File

@ -1,6 +1,6 @@
#pragma once #pragma once
#include <stdbool.h> #include <stdbool.h>
#include "libcodex.h" #include "libstorage.h"
extern void callback(int ret, char* msg, size_t len, void* resp); extern void callback(int ret, char* msg, size_t len, void* resp);

44
codex/debug.go → storage/debug.go
View File

@ -1,19 +1,19 @@
package codex package storage
/* /*
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
static int cGoCodexDebug(void* codexCtx, void* resp) { static int cGoStorageDebug(void* storageCtx, void* resp) {
return codex_debug(codexCtx, (CodexCallback) callback, resp); return storage_debug(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexLogLevel(void* codexCtx, char* logLevel, void* resp) { static int cGoStorageLogLevel(void* storageCtx, char* logLevel, void* resp) {
return codex_log_level(codexCtx, logLevel, (CodexCallback) callback, resp); return storage_log_level(storageCtx, logLevel, (StorageCallback) callback, resp);
} }
static int cGoCodexPeerDebug(void* codexCtx, char* peerId, void* resp) { static int cGoStoragePeerDebug(void* storageCtx, char* peerId, void* resp) {
return codex_peer_debug(codexCtx, peerId, (CodexCallback) callback, resp); return storage_peer_debug(storageCtx, peerId, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -40,7 +40,7 @@ type DebugInfo struct {
ID string `json:"id"` ID string `json:"id"`
// Peer info addresses // Peer info addresses
// Specified with `ListenAddresses` in `CodexConfig` // Specified with `ListenAddresses` in `StorageConfig`
Addrs []string `json:"addrs"` Addrs []string `json:"addrs"`
Spr string `json:"spr"` Spr string `json:"spr"`
@ -54,15 +54,15 @@ type PeerRecord struct {
Addresses []string `json:"addresses,omitempty"` Addresses []string `json:"addresses,omitempty"`
} }
// Debug retrieves debugging information from the Codex node. // Debug retrieves debugging information from the Logos Storage node.
func (node CodexNode) Debug() (DebugInfo, error) { func (node StorageNode) Debug() (DebugInfo, error) {
var info DebugInfo var info DebugInfo
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexDebug(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageDebug(node.ctx, bridge.resp) != C.RET_OK {
return info, bridge.callError("cGoCodexDebug") return info, bridge.callError("cGoStorageDebug")
} }
value, err := bridge.wait() value, err := bridge.wait()
@ -78,27 +78,27 @@ func (node CodexNode) Debug() (DebugInfo, error) {
// You can pass a plain level: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR, FATAL. // You can pass a plain level: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR, FATAL.
// The default level is TRACE. // The default level is TRACE.
// You can also use Chronicles topic directives. So for example if you want // You can also use Chronicles topic directives. So for example if you want
// to update the general level to INFO but want to see TRACE logs for the codexlib // to update the general level to INFO but want to see TRACE logs for the libstorage
// topic, you can pass "INFO,codexlib:TRACE". // topic, you can pass "INFO,libstorage:TRACE".
func (node CodexNode) UpdateLogLevel(logLevel string) error { func (node StorageNode) UpdateLogLevel(logLevel string) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cLogLevel = C.CString(string(logLevel)) var cLogLevel = C.CString(string(logLevel))
defer C.free(unsafe.Pointer(cLogLevel)) defer C.free(unsafe.Pointer(cLogLevel))
if C.cGoCodexLogLevel(node.ctx, cLogLevel, bridge.resp) != C.RET_OK { if C.cGoStorageLogLevel(node.ctx, cLogLevel, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexLogLevel") return bridge.callError("cGoStorageLogLevel")
} }
_, err := bridge.wait() _, err := bridge.wait()
return err return err
} }
// CodexPeerDebug retrieves the peer record for a given peer ID. // StoragePeerDebug retrieves the peer record for a given peer ID.
// This function is available only if the flag // This function is available only if the flag
// -d:codex_enable_api_debug_peers=true was set at build time. // -d:storage_enable_api_debug_peers=true was set at build time.
func (node CodexNode) CodexPeerDebug(peerId string) (PeerRecord, error) { func (node StorageNode) StoragePeerDebug(peerId string) (PeerRecord, error) {
var record PeerRecord var record PeerRecord
bridge := newBridgeCtx() bridge := newBridgeCtx()
@ -107,8 +107,8 @@ func (node CodexNode) CodexPeerDebug(peerId string) (PeerRecord, error) {
var cPeerId = C.CString(peerId) var cPeerId = C.CString(peerId)
defer C.free(unsafe.Pointer(cPeerId)) defer C.free(unsafe.Pointer(cPeerId))
if C.cGoCodexPeerDebug(node.ctx, cPeerId, bridge.resp) != C.RET_OK { if C.cGoStoragePeerDebug(node.ctx, cPeerId, bridge.resp) != C.RET_OK {
return record, bridge.callError("cGoCodexPeerDebug") return record, bridge.callError("cGoStoragePeerDebug")
} }
value, err := bridge.wait() value, err := bridge.wait()

38
codex/debug_test.go → storage/debug_test.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"os" "os"
@ -8,9 +8,9 @@ import (
) )
func TestDebug(t *testing.T) { func TestDebug(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
info, err := codex.Debug() info, err := storage.Debug()
if err != nil { if err != nil {
t.Fatalf("Debug call failed: %v", err) t.Fatalf("Debug call failed: %v", err)
} }
@ -26,13 +26,13 @@ func TestDebug(t *testing.T) {
} }
func TestUpdateLogLevel(t *testing.T) { func TestUpdateLogLevel(t *testing.T) {
tmpFile, err := os.CreateTemp("", "codex-log-*.log") tmpFile, err := os.CreateTemp("", "storage-log-*.log")
if err != nil { if err != nil {
t.Fatalf("Failed to create temp log file: %v", err) t.Fatalf("Failed to create temp log file: %v", err)
} }
defer os.Remove(tmpFile.Name()) defer os.Remove(tmpFile.Name())
node := newCodexNode(t, Config{ node := newStorageNode(t, Config{
LogLevel: "INFO", LogLevel: "INFO",
LogFile: tmpFile.Name(), LogFile: tmpFile.Name(),
LogFormat: LogFormatNoColors, LogFormat: LogFormatNoColors,
@ -53,7 +53,7 @@ func TestUpdateLogLevel(t *testing.T) {
} }
if err := node.Stop(); err != nil { if err := node.Stop(); err != nil {
t.Fatalf("Failed to stop Codex node: %v", err) t.Fatalf("Failed to stop Logos Storage node: %v", err)
} }
// Clear the file // Clear the file
@ -63,7 +63,7 @@ func TestUpdateLogLevel(t *testing.T) {
err = node.Start() err = node.Start()
if err != nil { if err != nil {
t.Fatalf("Failed to start Codex node: %v", err) t.Fatalf("Failed to start Logos Storage node: %v", err)
} }
content, err = os.ReadFile(tmpFile.Name()) content, err = os.ReadFile(tmpFile.Name())
@ -76,11 +76,11 @@ func TestUpdateLogLevel(t *testing.T) {
} }
} }
func TestCodexPeerDebug(t *testing.T) { func TestStoragePeerDebug(t *testing.T) {
var bootstrap, node1, node2 *CodexNode var bootstrap, node1, node2 *StorageNode
var err error var err error
bootstrap = newCodexNode(t, Config{ bootstrap = newStorageNode(t, Config{
DiscoveryPort: 8092, DiscoveryPort: 8092,
}) })
@ -91,12 +91,12 @@ func TestCodexPeerDebug(t *testing.T) {
bootstrapNodes := []string{spr} bootstrapNodes := []string{spr}
node1 = newCodexNode(t, Config{ node1 = newStorageNode(t, Config{
DiscoveryPort: 8090, DiscoveryPort: 8090,
BootstrapNodes: bootstrapNodes, BootstrapNodes: bootstrapNodes,
}) })
node2 = newCodexNode(t, Config{ node2 = newStorageNode(t, Config{
DiscoveryPort: 8091, DiscoveryPort: 8091,
BootstrapNodes: bootstrapNodes, BootstrapNodes: bootstrapNodes,
}) })
@ -108,7 +108,7 @@ func TestCodexPeerDebug(t *testing.T) {
var record PeerRecord var record PeerRecord
for range 10 { for range 10 {
record, err = node1.CodexPeerDebug(peerId) record, err = node1.StoragePeerDebug(peerId)
if err == nil { if err == nil {
break break
} }
@ -117,22 +117,22 @@ func TestCodexPeerDebug(t *testing.T) {
} }
if err != nil { if err != nil {
t.Fatalf("CodexPeerDebug call failed: %v", err) t.Fatalf("Logos StoragePeerDebug call failed: %v", err)
} }
if record.PeerId == "" { if record.PeerId == "" {
t.Fatalf("CodexPeerDebug call failed: %v", err) t.Fatalf("Logos StoragePeerDebug call failed: %v", err)
} }
if record.PeerId == "" { if record.PeerId == "" {
t.Error("CodexPeerDebug info PeerId is empty") t.Error("Logos StoragePeerDebug info PeerId is empty")
} }
if record.SeqNo == 0 { if record.SeqNo == 0 {
t.Error("CodexPeerDebug info SeqNo is empty") t.Error("Logos StoragePeerDebug info SeqNo is empty")
} }
if len(record.Addresses) == 0 { if len(record.Addresses) == 0 {
t.Error("CodexPeerDebug info Addresses is empty") t.Error("Logos StoragePeerDebug info Addresses is empty")
} }
if record.PeerId != peerId { if record.PeerId != peerId {
t.Errorf("CodexPeerDebug info PeerId (%s) does not match requested PeerId (%s)", record.PeerId, peerId) t.Errorf("Logos StoragePeerDebug info PeerId (%s) does not match requested PeerId (%s)", record.PeerId, peerId)
} }
} }

60
codex/download.go → storage/download.go
View File

@ -1,27 +1,27 @@
package codex package storage
/* /*
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
static int cGoCodexDownloadInit(void* codexCtx, char* cid, size_t chunkSize, bool local, void* resp) { static int cGoStorageDownloadInit(void* storageCtx, char* cid, size_t chunkSize, bool local, void* resp) {
return codex_download_init(codexCtx, cid, chunkSize, local, (CodexCallback) callback, resp); return storage_download_init(storageCtx, cid, chunkSize, local, (StorageCallback) callback, resp);
} }
static int cGoCodexDownloadChunk(void* codexCtx, char* cid, void* resp) { static int cGoStorageDownloadChunk(void* storageCtx, char* cid, void* resp) {
return codex_download_chunk(codexCtx, cid, (CodexCallback) callback, resp); return storage_download_chunk(storageCtx, cid, (StorageCallback) callback, resp);
} }
static int cGoCodexDownloadStream(void* codexCtx, char* cid, size_t chunkSize, bool local, const char* filepath, void* resp) { static int cGoStorageDownloadStream(void* storageCtx, char* cid, size_t chunkSize, bool local, const char* filepath, void* resp) {
return codex_download_stream(codexCtx, cid, chunkSize, local, filepath, (CodexCallback) callback, resp); return storage_download_stream(storageCtx, cid, chunkSize, local, filepath, (StorageCallback) callback, resp);
} }
static int cGoCodexDownloadCancel(void* codexCtx, char* cid, void* resp) { static int cGoStorageDownloadCancel(void* storageCtx, char* cid, void* resp) {
return codex_download_cancel(codexCtx, cid, (CodexCallback) callback, resp); return storage_download_cancel(storageCtx, cid, (StorageCallback) callback, resp);
} }
static int cGoCodexDownloadManifest(void* codexCtx, char* cid, void* resp) { static int cGoStorageDownloadManifest(void* storageCtx, char* cid, void* resp) {
return codex_download_manifest(codexCtx, cid, (CodexCallback) callback, resp); return storage_download_manifest(storageCtx, cid, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -37,7 +37,7 @@ import (
type OnDownloadProgressFunc func(read, total int, percent float64, err error) type OnDownloadProgressFunc func(read, total int, percent float64, err error)
// DownloadStreamOptions is used to download a file // DownloadStreamOptions is used to download a file
// in a streaming manner in Codex. // in a streaming manner in Logos Storage.
type DownloadStreamOptions = struct { type DownloadStreamOptions = struct {
// Filepath is the path destination used by DownloadStream. // Filepath is the path destination used by DownloadStream.
// If it is set, the content will be written into the specified // If it is set, the content will be written into the specified
@ -89,7 +89,7 @@ type DownloadInitOptions = struct {
} }
// Manifest is the object containing the information of // Manifest is the object containing the information of
// a file in Codex. // a file in Logos Storage.
type Manifest struct { type Manifest struct {
// Cid is the content identifier over the network // Cid is the content identifier over the network
Cid string Cid string
@ -113,18 +113,18 @@ type Manifest struct {
Protected bool `json:"protected"` Protected bool `json:"protected"`
} }
// DownloadManifest retrieves the Codex manifest from its cid. // DownloadManifest retrieves the Logos Storage manifest from its cid.
// The session identifier is the cid, i.e you cannot have multiple // The session identifier is the cid, i.e you cannot have multiple
// sessions for a cid. // sessions for a cid.
func (node CodexNode) DownloadManifest(cid string) (Manifest, error) { func (node StorageNode) DownloadManifest(cid string) (Manifest, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexDownloadManifest(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageDownloadManifest(node.ctx, cCid, bridge.resp) != C.RET_OK {
return Manifest{}, bridge.callError("cGoCodexDownloadManifest") return Manifest{}, bridge.callError("cGoStorageDownloadManifest")
} }
val, err := bridge.wait() val, err := bridge.wait()
@ -148,7 +148,7 @@ func (node CodexNode) DownloadManifest(cid string) (Manifest, error) {
// If options.writer is set, the data will be written into that writer. // If options.writer is set, the data will be written into that writer.
// The options filepath and writer are not mutually exclusive, i.e you can write // The options filepath and writer are not mutually exclusive, i.e you can write
// in different places in a same call. // in different places in a same call.
func (node CodexNode) DownloadStream(ctx context.Context, cid string, options DownloadStreamOptions) error { func (node StorageNode) DownloadStream(ctx context.Context, cid string, options DownloadStreamOptions) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -207,8 +207,8 @@ func (node CodexNode) DownloadStream(ctx context.Context, cid string, options Do
var cLocal = C.bool(options.Local) var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadStream(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK { if C.cGoStorageDownloadStream(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, cFilepath, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadLocal") return bridge.callError("cGoStorageDownloadLocal")
} }
// Create a done channel to signal the goroutine to stop // Create a done channel to signal the goroutine to stop
@ -255,7 +255,7 @@ func (node CodexNode) DownloadStream(ctx context.Context, cid string, options Do
// DownloadInit initializes the download process for a specific CID. // DownloadInit initializes the download process for a specific CID.
// This method should be used if you want to manage the download session // This method should be used if you want to manage the download session
// and the chunk downloads manually. // and the chunk downloads manually.
func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) error { func (node StorageNode) DownloadInit(cid string, options DownloadInitOptions) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -264,8 +264,8 @@ func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) erro
var cLocal = C.bool(options.Local) var cLocal = C.bool(options.Local)
if C.cGoCodexDownloadInit(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK { if C.cGoStorageDownloadInit(node.ctx, cCid, options.ChunkSize.toSizeT(), cLocal, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadInit") return bridge.callError("cGoStorageDownloadInit")
} }
_, err := bridge.wait() _, err := bridge.wait()
@ -277,9 +277,9 @@ func (node CodexNode) DownloadInit(cid string, options DownloadInitOptions) erro
// When using this method, you are managing at your own // When using this method, you are managing at your own
// the total size downloaded (use DownloadManifest to get the // the total size downloaded (use DownloadManifest to get the
// datasetSize). // datasetSize).
// When the download is complete, you need to call `CodexDownloadCancel` // When the download is complete, you need to call `StorageDownloadCancel`
// to free the resources. // to free the resources.
func (node CodexNode) DownloadChunk(cid string) ([]byte, error) { func (node StorageNode) DownloadChunk(cid string) ([]byte, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -292,8 +292,8 @@ func (node CodexNode) DownloadChunk(cid string) ([]byte, error) {
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexDownloadChunk(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageDownloadChunk(node.ctx, cCid, bridge.resp) != C.RET_OK {
return nil, bridge.callError("cGoCodexDownloadChunk") return nil, bridge.callError("cGoStorageDownloadChunk")
} }
if _, err := bridge.wait(); err != nil { if _, err := bridge.wait(); err != nil {
@ -306,15 +306,15 @@ func (node CodexNode) DownloadChunk(cid string) ([]byte, error) {
// DownloadCancel cancels a download session. // DownloadCancel cancels a download session.
// It can be only if the download session is managed manually. // It can be only if the download session is managed manually.
// It doesn't work with DownloadStream. // It doesn't work with DownloadStream.
func (node CodexNode) DownloadCancel(cid string) error { func (node StorageNode) DownloadCancel(cid string) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexDownloadCancel(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageDownloadCancel(node.ctx, cCid, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDownloadCancel") return bridge.callError("cGoStorageDownloadCancel")
} }
_, err := bridge.wait() _, err := bridge.wait()

48
codex/download_test.go → storage/download_test.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"context" "context"
@ -8,8 +8,8 @@ import (
) )
func TestDownloadStream(t *testing.T) { func TestDownloadStream(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, len := uploadHelper(t, codex) cid, len := uploadHelper(t, storage)
f, err := os.Create("testdata/hello.downloaded.txt") f, err := os.Create("testdata/hello.downloaded.txt")
if err != nil { if err != nil {
@ -33,7 +33,7 @@ func TestDownloadStream(t *testing.T) {
}, },
} }
if err := codex.DownloadStream(context.Background(), cid, opt); err != nil { if err := storage.DownloadStream(context.Background(), cid, opt); err != nil {
t.Fatal("Error happened:", err.Error()) t.Fatal("Error happened:", err.Error())
} }
@ -56,8 +56,8 @@ func TestDownloadStream(t *testing.T) {
} }
func TestDownloadStreamWithAutosize(t *testing.T) { func TestDownloadStreamWithAutosize(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, len := uploadHelper(t, codex) cid, len := uploadHelper(t, storage)
totalBytes := 0 totalBytes := 0
finalPercent := 0.0 finalPercent := 0.0
@ -73,7 +73,7 @@ func TestDownloadStreamWithAutosize(t *testing.T) {
}, },
} }
if err := codex.DownloadStream(context.Background(), cid, opt); err != nil { if err := storage.DownloadStream(context.Background(), cid, opt); err != nil {
t.Fatal("Error happened:", err.Error()) t.Fatal("Error happened:", err.Error())
} }
@ -87,10 +87,10 @@ func TestDownloadStreamWithAutosize(t *testing.T) {
} }
func TestDownloadStreamWithNotExisting(t *testing.T) { func TestDownloadStreamWithNotExisting(t *testing.T) {
codex := newCodexNode(t, Config{BlockRetries: 1}) storage := newStorageNode(t, Config{BlockRetries: 1})
opt := DownloadStreamOptions{} opt := DownloadStreamOptions{}
if err := codex.DownloadStream(context.Background(), "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", opt); err == nil { if err := storage.DownloadStream(context.Background(), "bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", opt); err == nil {
t.Fatal("Error expected when downloading non-existing cid") t.Fatal("Error expected when downloading non-existing cid")
} }
} }
@ -98,12 +98,12 @@ func TestDownloadStreamWithNotExisting(t *testing.T) {
func TestDownloadStreamCancelled(t *testing.T) { func TestDownloadStreamCancelled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadBigFileHelper(t, codex) cid, _ := uploadBigFileHelper(t, storage)
channelError := make(chan error, 1) channelError := make(chan error, 1)
go func() { go func() {
err := codex.DownloadStream(ctx, cid, DownloadStreamOptions{Local: true}) err := storage.DownloadStream(ctx, cid, DownloadStreamOptions{Local: true})
channelError <- err channelError <- err
}() }()
@ -120,15 +120,15 @@ func TestDownloadStreamCancelled(t *testing.T) {
} }
func TestDownloadManual(t *testing.T) { func TestDownloadManual(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
if err := codex.DownloadInit(cid, DownloadInitOptions{}); err != nil { if err := storage.DownloadInit(cid, DownloadInitOptions{}); err != nil {
t.Fatal("Error when initializing download:", err) t.Fatal("Error when initializing download:", err)
} }
var b strings.Builder var b strings.Builder
if chunk, err := codex.DownloadChunk(cid); err != nil { if chunk, err := storage.DownloadChunk(cid); err != nil {
t.Fatal("Error when downloading chunk:", err) t.Fatal("Error when downloading chunk:", err)
} else { } else {
b.Write(chunk) b.Write(chunk)
@ -139,16 +139,16 @@ func TestDownloadManual(t *testing.T) {
t.Fatalf("Expected data was \"Hello World!\" got %s", data) t.Fatalf("Expected data was \"Hello World!\" got %s", data)
} }
if err := codex.DownloadCancel(cid); err != nil { if err := storage.DownloadCancel(cid); err != nil {
t.Fatalf("Error when cancelling the download %s", err) t.Fatalf("Error when cancelling the download %s", err)
} }
} }
func TestDownloadManifest(t *testing.T) { func TestDownloadManifest(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
manifest, err := codex.DownloadManifest(cid) manifest, err := storage.DownloadManifest(cid)
if err != nil { if err != nil {
t.Fatal("Error when downloading manifest:", err) t.Fatal("Error when downloading manifest:", err)
} }
@ -159,9 +159,9 @@ func TestDownloadManifest(t *testing.T) {
} }
func TestDownloadManifestWithNotExistingCid(t *testing.T) { func TestDownloadManifestWithNotExistingCid(t *testing.T) {
codex := newCodexNode(t, Config{BlockRetries: 1}) storage := newStorageNode(t, Config{BlockRetries: 1})
manifest, err := codex.DownloadManifest("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") manifest, err := storage.DownloadManifest("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")
if err == nil { if err == nil {
t.Fatal("Error when downloading manifest:", err) t.Fatal("Error when downloading manifest:", err)
} }
@ -172,9 +172,9 @@ func TestDownloadManifestWithNotExistingCid(t *testing.T) {
} }
func TestDownloadInitWithNotExistingCid(t *testing.T) { func TestDownloadInitWithNotExistingCid(t *testing.T) {
codex := newCodexNode(t, Config{BlockRetries: 1}) storage := newStorageNode(t, Config{BlockRetries: 1})
if err := codex.DownloadInit("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", DownloadInitOptions{}); err == nil { if err := storage.DownloadInit("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", DownloadInitOptions{}); err == nil {
t.Fatal("expected error when initializing download for non-existent cid") t.Fatal("expected error when initializing download for non-existent cid")
} }
} }

136
codex/codex.go → storage/node.go
View File

@ -1,58 +1,58 @@
package codex package storage
/* /*
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
void libcodexNimMain(void); void libstorageNimMain(void);
static void codex_host_init_once(void){ static void storage_host_init_once(void){
static int done; static int done;
if (!__atomic_exchange_n(&done, 1, __ATOMIC_SEQ_CST)) libcodexNimMain(); if (!__atomic_exchange_n(&done, 1, __ATOMIC_SEQ_CST)) libstorageNimMain();
} }
// resp must be set != NULL in case interest on retrieving data from the callback // resp must be set != NULL in case interest on retrieving data from the callback
void callback(int ret, char* msg, size_t len, void* resp); void callback(int ret, char* msg, size_t len, void* resp);
static void* cGoCodexNew(const char* configJson, void* resp) { static void* cGoStorageNew(const char* configJson, void* resp) {
void* ret = codex_new(configJson, (CodexCallback) callback, resp); void* ret = storage_new(configJson, (StorageCallback) callback, resp);
return ret; return ret;
} }
static int cGoCodexStart(void* codexCtx, void* resp) { static int cGoStorageStart(void* storageCtx, void* resp) {
return codex_start(codexCtx, (CodexCallback) callback, resp); return storage_start(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexStop(void* codexCtx, void* resp) { static int cGoStorageStop(void* storageCtx, void* resp) {
return codex_stop(codexCtx, (CodexCallback) callback, resp); return storage_stop(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexClose(void* codexCtx, void* resp) { static int cGoStorageClose(void* storageCtx, void* resp) {
return codex_close(codexCtx, (CodexCallback) callback, resp); return storage_close(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexDestroy(void* codexCtx, void* resp) { static int cGoStorageDestroy(void* storageCtx, void* resp) {
return codex_destroy(codexCtx, (CodexCallback) callback, resp); return storage_destroy(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexVersion(void* codexCtx, void* resp) { static int cGoStorageVersion(void* storageCtx, void* resp) {
return codex_version(codexCtx, (CodexCallback) callback, resp); return storage_version(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexRevision(void* codexCtx, void* resp) { static int cGoStorageRevision(void* storageCtx, void* resp) {
return codex_revision(codexCtx, (CodexCallback) callback, resp); return storage_revision(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexRepo(void* codexCtx, void* resp) { static int cGoStorageRepo(void* storageCtx, void* resp) {
return codex_repo(codexCtx, (CodexCallback) callback, resp); return storage_repo(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexSpr(void* codexCtx, void* resp) { static int cGoStorageSpr(void* storageCtx, void* resp) {
return codex_spr(codexCtx, (CodexCallback) callback, resp); return storage_spr(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexPeerId(void* codexCtx, void* resp) { static int cGoStoragePeerId(void* storageCtx, void* resp) {
return codex_peer_id(codexCtx, (CodexCallback) callback, resp); return storage_peer_id(storageCtx, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -110,11 +110,11 @@ type Config struct {
// Default: 8008 // Default: 8008
MetricsPort int `json:"metrics-port,omitempty"` MetricsPort int `json:"metrics-port,omitempty"`
// The directory where codex will store configuration and data // The directory where Logos Storage will store configuration and data
// Default: // Default:
// $HOME\AppData\Roaming\Codex on Windows // $HOME\AppData\Roaming\Storage on Windows
// $HOME/Library/Application Support/Codex on macOS // $HOME/Library/Application Support/Storage on macOS
// $HOME/.cache/codex on Linux // $HOME/.cache/storage on Linux
DataDir string `json:"data-dir,omitempty"` DataDir string `json:"data-dir,omitempty"`
// Multi Addresses to listen on // Multi Addresses to listen on
@ -146,7 +146,7 @@ type Config struct {
NumThreads int `json:"num-threads,omitempty"` NumThreads int `json:"num-threads,omitempty"`
// Node agent string which is used as identifier in network // Node agent string which is used as identifier in network
// Default: "Codex" // Default: "Logos Storage"
AgentString string `json:"agent-string,omitempty"` AgentString string `json:"agent-string,omitempty"`
// Backend for main repo store (fs, sqlite, leveldb) // Backend for main repo store (fs, sqlite, leveldb)
@ -183,7 +183,7 @@ type Config struct {
LogFile string `json:"log-file,omitempty"` LogFile string `json:"log-file,omitempty"`
} }
type CodexNode struct { type StorageNode struct {
ctx unsafe.Pointer ctx unsafe.Pointer
} }
@ -201,12 +201,12 @@ func (c ChunkSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault()) return C.size_t(c.valOrDefault())
} }
// New creates a new Codex node with the provided configuration. // New creates a new Logos Storage node with the provided configuration.
// The node is not started automatically; you need to call CodexStart // The node is not started automatically; you need to call StorageStart
// to start it. // to start it.
// It returns a Codex node that can be used to interact // It returns a Logos Storage node that can be used to interact
// with the Codex network. // with the Logos Storage network.
func New(config Config) (*CodexNode, error) { func New(config Config) (*StorageNode, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -218,22 +218,22 @@ func New(config Config) (*CodexNode, error) {
cJsonConfig := C.CString(string(jsonConfig)) cJsonConfig := C.CString(string(jsonConfig))
defer C.free(unsafe.Pointer(cJsonConfig)) defer C.free(unsafe.Pointer(cJsonConfig))
ctx := C.cGoCodexNew(cJsonConfig, bridge.resp) ctx := C.cGoStorageNew(cJsonConfig, bridge.resp)
if _, err := bridge.wait(); err != nil { if _, err := bridge.wait(); err != nil {
return nil, bridge.err return nil, bridge.err
} }
return &CodexNode{ctx: ctx}, bridge.err return &StorageNode{ctx: ctx}, bridge.err
} }
// Start starts the Codex node. // Start starts the Logos Storage node.
func (node CodexNode) Start() error { func (node StorageNode) Start() error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexStart(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageStart(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexStart") return bridge.callError("cGoStorageStart")
} }
_, err := bridge.wait() _, err := bridge.wait()
@ -241,34 +241,34 @@ func (node CodexNode) Start() error {
} }
// StartAsync is the asynchronous version of Start. // StartAsync is the asynchronous version of Start.
func (node CodexNode) StartAsync(onDone func(error)) { func (node StorageNode) StartAsync(onDone func(error)) {
go func() { go func() {
err := node.Start() err := node.Start()
onDone(err) onDone(err)
}() }()
} }
// Stop stops the Codex node. // Stop stops the Logos Storage node.
func (node CodexNode) Stop() error { func (node StorageNode) Stop() error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexStop(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageStop(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexStop") return bridge.callError("cGoStorageStop")
} }
_, err := bridge.wait() _, err := bridge.wait()
return err return err
} }
// Destroy destroys the Codex node, freeing all resources. // Destroy destroys the Logos Storage node, freeing all resources.
// The node must be stopped before calling this method. // The node must be stopped before calling this method.
func (node CodexNode) Destroy() error { func (node StorageNode) Destroy() error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexClose(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageClose(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexClose") return bridge.callError("cGoStorageClose")
} }
_, err := bridge.wait() _, err := bridge.wait()
@ -276,8 +276,8 @@ func (node CodexNode) Destroy() error {
return err return err
} }
if C.cGoCodexDestroy(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageDestroy(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexDestroy") return bridge.callError("cGoStorageDestroy")
} }
// We don't wait for the bridge here. // We don't wait for the bridge here.
@ -288,58 +288,58 @@ func (node CodexNode) Destroy() error {
return nil return nil
} }
// Version returns the version of the Codex node. // Version returns the version of the Logos Storage node.
func (node CodexNode) Version() (string, error) { func (node StorageNode) Version() (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexVersion(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageVersion(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexVersion") return "", bridge.callError("cGoStorageVersion")
} }
return bridge.wait() return bridge.wait()
} }
func (node CodexNode) Revision() (string, error) { func (node StorageNode) Revision() (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexRevision(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageRevision(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexRevision") return "", bridge.callError("cGoStorageRevision")
} }
return bridge.wait() return bridge.wait()
} }
// Repo returns the path of the data dir folder. // Repo returns the path of the data dir folder.
func (node CodexNode) Repo() (string, error) { func (node StorageNode) Repo() (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexRepo(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageRepo(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexRepo") return "", bridge.callError("cGoStorageRepo")
} }
return bridge.wait() return bridge.wait()
} }
func (node CodexNode) Spr() (string, error) { func (node StorageNode) Spr() (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexSpr(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageSpr(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexSpr") return "", bridge.callError("cGoStorageSpr")
} }
return bridge.wait() return bridge.wait()
} }
func (node CodexNode) PeerId() (string, error) { func (node StorageNode) PeerId() (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexPeerId(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStoragePeerId(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexPeerId") return "", bridge.callError("cGoStoragePeerId")
} }
return bridge.wait() return bridge.wait()

160
storage/node_test.go Normal file
View File

@ -0,0 +1,160 @@
package storage
import (
"fmt"
"testing"
"time"
)
func TestStorageVersion(t *testing.T) {
config := defaultConfigHelper(t)
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Logos Storage node: %v", err)
}
version, err := node.Version()
if err != nil {
t.Fatalf("Failed to get Logos Storage version: %v", err)
}
if version == "" {
t.Fatal("Logos Storage version is empty")
}
t.Logf("Logos Storage version: %s", version)
}
func TestStorageRevision(t *testing.T) {
config := defaultConfigHelper(t)
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Logos Storage node: %v", err)
}
revision, err := node.Revision()
if err != nil {
t.Fatalf("Failed to get Logos Storage revision: %v", err)
}
if revision == "" {
t.Fatal("Logos Storage revision is empty")
}
t.Logf("Logos Storage revision: %s", revision)
}
func TestStorageRepo(t *testing.T) {
node := newStorageNode(t)
repo, err := node.Repo()
if err != nil {
t.Fatalf("Failed to get Logos Storage repo: %v", err)
}
if repo == "" {
t.Fatal("Logos Storage repo is empty")
}
t.Logf("Logos Storage repo: %s", repo)
}
func TestSpr(t *testing.T) {
node := newStorageNode(t)
spr, err := node.Spr()
if err != nil {
t.Fatalf("Failed to get Logos Storage SPR: %v", err)
}
if spr == "" {
t.Fatal("Logos Storage SPR is empty")
}
t.Logf("Logos Storage SPR: %s", spr)
}
func TestPeerId(t *testing.T) {
node := newStorageNode(t)
peerId, err := node.PeerId()
if err != nil {
t.Fatalf("Failed to get Logos Storage PeerId: %v", err)
}
if peerId == "" {
t.Fatal("Logos Storage PeerId is empty")
}
t.Logf("Logos Storage PeerId: %s", peerId)
}
func TestStorageQuota(t *testing.T) {
node := newStorageNode(t, Config{
StorageQuota: 1024 * 1024 * 1024, // 1GB
})
if node == nil {
t.Fatal("expected Logos Storage node to be created")
}
}
func TestCreateAndDestroyMultipleInstancesWithSameDatadir(t *testing.T) {
t.Skip("Enable when the PR https://github.com/logos-storage/logos-storage-nim/pull/1364 is merged into master.")
datadir := fmt.Sprintf("%s/special-test", t.TempDir())
config := Config{
DataDir: datadir,
LogFormat: LogFormatNoColors,
MetricsEnabled: false,
BlockRetries: 5,
Nat: "none",
}
for range 2 {
node, err := New(config)
if err != nil {
t.Fatalf("Failed to create Logos Storage node: %v", err)
}
if err := node.Start(); err != nil {
t.Fatalf("Failed to start Logos Storage node: %v", err)
}
if err := node.Stop(); err != nil {
t.Fatalf("Failed to stop Logos Storage node: %v", err)
}
if err := node.Destroy(); err != nil {
t.Fatalf("Failed to stop Logos Storage node after restart: %v", err)
}
time.Sleep(100 * time.Millisecond)
}
}
func TestNumThreads(t *testing.T) {
node := newStorageNode(t, Config{
NumThreads: 1,
})
if node == nil {
t.Fatal("expected Logos Storage node to be created")
}
}
func TestBlockTtl(t *testing.T) {
node := newStorageNode(t, Config{
BlockTtl: "10H",
})
if node == nil {
t.Fatal("expected Logos Storage node to be created")
}
}
func TestBlockMaintenanceInterval(t *testing.T) {
node := newStorageNode(t, Config{
BlockMaintenanceInterval: "10H",
})
if node == nil {
t.Fatal("expected Logos Storage node to be created")
}
}

18
codex/p2p.go → storage/p2p.go
View File

@ -1,11 +1,11 @@
package codex package storage
/* /*
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
static int cGoCodexConnect(void* codexCtx, char* peerId, const char** peerAddresses, uintptr_t peerAddressesSize, void* resp) { static int cGoStorageConnect(void* storageCtx, char* peerId, const char** peerAddresses, uintptr_t peerAddressesSize, void* resp) {
return codex_connect(codexCtx, peerId, peerAddresses, peerAddressesSize, (CodexCallback) callback, resp); return storage_connect(storageCtx, peerId, peerAddresses, peerAddressesSize, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -18,8 +18,8 @@ import (
// otherwise the `peerId` is used to invoke peer discovery, if it succeeds // otherwise the `peerId` is used to invoke peer discovery, if it succeeds
// the returned addresses will be used to dial. // the returned addresses will be used to dial.
// `peerAddresses` the listening addresses of the peers to dial, // `peerAddresses` the listening addresses of the peers to dial,
// eg the one specified with `ListenAddresses` in `CodexConfig`. // eg the one specified with `ListenAddresses` in `StorageConfig`.
func (node CodexNode) Connect(peerId string, peerAddresses []string) error { func (node StorageNode) Connect(peerId string, peerAddresses []string) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -33,12 +33,12 @@ func (node CodexNode) Connect(peerId string, peerAddresses []string) error {
defer C.free(unsafe.Pointer(cAddresses[i])) defer C.free(unsafe.Pointer(cAddresses[i]))
} }
if C.cGoCodexConnect(node.ctx, cPeerId, &cAddresses[0], C.uintptr_t(len(peerAddresses)), bridge.resp) != C.RET_OK { if C.cGoStorageConnect(node.ctx, cPeerId, &cAddresses[0], C.uintptr_t(len(peerAddresses)), bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexConnect") return bridge.callError("cGoStorageConnect")
} }
} else { } else {
if C.cGoCodexConnect(node.ctx, cPeerId, nil, 0, bridge.resp) != C.RET_OK { if C.cGoStorageConnect(node.ctx, cPeerId, nil, 0, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexConnect") return bridge.callError("cGoStorageConnect")
} }
} }

30
codex/p2p_test.go → storage/p2p_test.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"log" "log"
@ -6,27 +6,27 @@ import (
) )
func TestConnectWithAddress(t *testing.T) { func TestConnectWithAddress(t *testing.T) {
var node1, node2 *CodexNode var node1, node2 *StorageNode
var err error var err error
t.Cleanup(func() { t.Cleanup(func() {
if node1 != nil { if node1 != nil {
if err := node1.Stop(); err != nil { if err := node1.Stop(); err != nil {
t.Logf("cleanup codex1: %v", err) t.Logf("cleanup storage1: %v", err)
} }
if err := node1.Destroy(); err != nil { if err := node1.Destroy(); err != nil {
t.Logf("cleanup codex1: %v", err) t.Logf("cleanup storage1: %v", err)
} }
} }
if node2 != nil { if node2 != nil {
if err := node2.Stop(); err != nil { if err := node2.Stop(); err != nil {
t.Logf("cleanup codex2: %v", err) t.Logf("cleanup storage2: %v", err)
} }
if err := node2.Destroy(); err != nil { if err := node2.Destroy(); err != nil {
t.Logf("cleanup codex2: %v", err) t.Logf("cleanup storage2: %v", err)
} }
} }
}) })
@ -39,11 +39,11 @@ func TestConnectWithAddress(t *testing.T) {
Nat: "none", Nat: "none",
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create codex1: %v", err) t.Fatalf("Failed to create Logos Storage node 1: %v", err)
} }
if err := node1.Start(); err != nil { if err := node1.Start(); err != nil {
t.Fatalf("Failed to start codex1: %v", err) t.Fatalf("Failed to start Logos Storage node 1: %v", err)
} }
node2, err = New(Config{ node2, err = New(Config{
@ -53,11 +53,11 @@ func TestConnectWithAddress(t *testing.T) {
DiscoveryPort: 8091, DiscoveryPort: 8091,
}) })
if err != nil { if err != nil {
t.Fatalf("Failed to create codex2: %v", err) t.Fatalf("Failed to create Logos Storage node 2: %v", err)
} }
if err := node2.Start(); err != nil { if err := node2.Start(); err != nil {
t.Fatalf("Failed to start codex2: %v", err) t.Fatalf("Failed to start Logos Storage node 2: %v", err)
} }
info2, err := node2.Debug() info2, err := node2.Debug()
@ -70,11 +70,11 @@ func TestConnectWithAddress(t *testing.T) {
} }
} }
func TestCodexWithPeerId(t *testing.T) { func TestStorageWithPeerId(t *testing.T) {
var bootstrap, node1, node2 *CodexNode var bootstrap, node1, node2 *StorageNode
var err error var err error
bootstrap = newCodexNode(t, Config{ bootstrap = newStorageNode(t, Config{
DiscoveryPort: 8092, DiscoveryPort: 8092,
}) })
@ -85,12 +85,12 @@ func TestCodexWithPeerId(t *testing.T) {
bootstrapNodes := []string{spr} bootstrapNodes := []string{spr}
node1 = newCodexNode(t, Config{ node1 = newStorageNode(t, Config{
DiscoveryPort: 8090, DiscoveryPort: 8090,
BootstrapNodes: bootstrapNodes, BootstrapNodes: bootstrapNodes,
}) })
node2 = newCodexNode(t, Config{ node2 = newStorageNode(t, Config{
DiscoveryPort: 8091, DiscoveryPort: 8091,
BootstrapNodes: bootstrapNodes, BootstrapNodes: bootstrapNodes,
}) })

60
codex/storage.go → storage/storage.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"encoding/json" "encoding/json"
@ -9,24 +9,24 @@ import (
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
static int cGoCodexStorageList(void* codexCtx, void* resp) { static int cGoStorageStorageList(void* storageCtx, void* resp) {
return codex_storage_list(codexCtx, (CodexCallback) callback, resp); return storage_list(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexStorageFetch(void* codexCtx, char* cid, void* resp) { static int cGoStorageStorageFetch(void* storageCtx, char* cid, void* resp) {
return codex_storage_fetch(codexCtx, cid, (CodexCallback) callback, resp); return storage_fetch(storageCtx, cid, (StorageCallback) callback, resp);
} }
static int cGoCodexStorageSpace(void* codexCtx, void* resp) { static int cGoStorageStorageSpace(void* storageCtx, void* resp) {
return codex_storage_space(codexCtx, (CodexCallback) callback, resp); return storage_space(storageCtx, (StorageCallback) callback, resp);
} }
static int cGoCodexStorageDelete(void* codexCtx, char* cid, void* resp) { static int cGoStorageStorageDelete(void* storageCtx, char* cid, void* resp) {
return codex_storage_delete(codexCtx, cid, (CodexCallback) callback, resp); return storage_delete(storageCtx, cid, (StorageCallback) callback, resp);
} }
static int cGoCodexStorageExists(void* codexCtx, char* cid, void* resp) { static int cGoStorageStorageExists(void* storageCtx, char* cid, void* resp) {
return codex_storage_exists(codexCtx, cid, (CodexCallback) callback, resp); return storage_exists(storageCtx, cid, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -41,27 +41,27 @@ type Space struct {
TotalBlocks int `json:"totalBlocks"` TotalBlocks int `json:"totalBlocks"`
// QuotaMaxBytes is the maximum storage space (in bytes) available // QuotaMaxBytes is the maximum storage space (in bytes) available
// for the node in Codex's local repository. // for the node in Logos Storage's local repository.
QuotaMaxBytes int64 `json:"quotaMaxBytes"` QuotaMaxBytes int64 `json:"quotaMaxBytes"`
// QuotaUsedBytes is the mount of storage space (in bytes) currently used // QuotaUsedBytes is the mount of storage space (in bytes) currently used
// for storing files in Codex's local repository. // for storing files in Logos Storage's local repository.
QuotaUsedBytes int64 `json:"quotaUsedBytes"` QuotaUsedBytes int64 `json:"quotaUsedBytes"`
// QuotaReservedBytes is the amount of storage reserved (in bytes) in the // QuotaReservedBytes is the amount of storage reserved (in bytes) in the
// Codex's local repository for future use when storage requests will be picked // Logos Storage's local repository for future use when storage requests will be picked
// up and hosted by the node using node's availabilities. // up and hosted by the node using node's availabilities.
// This does not include the storage currently in use. // This does not include the storage currently in use.
QuotaReservedBytes int64 `json:"quotaReservedBytes"` QuotaReservedBytes int64 `json:"quotaReservedBytes"`
} }
// Manifests returns the list of all manifests stored by the Codex node. // Manifests returns the list of all manifests stored by the Logos Storage node.
func (node CodexNode) Manifests() ([]Manifest, error) { func (node StorageNode) Manifests() ([]Manifest, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexStorageList(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageStorageList(node.ctx, bridge.resp) != C.RET_OK {
return nil, bridge.callError("cGoCodexStorageList") return nil, bridge.callError("cGoStorageStorageList")
} }
value, err := bridge.wait() value, err := bridge.wait()
if err != nil { if err != nil {
@ -84,15 +84,15 @@ func (node CodexNode) Manifests() ([]Manifest, error) {
} }
// Fetch download a file from the network and store it to the local node. // Fetch download a file from the network and store it to the local node.
func (node CodexNode) Fetch(cid string) (Manifest, error) { func (node StorageNode) Fetch(cid string) (Manifest, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexStorageFetch(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageStorageFetch(node.ctx, cCid, bridge.resp) != C.RET_OK {
return Manifest{}, bridge.callError("cGoCodexStorageFetch") return Manifest{}, bridge.callError("cGoStorageStorageFetch")
} }
value, err := bridge.wait() value, err := bridge.wait()
@ -111,14 +111,14 @@ func (node CodexNode) Fetch(cid string) (Manifest, error) {
} }
// Space returns information about the storage space used and available. // Space returns information about the storage space used and available.
func (node CodexNode) Space() (Space, error) { func (node StorageNode) Space() (Space, error) {
var space Space var space Space
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
if C.cGoCodexStorageSpace(node.ctx, bridge.resp) != C.RET_OK { if C.cGoStorageStorageSpace(node.ctx, bridge.resp) != C.RET_OK {
return space, bridge.callError("cGoCodexStorageSpace") return space, bridge.callError("cGoStorageStorageSpace")
} }
value, err := bridge.wait() value, err := bridge.wait()
@ -132,15 +132,15 @@ func (node CodexNode) Space() (Space, error) {
// Deletes either a single block or an entire dataset // Deletes either a single block or an entire dataset
// from the local node. Does nothing if the dataset is not locally available. // from the local node. Does nothing if the dataset is not locally available.
func (node CodexNode) Delete(cid string) error { func (node StorageNode) Delete(cid string) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexStorageDelete(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageStorageDelete(node.ctx, cCid, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexStorageDelete") return bridge.callError("cGoStorageStorageDelete")
} }
_, err := bridge.wait() _, err := bridge.wait()
@ -148,15 +148,15 @@ func (node CodexNode) Delete(cid string) error {
} }
// Exists checks if a given cid exists in the local storage. // Exists checks if a given cid exists in the local storage.
func (node CodexNode) Exists(cid string) (bool, error) { func (node StorageNode) Exists(cid string) (bool, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cCid = C.CString(cid) var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid)) defer C.free(unsafe.Pointer(cCid))
if C.cGoCodexStorageExists(node.ctx, cCid, bridge.resp) != C.RET_OK { if C.cGoStorageStorageExists(node.ctx, cCid, bridge.resp) != C.RET_OK {
return false, bridge.callError("cGoCodexStorageExists") return false, bridge.callError("cGoStorageStorageExists")
} }
result, err := bridge.wait() result, err := bridge.wait()

48
codex/storage_test.go → storage/storage_test.go
View File

@ -1,11 +1,11 @@
package codex package storage
import "testing" import "testing"
func TestManifests(t *testing.T) { func TestManifests(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
manifests, err := codex.Manifests() manifests, err := storage.Manifests()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -14,9 +14,9 @@ func TestManifests(t *testing.T) {
t.Fatal("expected manifests to be empty") t.Fatal("expected manifests to be empty")
} }
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
manifests, err = codex.Manifests() manifests, err = storage.Manifests()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -33,9 +33,9 @@ func TestManifests(t *testing.T) {
} }
func TestSpace(t *testing.T) { func TestSpace(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
space, err := codex.Space() space, err := storage.Space()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -56,9 +56,9 @@ func TestSpace(t *testing.T) {
t.Fatal("expected quota reserved bytes to be non-zero") t.Fatal("expected quota reserved bytes to be non-zero")
} }
uploadHelper(t, codex) uploadHelper(t, storage)
space, err = codex.Space() space, err = storage.Space()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -73,31 +73,31 @@ func TestSpace(t *testing.T) {
} }
func TestFetch(t *testing.T) { func TestFetch(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
_, err := codex.Fetch(cid) _, err := storage.Fetch(cid)
if err != nil { if err != nil {
t.Fatal("expected error when fetching non-existent manifest") t.Fatal("expected error when fetching non-existent manifest")
} }
} }
func TestFetchCidDoesNotExist(t *testing.T) { func TestFetchCidDoesNotExist(t *testing.T) {
codex := newCodexNode(t, Config{BlockRetries: 1}) storage := newStorageNode(t, Config{BlockRetries: 1})
_, err := codex.Fetch("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku") _, err := storage.Fetch("bafybeihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku")
if err == nil { if err == nil {
t.Fatal("expected error when fetching non-existent manifest") t.Fatal("expected error when fetching non-existent manifest")
} }
} }
func TestDelete(t *testing.T) { func TestDelete(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
manifests, err := codex.Manifests() manifests, err := storage.Manifests()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -105,12 +105,12 @@ func TestDelete(t *testing.T) {
t.Fatal("expected manifests to be empty after deletion") t.Fatal("expected manifests to be empty after deletion")
} }
err = codex.Delete(cid) err = storage.Delete(cid)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
manifests, err = codex.Manifests() manifests, err = storage.Manifests()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -121,11 +121,11 @@ func TestDelete(t *testing.T) {
} }
func TestExists(t *testing.T) { func TestExists(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
cid, _ := uploadHelper(t, codex) cid, _ := uploadHelper(t, storage)
exists, err := codex.Exists(cid) exists, err := storage.Exists(cid)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -133,12 +133,12 @@ func TestExists(t *testing.T) {
t.Fatal("expected cid to exist") t.Fatal("expected cid to exist")
} }
err = codex.Delete(cid) err = storage.Delete(cid)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
exists, err = codex.Exists(cid) exists, err = storage.Exists(cid)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }

0
codex/testdata/hello.txt → storage/testdata/hello.txt vendored
View File

20
codex/testutil.go → storage/testutil.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"bytes" "bytes"
@ -18,7 +18,7 @@ func defaultConfigHelper(t *testing.T) Config {
} }
} }
func newCodexNode(t *testing.T, opts ...Config) *CodexNode { func newStorageNode(t *testing.T, opts ...Config) *StorageNode {
config := defaultConfigHelper(t) config := defaultConfigHelper(t)
if len(opts) > 0 { if len(opts) > 0 {
@ -63,33 +63,33 @@ func newCodexNode(t *testing.T, opts ...Config) *CodexNode {
node, err := New(config) node, err := New(config)
if err != nil { if err != nil {
t.Fatalf("Failed to create Codex node: %v", err) t.Fatalf("Failed to create Logos Storage node: %v", err)
} }
err = node.Start() err = node.Start()
if err != nil { if err != nil {
t.Fatalf("Failed to start Codex node: %v", err) t.Fatalf("Failed to start Logos Storage node: %v", err)
} }
t.Cleanup(func() { t.Cleanup(func() {
if err := node.Stop(); err != nil { if err := node.Stop(); err != nil {
t.Logf("cleanup codex: %v", err) t.Logf("cleanup storage: %v", err)
} }
if err := node.Destroy(); err != nil { if err := node.Destroy(); err != nil {
t.Logf("cleanup codex: %v", err) t.Logf("cleanup storage: %v", err)
} }
}) })
return node return node
} }
func uploadHelper(t *testing.T, codex *CodexNode) (string, int) { func uploadHelper(t *testing.T, storage *StorageNode) (string, int) {
t.Helper() t.Helper()
buf := bytes.NewBuffer([]byte("Hello World!")) buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len() len := buf.Len()
cid, err := codex.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt"}, buf) cid, err := storage.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt"}, buf)
if err != nil { if err != nil {
t.Fatalf("Error happened during upload: %v\n", err) t.Fatalf("Error happened during upload: %v\n", err)
} }
@ -97,13 +97,13 @@ func uploadHelper(t *testing.T, codex *CodexNode) (string, int) {
return cid, len return cid, len
} }
func uploadBigFileHelper(t *testing.T, codex *CodexNode) (string, int) { func uploadBigFileHelper(t *testing.T, storage *StorageNode) (string, int) {
t.Helper() t.Helper()
len := 1024 * 1024 * 50 len := 1024 * 1024 * 50
buf := bytes.NewBuffer(make([]byte, len)) buf := bytes.NewBuffer(make([]byte, len))
cid, err := codex.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt"}, buf) cid, err := storage.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt"}, buf)
if err != nil { if err != nil {
t.Fatalf("Error happened during upload: %v\n", err) t.Fatalf("Error happened during upload: %v\n", err)
} }

70
codex/upload.go → storage/upload.go
View File

@ -1,27 +1,27 @@
package codex package storage
/* /*
#include "bridge.h" #include "bridge.h"
#include <stdlib.h> #include <stdlib.h>
static int cGoCodexUploadInit(void* codexCtx, char* filepath, size_t chunkSize, void* resp) { static int cGoStorageUploadInit(void* storageCtx, char* filepath, size_t chunkSize, void* resp) {
return codex_upload_init(codexCtx, filepath, chunkSize, (CodexCallback) callback, resp); return storage_upload_init(storageCtx, filepath, chunkSize, (StorageCallback) callback, resp);
} }
static int cGoCodexUploadChunk(void* codexCtx, char* sessionId, const uint8_t* chunk, size_t len, void* resp) { static int cGoStorageUploadChunk(void* storageCtx, char* sessionId, const uint8_t* chunk, size_t len, void* resp) {
return codex_upload_chunk(codexCtx, sessionId, chunk, len, (CodexCallback) callback, resp); return storage_upload_chunk(storageCtx, sessionId, chunk, len, (StorageCallback) callback, resp);
} }
static int cGoCodexUploadFinalize(void* codexCtx, char* sessionId, void* resp) { static int cGoStorageUploadFinalize(void* storageCtx, char* sessionId, void* resp) {
return codex_upload_finalize(codexCtx, sessionId, (CodexCallback) callback, resp); return storage_upload_finalize(storageCtx, sessionId, (StorageCallback) callback, resp);
} }
static int cGoCodexUploadCancel(void* codexCtx, char* sessionId, void* resp) { static int cGoStorageUploadCancel(void* storageCtx, char* sessionId, void* resp) {
return codex_upload_cancel(codexCtx, sessionId, (CodexCallback) callback, resp); return storage_upload_cancel(storageCtx, sessionId, (StorageCallback) callback, resp);
} }
static int cGoCodexUploadFile(void* codexCtx, char* sessionId, void* resp) { static int cGoStorageUploadFile(void* storageCtx, char* sessionId, void* resp) {
return codex_upload_file(codexCtx, sessionId, (CodexCallback) callback, resp); return storage_upload_file(storageCtx, sessionId, (StorageCallback) callback, resp);
} }
*/ */
import "C" import "C"
@ -45,7 +45,7 @@ type UploadOptions struct {
// It is used to detect the mimetype. // It is used to detect the mimetype.
Filepath string Filepath string
// ChunkSize is the size of each upload chunk, passed as `blockSize` to the Codex node // ChunkSize is the size of each upload chunk, passed as `blockSize` to the Logos Storage node
// store. Default is to 64 KB. // store. Default is to 64 KB.
ChunkSize ChunkSize ChunkSize ChunkSize
@ -82,26 +82,26 @@ func getReaderSize(r io.Reader) int64 {
// It returns a session ID that can be used for subsequent upload operations. // It returns a session ID that can be used for subsequent upload operations.
// This function is called by UploadReader and UploadFile internally. // This function is called by UploadReader and UploadFile internally.
// You should use this function only if you need to manage the upload session manually. // You should use this function only if you need to manage the upload session manually.
func (node CodexNode) UploadInit(options *UploadOptions) (string, error) { func (node StorageNode) UploadInit(options *UploadOptions) (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cFilename = C.CString(options.Filepath) var cFilename = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilename)) defer C.free(unsafe.Pointer(cFilename))
if C.cGoCodexUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK { if C.cGoStorageUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexUploadInit") return "", bridge.callError("cGoStorageUploadInit")
} }
return bridge.wait() return bridge.wait()
} }
// UploadChunk uploads a chunk of data to the Codex node. // UploadChunk uploads a chunk of data to the Logos Storage node.
// It takes the session ID returned by UploadInit // It takes the session ID returned by UploadInit
// and a byte slice containing the chunk data. // and a byte slice containing the chunk data.
// This function is called by UploadReader internally. // This function is called by UploadReader internally.
// You should use this function only if you need to manage the upload session manually. // You should use this function only if you need to manage the upload session manually.
func (node CodexNode) UploadChunk(sessionId string, chunk []byte) error { func (node StorageNode) UploadChunk(sessionId string, chunk []byte) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -113,8 +113,8 @@ func (node CodexNode) UploadChunk(sessionId string, chunk []byte) error {
cChunkPtr = (*C.uint8_t)(unsafe.Pointer(&chunk[0])) cChunkPtr = (*C.uint8_t)(unsafe.Pointer(&chunk[0]))
} }
if C.cGoCodexUploadChunk(node.ctx, cSessionId, cChunkPtr, C.size_t(len(chunk)), bridge.resp) != C.RET_OK { if C.cGoStorageUploadChunk(node.ctx, cSessionId, cChunkPtr, C.size_t(len(chunk)), bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexUploadChunk") return bridge.callError("cGoStorageUploadChunk")
} }
_, err := bridge.wait() _, err := bridge.wait()
@ -125,15 +125,15 @@ func (node CodexNode) UploadChunk(sessionId string, chunk []byte) error {
// It takes the session ID returned by UploadInit. // It takes the session ID returned by UploadInit.
// This function is called by UploadReader and UploadFile internally. // This function is called by UploadReader and UploadFile internally.
// You should use this function only if you need to manage the upload session manually. // You should use this function only if you need to manage the upload session manually.
func (node CodexNode) UploadFinalize(sessionId string) (string, error) { func (node StorageNode) UploadFinalize(sessionId string) (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cSessionId = C.CString(sessionId) var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId)) defer C.free(unsafe.Pointer(cSessionId))
if C.cGoCodexUploadFinalize(node.ctx, cSessionId, bridge.resp) != C.RET_OK { if C.cGoStorageUploadFinalize(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexUploadFinalize") return "", bridge.callError("cGoStorageUploadFinalize")
} }
return bridge.wait() return bridge.wait()
@ -142,31 +142,31 @@ func (node CodexNode) UploadFinalize(sessionId string) (string, error) {
// UploadCancel cancels an ongoing upload session. // UploadCancel cancels an ongoing upload session.
// It can be only if the upload session is managed manually. // It can be only if the upload session is managed manually.
// It doesn't work with UploadFile. // It doesn't work with UploadFile.
func (node CodexNode) UploadCancel(sessionId string) error { func (node StorageNode) UploadCancel(sessionId string) error {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
var cSessionId = C.CString(sessionId) var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId)) defer C.free(unsafe.Pointer(cSessionId))
if C.cGoCodexUploadCancel(node.ctx, cSessionId, bridge.resp) != C.RET_OK { if C.cGoStorageUploadCancel(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return bridge.callError("cGoCodexUploadCancel") return bridge.callError("cGoStorageUploadCancel")
} }
_, err := bridge.wait() _, err := bridge.wait()
return err return err
} }
// UploadReader uploads data from an io.Reader to the Codex node. // UploadReader uploads data from an io.Reader to the Logos Storage node.
// It takes the upload options and the reader as parameters. // It takes the upload options and the reader as parameters.
// It returns the CID of the uploaded file or an error. // It returns the CID of the uploaded file or an error.
// //
// Internally, it calls: // Internally, it calls:
// - UploadInit to create the upload session. // - UploadInit to create the upload session.
// - UploadChunk to upload a chunk to codex. // - UploadChunk to upload a chunk to storage.
// - UploadFinalize to finalize the upload session. // - UploadFinalize to finalize the upload session.
// - UploadCancel if an error occurs. // - UploadCancel if an error occurs.
func (node CodexNode) UploadReader(ctx context.Context, options UploadOptions, r io.Reader) (string, error) { func (node StorageNode) UploadReader(ctx context.Context, options UploadOptions, r io.Reader) (string, error) {
sessionId, err := node.UploadInit(&options) sessionId, err := node.UploadInit(&options)
if err != nil { if err != nil {
return "", err return "", err
@ -235,20 +235,20 @@ func (node CodexNode) UploadReader(ctx context.Context, options UploadOptions, r
} }
// UploadReaderAsync is the asynchronous version of UploadReader using a goroutine. // UploadReaderAsync is the asynchronous version of UploadReader using a goroutine.
func (node CodexNode) UploadReaderAsync(ctx context.Context, options UploadOptions, r io.Reader, onDone func(cid string, err error)) { func (node StorageNode) UploadReaderAsync(ctx context.Context, options UploadOptions, r io.Reader, onDone func(cid string, err error)) {
go func() { go func() {
cid, err := node.UploadReader(ctx, options, r) cid, err := node.UploadReader(ctx, options, r)
onDone(cid, err) onDone(cid, err)
}() }()
} }
// UploadFile uploads a file to the Codex node. // UploadFile uploads a file to the Logos Storage node.
// It takes the upload options as parameter. // It takes the upload options as parameter.
// It returns the CID of the uploaded file or an error. // It returns the CID of the uploaded file or an error.
// //
// The options parameter contains the following fields: // The options parameter contains the following fields:
// - filepath: the full path of the file to upload. // - filepath: the full path of the file to upload.
// - chunkSize: the size of each upload chunk, passed as `blockSize` to the Codex node // - chunkSize: the size of each upload chunk, passed as `blockSize` to the Logos Storage node
// store. Default is to 64 KB. // store. Default is to 64 KB.
// - onProgress: a callback function that is called after each chunk is uploaded with: // - onProgress: a callback function that is called after each chunk is uploaded with:
// - read: the number of bytes read in the last chunk. // - read: the number of bytes read in the last chunk.
@ -262,7 +262,7 @@ func (node CodexNode) UploadReaderAsync(ctx context.Context, options UploadOptio
// is sent to the stream. // is sent to the stream.
// //
// Internally, it calls UploadInit to create the upload session. // Internally, it calls UploadInit to create the upload session.
func (node CodexNode) UploadFile(ctx context.Context, options UploadOptions) (string, error) { func (node StorageNode) UploadFile(ctx context.Context, options UploadOptions) (string, error) {
bridge := newBridgeCtx() bridge := newBridgeCtx()
defer bridge.free() defer bridge.free()
@ -303,8 +303,8 @@ func (node CodexNode) UploadFile(ctx context.Context, options UploadOptions) (st
var cSessionId = C.CString(sessionId) var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId)) defer C.free(unsafe.Pointer(cSessionId))
if C.cGoCodexUploadFile(node.ctx, cSessionId, bridge.resp) != C.RET_OK { if C.cGoStorageUploadFile(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoCodexUploadFile") return "", bridge.callError("cGoStorageUploadFile")
} }
// Create a done channel to signal the goroutine to stop // Create a done channel to signal the goroutine to stop
@ -349,7 +349,7 @@ func (node CodexNode) UploadFile(ctx context.Context, options UploadOptions) (st
} }
// UploadFileAsync is the asynchronous version of UploadFile using a goroutine. // UploadFileAsync is the asynchronous version of UploadFile using a goroutine.
func (node CodexNode) UploadFileAsync(ctx context.Context, options UploadOptions, onDone func(cid string, err error)) { func (node StorageNode) UploadFileAsync(ctx context.Context, options UploadOptions, onDone func(cid string, err error)) {
go func() { go func() {
cid, err := node.UploadFile(ctx, options) cid, err := node.UploadFile(ctx, options)
onDone(cid, err) onDone(cid, err)

32
codex/upload_test.go → storage/upload_test.go
View File

@ -1,4 +1,4 @@
package codex package storage
import ( import (
"bytes" "bytes"
@ -11,13 +11,13 @@ import (
const expectedCID = "zDvZRwzmAkhzDRPH5EW242gJBNZ2T7aoH2v1fVH66FxXL4kSbvyM" const expectedCID = "zDvZRwzmAkhzDRPH5EW242gJBNZ2T7aoH2v1fVH66FxXL4kSbvyM"
func TestUploadReader(t *testing.T) { func TestUploadReader(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
totalBytes := 0 totalBytes := 0
finalPercent := 0.0 finalPercent := 0.0
buf := bytes.NewBuffer([]byte("Hello World!")) buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len() len := buf.Len()
cid, err := codex.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt", OnProgress: func(read, total int, percent float64, err error) { cid, err := storage.UploadReader(context.Background(), UploadOptions{Filepath: "hello.txt", OnProgress: func(read, total int, percent float64, err error) {
if err != nil { if err != nil {
log.Fatalf("Error happened during upload: %v\n", err) log.Fatalf("Error happened during upload: %v\n", err)
} }
@ -46,12 +46,12 @@ func TestUploadReader(t *testing.T) {
func TestUploadReaderCancel(t *testing.T) { func TestUploadReaderCancel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
codex := newCodexNode(t) storage := newStorageNode(t)
buf := bytes.NewBuffer(make([]byte, 1024*1024*10)) buf := bytes.NewBuffer(make([]byte, 1024*1024*10))
channelErr := make(chan error, 1) channelErr := make(chan error, 1)
go func() { go func() {
_, e := codex.UploadReader(ctx, UploadOptions{Filepath: "hello.txt"}, buf) _, e := storage.UploadReader(ctx, UploadOptions{Filepath: "hello.txt"}, buf)
channelErr <- e channelErr <- e
}() }()
@ -68,7 +68,7 @@ func TestUploadReaderCancel(t *testing.T) {
} }
func TestUploadFile(t *testing.T) { func TestUploadFile(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
totalBytes := 0 totalBytes := 0
finalPercent := 0.0 finalPercent := 0.0
@ -86,7 +86,7 @@ func TestUploadFile(t *testing.T) {
finalPercent = percent finalPercent = percent
}} }}
cid, err := codex.UploadFile(context.Background(), options) cid, err := storage.UploadFile(context.Background(), options)
if err != nil { if err != nil {
t.Fatalf("UploadReader failed: %v", err) t.Fatalf("UploadReader failed: %v", err)
} }
@ -119,11 +119,11 @@ func TestUploadFileCancel(t *testing.T) {
} }
tmpFile.Close() tmpFile.Close()
codex := newCodexNode(t) storage := newStorageNode(t)
channelError := make(chan error, 1) channelError := make(chan error, 1)
go func() { go func() {
_, err := codex.UploadFile(ctx, UploadOptions{Filepath: tmpFile.Name()}) _, err := storage.UploadFile(ctx, UploadOptions{Filepath: tmpFile.Name()})
channelError <- err channelError <- err
}() }()
@ -140,11 +140,11 @@ func TestUploadFileCancel(t *testing.T) {
} }
func TestUploadFileNoProgress(t *testing.T) { func TestUploadFileNoProgress(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
options := UploadOptions{Filepath: "./testdata/doesnt_exist.txt"} options := UploadOptions{Filepath: "./testdata/doesnt_exist.txt"}
cid, err := codex.UploadFile(context.Background(), options) cid, err := storage.UploadFile(context.Background(), options)
if err == nil { if err == nil {
t.Fatalf("UploadReader should have failed") t.Fatalf("UploadReader should have failed")
} }
@ -155,24 +155,24 @@ func TestUploadFileNoProgress(t *testing.T) {
} }
func TestManualUpload(t *testing.T) { func TestManualUpload(t *testing.T) {
codex := newCodexNode(t) storage := newStorageNode(t)
sessionId, err := codex.UploadInit(&UploadOptions{Filepath: "hello.txt"}) sessionId, err := storage.UploadInit(&UploadOptions{Filepath: "hello.txt"})
if err != nil { if err != nil {
log.Fatal("Error happened:", err.Error()) log.Fatal("Error happened:", err.Error())
} }
err = codex.UploadChunk(sessionId, []byte("Hello ")) err = storage.UploadChunk(sessionId, []byte("Hello "))
if err != nil { if err != nil {
log.Fatal("Error happened:", err.Error()) log.Fatal("Error happened:", err.Error())
} }
err = codex.UploadChunk(sessionId, []byte("World!")) err = storage.UploadChunk(sessionId, []byte("World!"))
if err != nil { if err != nil {
log.Fatal("Error happened:", err.Error()) log.Fatal("Error happened:", err.Error())
} }
cid, err := codex.UploadFinalize(sessionId) cid, err := storage.UploadFinalize(sessionId)
if err != nil { if err != nil {
log.Fatal("Error happened:", err.Error()) log.Fatal("Error happened:", err.Error())
} }

1
vendor/logos-storage-nim vendored Submodule

@ -0,0 +1 @@
Subproject commit 60861d6af841d96085db366d44c1d543b7659fa5

1
vendor/nim-codex vendored

@ -1 +0,0 @@
Subproject commit 2b9fc1eb554e5eee43b8a815084fb8c61687ada9