logos-storage/logos-storage-nim
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim.git synced 2026-01-02 13:33:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 50df458ec406bab8857e856bc8edfc5bee2dfe14 into 60861d6af841d96085db366d44c1d543b7659fa5

Browse Source
This commit is contained in:
Arnaud 2025-12-19 11:05:25 +00:00 committed by GitHub
commit 4a71f658fa
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 1900 additions and 1067 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

134
.github/workflows/artifacts.yml vendored Normal file
View File

@ -0,0 +1,134 @@
name: Artifacts
on:
push:
tags:
- "v*"
workflow_dispatch:
jobs:
build:
runs-on: ${{ matrix.target.os }}
strategy:
matrix:
target:
- os: ubuntu-latest
cpu: amd64
lib_ext: so
- os: ubuntu-24.04-arm
cpu: arm64
lib_ext: so
- os: macos-latest
lib_ext: dylib
cpu: arm64
- os: windows-latest
cpu: amd64
lib_ext: dll
steps:
- name: Check out sources
uses: actions/checkout@v4
with:
submodules: recursive
- name: Install Rust 1.85.0
if: matrix.target.os != 'windows-latest'
uses: dtolnay/rust-toolchain@master
with:
toolchain: 1.85.0
- name: Record submodule commit
run: git -C . rev-parse HEAD > .storage-commit
- name: Cache libstorage build
id: cache-libstorage
uses: actions/cache@v4
with:
path: ./build
key: ${{ runner.os }}-libstorage-${{ hashFiles('.storage-commit') }}
- name: MSYS2 (Windows amd64)
if: matrix.target.os == 'windows-latest' && matrix.target.cpu == 'amd64'
uses: msys2/setup-msys2@v2
with:
path-type: inherit
msystem: UCRT64
install: >-
base-devel
git
mingw-w64-ucrt-x86_64-toolchain
mingw-w64-ucrt-x86_64-cmake
mingw-w64-ucrt-x86_64-ntldd-git
mingw-w64-ucrt-x86_64-rust
- name: Build libstorage (Linux)
if: matrix.target.lib_ext == 'so' && steps.cache-libstorage.outputs.cache-hit != 'true'
run: |
make update
make libstorage
- name: Build libstorage (MacOS)
if: matrix.target.os == 'macos-latest' && steps.cache-libstorage.outputs.cache-hit != 'true'
run: |
make update
STORAGE_LIB_PARAMS="--passL:\"-Wl,-install_name,@rpath/libstorage.dylib\"" make libstorage
- name: Build libstorage (Windows)
if: matrix.target.os == 'windows-latest' && steps.cache-libstorage.outputs.cache-hit != 'true'
shell: msys2 {0}
run: |
pacman -Sy --noconfirm make
git config --global core.symlinks false
make update
make libstorage
- name: Package artifacts Linux
if: matrix.target.os == 'ubuntu-latest' || matrix.target.os == 'ubuntu-24.04-arm'
run: |
sudo apt-get update && sudo apt-get install -y zip
ZIPFILE=storage-linux-${{ matrix.target.cpu }}.zip
zip -j $ZIPFILE .build/libstorage.${{ matrix.target.lib_ext }} ./library/libstorage.h
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Package artifacts MacOS
if: matrix.target.os == 'macos-latest'
run: |
ZIPFILE=storage-macos-${{ matrix.target.cpu }}.zip
zip -j $ZIPFILE ./build/libstorage.${{ matrix.target.lib_ext }} ./library/libstorage.h
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Package artifacts (Windows)
if: matrix.target.os == 'windows-latest'
shell: msys2 {0}
run: |
ZIPFILE=storage-windows-${{ matrix.target.cpu }}.zip
(cd ./build && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libstorage.dll)
(cd ./library && 7z a -tzip "${GITHUB_WORKSPACE}/${ZIPFILE}" libstorage.h)
echo "ZIPFILE=$ZIPFILE" >> $GITHUB_ENV
- name: Upload artifacts
uses: actions/upload-artifact@v4
with:
name: ${{ env.ZIPFILE }}
path: ${{ env.ZIPFILE }}
if-no-files-found: error
publish-release:
needs: build
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/tags/')
steps:
- name: Check out sources
uses: actions/checkout@v4
- name: Download artifacts
uses: actions/download-artifact@v5
with:
path: dist
- name: Create release
uses: softprops/action-gh-release@v2
with:
files: dist/**
draft: true

37
.github/workflows/ci.yml vendored
View File

@ -84,3 +84,40 @@ jobs:
name: codecov-umbrella
token: ${{ secrets.CODECOV_TOKEN }}
verbose: true
cbinding:
runs-on: ubuntu-latest
steps:
- name: Checkout sources
uses: actions/checkout@v4
with:
submodules: recursive
ref: ${{ github.event.pull_request.head.sha }}
- name: Setup Nimbus Build System
uses: ./.github/actions/nimbus-build-system
with:
os: linux
nim_version: ${{ env.nim_version }}
- name: Record submodule commit
run: git -C . rev-parse HEAD > .storage-commit
- name: Cache libstorage build
id: cache-libstorage
uses: actions/cache@v4
with:
path: ./build
key: ${{ runner.os }}-libstorage-${{ hashFiles('.storage-commit') }}
- name: C Binding build
if: steps.cache-libstorage.outputs.cache-hit != 'true'
run: |
make -j${ncpu} update
make -j${ncpu} libstorage
- name: C Binding test
run: |
cd examples/c
gcc -o storage storage.c -L../../build -lstorage -Wl,-rpath,../../ -pthread
LD_LIBRARY_PATH=../../build ./storage

3
.gitignore vendored
View File

@ -47,3 +47,6 @@ nim.cfg
tests/integration/logs
data/
examples/c/data-dir
examples/c/downloaded_hello.txt

0
examples/golang/hello.txt → examples/c/hello_world.txt
View File

908
examples/c/storage.c Normal file
View File

@ -0,0 +1,908 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <unistd.h>
#include "../../library/libstorage.h"
#define MAX_RETRIES 500
typedef struct
{
int ret;
char *msg;
char *chunk;
size_t len;
} Resp;
static Resp *alloc_resp(void)
{
Resp *r = (Resp *)calloc(1, sizeof(Resp));
r->ret = -1;
return r;
}
static void free_resp(Resp *r)
{
if (!r)
{
return;
}
if (r->msg)
{
free(r->msg);
}
if (r->chunk)
{
free(r->chunk);
}
free(r);
}
static int get_ret(Resp *r)
{
if (!r)
{
return RET_ERR;
}
return r->ret;
}
static void wait_resp(Resp *r)
{
int retries = 0;
while (get_ret(r) == -1 && retries < MAX_RETRIES)
{
usleep(1000 * 100); // 100 ms
retries++;
}
}
static int is_resp_ok(Resp *r, char **res)
{
if (!r)
{
return RET_ERR;
}
wait_resp(r);
int ret = (r->ret == RET_OK) ? RET_OK : RET_ERR;
if (res)
{
*res = NULL;
}
if (r->chunk)
{
*res = strdup(r->chunk);
}
else if (res && r->msg)
{
*res = strdup(r->msg);
}
free_resp(r);
return ret;
}
static void callback(int ret, const char *msg, size_t len, void *userData)
{
Resp *r = (Resp *)userData;
if (!r)
{
return;
}
r->ret = ret;
if (r->msg)
{
free(r->msg);
r->msg = NULL;
r->len = 0;
}
if (ret == RET_PROGRESS && msg && len > 0 && r->chunk)
{
memcpy(r->chunk, msg, len);
r->chunk[len] = '\0';
r->len = len;
}
if (msg && len > 0)
{
r->msg = (char *)malloc(len + 1);
if (!r->msg)
{
r->len = 0;
return;
}
memcpy(r->msg, msg, len);
r->msg[len] = '\0';
r->len = len;
}
else
{
r->msg = NULL;
r->len = 0;
}
}
static int read_file(const char *filepath, char **res)
{
FILE *file;
char c;
char content[100];
file = fopen(filepath, "r");
if (file == NULL)
{
return RET_ERR;
}
fgets(content, 100, file);
*res = strdup(content);
fclose(file);
return RET_OK;
}
int setup(void **storage_ctx)
{
// Initialize Nim runtime
extern void libstorageNimMain(void);
libstorageNimMain();
Resp *r = alloc_resp();
const char *cfg = "{\"log-level\":\"WARN\",\"data-dir\":\"./data-dir\"}";
void *ctx = storage_new(cfg, (StorageCallback)callback, r);
if (!ctx)
{
free_resp(r);
return RET_ERR;
}
wait_resp(r);
if (r->ret != RET_OK)
{
free_resp(r);
return RET_ERR;
}
(*storage_ctx) = ctx;
free_resp(r);
return RET_OK;
}
int start(void *storage_ctx)
{
Resp *r = alloc_resp();
if (storage_start(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
return is_resp_ok(r, NULL);
}
int cleanup(void *storage_ctx)
{
Resp *r = alloc_resp();
// Stop node
if (storage_stop(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, NULL) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
// Close node
if (storage_close(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, NULL) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
// Destroy node
// No need to wait here as storage_destroy is synchronous
if (storage_destroy(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
free_resp(r);
return RET_OK;
}
int check_version(void *storage_ctx)
{
char *res = NULL;
Resp *r = alloc_resp();
// No need to wait here as storage_version is synchronous
if (storage_version(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
free_resp(r);
return RET_OK;
}
int check_repo(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_repo(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
if (strcmp(res, "./data-dir") != 0)
{
printf("repo mismatch: %s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_debug(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_debug(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
// Simple check to ensure the response contains spr
if (strstr(res, "spr") == NULL)
{
fprintf(stderr, "debug content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_spr(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_spr(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
if (strstr(res, "spr") == NULL)
{
fprintf(stderr, "spr content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_peer_id(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_peer_id(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
return is_resp_ok(r, &res);
}
int update_log_level(void *storage_ctx, const char *log_level)
{
char *res = NULL;
Resp *r = alloc_resp();
if (storage_log_level(storage_ctx, log_level, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
return is_resp_ok(r, NULL);
}
int check_upload_chunk(void *storage_ctx, const char *filepath)
{
Resp *r = alloc_resp();
char *res = NULL;
char *session_id = NULL;
const char *payload = "hello world";
size_t chunk_size = strlen(payload);
if (storage_upload_init(storage_ctx, filepath, chunk_size, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, &session_id) != RET_OK)
{
return RET_ERR;
}
uint8_t *chunk = malloc(chunk_size);
if (!chunk)
{
free(session_id);
return RET_ERR;
}
memcpy(chunk, payload, chunk_size);
r = alloc_resp();
if (storage_upload_chunk(storage_ctx, session_id, chunk, chunk_size, (StorageCallback)callback, r) != RET_OK)
{
free(session_id);
free_resp(r);
free(chunk);
return RET_ERR;
}
if (is_resp_ok(r, NULL) != RET_OK)
{
free(session_id);
free(chunk);
return RET_ERR;
}
free(chunk);
r = alloc_resp();
if (storage_upload_finalize(storage_ctx, session_id, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
free(session_id);
return RET_ERR;
}
free(session_id);
int ret = is_resp_ok(r, &res);
if (res == NULL || strlen(res) == 0)
{
fprintf(stderr, "CID is missing\n");
ret = RET_ERR;
}
free(res);
return ret;
}
int upload_cancel(void *storage_ctx)
{
Resp *r = alloc_resp();
char *session_id = NULL;
size_t chunk_size = 64 * 1024;
if (storage_upload_init(storage_ctx, "hello.txt", chunk_size, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, &session_id) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
if (storage_upload_cancel(storage_ctx, session_id, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
free(session_id);
return RET_ERR;
}
free(session_id);
return is_resp_ok(r, NULL);
}
int check_upload_file(void *storage_ctx, const char *filepath, char **res)
{
Resp *r = alloc_resp();
char *session_id = NULL;
size_t chunk_size = 64 * 1024;
if (storage_upload_init(storage_ctx, filepath, chunk_size, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, &session_id) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
if (storage_upload_file(storage_ctx, session_id, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
free(session_id);
return RET_ERR;
}
free(session_id);
int ret = is_resp_ok(r, res);
if (res == NULL || strlen(*res) == 0)
{
fprintf(stderr, "CID is missing\n");
return RET_ERR;
}
return ret;
}
int check_download_stream(void *storage_ctx, const char *cid, const char *filepath)
{
Resp *r = alloc_resp();
char *res = NULL;
size_t chunk_size = 64 * 1024;
bool local = true;
if (storage_download_init(storage_ctx, cid, chunk_size, local, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, NULL) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
r->chunk = malloc(chunk_size + 1);
if (storage_download_stream(storage_ctx, cid, chunk_size, local, filepath, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
if (strncmp(res, "Hello World!", strlen("Hello World!")) != 0)
{
fprintf(stderr, "downloaded content mismatch, res:%s\n", res);
ret = RET_ERR;
}
if (read_file("downloaded_hello.txt", &res) != RET_OK)
{
fprintf(stderr, "read downloaded file failed\n");
ret = RET_ERR;
}
if (strncmp(res, "Hello World!", strlen("Hello World!")) != 0)
{
fprintf(stderr, "downloaded content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_download_chunk(void *storage_ctx, const char *cid)
{
Resp *r = alloc_resp();
char *res = NULL;
size_t chunk_size = 64 * 1024;
bool local = true;
if (storage_download_init(storage_ctx, cid, chunk_size, local, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
if (is_resp_ok(r, NULL) != RET_OK)
{
return RET_ERR;
}
r = alloc_resp();
r->chunk = malloc(chunk_size + 1);
if (storage_download_chunk(storage_ctx, cid, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
if (strncmp(res, "Hello World!", strlen("Hello World!")) != 0)
{
fprintf(stderr, "downloaded chunk content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_download_cancel(void *storage_ctx, const char *cid)
{
Resp *r = alloc_resp();
if (storage_download_cancel(storage_ctx, cid, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
return is_resp_ok(r, NULL);
}
int check_download_manifest(void *storage_ctx, const char *cid)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_download_manifest(storage_ctx, cid, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
const char *expected_manifest = "{\"treeCid\":\"zDzSvJTf8JYwvysKPmG7BtzpbiAHfuwFMRphxm4hdvnMJ4XPJjKX\",\"datasetSize\":12,\"blockSize\":65536,\"filename\":\"hello_world.txt\",\"mimetype\":\"text/plain\",\"protected\":false}";
if (strncmp(res, expected_manifest, strlen(expected_manifest)) != 0)
{
fprintf(stderr, "downloaded manifest content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_list(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_list(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
const char *expected_manifest = "{\"treeCid\":\"zDzSvJTf8JYwvysKPmG7BtzpbiAHfuwFMRphxm4hdvnMJ4XPJjKX\",\"datasetSize\":12,\"blockSize\":65536,\"filename\":\"hello_world.txt\",\"mimetype\":\"text/plain\",\"protected\":false}";
if (strstr(res, expected_manifest) == NULL)
{
fprintf(stderr, "downloaded manifest content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_space(void *storage_ctx)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_space(storage_ctx, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
// Simple check to ensure the response contains totalBlocks
if (strstr(res, "totalBlocks") == NULL)
{
fprintf(stderr, "list content mismatch, res:%s\n", res);
ret = RET_ERR;
}
free(res);
return ret;
}
int check_exists(void *storage_ctx, const char *cid, bool expected)
{
Resp *r = alloc_resp();
char *res = NULL;
if (storage_exists(storage_ctx, cid, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
int ret = is_resp_ok(r, &res);
if (expected)
{
if (strcmp(res, "true") != 0)
{
fprintf(stderr, "exists content mismatch, res:%s\n", res);
ret = RET_ERR;
}
}
else
{
if (strcmp(res, "false") != 0)
{
fprintf(stderr, "exists content mismatch, res:%s\n", res);
ret = RET_ERR;
}
}
free(res);
return ret;
}
int check_delete(void *storage_ctx, const char *cid)
{
Resp *r = alloc_resp();
if (storage_delete(storage_ctx, cid, (StorageCallback)callback, r) != RET_OK)
{
free_resp(r);
return RET_ERR;
}
return is_resp_ok(r, NULL);
}
// TODO: implement check_fetch
// It is a bit complicated because it requires two nodes
// connected together to fetch from peers.
// A good idea would be to use connect function using addresses.
// This test will be quite important when the block engine is re-implemented.
int check_fetch(void *storage_ctx, const char *cid)
{
return RET_OK;
}
int main(void)
{
void *storage_ctx = NULL;
char *res = NULL;
char *cid = NULL;
if (setup(&storage_ctx) != RET_OK)
{
fprintf(stderr, "setup failed\n");
return RET_ERR;
}
if (check_version(storage_ctx) != RET_OK)
{
fprintf(stderr, "check version failed\n");
return RET_ERR;
}
if (start(storage_ctx) != RET_OK)
{
fprintf(stderr, "start failed\n");
return RET_ERR;
}
if (check_repo(storage_ctx) != RET_OK)
{
fprintf(stderr, "check repo failed\n");
return RET_ERR;
}
if (check_debug(storage_ctx) != RET_OK)
{
fprintf(stderr, "check debug failed\n");
return RET_ERR;
}
if (check_spr(storage_ctx) != RET_OK)
{
fprintf(stderr, "check spr failed\n");
return RET_ERR;
}
if (check_peer_id(storage_ctx) != RET_OK)
{
fprintf(stderr, "check peer_id failed\n");
return RET_ERR;
}
if (check_upload_chunk(storage_ctx, "hello_world.txt") != RET_OK)
{
fprintf(stderr, "upload chunk failed\n");
return RET_ERR;
}
if (upload_cancel(storage_ctx) != RET_OK)
{
fprintf(stderr, "upload cancel failed\n");
return RET_ERR;
}
char *path = realpath("hello_world.txt", NULL);
if (!path)
{
fprintf(stderr, "realpath failed\n");
return RET_ERR;
}
if (check_upload_file(storage_ctx, path, &cid) != RET_OK)
{
fprintf(stderr, "upload file failed\n");
free(path);
return RET_ERR;
}
free(path);
if (check_download_stream(storage_ctx, cid, "downloaded_hello.txt") != RET_OK)
{
fprintf(stderr, "download stream failed\n");
free(cid);
return RET_ERR;
}
if (check_download_chunk(storage_ctx, cid) != RET_OK)
{
fprintf(stderr, "download chunk failed\n");
free(cid);
return RET_ERR;
}
if (check_download_cancel(storage_ctx, cid) != RET_OK)
{
fprintf(stderr, "download cancel failed\n");
free(cid);
return RET_ERR;
}
if (check_download_manifest(storage_ctx, cid) != RET_OK)
{
fprintf(stderr, "download manifest failed\n");
free(cid);
return RET_ERR;
}
if (check_list(storage_ctx) != RET_OK)
{
fprintf(stderr, "list failed\n");
free(cid);
return RET_ERR;
}
if (check_space(storage_ctx) != RET_OK)
{
fprintf(stderr, "space failed\n");
free(cid);
return RET_ERR;
}
if (check_exists(storage_ctx, cid, true) != RET_OK)
{
fprintf(stderr, "exists failed\n");
free(cid);
return RET_ERR;
}
if (check_delete(storage_ctx, cid) != RET_OK)
{
fprintf(stderr, "delete failed\n");
free(cid);
return RET_ERR;
}
if (check_exists(storage_ctx, cid, false) != RET_OK)
{
fprintf(stderr, "exists failed\n");
free(cid);
return RET_ERR;
}
free(cid);
if (update_log_level(storage_ctx, "INFO") != RET_OK)
{
fprintf(stderr, "update log level failed\n");
return RET_ERR;
}
if (cleanup(storage_ctx) != RET_OK)
{
fprintf(stderr, "cleanup failed\n");
return RET_ERR;
}
return RET_OK;
}

24
examples/golang/README.md
View File

@ -1,24 +0,0 @@
## Pre-requisite
libstorage.so is needed to be compiled and present in build folder.
## Compilation
From the Logos Storage root folder:
```code
go build -o storage-go examples/golang/storage.go
```
## Run
From the storage root folder:
```code
export LD_LIBRARY_PATH=build
```
```code
./storage-go
```

885
examples/golang/storage.go
View File

@ -1,885 +0,0 @@
package main
/*
#cgo LDFLAGS: -L../../build/ -lstorage
#cgo LDFLAGS: -L../../ -Wl,-rpath,../../
#include <stdbool.h>
#include <stdlib.h>
#include "../../library/libstorage.h"
typedef struct {
int ret;
char* msg;
size_t len;
uintptr_t h;
} Resp;
static void* allocResp(uintptr_t h) {
Resp* r = (Resp*)calloc(1, sizeof(Resp));
r->h = h;
return r;
}
static void freeResp(void* resp) {
if (resp != NULL) {
free(resp);
}
}
static int getRet(void* resp) {
if (resp == NULL) {
return 0;
}
Resp* m = (Resp*) resp;
return m->ret;
}
void libstorageNimMain(void);
static void storage_host_init_once(void){
static int done;
if (!__atomic_exchange_n(&done, 1, __ATOMIC_SEQ_CST)) libstorageNimMain();
}
// resp must be set != NULL in case interest on retrieving data from the callback
void callback(int ret, char* msg, size_t len, void* resp);
static void* cGoStorageNew(const char* configJson, void* resp) {
void* ret = storage_new(configJson, (StorageCallback) callback, resp);
return ret;
}
static int cGoStorageStart(void* storageCtx, void* resp) {
return storage_start(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageStop(void* storageCtx, void* resp) {
return storage_stop(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageClose(void* storageCtx, void* resp) {
return storage_close(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageDestroy(void* storageCtx, void* resp) {
return storage_destroy(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageVersion(void* storageCtx, void* resp) {
return storage_version(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageRevision(void* storageCtx, void* resp) {
return storage_revision(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageRepo(void* storageCtx, void* resp) {
return storage_repo(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageSpr(void* storageCtx, void* resp) {
return storage_spr(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStoragePeerId(void* storageCtx, void* resp) {
return storage_peer_id(storageCtx, (StorageCallback) callback, resp);
}
static int cGoStorageUploadInit(void* storageCtx, char* filepath, size_t chunkSize, void* resp) {
return storage_upload_init(storageCtx, filepath, chunkSize, (StorageCallback) callback, resp);
}
static int cGoStorageUploadChunk(void* storageCtx, char* sessionId, const uint8_t* chunk, size_t len, void* resp) {
return storage_upload_chunk(storageCtx, sessionId, chunk, len, (StorageCallback) callback, resp);
}
static int cGoStorageUploadFinalize(void* storageCtx, char* sessionId, void* resp) {
return storage_upload_finalize(storageCtx, sessionId, (StorageCallback) callback, resp);
}
static int cGoStorageUploadCancel(void* storageCtx, char* sessionId, void* resp) {
return storage_upload_cancel(storageCtx, sessionId, (StorageCallback) callback, resp);
}
static int cGoStorageUploadFile(void* storageCtx, char* sessionId, void* resp) {
return storage_upload_file(storageCtx, sessionId, (StorageCallback) callback, resp);
}
static int cGoStorageLogLevel(void* storageCtx, char* logLevel, void* resp) {
return storage_log_level(storageCtx, logLevel, (StorageCallback) callback, resp);
}
static int cGoStorageExists(void* storageCtx, char* cid, void* resp) {
return storage_exists(storageCtx, cid, (StorageCallback) callback, resp);
}
*/
import "C"
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"runtime/cgo"
"sync"
"syscall"
"unsafe"
)
type LogFormat string
const (
LogFormatAuto LogFormat = "auto"
LogFormatColors LogFormat = "colors"
LogFormatNoColors LogFormat = "nocolors"
LogFormatJSON LogFormat = "json"
)
type RepoKind string
const (
FS RepoKind = "fs"
SQLite RepoKind = "sqlite"
LevelDb RepoKind = "leveldb"
)
const defaultBlockSize = 1024 * 64
type Config struct {
// Default: INFO
LogLevel string `json:"log-level,omitempty"`
// Specifies what kind of logs should be written to stdout
// Default: auto
LogFormat LogFormat `json:"log-format,omitempty"`
// Enable the metrics server
// Default: false
MetricsEnabled bool `json:"metrics,omitempty"`
// Listening address of the metrics server
// Default: 127.0.0.1
MetricsAddress string `json:"metrics-address,omitempty"`
// Listening HTTP port of the metrics server
// Default: 8008
MetricsPort int `json:"metrics-port,omitempty"`
// The directory where logos storage will store configuration and data
// Default:
// $HOME\AppData\Roaming\Logos Storage on Windows
// $HOME/Library/Application Support/Logos Storage on macOS
// $HOME/.cache/logos_storage on Linux
DataDir string `json:"data-dir,omitempty"`
// Multi Addresses to listen on
// Default: ["/ip4/0.0.0.0/tcp/0"]
ListenAddrs []string `json:"listen-addrs,omitempty"`
// Specify method to use for determining public address.
// Must be one of: any, none, upnp, pmp, extip:<IP>
// Default: any
Nat string `json:"nat,omitempty"`
// Discovery (UDP) port
// Default: 8090
DiscoveryPort int `json:"disc-port,omitempty"`
// Source of network (secp256k1) private key file path or name
// Default: "key"
NetPrivKeyFile string `json:"net-privkey,omitempty"`
// Specifies one or more bootstrap nodes to use when connecting to the network.
BootstrapNodes []string `json:"bootstrap-node,omitempty"`
// The maximum number of peers to connect to.
// Default: 160
MaxPeers int `json:"max-peers,omitempty"`
// Number of worker threads (\"0\" = use as many threads as there are CPU cores available)
// Default: 0
NumThreads int `json:"num-threads,omitempty"`
// Node agent string which is used as identifier in network
// Default: "Logos Storage"
AgentString string `json:"agent-string,omitempty"`
// Backend for main repo store (fs, sqlite, leveldb)
// Default: fs
RepoKind RepoKind `json:"repo-kind,omitempty"`
// The size of the total storage quota dedicated to the node
// Default: 20 GiBs
StorageQuota int `json:"storage-quota,omitempty"`
// Default block timeout in seconds - 0 disables the ttl
// Default: 30 days
BlockTtl int `json:"block-ttl,omitempty"`
// Time interval in seconds - determines frequency of block
// maintenance cycle: how often blocks are checked for expiration and cleanup
// Default: 10 minutes
BlockMaintenanceInterval int `json:"block-mi,omitempty"`
// Number of blocks to check every maintenance cycle
// Default: 1000
BlockMaintenanceNumberOfBlocks int `json:"block-mn,omitempty"`
// Number of times to retry fetching a block before giving up
// Default: 3000
BlockRetries int `json:"block-retries,omitempty"`
// The size of the block cache, 0 disables the cache -
// might help on slow hardrives
// Default: 0
CacheSize int `json:"cache-size,omitempty"`
// Default: "" (no log file)
LogFile string `json:"log-file,omitempty"`
}
type StorageNode struct {
ctx unsafe.Pointer
}
type ChunkSize int
func (c ChunkSize) valOrDefault() int {
if c == 0 {
return defaultBlockSize
}
return int(c)
}
func (c ChunkSize) toSizeT() C.size_t {
return C.size_t(c.valOrDefault())
}
// bridgeCtx is used for managing the C-Go bridge calls.
// It contains a wait group for synchronizing the calls,
// a cgo.Handle for passing context to the C code,
// a response pointer for receiving data from the C code,
// and fields for storing the result and error of the call.
type bridgeCtx struct {
wg *sync.WaitGroup
h cgo.Handle
resp unsafe.Pointer
result string
err error
// Callback used for receiving progress updates during upload/download.
//
// For the upload, the bytes parameter indicates the number of bytes uploaded.
// If the chunk size is superior or equal to the blocksize (passed in init function),
// the callback will be called when a block is put in the store.
// Otherwise, it will be called when a chunk is pushed into the stream.
//
// For the download, the bytes is the size of the chunk received, and the chunk
// is the actual chunk of data received.
onProgress func(bytes int, chunk []byte)
}
// newBridgeCtx creates a new bridge context for managing C-Go calls.
// The bridge context is initialized with a wait group and a cgo.Handle.
func newBridgeCtx() *bridgeCtx {
bridge := &bridgeCtx{}
bridge.wg = &sync.WaitGroup{}
bridge.wg.Add(1)
bridge.h = cgo.NewHandle(bridge)
bridge.resp = C.allocResp(C.uintptr_t(uintptr(bridge.h)))
return bridge
}
// callError creates an error message for a failed C-Go call.
func (b *bridgeCtx) callError(name string) error {
return fmt.Errorf("failed the call to %s returned code %d", name, C.getRet(b.resp))
}
// free releases the resources associated with the bridge context,
// including the cgo.Handle and the response pointer.
func (b *bridgeCtx) free() {
if b.h > 0 {
b.h.Delete()
b.h = 0
}
if b.resp != nil {
C.freeResp(b.resp)
b.resp = nil
}
}
// callback is the function called by the C code to communicate back to Go.
// It handles progress updates, successful completions, and errors.
// The function uses the response pointer to retrieve the bridge context
// and update its state accordingly.
//
//export callback
func callback(ret C.int, msg *C.char, len C.size_t, resp unsafe.Pointer) {
if resp == nil {
return
}
m := (*C.Resp)(resp)
m.ret = ret
m.msg = msg
m.len = len
if m.h == 0 {
return
}
h := cgo.Handle(m.h)
if h == 0 {
return
}
if v, ok := h.Value().(*bridgeCtx); ok {
switch ret {
case C.RET_PROGRESS:
if v.onProgress == nil {
return
}
if msg != nil {
chunk := C.GoBytes(unsafe.Pointer(msg), C.int(len))
v.onProgress(int(C.int(len)), chunk)
} else {
v.onProgress(int(C.int(len)), nil)
}
case C.RET_OK:
retMsg := C.GoStringN(msg, C.int(len))
v.result = retMsg
v.err = nil
if v.wg != nil {
v.wg.Done()
}
case C.RET_ERR:
retMsg := C.GoStringN(msg, C.int(len))
v.err = errors.New(retMsg)
if v.wg != nil {
v.wg.Done()
}
}
}
}
// wait waits for the bridge context to complete its operation.
// It returns the result and error of the operation.
func (b *bridgeCtx) wait() (string, error) {
b.wg.Wait()
return b.result, b.err
}
type OnUploadProgressFunc func(read, total int, percent float64, err error)
type UploadOptions struct {
// Filepath can be the full path when using UploadFile
// otherwise the file name.
// It is used to detect the mimetype.
Filepath string
// ChunkSize is the size of each upload chunk, passed as `blockSize` to the Logos Storage node
// store. Default is to 64 KB.
ChunkSize ChunkSize
// OnProgress is a callback function that is called after each chunk is uploaded with:
// - read: the number of bytes read in the last chunk.
// - total: the total number of bytes read so far.
// - percent: the percentage of the total file size that has been uploaded. It is
// determined from a `stat` call if it is a file and from the length of the buffer
// if it is a buffer. Otherwise, it is 0.
// - err: an error, if one occurred.
//
// If the chunk size is more than the `chunkSize` parameter, the callback is called
// after the block is actually stored in the block store. Otherwise, it is called
// after the chunk is sent to the stream.
OnProgress OnUploadProgressFunc
}
func getReaderSize(r io.Reader) int64 {
switch v := r.(type) {
case *os.File:
stat, err := v.Stat()
if err != nil {
return 0
}
return stat.Size()
case *bytes.Buffer:
return int64(v.Len())
default:
return 0
}
}
// New creates a new Logos Storage node with the provided configuration.
// The node is not started automatically; you need to call StorageStart
// to start it.
// It returns a Logos Storage node that can be used to interact
// with the Logos Storage network.
func New(config Config) (*StorageNode, error) {
bridge := newBridgeCtx()
defer bridge.free()
jsonConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
cJsonConfig := C.CString(string(jsonConfig))
defer C.free(unsafe.Pointer(cJsonConfig))
ctx := C.cGoStorageNew(cJsonConfig, bridge.resp)
if _, err := bridge.wait(); err != nil {
return nil, bridge.err
}
return &StorageNode{ctx: ctx}, bridge.err
}
// Start starts the Logos Storage node.
func (node StorageNode) Start() error {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageStart(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageStart")
}
_, err := bridge.wait()
return err
}
// StartAsync is the asynchronous version of Start.
func (node StorageNode) StartAsync(onDone func(error)) {
go func() {
err := node.Start()
onDone(err)
}()
}
// Stop stops the Logos Storage node.
func (node StorageNode) Stop() error {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageStop(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageStop")
}
_, err := bridge.wait()
return err
}
// Destroy destroys the Logos Storage node, freeing all resources.
// The node must be stopped before calling this method.
func (node StorageNode) Destroy() error {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageClose(node.ctx, bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageClose")
}
_, err := bridge.wait()
if err != nil {
return err
}
if C.cGoStorageDestroy(node.ctx, bridge.resp) != C.RET_OK {
return errors.New("Failed to destroy the Logos Storage node.")
}
return err
}
// Version returns the version of the Logos Storage node.
func (node StorageNode) Version() (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageVersion(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageVersion")
}
return bridge.wait()
}
func (node StorageNode) Revision() (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageRevision(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageRevision")
}
return bridge.wait()
}
// Repo returns the path of the data dir folder.
func (node StorageNode) Repo() (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageRepo(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageRepo")
}
return bridge.wait()
}
func (node StorageNode) Spr() (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStorageSpr(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageSpr")
}
return bridge.wait()
}
func (node StorageNode) PeerId() (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if C.cGoStoragePeerId(node.ctx, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStoragePeerId")
}
return bridge.wait()
}
// UploadInit initializes a new upload session.
// It returns a session ID that can be used for subsequent upload operations.
// This function is called by UploadReader and UploadFile internally.
// You should use this function only if you need to manage the upload session manually.
func (node StorageNode) UploadInit(options *UploadOptions) (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
var cFilename = C.CString(options.Filepath)
defer C.free(unsafe.Pointer(cFilename))
if C.cGoStorageUploadInit(node.ctx, cFilename, options.ChunkSize.toSizeT(), bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageUploadInit")
}
return bridge.wait()
}
// UploadChunk uploads a chunk of data to the Logos Storage node.
// It takes the session ID returned by UploadInit
// and a byte slice containing the chunk data.
// This function is called by UploadReader internally.
// You should use this function only if you need to manage the upload session manually.
func (node StorageNode) UploadChunk(sessionId string, chunk []byte) error {
bridge := newBridgeCtx()
defer bridge.free()
var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId))
var cChunkPtr *C.uint8_t
if len(chunk) > 0 {
cChunkPtr = (*C.uint8_t)(unsafe.Pointer(&chunk[0]))
}
if C.cGoStorageUploadChunk(node.ctx, cSessionId, cChunkPtr, C.size_t(len(chunk)), bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageUploadChunk")
}
_, err := bridge.wait()
return err
}
// UploadFinalize finalizes the upload session and returns the CID of the uploaded file.
// It takes the session ID returned by UploadInit.
// This function is called by UploadReader and UploadFile internally.
// You should use this function only if you need to manage the upload session manually.
func (node StorageNode) UploadFinalize(sessionId string) (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId))
if C.cGoStorageUploadFinalize(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageUploadFinalize")
}
return bridge.wait()
}
// UploadCancel cancels an ongoing upload session.
// It can be only if the upload session is managed manually.
// It doesn't work with UploadFile.
func (node StorageNode) UploadCancel(sessionId string) error {
bridge := newBridgeCtx()
defer bridge.free()
var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId))
if C.cGoStorageUploadCancel(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageUploadCancel")
}
_, err := bridge.wait()
return err
}
// UploadReader uploads data from an io.Reader to the Logos Storage node.
// It takes the upload options and the reader as parameters.
// It returns the CID of the uploaded file or an error.
//
// Internally, it calls:
// - UploadInit to create the upload session.
// - UploadChunk to upload a chunk to Logos Storage.
// - UploadFinalize to finalize the upload session.
// - UploadCancel if an error occurs.
func (node StorageNode) UploadReader(options UploadOptions, r io.Reader) (string, error) {
sessionId, err := node.UploadInit(&options)
if err != nil {
return "", err
}
buf := make([]byte, options.ChunkSize.valOrDefault())
total := 0
var size int64
if options.OnProgress != nil {
size = getReaderSize(r)
}
for {
n, err := r.Read(buf)
if err == io.EOF {
break
}
if err != nil {
if cancelErr := node.UploadCancel(sessionId); cancelErr != nil {
return "", fmt.Errorf("failed to upload chunk %v and failed to cancel upload session %v", err, cancelErr)
}
return "", err
}
if n == 0 {
break
}
if err := node.UploadChunk(sessionId, buf[:n]); err != nil {
if cancelErr := node.UploadCancel(sessionId); cancelErr != nil {
return "", fmt.Errorf("failed to upload chunk %v and failed to cancel upload session %v", err, cancelErr)
}
return "", err
}
total += n
if options.OnProgress != nil && size > 0 {
percent := float64(total) / float64(size) * 100.0
// The last block could be a bit over the size due to padding
// on the chunk size.
if percent > 100.0 {
percent = 100.0
}
options.OnProgress(n, total, percent, nil)
} else if options.OnProgress != nil {
options.OnProgress(n, total, 0, nil)
}
}
return node.UploadFinalize(sessionId)
}
// UploadReaderAsync is the asynchronous version of UploadReader using a goroutine.
func (node StorageNode) UploadReaderAsync(options UploadOptions, r io.Reader, onDone func(cid string, err error)) {
go func() {
cid, err := node.UploadReader(options, r)
onDone(cid, err)
}()
}
// UploadFile uploads a file to the Logos Storage node.
// It takes the upload options as parameter.
// It returns the CID of the uploaded file or an error.
//
// The options parameter contains the following fields:
// - filepath: the full path of the file to upload.
// - chunkSize: the size of each upload chunk, passed as `blockSize` to the Logos Storage node
// store. Default is to 64 KB.
// - onProgress: a callback function that is called after each chunk is uploaded with:
// - read: the number of bytes read in the last chunk.
// - total: the total number of bytes read so far.
// - percent: the percentage of the total file size that has been uploaded. It is
// determined from a `stat` call.
// - err: an error, if one occurred.
//
// If the chunk size is more than the `chunkSize` parameter, the callback is called after
// the block is actually stored in the block store. Otherwise, it is called after the chunk
// is sent to the stream.
//
// Internally, it calls UploadInit to create the upload session.
func (node StorageNode) UploadFile(options UploadOptions) (string, error) {
bridge := newBridgeCtx()
defer bridge.free()
if options.OnProgress != nil {
stat, err := os.Stat(options.Filepath)
if err != nil {
return "", err
}
size := stat.Size()
total := 0
if size > 0 {
bridge.onProgress = func(read int, _ []byte) {
if read == 0 {
return
}
total += read
percent := float64(total) / float64(size) * 100.0
// The last block could be a bit over the size due to padding
// on the chunk size.
if percent > 100.0 {
percent = 100.0
}
options.OnProgress(read, int(size), percent, nil)
}
}
}
sessionId, err := node.UploadInit(&options)
if err != nil {
return "", err
}
var cSessionId = C.CString(sessionId)
defer C.free(unsafe.Pointer(cSessionId))
if C.cGoStorageUploadFile(node.ctx, cSessionId, bridge.resp) != C.RET_OK {
return "", bridge.callError("cGoStorageUploadFile")
}
return bridge.wait()
}
// UploadFileAsync is the asynchronous version of UploadFile using a goroutine.
func (node StorageNode) UploadFileAsync(options UploadOptions, onDone func(cid string, err error)) {
go func() {
cid, err := node.UploadFile(options)
onDone(cid, err)
}()
}
func (node StorageNode) UpdateLogLevel(logLevel string) error {
bridge := newBridgeCtx()
defer bridge.free()
var cLogLevel = C.CString(string(logLevel))
defer C.free(unsafe.Pointer(cLogLevel))
if C.cGoStorageLogLevel(node.ctx, cLogLevel, bridge.resp) != C.RET_OK {
return bridge.callError("cGoStorageLogLevel")
}
_, err := bridge.wait()
return err
}
func (node StorageNode) Exists(cid string) (bool, error) {
bridge := newBridgeCtx()
defer bridge.free()
var cCid = C.CString(cid)
defer C.free(unsafe.Pointer(cCid))
if C.cGoStorageExists(node.ctx, cCid, bridge.resp) != C.RET_OK {
return false, bridge.callError("cGoStorageUploadCancel")
}
result, err := bridge.wait()
return result == "true", err
}
func main() {
dataDir := os.TempDir() + "/data-dir"
node, err := New(Config{
BlockRetries: 5,
LogLevel: "WARN",
DataDir: dataDir,
})
if err != nil {
log.Fatalf("Failed to create Logos Storage node: %v", err)
}
defer os.RemoveAll(dataDir)
if err := node.Start(); err != nil {
log.Fatalf("Failed to start Logos Storage node: %v", err)
}
log.Println("Logos Storage node started")
version, err := node.Version()
if err != nil {
log.Fatalf("Failed to get Logos Storage version: %v", err)
}
log.Printf("Logos Storage version: %s", version)
err = node.UpdateLogLevel("ERROR")
if err != nil {
log.Fatalf("Failed to update log level: %v", err)
}
cid := "zDvZRwzmAkhzDRPH5EW242gJBNZ2T7aoH2v1fVH66FxXL4kSbvyM"
exists, err := node.Exists(cid)
if err != nil {
log.Fatalf("Failed to check data existence: %v", err)
}
if exists {
log.Fatalf("The data should not exist")
}
buf := bytes.NewBuffer([]byte("Hello World!"))
len := buf.Len()
cid, err = node.UploadReader(UploadOptions{Filepath: "hello.txt"}, buf)
if err != nil {
log.Fatalf("Failed to upload data: %v", err)
}
log.Printf("Uploaded data with CID: %s (size: %d bytes)", cid, len)
exists, err = node.Exists(cid)
if err != nil {
log.Fatalf("Failed to check data existence: %v", err)
}
if !exists {
log.Fatalf("The data should exist")
}
// Wait for a SIGINT or SIGTERM signal
ch := make(chan os.Signal, 1)
signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)
<-ch
if err := node.Stop(); err != nil {
log.Fatalf("Failed to stop Storage node: %v", err)
}
log.Println("Logos Storage node stopped")
if err := node.Destroy(); err != nil {
log.Fatalf("Failed to destroy Logos Storage node: %v", err)
}
}

468
library/README.md
View File

@ -34,4 +34,470 @@ sequenceDiagram
Ctx-->>C: forward callback
C-->>Go: forward callback
Go-->>App: done
```
```
## C API
C-exported interface for the Logos Storage shared library.
This API provides a C-compatible interface to the internal Nim implementation of Logos Storage.
Unless explicitly stated otherwise, all functions are asynchronous and execute their work on a separate thread, returning results via the provided callback. The `int` return value is the synchronous status of dispatch:
- `RET_OK`: job dispatched to the worker thread
- `RET_ERR`: immediate failure
- `RET_MISSING_CALLBACK`: callback is missing
Some functions may emit progress updates via the callback using `RET_PROGRESS`, and finally complete with `RET_OK` or `RET_ERR`.
For upload/download streaming, `msg` contains a chunk of data and `len` the chunk length.
---
## Types
### `StorageCallback`
```c
typedef void (*StorageCallback)(int callerRet, const char *msg, size_t len, void *userData);
```
---
## Return codes
```c
#define RET_OK 0
#define RET_ERR 1
#define RET_MISSING_CALLBACK 2
#define RET_PROGRESS 3
```
---
## Context lifecycle
### `storage_new`
Create a new instance of a Logos Storage node.
```c
void *storage_new(
const char *configJson,
StorageCallback callback,
void *userData
);
```
- `configJson`: JSON string with configuration overwriting defaults
- Returns an opaque context pointer used for subsequent calls
Typical usage:
- `storage_new(...)`
- `storage_start(...)`
- `storage_stop(...)`
- `storage_destroy(...)`
---
### `storage_start`
Start the Logos Storage node (can be started/stopped multiple times).
```c
int storage_start(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_stop`
Stop the Logos Storage node (can be started/stopped multiple times).
```c
int storage_stop(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_close`
Close the node and release resources before destruction.
```c
int storage_close(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_destroy`
Destroy the node instance and free associated resources. Node must be stopped and closed.
```c
int storage_destroy(void *ctx, StorageCallback callback, void *userData);
```
---
## Version
### `storage_version`
Get the Logos Storage version string.
Does not require the node to be started and does not involve a thread call.
```c
int storage_version(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_revision`
Get the Logos Storage contracts revision.
Does not require the node to be started and does not involve a thread call.
```c
int storage_revision(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_repo`
Get the repo (data-dir) used by the node.
```c
int storage_repo(void *ctx, StorageCallback callback, void *userData);
```
---
## Debug
### `storage_debug`
Retrieve debug information (JSON).
```c
int storage_debug(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_spr`
Get the node's Signed Peer Record (SPR).
```c
int storage_spr(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_peer_id`
Get the node's peer ID (libp2p Peer Identity).
```c
int storage_peer_id(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_peer_debug`
Request debug information for a given peer ID.
Only available if compiled with `storage_enable_api_debug_peers`.
```c
int storage_peer_debug(void *ctx, const char *peerId, StorageCallback callback, void *userData);
```
---
## Logging
### `storage_log_level`
Set the log level at run time.
`logLevel` can be: `TRACE`, `DEBUG`, `INFO`, `NOTICE`, `WARN`, `ERROR`, `FATAL`.
```c
int storage_log_level(
void *ctx,
const char *logLevel,
StorageCallback callback,
void *userData
);
```
---
## Networking
### `storage_connect`
Connect to a peer by using `peerAddresses` if provided, otherwise use `peerId`.
Note that the `peerId` has to be advertised in the DHT for this to work.
```c
int storage_connect(
void *ctx,
const char *peerId,
const char **peerAddresses,
size_t peerAddressesSize,
StorageCallback callback,
void *userData
);
```
---
## Upload
### `storage_upload_init`
Initialize an upload session for a file.
- `filepath`: absolute path for file upload; for chunk uploads it's the file name. The metadata filename and mime type are derived from this value.
- `chunkSize`: chunk size for upload (default: `1024 * 64` bytes)
- Callback returns the `sessionId`
```c
int storage_upload_init(
void *ctx,
const char *filepath,
size_t chunkSize,
StorageCallback callback,
void *userData
);
```
---
### `storage_upload_chunk`
Upload a chunk for the given `sessionId`.
```c
int storage_upload_chunk(
void *ctx,
const char *sessionId,
const uint8_t *chunk,
size_t len,
StorageCallback callback,
void *userData
);
```
---
### `storage_upload_finalize`
Finalize an upload session identified by `sessionId`.
Callback returns the `cid` of the uploaded content.
```c
int storage_upload_finalize(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData
);
```
---
### `storage_upload_cancel`
Cancel an ongoing upload session.
```c
int storage_upload_cancel(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData
);
```
---
### `storage_upload_file`
Upload the file defined as `filepath` in the init method.
- Callback may be called with `RET_PROGRESS` during upload (depending on chunk size constraints)
- Callback returns the `cid` of the uploaded content
```c
int storage_upload_file(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData
);
```
---
## Download API
### `storage_download_init`
Initialize a download for `cid`.
- `chunkSize`: chunk size for download (default: `1024 * 64` bytes)
- `local`: attempt local store retrieval only
```c
int storage_download_init(
void *ctx,
const char *cid,
size_t chunkSize,
bool local,
StorageCallback callback,
void *userData
);
```
---
### `storage_download_stream`
Perform a streaming download for `cid`. Init must have been called prior.
- If `filepath` is provided, content is written to that file.
- Callback may be called with `RET_PROGRESS` updates during download.
- `local` indicates whether to attempt local store retrieval only.
```c
int storage_download_stream(
void *ctx,
const char *cid,
size_t chunkSize,
bool local,
const char *filepath,
StorageCallback callback,
void *userData
);
```
---
### `storage_download_chunk`
Download a chunk for the given `cid`. Init must have been called prior.
Chunk returned via callback using `RET_PROGRESS`.
```c
int storage_download_chunk(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData
);
```
---
### `storage_download_cancel`
Cancel an ongoing download for `cid`.
```c
int storage_download_cancel(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData
);
```
---
### `storage_download_manifest`
Retrieve the manifest for the given `cid` (JSON).
```c
int storage_download_manifest(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData
);
```
---
## Storage operations
### `storage_list`
Retrieve the list of manifests stored in the node.
```c
int storage_list(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_space`
Retrieve storage space information (JSON).
```c
int storage_space(void *ctx, StorageCallback callback, void *userData);
```
---
### `storage_delete`
Delete the content identified by `cid`.
```c
int storage_delete(void *ctx, const char *cid, StorageCallback callback, void *userData);
```
---
### `storage_fetch`
Fetch content identified by `cid` from the network into local store
in background. The callback will not receive progress updates.
```c
int storage_fetch(void *ctx, const char *cid, StorageCallback callback, void *userData);
```
---
### `storage_exists`
Check if content identified by `cid` exists in local store.
```c
int storage_exists(void *ctx, const char *cid, StorageCallback callback, void *userData);
```
### `storage_set_event_callback`
Not used currently. Reserved for future use to set an event callback.
```c
void storage_set_event_callback(void *ctx, StorageCallback callback, void *userData);
```
---
## Go wrapper
A Go wrapper is available [here](https://github.com/logos-storage/logos-storage-go-bindings).
## Rust Wrapper
A Rust wrapper is available [here](https://github.com/nipsysdev/codex-rust-bindings).

508
library/libstorage.h
View File

@ -1,203 +1,397 @@
/**
* libstorage.h - C Interface for Example Library
*
* This header provides the public API for libstorage
*
* To see the auto-generated header by Nim, run `make libstorage` from the
* repository root. The generated file will be created at:
* nimcache/release/libstorage/libstorage.h
*/
* libstorage.h - C-exported interface for the Storage shared library
*
* This file implements the public C API for libstorage. It acts as the bridge
* between C programs and the internal Nim implementation.
*
* Unless it is explicitly stated otherwise, all functions are asynchronous and execute
* their work on a separate thread, returning results via the provided callback. The
* result code of the function represents the synchronous status of the call itself:
* returning RET_OK if the job has been dispatched to the thread, and RET_ERR in case
* of immediate failure.
*
* The callback function is invoked with the result of the operation, including
* any data or error messages. If the call was successful, `callerRet` will be RET_OK,
* and `msg` will contain the result data. If there was an error, `callerRet` will be RET_ERR,
* and `msg` will contain the error message.
*
* When a function supports progress updates, it may invoke the callback multiple times:
* first with RET_PROGRESS and progress information, and finally with RET_OK or RET_ERR
* upon completion. The msg parameter will a chunk of data for upload and download operations.
*
* `userData` is a pointer provided by the caller that is passed back to the callback
* for context.
*/
#ifndef __libstorage__
#define __libstorage__
#include <stddef.h>
#include <stdint.h>
#include <stdbool.h>
// The possible returned values for the functions that return int
#define RET_OK 0
#define RET_ERR 1
#define RET_MISSING_CALLBACK 2
#define RET_PROGRESS 3
#define RET_OK 0
#define RET_ERR 1
#define RET_MISSING_CALLBACK 2
// RET_PROGRESS is used to indicate that the callback is being
// with progress updates.
#define RET_PROGRESS 3
#ifdef __cplusplus
extern "C" {
extern "C"
{
#endif
typedef void (*StorageCallback) (int callerRet, const char* msg, size_t len, void* userData);
typedef void (*StorageCallback)(int callerRet, const char *msg, size_t len, void *userData);
void* storage_new(
const char* configJson,
StorageCallback callback,
void* userData);
// Create a new instance of a Logos Storage node.
// `configJson` is a JSON string with the configuration overwriting defaults.
// Returns a pointer to the StorageContext used to interact with the node.
//
// Typical usage:
// ctx = storage_new(configJson, myCallback, myUserData);
// storage_start(ctx, ...);
// ...
// storage_stop(ctx, ...);
// storage_destroy(ctx, ...);
void *storage_new(
const char *configJson,
StorageCallback callback,
void *userData);
int storage_version(
void* ctx,
StorageCallback callback,
void* userData);
// Get the Logos Storage version string.
// This call does not require the node to be started and
// does not involve a thread call.
int storage_version(
void *ctx,
StorageCallback callback,
void *userData);
int storage_revision(
void* ctx,
StorageCallback callback,
void* userData);
// Get the Logos Storage contracts revision.
// This call does not require the node to be started and
// does not involve a thread call.
int storage_revision(
void *ctx,
StorageCallback callback,
void *userData);
int storage_repo(
void* ctx,
StorageCallback callback,
void* userData);
// Get the repo (data-dir) used by the node.
int storage_repo(
void *ctx,
StorageCallback callback,
void *userData);
int storage_debug(
void* ctx,
StorageCallback callback,
void* userData);
// Retrieve debug information (JSON).
//
// Here is an example of the returned JSON structure:
// {
// "id": "...",
// "addrs": ["..."],
// "spt": "",
// "announceAddresses": ["..."],
// "table": {
// "localNode": "",
// "nodes": [
// {
// "nodeId": "...",
// "peerId": "...",
// "record": "...",
// "address": "...",
// "seen": true,
// }
// ]
// }
int storage_debug(
void *ctx,
StorageCallback callback,
void *userData);
int storage_spr(
void* ctx,
StorageCallback callback,
void* userData);
/// Get the node's (Signed Peer Record)
int storage_spr(
void *ctx,
StorageCallback callback,
void *userData);
int storage_peer_id(
void* ctx,
StorageCallback callback,
void* userData);
// Get the node's peer ID.
// Peer Identity reference as specified at
// https://docs.libp2p.io/concepts/fundamentals/peers/
int storage_peer_id(
void *ctx,
StorageCallback callback,
void *userData);
int storage_log_level(
void* ctx,
const char* logLevel,
StorageCallback callback,
void* userData);
// Set the log level at run time.
// `logLevel` can be one of:
// TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL
int storage_log_level(
void *ctx,
const char *logLevel,
StorageCallback callback,
void *userData);
int storage_connect(
void* ctx,
const char* peerId,
const char** peerAddresses,
size_t peerAddressesSize,
StorageCallback callback,
void* userData);
// Connect to a peer by using `peerAddresses` if provided, otherwise use `peerId`.
// Note that the `peerId` has to be advertised in the DHT for this to work.
int storage_connect(
void *ctx,
const char *peerId,
const char **peerAddresses,
size_t peerAddressesSize,
StorageCallback callback,
void *userData);
int storage_peer_debug(
void* ctx,
const char* peerId,
StorageCallback callback,
void* userData);
// Request debug information for a given peer ID.
// This api is only available if the library was compiled with
// `storage_enable_api_debug_peers` argument.
//
// Here is an example of the returned JSON structure:
// {
// "peerId": "...",
// "seqNo": 0,
// "addresses": [],
// }
int storage_peer_debug(
void *ctx,
const char *peerId,
StorageCallback callback,
void *userData);
// Initialize an upload session for a file.
// `filepath` for a file upload, this is the absolute path to the file
// to be uploaded. For an upload using chunks, this is the name of the file.
// The metadata filename and mime type are derived from this value.
//
// `chunkSize` defines the size of each chunk to be used during upload.
// The default value is the default block size 1024 * 64 bytes.
//
// The callback returns the `sessionId` for the download session created.
//
// Typical usage:
// storage_upload_init(ctx, filepath, chunkSize, myCallback, myUserData);
// ...
// storage_upload_chunk(ctx, sessionId, chunk, len, myCallback, myUserData);
// ...
// storage_upload_finalize(ctx, sessionId, myCallback, myUserData);
int storage_upload_init(
void *ctx,
const char *filepath,
size_t chunkSize,
StorageCallback callback,
void *userData);
int storage_upload_init(
void* ctx,
const char* filepath,
size_t chunkSize,
StorageCallback callback,
void* userData);
// Upload a chunk for the given `sessionId`.
int storage_upload_chunk(
void *ctx,
const char *sessionId,
const uint8_t *chunk,
size_t len,
StorageCallback callback,
void *userData);
int storage_upload_chunk(
void* ctx,
const char* sessionId,
const uint8_t* chunk,
size_t len,
StorageCallback callback,
void* userData);
// Finalize an upload session identified by `sessionId`.
// The callback returns the `cid` of the uploaded content.
int storage_upload_finalize(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData);
int storage_upload_finalize(
void* ctx,
const char* sessionId,
StorageCallback callback,
void* userData);
// Cancel an ongoing upload session.
int storage_upload_cancel(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData);
int storage_upload_cancel(
void* ctx,
const char* sessionId,
StorageCallback callback,
void* userData);
// Upload the file defined as `filepath` in the init method.
// The callback will be called with RET_PROGRESS updates during the upload,
// if the chunk size is equal or greater than the session chunkSize.
//
// The callback returns the `cid` of the uploaded content.
//
// Typical usage:
// storage_upload_init(ctx, filepath, chunkSize, myCallback, myUserData);
// ...
// storage_upload_file(ctx, sessionId, myCallback, myUserData);
int storage_upload_file(
void *ctx,
const char *sessionId,
StorageCallback callback,
void *userData);
int storage_upload_file(
void* ctx,
const char* sessionId,
StorageCallback callback,
void* userData);
// Initialize a download for `cid`.
// `chunkSize` defines the size of each chunk to be used during download.
// The default value is the default block size 1024 * 64 bytes.
// `local` indicates whether to attempt local store retrieval only.
//
// Typical usage:
// storage_download_init(ctx, cid, chunkSize, local, myCallback, myUserData);
// ...
// storage_download_stream(ctx, cid, filepath, myCallback, myUserData);
int storage_download_init(
void *ctx,
const char *cid,
size_t chunkSize,
bool local,
StorageCallback callback,
void *userData);
int storage_download_stream(
void* ctx,
const char* cid,
size_t chunkSize,
bool local,
const char* filepath,
StorageCallback callback,
void* userData);
// Perform a streaming download for `cid`.
// The init method must have been called prior to this.
// If filepath is provided, the content will be written to that file.
// The callback will be called with RET_PROGRESS updates during the download/
// `local` indicates whether to attempt local store retrieval only.
//
// Typical usage:
// storage_download_init(ctx, cid, chunkSize, local, myCallback, myUserData);
// ...
// storage_download_stream(ctx, cid, filepath, myCallback, myUserData);
int storage_download_stream(
void *ctx,
const char *cid,
size_t chunkSize,
bool local,
const char *filepath,
StorageCallback callback,
void *userData);
int storage_download_init(
void* ctx,
const char* cid,
size_t chunkSize,
bool local,
StorageCallback callback,
void* userData);
// Download a chunk for the given `cid`.
// The init method must have been called prior to this.
// The chunk will be returned via the callback using `RET_PROGRESS`.
int storage_download_chunk(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_download_chunk(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Cancel an ongoing download for `cid`.
int storage_download_cancel(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_download_cancel(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Retrieve the manifest for the given `cid`.
//
// Here is an example of the returned manifest JSON structure:
// {
// "treeCid": "zDzSvJTf8JYwvysKPmG7BtzpbiAHfuwFMRphxm4hdvnMJ4XPJjKX",
// "datasetSize": 123456,
// "blockSize": 65536,
// "filename": "example.txt",
// "mimetype": "text/plain",
// "protected": false
// }
int storage_download_manifest(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_download_manifest(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Retrieve the list of the manifests stored in the node.
int storage_list(
void *ctx,
StorageCallback callback,
void *userData);
int storage_list(
void* ctx,
StorageCallback callback,
void* userData);
// Retrieve the storage space information.
//
// Here is an example of the returned JSON structure:
// {
// "totalBlocks": 100000,
// "quotaMaxBytes": 0,
// "quotaUsedBytes": 0,
// "quotaReservedBytes": 0
// }
int storage_space(
void *ctx,
StorageCallback callback,
void *userData);
int storage_space(
void* ctx,
StorageCallback callback,
void* userData);
// Delete the content identified by `cid`.
int storage_delete(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_delete(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Fetch the content identified by `cid` from the network into
// local store.
// The download is done in background so the callback
// will not receive progress updates.
int storage_fetch(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_fetch(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Check if the content identified by `cid` exists in local store.
int storage_exists(
void *ctx,
const char *cid,
StorageCallback callback,
void *userData);
int storage_exists(
void* ctx,
const char* cid,
StorageCallback callback,
void* userData);
// Start the Logos Storage node.
// The node can be started and stopped multiple times.
//
// Typical usage:
// ctx = storage_new(configJson, myCallback, myUserData);
// storage_start(ctx, ...);
// ...
// storage_stop(ctx, ...);
// storage_destroy(ctx, ...);
int storage_start(void *ctx,
StorageCallback callback,
void *userData);
int storage_start(void* ctx,
StorageCallback callback,
void* userData);
// Stop the Logos Storage node.
// The node can be started and stopped multiple times.
//
// Typical usage:
// ctx = storage_new(configJson, myCallback, myUserData);
// storage_start(ctx, ...);
// ...
// storage_stop(ctx, ...);
// storage_destroy(ctx, ...);
int storage_stop(void *ctx,
StorageCallback callback,
void *userData);
int storage_stop(void* ctx,
StorageCallback callback,
void* userData);
// Close the Logos Storage node.
// Use this to release resources before destroying the node.
//
// Typical usage:
// ctx = storage_new(configJson, myCallback, myUserData);
// storage_start(ctx, ...);
// ...
// storage_stop(ctx, ...);
// storage_close(ctx, ...);
int storage_close(void *ctx,
StorageCallback callback,
void *userData);
int storage_close(void* ctx,
StorageCallback callback,
void* userData);
// Destroys an instance of a Logos Storage node.
// This will free all resources associated with the node.
// The node must be stopped and closed before calling this function.
//
// Typical usage:
// ctx = storage_new(configJson, myCallback, myUserData);
// storage_start(ctx, ...);
// ...
// storage_stop(ctx, ...);
// storage_close(ctx, ...);
// storage_destroy(ctx, ...);
int storage_destroy(void *ctx,
StorageCallback callback,
void *userData);
// Destroys an instance of a Logos Storage node created with storage_new
int storage_destroy(void* ctx,
StorageCallback callback,
void* userData);
void storage_set_event_callback(void* ctx,
StorageCallback callback,
void* userData);
// Not used currently.
// Reserved for future use to set an event callback.
void storage_set_event_callback(void *ctx,
StorageCallback callback,
void *userData);
#ifdef __cplusplus
}