mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-06-26 12:29:30 +00:00
test: adds libstorage wrapper and the convenient scripting
This commit is contained in:
Marcin Czenko
parent
358075cfef
commit
14c68d75c6
2
examples/cpp/.gitignore
vendored
Normal file
2
examples/cpp/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
storage_example
|
||||
cpp-example-data/
|
||||
45
examples/cpp/Makefile
Normal file
45
examples/cpp/Makefile
Normal file
@ -0,0 +1,45 @@
|
||||
# Makefile for building C++ examples against Logos Storage C bindings
|
||||
#
|
||||
# Usage (from this directory):
|
||||
# make
|
||||
#
|
||||
# This produces ./storage_example
|
||||
#
|
||||
# The example links against the library built by the main project:
|
||||
# ../../build/libstorage.so
|
||||
#
|
||||
# You may need to run:
|
||||
# LD_LIBRARY_PATH=../../build ./storage_example
|
||||
#
|
||||
# Or build with rpath so the binary finds the library automatically.
|
||||
|
||||
CXX := g++
|
||||
CXXFLAGS := -std=c++17 -Wall -Wextra -O2 -I../../library
|
||||
LIBSTORAGE := ../../build/libstorage.so
|
||||
LDFLAGS := $(LIBSTORAGE) -pthread
|
||||
|
||||
# Use rpath so the resulting binary can find libstorage.so relative to itself
|
||||
# when run from the examples/cpp directory or nearby.
|
||||
RPATH := -Wl,-rpath,'$$ORIGIN/../../build'
|
||||
|
||||
TARGET := storage_example
|
||||
SRC := storage_example.cpp
|
||||
|
||||
.PHONY: all clean run
|
||||
|
||||
all: $(TARGET)
|
||||
|
||||
$(TARGET): $(SRC) $(LIBSTORAGE)
|
||||
$(CXX) $(CXXFLAGS) $(SRC) -o $(TARGET) $(LDFLAGS) $(RPATH)
|
||||
|
||||
$(LIBSTORAGE):
|
||||
@echo "Missing $(LIBSTORAGE). Run 'make libstorage' from the repository root first." >&2
|
||||
@exit 1
|
||||
|
||||
clean:
|
||||
rm -f $(TARGET)
|
||||
rm -rf cpp-example-data
|
||||
|
||||
# Convenience target: build + run with correct library path
|
||||
run: $(TARGET)
|
||||
LD_LIBRARY_PATH=../../build ./$(TARGET)
|
||||
430
examples/cpp/README.md
Normal file
430
examples/cpp/README.md
Normal file
@ -0,0 +1,430 @@
|
||||
# Using Logos Storage from C and C++
|
||||
|
||||
This directory contains **ready-to-build C++ examples** showing how to use the Logos Storage C bindings from a C++ application.
|
||||
|
||||
Everything here is self-contained under `examples/cpp/`. No existing files in the repository were modified.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# From the repository root, make sure the C library is built
|
||||
make libstorage
|
||||
|
||||
# Go to the C++ example
|
||||
cd examples/cpp
|
||||
|
||||
# Build
|
||||
make
|
||||
|
||||
# Run (the library is loaded from the build directory)
|
||||
LD_LIBRARY_PATH=../../build ./storage_example
|
||||
```
|
||||
|
||||
Or simply:
|
||||
|
||||
```bash
|
||||
make run # (from examples/cpp)
|
||||
```
|
||||
|
||||
## High-Level Mental Model
|
||||
|
||||
The Logos Storage library exposes a **stable C ABI**. All heavy work happens inside Nim on a dedicated worker thread.
|
||||
|
||||
Key concepts:
|
||||
|
||||
- **`void* ctx`** — Opaque handle to a `StorageContext`. You get it from `storage_new()` and pass it to every other call. Think of it as "the node".
|
||||
- **Most APIs are asynchronous** — You call a function (e.g. `storage_start`, `storage_get_metrics`). It returns immediately with a dispatch status (`RET_OK`, `RET_ERR`, ...). The real result arrives later via your `StorageCallback`.
|
||||
- **`StorageCallback`** — A C function pointer with this signature:
|
||||
|
||||
```c
|
||||
typedef void (*StorageCallback)(int callerRet, const char *msg, size_t len, void *userData);
|
||||
```
|
||||
|
||||
- `callerRet` is one of `RET_OK`, `RET_ERR`, `RET_PROGRESS`, `RET_MISSING_CALLBACK`.
|
||||
- `msg` / `len` contain the payload (usually JSON) or an error string.
|
||||
- `userData` is whatever pointer **you** passed in — the library just hands it back.
|
||||
|
||||
- **Callbacks run on the worker thread** — They must be fast and non-blocking. Do not do heavy work, I/O, or call back into libstorage from inside the callback.
|
||||
- **`userData` is caller-owned** — You decide what it points to (a struct, a C++ object, a promise, etc.). The library never frees it.
|
||||
|
||||
## Lifecycle (the only correct order)
|
||||
|
||||
```text
|
||||
storage_new(...) -> gives you ctx (result comes via callback)
|
||||
|
|
||||
v
|
||||
storage_start(ctx, ...) -> start the node (async)
|
||||
|
|
||||
v
|
||||
... do useful work (repo, peer_id, get_metrics, upload, download, etc.) ...
|
||||
|
|
||||
v
|
||||
storage_stop(ctx, ...)
|
||||
storage_close(ctx, ...)
|
||||
storage_destroy(ctx) -> synchronous, no callback needed
|
||||
```
|
||||
|
||||
You can `start` / `stop` the same context multiple times. Always `stop` + `close` before `destroy`.
|
||||
|
||||
## Return Codes
|
||||
|
||||
| Constant | Value | Meaning |
|
||||
|--------------------|-------|---------|
|
||||
| `RET_OK` | 0 | Operation dispatched successfully (or completed for sync calls) |
|
||||
| `RET_ERR` | 1 | Immediate failure or error reported via callback |
|
||||
| `RET_MISSING_CALLBACK` | 2 | You forgot to pass a callback for an async function |
|
||||
| `RET_PROGRESS` | 3 | Intermediate progress (used by upload/download streaming) |
|
||||
|
||||
Only `RET_OK` and `RET_ERR` are terminal for normal calls.
|
||||
|
||||
## Synchronous vs Asynchronous Calls
|
||||
|
||||
**Synchronous (no callback, result returned directly):**
|
||||
|
||||
- `storage_version(ctx)` → `char*` (you must `free()` it)
|
||||
- `storage_revision(ctx)` → `char*` (you must `free()` it)
|
||||
- `storage_destroy(ctx)`
|
||||
|
||||
**Asynchronous (result via callback):**
|
||||
|
||||
Everything else: `storage_start`, `storage_stop`, `storage_repo`, `storage_peer_id`, `storage_get_metrics`, `storage_list`, `storage_space`, upload/download APIs, etc.
|
||||
|
||||
## How to Wait for an Async Result from C++
|
||||
|
||||
The classic C pattern (see `tests/cbindings/storage.c`) uses a `pthread_mutex` + `pthread_cond_t` struct called `Resp`.
|
||||
|
||||
In C++ we do the modern equivalent:
|
||||
|
||||
```cpp
|
||||
class StorageResponse {
|
||||
// mutex + condition_variable
|
||||
// setResult(...) called from the C callback
|
||||
// wait(timeout) on the calling thread
|
||||
};
|
||||
```
|
||||
|
||||
See `storage_example.cpp` for a clean, production-style implementation (`StorageResponse` + `cCallback`).
|
||||
|
||||
Typical call pattern:
|
||||
|
||||
```cpp
|
||||
StorageResponse resp;
|
||||
if (storage_repo(ctx, cCallback, &resp) != RET_OK) { /* dispatch failed */ }
|
||||
|
||||
if (!resp.wait(std::chrono::seconds(30)) || resp.status() != RET_OK) {
|
||||
// handle error
|
||||
}
|
||||
std::cout << "Repo: " << resp.data() << "\n";
|
||||
```
|
||||
|
||||
## Important Rules & Gotchas (especially when coming from other languages)
|
||||
|
||||
1. **Call `libstorageNimMain()` exactly once** before any other function. This initializes the Nim runtime.
|
||||
2. **Free strings returned by `storage_version` / `storage_revision`** with `std::free` (or `free`).
|
||||
3. **Progress callbacks (`RET_PROGRESS`)** are *not* terminal. Only mark completion when you receive `RET_OK` or `RET_ERR`.
|
||||
4. **The worker thread owns the callback invocation.** Any objects you touch from the callback must be thread-safe or protected.
|
||||
5. **JSON is the lingua franca.** Most responses (`debug`, `metrics`, `list`, manifests, etc.) are JSON strings.
|
||||
6. **Metrics are currently process-global.** `storage_get_metrics` returns data from the single `defaultRegistry`. If you create multiple `ctx` instances in the same process they share the same metric set.
|
||||
7. **Config is a JSON string.** See the main `openapi.yaml` or existing tests for the schema. Minimal useful example:
|
||||
|
||||
```json
|
||||
{"log-level":"WARN", "data-dir":"./my-data"}
|
||||
```
|
||||
|
||||
8. **Build & runtime linking.** You must link against `libstorage.so` (or the static `.a`) and make sure the dynamic linker can find it at runtime (`LD_LIBRARY_PATH`, rpath, or install it in a standard location).
|
||||
|
||||
## Detailed Code Walkthrough
|
||||
|
||||
This section walks through `storage_example.cpp` in detail. The goal is to make it easy to understand exactly how a C++ application interacts with the C API, especially if you switch languages frequently.
|
||||
|
||||
The full source is in `storage_example.cpp`. We will go through it section by section.
|
||||
|
||||
### 1. Includes and Nim Runtime Declaration
|
||||
|
||||
```cpp
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
extern "C" {
|
||||
#include "libstorage.h"
|
||||
|
||||
// Forward declaration of the Nim runtime initializer.
|
||||
// Must be called once before any other libstorage call.
|
||||
extern void libstorageNimMain(void);
|
||||
}
|
||||
```
|
||||
|
||||
**Key points:**
|
||||
- We include the C header inside an `extern "C"` block so the C++ compiler knows the functions have C linkage.
|
||||
- `libstorageNimMain()` is declared here because it is not part of the public header (it is generated by Nim). It **must** be called exactly once before using any other API. It initializes the Nim garbage collector and runtime.
|
||||
|
||||
### 2. The `StorageResponse` Class — Bridging the Callback Model
|
||||
|
||||
This is the most important piece for comfortable C++ usage.
|
||||
|
||||
```cpp
|
||||
class StorageResponse {
|
||||
public:
|
||||
StorageResponse() = default;
|
||||
|
||||
// Not copyable (owns synchronization primitives)
|
||||
StorageResponse(const StorageResponse&) = delete;
|
||||
StorageResponse& operator=(const StorageResponse&) = delete;
|
||||
|
||||
// Called from the C callback (runs on the libstorage worker thread).
|
||||
void setResult(int callerRet, const char* msg, size_t len) {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
|
||||
if (msg && len > 0) {
|
||||
result_.assign(msg, len);
|
||||
} else {
|
||||
result_.clear();
|
||||
}
|
||||
status_ = callerRet;
|
||||
done_ = true;
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
bool wait(std::chrono::milliseconds timeout = std::chrono::seconds(60)) {
|
||||
std::unique_lock<std::mutex> lock(mtx_);
|
||||
return cv_.wait_for(lock, timeout, [this] { return done_; });
|
||||
}
|
||||
|
||||
int status() const { ... }
|
||||
std::string data() const { ... }
|
||||
bool isDone() const { ... }
|
||||
|
||||
private:
|
||||
mutable std::mutex mtx_;
|
||||
std::condition_variable cv_;
|
||||
bool done_ = false;
|
||||
int status_ = -1;
|
||||
std::string result_;
|
||||
};
|
||||
```
|
||||
|
||||
**Why this design?**
|
||||
|
||||
- The C API is callback-based. The library calls your `StorageCallback` later, possibly from another thread.
|
||||
- We turn the fire-and-forget + callback model into a simple "call → wait → read result" pattern that feels natural in C++.
|
||||
- `setResult` is called from the worker thread → we protect everything with a mutex.
|
||||
- We **copy** the message into `std::string result_` inside `setResult`. This is important because the `msg` pointer is only valid for the duration of the callback.
|
||||
- Deleted copy constructor/assignment: the class owns a `std::mutex` and `std::condition_variable`, which are not copyable.
|
||||
- `wait()` uses `wait_for` with a timeout as a safety net (the C test harness uses a similar retry-based wait).
|
||||
|
||||
This class is the C++ equivalent of the `Resp` struct + `alloc_resp`/`wait_resp`/`is_resp_ok` functions in `tests/cbindings/storage.c`.
|
||||
|
||||
### 3. The C Callback Adapter
|
||||
|
||||
```cpp
|
||||
static void cCallback(int callerRet, const char* msg, size_t len, void* userData) {
|
||||
if (auto* resp = static_cast<StorageResponse*>(userData)) {
|
||||
resp->setResult(callerRet, msg, len);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This is the actual function we register with every async call.
|
||||
|
||||
- It has the exact signature required by `StorageCallback`.
|
||||
- It casts `userData` back to `StorageResponse*` (the pointer we passed when calling `storage_xxx`).
|
||||
- It is deliberately tiny — remember the rule: **callbacks must be fast and non-blocking**.
|
||||
|
||||
### 4. Small Utility Functions
|
||||
|
||||
```cpp
|
||||
static bool isOk(int ret) { return ret == RET_OK; }
|
||||
|
||||
static void printSection(const std::string& title) { ... }
|
||||
|
||||
static void printJsonExcerpt(const std::string& json, size_t maxLen = 300) { ... }
|
||||
```
|
||||
|
||||
These are just for readability. `isOk` makes the main logic easier to follow. The print helpers keep the output clean during the example run.
|
||||
|
||||
### 5. `main()` — Step by Step
|
||||
|
||||
#### Step 5.1: Runtime initialization and node creation
|
||||
|
||||
```cpp
|
||||
int main() {
|
||||
libstorageNimMain(); // Required first step
|
||||
|
||||
const char* configJson = "{\"log-level\":\"INFO\","
|
||||
"\"data-dir\":\"./cpp-example-data\","
|
||||
"\"metrics\":false}";
|
||||
|
||||
StorageResponse newResp;
|
||||
void* ctx = storage_new(configJson, cCallback, &newResp);
|
||||
|
||||
if (!ctx) { /* handle immediate failure */ }
|
||||
|
||||
if (!newResp.wait()) { /* timeout */ }
|
||||
if (!isOk(newResp.status())) { /* creation failed */ }
|
||||
```
|
||||
|
||||
Important observations:
|
||||
- `storage_new` returns the context pointer **synchronously**, but the actual initialization result comes through the callback (just like in the C test).
|
||||
- We pass `&newResp` as `userData`. The library will call `cCallback(..., &newResp)`.
|
||||
- Always check the immediate return value **and** the result that arrives via the callback.
|
||||
|
||||
#### Step 5.2: Starting the node
|
||||
|
||||
```cpp
|
||||
StorageResponse startResp;
|
||||
if (!isOk(storage_start(ctx, cCallback, &startResp))) {
|
||||
// dispatch failed immediately
|
||||
storage_destroy(ctx);
|
||||
return 1;
|
||||
}
|
||||
if (!startResp.wait() || !isOk(startResp.status())) {
|
||||
// start failed asynchronously
|
||||
}
|
||||
```
|
||||
|
||||
Note how we create a **fresh** `StorageResponse` for every async operation. This is the recommended pattern.
|
||||
|
||||
#### Step 5.3: Synchronous calls
|
||||
|
||||
```cpp
|
||||
char* ver = storage_version(ctx);
|
||||
if (ver) {
|
||||
std::cout << "Version: " << ver << "\n";
|
||||
std::free(ver); // Important: caller must free
|
||||
}
|
||||
```
|
||||
|
||||
Only a few functions are synchronous:
|
||||
- `storage_version`
|
||||
- `storage_revision`
|
||||
- `storage_destroy`
|
||||
|
||||
Everything else goes through the callback mechanism.
|
||||
|
||||
#### Step 5.4: Async information queries (the common pattern)
|
||||
|
||||
```cpp
|
||||
// Repository
|
||||
StorageResponse repoResp;
|
||||
if (!isOk(storage_repo(ctx, cCallback, &repoResp))) {
|
||||
// dispatch error
|
||||
} else if (repoResp.wait() && isOk(repoResp.status())) {
|
||||
std::cout << "Repo: " << repoResp.data() << "\n";
|
||||
}
|
||||
|
||||
// Same pattern for peer_id and metrics
|
||||
StorageResponse metricsResp;
|
||||
storage_get_metrics(ctx, cCallback, &metricsResp);
|
||||
metricsResp.wait();
|
||||
printJsonExcerpt(metricsResp.data());
|
||||
```
|
||||
|
||||
This is the core usage pattern you will repeat for almost every API:
|
||||
1. Create a `StorageResponse` on the stack.
|
||||
2. Call the `storage_*` function, passing `cCallback` and `&yourResp`.
|
||||
3. Check the immediate return code.
|
||||
4. Call `.wait()`.
|
||||
5. Check `.status()` and read `.data()`.
|
||||
|
||||
#### Step 5.5: Metrics specifically
|
||||
|
||||
The `storage_get_metrics` call returns JSON in the Logos openmetrics format (with `name`, `type`, `help`, `value`, `labels`). The example prints an excerpt and does a simple sanity check for known metrics.
|
||||
|
||||
#### Step 5.6: Shutdown sequence
|
||||
|
||||
```cpp
|
||||
StorageResponse stopResp;
|
||||
if (isOk(storage_stop(ctx, cCallback, &stopResp))) {
|
||||
stopResp.wait();
|
||||
}
|
||||
|
||||
StorageResponse closeResp;
|
||||
if (isOk(storage_close(ctx, cCallback, &closeResp))) {
|
||||
closeResp.wait();
|
||||
}
|
||||
|
||||
if (storage_destroy(ctx) != RET_OK) { ... }
|
||||
```
|
||||
|
||||
Order matters:
|
||||
- `stop` (async)
|
||||
- `close` (async)
|
||||
- `destroy` (synchronous — no callback)
|
||||
|
||||
You should always stop + close before destroy.
|
||||
|
||||
### Design Observations & Lessons
|
||||
|
||||
- **One `StorageResponse` per operation** — Reusing the same object for multiple calls is possible but error-prone (you would need to reset `done_`). Creating a new one per call is clearer and safer.
|
||||
- **No RAII wrapper for `ctx`** in this example — We do manual lifecycle to keep the example simple and explicit. In real code you would probably wrap `ctx` in a class that calls `stop`/`close`/`destroy` in its destructor.
|
||||
- **Error handling is deliberately repetitive** — This mirrors the reality of the C API. In a larger project you would likely build a small wrapper layer on top of `StorageResponse`.
|
||||
- **Threading model is hidden but important** — Your main thread mostly blocks on condition variables. The real work and the callbacks happen on the libstorage worker thread.
|
||||
- **Memory ownership** — Strings returned by `storage_version`/`storage_revision` must be freed by you. Data delivered via callbacks is copied by `StorageResponse`, so you don't have to worry about the original buffer lifetime.
|
||||
|
||||
This pattern (small response object + single C callback trampoline + per-call instances) is very close to what the original C test harness does, just expressed in idiomatic C++.
|
||||
|
||||
## Illustrative Snippets (Quick Reference)
|
||||
|
||||
(kept for quick copy-paste)
|
||||
|
||||
### Creating a node and starting it
|
||||
|
||||
```cpp
|
||||
libstorageNimMain();
|
||||
|
||||
const char* cfg = R"({"log-level":"INFO","data-dir":"./data"})";
|
||||
|
||||
StorageResponse initResp;
|
||||
void* ctx = storage_new(cfg, cCallback, &initResp);
|
||||
initResp.wait();
|
||||
|
||||
StorageResponse startResp;
|
||||
storage_start(ctx, cCallback, &startResp);
|
||||
startResp.wait();
|
||||
```
|
||||
|
||||
### Getting metrics
|
||||
|
||||
```cpp
|
||||
StorageResponse m;
|
||||
storage_get_metrics(ctx, cCallback, &m);
|
||||
m.wait();
|
||||
|
||||
if (m.status() == RET_OK) {
|
||||
// m.data() contains JSON with name, type, help, value, labels
|
||||
}
|
||||
```
|
||||
|
||||
### Clean shutdown
|
||||
|
||||
```cpp
|
||||
storage_stop(ctx, cCallback, &stopResp); stopResp.wait();
|
||||
storage_close(ctx, cCallback, &closeResp); closeResp.wait();
|
||||
storage_destroy(ctx); // synchronous
|
||||
```
|
||||
|
||||
## Directory Contents
|
||||
|
||||
- `storage_example.cpp` — Full working C++ program with `StorageResponse` helper
|
||||
- `Makefile` — Simple, copy-paste friendly build system
|
||||
- `README.md` — This file (mental model + detailed code walkthrough)
|
||||
|
||||
## Further Reading (from the main tree)
|
||||
|
||||
- `library/libstorage.h` — The authoritative C header (all signatures and comments).
|
||||
- `tests/cbindings/storage.c` — The reference C implementation. This is the best place to see the exact waiting + callback pattern the C++ code is modeled after.
|
||||
- `library/storage_context.nim` and `library/storage_thread_requests/storage_thread_request.nim` — If you want to understand the worker thread and "callbacks run on the worker thread" rule.
|
||||
|
||||
## Notes for Polyglot Developers
|
||||
|
||||
If you are coming from other languages:
|
||||
|
||||
- The binding is intentionally thin. Most complexity lives in Nim.
|
||||
- The async + callback + userData model is very common in C libraries that have to cross thread boundaries (similar to many libuv-style or Java JNI callback patterns).
|
||||
- For production use you will almost certainly want a small C++ wrapper class that owns the `ctx` and provides RAII + futures or coroutines.
|
||||
|
||||
Happy hacking!
|
||||
271
examples/cpp/storage_example.cpp
Normal file
271
examples/cpp/storage_example.cpp
Normal file
@ -0,0 +1,271 @@
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
|
||||
extern "C" {
|
||||
#include "libstorage.h"
|
||||
|
||||
// Forward declaration of the Nim runtime initializer.
|
||||
// Must be called once before any other libstorage call.
|
||||
extern void libstorageNimMain(void);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// C++-friendly response helper.
|
||||
// Mirrors the pattern from tests/cbindings/storage.c but uses modern C++.
|
||||
// The C callback receives a pointer to an instance of this class as userData.
|
||||
// -----------------------------------------------------------------------------
|
||||
class StorageResponse {
|
||||
public:
|
||||
StorageResponse() = default;
|
||||
|
||||
// Not copyable (owns synchronization primitives)
|
||||
StorageResponse(const StorageResponse&) = delete;
|
||||
StorageResponse& operator=(const StorageResponse&) = delete;
|
||||
|
||||
// Called from the C callback (runs on the libstorage worker thread).
|
||||
// We copy the data here so the main thread can safely read it later.
|
||||
void setResult(int callerRet, const char* msg, size_t len) {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
|
||||
if (callerRet == RET_PROGRESS) {
|
||||
progressCount_ += 1;
|
||||
if (msg && len > 0) {
|
||||
lastProgress_.assign(msg, len);
|
||||
} else {
|
||||
lastProgress_.clear();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg && len > 0) {
|
||||
result_.assign(msg, len);
|
||||
} else {
|
||||
result_.clear();
|
||||
}
|
||||
status_ = callerRet;
|
||||
done_ = true;
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
// Block until the operation completes or timeout.
|
||||
// Returns true if we got a terminal response (RET_OK or RET_ERR).
|
||||
bool wait(std::chrono::milliseconds timeout = std::chrono::seconds(60)) {
|
||||
std::unique_lock<std::mutex> lock(mtx_);
|
||||
return cv_.wait_for(lock, timeout, [this] { return done_; });
|
||||
}
|
||||
|
||||
int status() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return status_;
|
||||
}
|
||||
|
||||
// Returns the payload (JSON for most info calls, error text on error).
|
||||
std::string data() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return result_;
|
||||
}
|
||||
|
||||
bool isDone() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return done_;
|
||||
}
|
||||
|
||||
size_t progressCount() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return progressCount_;
|
||||
}
|
||||
|
||||
std::string lastProgress() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return lastProgress_;
|
||||
}
|
||||
|
||||
private:
|
||||
mutable std::mutex mtx_;
|
||||
std::condition_variable cv_;
|
||||
bool done_ = false;
|
||||
int status_ = -1;
|
||||
std::string result_;
|
||||
size_t progressCount_ = 0;
|
||||
std::string lastProgress_;
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// The single C callback that the library will invoke for every async operation.
|
||||
// We store the caller's StorageResponse* in userData.
|
||||
// This runs on the internal worker thread — keep it fast and non-blocking.
|
||||
// -----------------------------------------------------------------------------
|
||||
static void cCallback(int callerRet, const char* msg, size_t len, void* userData) {
|
||||
if (auto* resp = static_cast<StorageResponse*>(userData)) {
|
||||
resp->setResult(callerRet, msg, len);
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// Small helpers
|
||||
// -----------------------------------------------------------------------------
|
||||
static bool isOk(int ret) {
|
||||
return ret == RET_OK;
|
||||
}
|
||||
|
||||
static void printSection(const std::string& title) {
|
||||
std::cout << "\n=== " << title << " ===\n";
|
||||
}
|
||||
|
||||
static void printJsonExcerpt(const std::string& json, size_t maxLen = 300) {
|
||||
if (json.empty()) {
|
||||
std::cout << "(empty)\n";
|
||||
return;
|
||||
}
|
||||
if (json.size() <= maxLen) {
|
||||
std::cout << json << "\n";
|
||||
} else {
|
||||
std::cout << json.substr(0, maxLen) << "...\n";
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
// main - demonstrates a typical C++ client usage pattern
|
||||
// -----------------------------------------------------------------------------
|
||||
int main() {
|
||||
// 1. Initialize the Nim runtime. Must be done once per process.
|
||||
libstorageNimMain();
|
||||
|
||||
printSection("Creating storage node");
|
||||
|
||||
// Use a dedicated data directory for this example.
|
||||
const char* configJson =
|
||||
"{\"log-level\":\"INFO\","
|
||||
"\"data-dir\":\"./cpp-example-data\","
|
||||
"\"metrics\":false}";
|
||||
|
||||
StorageResponse newResp;
|
||||
void* ctx = storage_new(configJson, cCallback, &newResp);
|
||||
|
||||
if (!ctx) {
|
||||
std::cerr << "storage_new returned null context\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!newResp.wait()) {
|
||||
std::cerr << "Timed out waiting for storage_new\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!isOk(newResp.status())) {
|
||||
std::cerr << "Failed to create node: " << newResp.data() << "\n";
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "Node context created successfully.\n";
|
||||
|
||||
// 2. Start the node (async)
|
||||
printSection("Starting node");
|
||||
|
||||
StorageResponse startResp;
|
||||
if (!isOk(storage_start(ctx, cCallback, &startResp))) {
|
||||
std::cerr << "storage_start dispatch failed\n";
|
||||
storage_destroy(ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!startResp.wait()) {
|
||||
std::cerr << "Timed out waiting for start\n";
|
||||
storage_destroy(ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!isOk(startResp.status())) {
|
||||
std::cerr << "Start failed: " << startResp.data() << "\n";
|
||||
storage_destroy(ctx);
|
||||
return 1;
|
||||
}
|
||||
|
||||
std::cout << "Node started.\n";
|
||||
|
||||
// 3. Synchronous calls (no callback needed)
|
||||
printSection("Version / Revision (synchronous)");
|
||||
|
||||
char* ver = storage_version(ctx);
|
||||
if (ver) {
|
||||
std::cout << "Version: " << ver << "\n";
|
||||
std::free(ver);
|
||||
}
|
||||
|
||||
char* rev = storage_revision(ctx);
|
||||
if (rev) {
|
||||
std::cout << "Revision: " << rev << "\n";
|
||||
std::free(rev);
|
||||
}
|
||||
|
||||
// 4. Async info queries (very common pattern)
|
||||
printSection("Repository (data-dir)");
|
||||
|
||||
StorageResponse repoResp;
|
||||
if (!isOk(storage_repo(ctx, cCallback, &repoResp))) {
|
||||
std::cerr << "storage_repo dispatch failed\n";
|
||||
} else if (repoResp.wait() && isOk(repoResp.status())) {
|
||||
std::cout << "Repo: " << repoResp.data() << "\n";
|
||||
} else {
|
||||
std::cerr << "Failed to get repo: " << repoResp.data() << "\n";
|
||||
}
|
||||
|
||||
printSection("Peer ID");
|
||||
|
||||
StorageResponse peerResp;
|
||||
if (!isOk(storage_peer_id(ctx, cCallback, &peerResp))) {
|
||||
std::cerr << "storage_peer_id dispatch failed\n";
|
||||
} else if (peerResp.wait() && isOk(peerResp.status())) {
|
||||
std::cout << "PeerId: " << peerResp.data() << "\n";
|
||||
} else {
|
||||
std::cerr << "Failed to get peer id: " << peerResp.data() << "\n";
|
||||
}
|
||||
|
||||
printSection("Metrics (storage_get_metrics)");
|
||||
|
||||
StorageResponse metricsResp;
|
||||
if (!isOk(storage_get_metrics(ctx, cCallback, &metricsResp))) {
|
||||
std::cerr << "storage_get_metrics dispatch failed\n";
|
||||
} else if (metricsResp.wait() && isOk(metricsResp.status())) {
|
||||
std::cout << "Metrics JSON excerpt:\n";
|
||||
printJsonExcerpt(metricsResp.data());
|
||||
// A very basic sanity check that we got something useful
|
||||
if (metricsResp.data().find("libp2p_") == std::string::npos) {
|
||||
std::cout << "(Note: did not see a libp2p metric in the response)\n";
|
||||
}
|
||||
} else {
|
||||
std::cerr << "Failed to get metrics: " << metricsResp.data() << "\n";
|
||||
}
|
||||
|
||||
// 5. Clean shutdown sequence (stop + close + destroy)
|
||||
printSection("Stopping node");
|
||||
|
||||
StorageResponse stopResp;
|
||||
if (isOk(storage_stop(ctx, cCallback, &stopResp))) {
|
||||
stopResp.wait();
|
||||
}
|
||||
|
||||
printSection("Closing node");
|
||||
|
||||
StorageResponse closeResp;
|
||||
if (isOk(storage_close(ctx, cCallback, &closeResp))) {
|
||||
closeResp.wait();
|
||||
}
|
||||
|
||||
printSection("Destroying node");
|
||||
|
||||
// storage_destroy is synchronous and does not use a callback.
|
||||
if (storage_destroy(ctx) != RET_OK) {
|
||||
std::cerr << "storage_destroy reported an error\n";
|
||||
} else {
|
||||
std::cout << "Node destroyed.\n";
|
||||
}
|
||||
|
||||
printSection("Done");
|
||||
std::cout << "Example finished successfully.\n";
|
||||
return 0;
|
||||
}
|
||||
@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
|
||||
BINARY="${STORAGE_BINARY:-${ROOT_DIR}/build/storage}"
|
||||
DATA_DIR="${STORAGE_DATA_DIR:-${HOME}/.logos/storage/local-node}"
|
||||
LOG_LEVEL="${STORAGE_LOG_LEVEL:-info}"
|
||||
LISTEN_PORT="${STORAGE_LISTEN_PORT:-8071}"
|
||||
DISC_PORT="${STORAGE_DISC_PORT:-8091}"
|
||||
API_BINDADDR="${STORAGE_API_BINDADDR:-127.0.0.1}"
|
||||
API_PORT="${STORAGE_API_PORT:-8080}"
|
||||
NETWORK="${STORAGE_NETWORK:-logos.test}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [options] [-- extra storage args]
|
||||
|
||||
Start a local Logos Storage node in the foreground so logs are visible.
|
||||
|
||||
Options:
|
||||
--binary <path> Storage binary [$BINARY]
|
||||
--data-dir <path> Data directory [$DATA_DIR]
|
||||
--log-level <level> Log level: trace, debug, info, notice, warn, error [$LOG_LEVEL]
|
||||
--listen-port <port> Local libp2p TCP listen port [$LISTEN_PORT]
|
||||
--disc-port <port> Local discovery UDP port [$DISC_PORT]
|
||||
--api-bindaddr <ip> REST API bind address [$API_BINDADDR]
|
||||
--api-port <port> REST API port [$API_PORT]
|
||||
--network <name> Network preset [$NETWORK]
|
||||
-h, --help Show this help.
|
||||
|
||||
Environment overrides:
|
||||
STORAGE_BINARY, STORAGE_DATA_DIR, STORAGE_LOG_LEVEL, STORAGE_LISTEN_PORT,
|
||||
STORAGE_DISC_PORT, STORAGE_API_BINDADDR, STORAGE_API_PORT, STORAGE_NETWORK
|
||||
|
||||
Examples:
|
||||
$0
|
||||
$0 --log-level debug
|
||||
$0 --data-dir /tmp/logos-storage-local --listen-port 8072 --disc-port 8092
|
||||
$0 --log-level trace -- --metrics --metrics-address=127.0.0.1
|
||||
|
||||
The node runs in the foreground. Press Ctrl-C to stop it.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--binary)
|
||||
BINARY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--data-dir)
|
||||
DATA_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--log-level)
|
||||
LOG_LEVEL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--listen-port)
|
||||
LISTEN_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--disc-port)
|
||||
DISC_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--api-bindaddr)
|
||||
API_BINDADDR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--api-port)
|
||||
API_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--network)
|
||||
NETWORK="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
printf 'error: unknown option: %s\n\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -n "$BINARY" ]] || { printf 'error: --binary cannot be empty\n' >&2; exit 1; }
|
||||
[[ -x "$BINARY" ]] || { printf 'error: storage binary not executable: %s\n' "$BINARY" >&2; exit 1; }
|
||||
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
printf 'Starting local Logos Storage node\n'
|
||||
printf ' binary: %s\n' "$BINARY"
|
||||
printf ' data dir: %s\n' "$DATA_DIR"
|
||||
printf ' log level: %s\n' "$LOG_LEVEL"
|
||||
printf ' listen TCP: %s\n' "$LISTEN_PORT"
|
||||
printf ' discovery: %s/udp\n' "$DISC_PORT"
|
||||
printf ' REST API: %s:%s\n' "$API_BINDADDR" "$API_PORT"
|
||||
printf ' network: %s\n' "$NETWORK"
|
||||
printf '\n'
|
||||
|
||||
exec "$BINARY" \
|
||||
--data-dir="$DATA_DIR" \
|
||||
--log-level="$LOG_LEVEL" \
|
||||
--listen-port="$LISTEN_PORT" \
|
||||
--disc-port="$DISC_PORT" \
|
||||
--api-bindaddr="$API_BINDADDR" \
|
||||
--api-port="$API_PORT" \
|
||||
--network="$NETWORK" \
|
||||
"$@"
|
||||
@ -1,369 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE_SSH_HOST="${REMOTE_SSH_HOST:-storage@172.235.163.25}"
|
||||
REMOTE_API_PORT="${REMOTE_API_PORT:-18080}"
|
||||
LOCAL_API_PORT="${LOCAL_API_PORT:-8080}"
|
||||
TUNNEL_CONTROL_PATH="${TUNNEL_CONTROL_PATH:-/tmp/logos-storage-tunnel-${USER:-user}.ctl}"
|
||||
CID_STATE_FILE="${CID_STATE_FILE:-${HOME}/.logos/storage/test/cids.log}"
|
||||
TEST_FILES_DIR="${TEST_FILES_DIR:-${HOME}/.logos/storage/test/files}"
|
||||
|
||||
LOCAL_API="http://127.0.0.1:${LOCAL_API_PORT}/api/storage/v1"
|
||||
REMOTE_API="http://127.0.0.1:${REMOTE_API_PORT}/api/storage/v1"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 <command> [options]
|
||||
|
||||
Environment:
|
||||
REMOTE_SSH_HOST SSH host for the remote node [$REMOTE_SSH_HOST]
|
||||
REMOTE_API_PORT Local tunnel port for remote API [$REMOTE_API_PORT]
|
||||
LOCAL_API_PORT Local node API port [$LOCAL_API_PORT]
|
||||
TUNNEL_CONTROL_PATH SSH control socket path [$TUNNEL_CONTROL_PATH]
|
||||
CID_STATE_FILE Upload history log [$CID_STATE_FILE]
|
||||
TEST_FILES_DIR Default generated test file directory [$TEST_FILES_DIR]
|
||||
|
||||
Targets:
|
||||
local API at $LOCAL_API
|
||||
remote API through SSH tunnel at $REMOTE_API
|
||||
|
||||
Commands:
|
||||
help
|
||||
Show this help.
|
||||
|
||||
tunnel start|stop|status
|
||||
Manage SSH tunnel to the remote node API.
|
||||
|
||||
make-file <size> [output-file]
|
||||
Create random content with dd. Example: make-file 10M /tmp/logos-10M.bin
|
||||
|
||||
upload <target> <file>
|
||||
Upload a file to target node and print the returned CID. Appends CID to
|
||||
CID_STATE_FILE. Example: upload remote /tmp/logos-10M.bin
|
||||
|
||||
upload-random <target> <size> [--keep]
|
||||
Create a temporary random file, upload it, print CID. Deletes temp file
|
||||
unless --keep is passed. Example: upload-random remote 10M
|
||||
|
||||
last-cid [target]
|
||||
Print the most recent CID from CID_STATE_FILE, optionally filtered by
|
||||
target.
|
||||
|
||||
list <target>
|
||||
List manifest CIDs stored locally by target node.
|
||||
|
||||
delete <target> <cid>
|
||||
Delete CID from target node local storage.
|
||||
|
||||
delete-all <target> --yes
|
||||
Delete every CID returned by list from target node local storage.
|
||||
|
||||
exists <target> <cid>
|
||||
Check whether target node has CID locally.
|
||||
|
||||
space <target>
|
||||
Show target node storage space information.
|
||||
|
||||
peerid <target>
|
||||
Show target node peer ID.
|
||||
|
||||
fetch-local <cid> [--wait]
|
||||
Ask local node to fetch CID from the network. With --wait, poll progress
|
||||
until the background download is inactive or complete.
|
||||
|
||||
stream-local <cid> <output-file>
|
||||
Stream CID from the network through local node into output-file.
|
||||
|
||||
Examples:
|
||||
$0 tunnel start
|
||||
$0 upload-random remote 10M
|
||||
$0 fetch-local <CID> --wait
|
||||
$0 stream-local <CID> /tmp/downloaded.bin
|
||||
$0 delete remote <CID>
|
||||
$0 delete-all local --yes
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'error: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
|
||||
}
|
||||
|
||||
check_common_deps() {
|
||||
need curl
|
||||
need jq
|
||||
}
|
||||
|
||||
tunnel_status() {
|
||||
ssh -S "$TUNNEL_CONTROL_PATH" -O check "$REMOTE_SSH_HOST" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
tunnel_start() {
|
||||
need ssh
|
||||
if tunnel_status; then
|
||||
printf 'tunnel already running: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \
|
||||
"$REMOTE_API_PORT" "$REMOTE_SSH_HOST"
|
||||
return 0
|
||||
fi
|
||||
|
||||
ssh \
|
||||
-M \
|
||||
-S "$TUNNEL_CONTROL_PATH" \
|
||||
-fN \
|
||||
-L "127.0.0.1:${REMOTE_API_PORT}:127.0.0.1:8080" \
|
||||
"$REMOTE_SSH_HOST"
|
||||
|
||||
printf 'started tunnel: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \
|
||||
"$REMOTE_API_PORT" "$REMOTE_SSH_HOST"
|
||||
}
|
||||
|
||||
tunnel_stop() {
|
||||
need ssh
|
||||
if tunnel_status; then
|
||||
ssh -S "$TUNNEL_CONTROL_PATH" -O exit "$REMOTE_SSH_HOST" >/dev/null
|
||||
printf 'stopped tunnel\n'
|
||||
else
|
||||
printf 'tunnel not running\n'
|
||||
fi
|
||||
}
|
||||
|
||||
target_api() {
|
||||
case "${1:-}" in
|
||||
local)
|
||||
printf '%s\n' "$LOCAL_API"
|
||||
;;
|
||||
remote)
|
||||
tunnel_start >/dev/null
|
||||
printf '%s\n' "$REMOTE_API"
|
||||
;;
|
||||
*)
|
||||
die "target must be 'local' or 'remote'"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
make_file() {
|
||||
local size="${1:-}"
|
||||
local out="${2:-}"
|
||||
[[ -n "$size" ]] || die 'make-file requires <size>'
|
||||
if [[ -z "$out" ]]; then
|
||||
mkdir -p "$TEST_FILES_DIR"
|
||||
out="${TEST_FILES_DIR}/logos-test-${size}-$(date -u +%Y%m%dT%H%M%SZ).bin"
|
||||
fi
|
||||
dd if=/dev/urandom of="$out" bs="$size" count=1 status=progress
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
upload_file() {
|
||||
check_common_deps
|
||||
local target="${1:-}"
|
||||
local file="${2:-}"
|
||||
[[ -n "$file" ]] || die 'upload requires <target> <file>'
|
||||
[[ -f "$file" ]] || die "file not found: $file"
|
||||
[[ -z "${3:-}" ]] || die 'upload does not accept extra options'
|
||||
|
||||
local api cid
|
||||
api="$(target_api "$target")"
|
||||
cid="$(curl -fsS \
|
||||
-H 'Content-Type: application/octet-stream' \
|
||||
--data-binary "@${file}" \
|
||||
"${api}/data")"
|
||||
|
||||
printf '%s\n' "$cid"
|
||||
mkdir -p "$(dirname "$CID_STATE_FILE")"
|
||||
printf '%s %s %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$target" "$cid" "$file" >> "$CID_STATE_FILE"
|
||||
}
|
||||
|
||||
upload_random() {
|
||||
local target="${1:-}"
|
||||
local size="${2:-}"
|
||||
local keep=false
|
||||
[[ -n "$target" && -n "$size" ]] || die 'upload-random requires <target> <size> [--keep]'
|
||||
|
||||
shift 2
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--keep)
|
||||
keep=true
|
||||
;;
|
||||
*)
|
||||
die "unknown upload-random option: $1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
local tmp cid
|
||||
tmp="$(mktemp "${TMPDIR:-/tmp}/logos-storage-test.XXXXXX")"
|
||||
dd if=/dev/urandom of="$tmp" bs="$size" count=1 status=progress >&2
|
||||
cid="$(upload_file "$target" "$tmp")"
|
||||
printf '%s\n' "$cid"
|
||||
|
||||
if [[ "$keep" == true ]]; then
|
||||
printf 'kept file: %s\n' "$tmp" >&2
|
||||
else
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
}
|
||||
|
||||
last_cid() {
|
||||
local target=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
local|remote)
|
||||
[[ -z "$target" ]] || die 'target specified more than once'
|
||||
target="$1"
|
||||
;;
|
||||
*)
|
||||
die "unknown last-cid option: $1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[[ -f "$CID_STATE_FILE" ]] || die "CID state file not found: $CID_STATE_FILE"
|
||||
|
||||
local cid
|
||||
if [[ -n "$target" ]]; then
|
||||
cid="$(awk -v target="$target" '$2 == target { cid = $3 } END { print cid }' "$CID_STATE_FILE")"
|
||||
else
|
||||
cid="$(awk 'NF >= 3 { cid = $3 } END { print cid }' "$CID_STATE_FILE")"
|
||||
fi
|
||||
|
||||
[[ -n "$cid" ]] || die 'no matching CID found'
|
||||
printf '%s\n' "$cid"
|
||||
}
|
||||
|
||||
list_cids() {
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "${1:-}")"
|
||||
curl -fsS "${api}/data" | jq -r '.content[]?.cid'
|
||||
}
|
||||
|
||||
delete_cid() {
|
||||
check_common_deps
|
||||
local target="${1:-}"
|
||||
local cid="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'delete requires <target> <cid>'
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS -X DELETE "${api}/data/${cid}" >/dev/null
|
||||
printf 'deleted %s from %s\n' "$cid" "$target"
|
||||
}
|
||||
|
||||
delete_all() {
|
||||
local target="${1:-}"
|
||||
local yes="${2:-}"
|
||||
[[ "$yes" == '--yes' ]] || die 'delete-all requires --yes'
|
||||
|
||||
local cid count=0
|
||||
while IFS= read -r cid; do
|
||||
[[ -n "$cid" ]] || continue
|
||||
delete_cid "$target" "$cid"
|
||||
count=$((count + 1))
|
||||
done < <(list_cids "$target")
|
||||
printf 'deleted %d CID(s) from %s\n' "$count" "$target"
|
||||
}
|
||||
|
||||
simple_get() {
|
||||
check_common_deps
|
||||
local target="${1:-}"
|
||||
local path="${2:-}"
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS "${api}/${path}"
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
exists_cid() {
|
||||
local target="${1:-}"
|
||||
local cid="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'exists requires <target> <cid>'
|
||||
simple_get "$target" "data/${cid}/exists"
|
||||
}
|
||||
|
||||
fetch_local() {
|
||||
check_common_deps
|
||||
local cid="${1:-}"
|
||||
local wait="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'fetch-local requires <cid> [--wait]'
|
||||
[[ -z "$wait" || "$wait" == '--wait' ]] || die 'only supported option is --wait'
|
||||
|
||||
local response download_id
|
||||
response="$(curl -fsS -X POST "${LOCAL_API}/data/${cid}/network")"
|
||||
printf '%s\n' "$response" | jq .
|
||||
|
||||
if [[ "$wait" != '--wait' ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
download_id="$(printf '%s\n' "$response" | jq -r '.downloadId // empty')"
|
||||
[[ -n "$download_id" ]] || die 'response did not contain downloadId'
|
||||
|
||||
while true; do
|
||||
local progress active received total
|
||||
progress="$(curl -fsS "${LOCAL_API}/data/${cid}/network/progress/${download_id}")"
|
||||
printf '%s\n' "$progress" | jq .
|
||||
active="$(printf '%s\n' "$progress" | jq -r '.active')"
|
||||
received="$(printf '%s\n' "$progress" | jq -r '.received // 0')"
|
||||
total="$(printf '%s\n' "$progress" | jq -r '.total // 0')"
|
||||
if [[ "$active" != 'true' || ( "$total" != '0' && "$received" == "$total" ) ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
stream_local() {
|
||||
local cid="${1:-}"
|
||||
local out="${2:-}"
|
||||
[[ -n "$cid" && -n "$out" ]] || die 'stream-local requires <cid> <output-file>'
|
||||
curl -fL "${LOCAL_API}/data/${cid}/network/stream" -o "$out"
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
cmd="${1:-help}"
|
||||
shift || true
|
||||
|
||||
case "$cmd" in
|
||||
help|--help|-h)
|
||||
usage
|
||||
;;
|
||||
tunnel)
|
||||
case "${1:-}" in
|
||||
start) tunnel_start ;;
|
||||
stop) tunnel_stop ;;
|
||||
status)
|
||||
if tunnel_status; then
|
||||
printf 'tunnel running\n'
|
||||
else
|
||||
printf 'tunnel stopped\n'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*) die 'usage: tunnel start|stop|status' ;;
|
||||
esac
|
||||
;;
|
||||
make-file) make_file "$@" ;;
|
||||
upload) upload_file "$@" ;;
|
||||
upload-random) upload_random "$@" ;;
|
||||
last-cid) last_cid "$@" ;;
|
||||
list) list_cids "$@" ;;
|
||||
delete) delete_cid "$@" ;;
|
||||
delete-all) delete_all "$@" ;;
|
||||
exists) exists_cid "$@" ;;
|
||||
space) simple_get "${1:-}" 'space' ;;
|
||||
peerid) simple_get "${1:-}" 'peerid' ;;
|
||||
fetch-local) fetch_local "$@" ;;
|
||||
stream-local) stream_local "$@" ;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
12
tools/libstorage-cpp/.gitignore
vendored
Normal file
12
tools/libstorage-cpp/.gitignore
vendored
Normal file
@ -0,0 +1,12 @@
|
||||
storage_lib_cli
|
||||
storage_lib
|
||||
storage_lib_ctl
|
||||
storage-lib-cli-data/
|
||||
storage-lib-data/
|
||||
storage_lib.sock
|
||||
daemon.log
|
||||
downloaded*
|
||||
*.download
|
||||
*.download.*
|
||||
*.roundtrip
|
||||
*.out.*
|
||||
34
tools/libstorage-cpp/Makefile
Normal file
34
tools/libstorage-cpp/Makefile
Normal file
@ -0,0 +1,34 @@
|
||||
CXX := g++
|
||||
CXXFLAGS := -std=c++17 -Wall -Wextra -O2 -I../../library
|
||||
LIBSTORAGE := ../../build/libstorage.so
|
||||
LDFLAGS := $(LIBSTORAGE) -pthread
|
||||
RPATH := -Wl,-rpath,'$$ORIGIN/../../build'
|
||||
|
||||
TARGETS := storage_lib_cli storage_lib storage_lib_ctl
|
||||
COMMON_SRCS := storage_client.cpp
|
||||
|
||||
.PHONY: all clean run
|
||||
|
||||
all: $(TARGETS)
|
||||
|
||||
storage_lib_cli: storage_lib_cli.cpp $(COMMON_SRCS) storage_client.hpp $(LIBSTORAGE)
|
||||
$(CXX) $(CXXFLAGS) storage_lib_cli.cpp $(COMMON_SRCS) -o $@ $(LDFLAGS) $(RPATH)
|
||||
|
||||
storage_lib: storage_lib.cpp $(COMMON_SRCS) storage_client.hpp $(LIBSTORAGE)
|
||||
$(CXX) $(CXXFLAGS) storage_lib.cpp $(COMMON_SRCS) -o $@ $(LDFLAGS) $(RPATH)
|
||||
|
||||
storage_lib_ctl: storage_lib_ctl.cpp
|
||||
$(CXX) $(CXXFLAGS) storage_lib_ctl.cpp -o $@
|
||||
|
||||
$(LIBSTORAGE):
|
||||
@echo "Missing $(LIBSTORAGE). Run 'make libstorage' from the repository root first." >&2
|
||||
@exit 1
|
||||
|
||||
run: storage_lib_cli
|
||||
LD_LIBRARY_PATH=../../build ./storage_lib_cli info
|
||||
|
||||
clean:
|
||||
rm -f $(TARGETS)
|
||||
rm -rf storage-lib-cli-data storage-lib-data
|
||||
rm -f storage_lib.sock daemon.log
|
||||
rm -f downloaded* *.download *.download.* *.roundtrip *.out.*
|
||||
111
tools/libstorage-cpp/README.md
Normal file
111
tools/libstorage-cpp/README.md
Normal file
@ -0,0 +1,111 @@
|
||||
# libstorage C++ CLI
|
||||
|
||||
This directory contains C++ tools around `library/libstorage.h`.
|
||||
|
||||
- `storage_lib_cli`: one-shot command-line wrapper.
|
||||
- `storage_lib`: daemon-like libstorage process using Unix socket IPC.
|
||||
- `storage_lib_ctl`: small client that sends one command to `storage_lib`.
|
||||
|
||||
Build libstorage first from the repository root:
|
||||
|
||||
```bash
|
||||
make libstorage
|
||||
```
|
||||
|
||||
If the local compiler hits the known secp256k1 `-march=native` issue on Linux amd64, use:
|
||||
|
||||
```bash
|
||||
make libstorage NIMFLAGS="-d:disableMarchNative"
|
||||
```
|
||||
|
||||
Then build the CLI:
|
||||
|
||||
```bash
|
||||
cd tools/libstorage-cpp
|
||||
make
|
||||
```
|
||||
|
||||
`--timeout-ms 0` means wait indefinitely for libstorage async operations.
|
||||
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
./storage_lib_cli info
|
||||
./storage_lib_cli spr
|
||||
./storage_lib_cli debug
|
||||
./storage_lib_cli list
|
||||
./storage_lib_cli upload ./payload.txt
|
||||
./storage_lib_cli --local download <cid> ./downloaded.txt
|
||||
./storage_lib_cli --local roundtrip ./payload.txt ./payload.roundtrip
|
||||
./storage_lib_cli --local repeat-roundtrip ./payload.txt ./payload.out 10
|
||||
./storage_lib_cli upload-many ./payload.txt 10
|
||||
./storage_lib_cli --local download-many <cid> ./payload.download 10
|
||||
```
|
||||
|
||||
Options must come before the command. Some libstorage networking diagnostics are
|
||||
currently printed directly by the library, so scripts should treat the final line
|
||||
as the command result for commands such as `upload`.
|
||||
|
||||
Use `make clean` to remove the binary and the default data directory.
|
||||
|
||||
## Daemon Mode
|
||||
|
||||
Start the libstorage daemon:
|
||||
|
||||
```bash
|
||||
./storage_lib \
|
||||
--socket ./storage_lib.sock \
|
||||
--data-dir ./storage-lib-data \
|
||||
--log-level WARN \
|
||||
--listen-port 8071 \
|
||||
--disc-port 8091 \
|
||||
--network logos.test \
|
||||
--timeout-ms 0
|
||||
```
|
||||
|
||||
Send commands with `storage_lib_ctl`:
|
||||
|
||||
```bash
|
||||
./storage_lib_ctl --socket ./storage_lib.sock info
|
||||
./storage_lib_ctl --socket ./storage_lib.sock spr
|
||||
./storage_lib_ctl --socket ./storage_lib.sock debug
|
||||
./storage_lib_ctl --socket ./storage_lib.sock upload README.md
|
||||
./storage_lib_ctl --socket ./storage_lib.sock manifest <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock exists <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock download <cid> ./downloaded.md true
|
||||
./storage_lib_ctl --socket ./storage_lib.sock delete <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock shutdown
|
||||
```
|
||||
|
||||
If `--socket` is omitted, `storage_lib_ctl` uses `STORAGE_LIB_SOCKET` when set,
|
||||
otherwise `~/.logos/storage/libstorage/storage_lib.sock`.
|
||||
|
||||
The IPC request protocol is one whitespace-separated command line per connection.
|
||||
Paths with spaces are not supported yet. Responses are one JSON line:
|
||||
|
||||
```json
|
||||
{"ok":true,"result":"..."}
|
||||
```
|
||||
|
||||
or:
|
||||
|
||||
```json
|
||||
{"ok":false,"error":"..."}
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
- `info`: prints version, revision, repo, and peer id.
|
||||
- `spr`: prints the node signed peer record.
|
||||
- `debug`: prints debug JSON.
|
||||
- `connect <peer-id> [addr...]`: connects to a peer using optional multiaddresses.
|
||||
- `manifest <cid>`: prints manifest JSON.
|
||||
- `delete <cid>`: deletes locally stored content.
|
||||
- `fetch <cid>`: fetches content into the local store.
|
||||
- `roundtrip <in> <out>`: uploads one file, downloads it, compares bytes, and prints the cid.
|
||||
- `repeat-roundtrip <in> <out-prefix> <count>`: repeats roundtrip in one node session.
|
||||
- `upload-many <file> <count>`: uploads the same file repeatedly in one node session.
|
||||
- `download-many <cid> <out-prefix> <count>`: downloads the same cid repeatedly in one node session.
|
||||
|
||||
Options must appear before the command, for example `--local roundtrip`, not
|
||||
`roundtrip --local`.
|
||||
289
tools/libstorage-cpp/storage_client.cpp
Normal file
289
tools/libstorage-cpp/storage_client.cpp
Normal file
@ -0,0 +1,289 @@
|
||||
#include "storage_client.hpp"
|
||||
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <utility>
|
||||
|
||||
extern "C" {
|
||||
extern void libstorageNimMain(void);
|
||||
}
|
||||
|
||||
namespace {
|
||||
void ensureNimRuntime() {
|
||||
static std::once_flag once;
|
||||
std::call_once(once, [] { libstorageNimMain(); });
|
||||
}
|
||||
|
||||
bool isOk(int ret) {
|
||||
return ret == RET_OK;
|
||||
}
|
||||
|
||||
std::string takeCString(char* value) {
|
||||
if (!value) {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string result(value);
|
||||
std::free(value);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
void StorageResponse::setResult(int callerRet, const char* msg, size_t len) {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
|
||||
if (callerRet == RET_PROGRESS) {
|
||||
progressCount_ += 1;
|
||||
if (msg && len > 0) {
|
||||
lastProgress_.assign(msg, len);
|
||||
} else {
|
||||
lastProgress_.clear();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (msg && len > 0) {
|
||||
result_.assign(msg, len);
|
||||
} else {
|
||||
result_.clear();
|
||||
}
|
||||
|
||||
status_ = callerRet;
|
||||
done_ = true;
|
||||
cv_.notify_one();
|
||||
}
|
||||
|
||||
bool StorageResponse::wait(std::chrono::milliseconds timeout) {
|
||||
std::unique_lock<std::mutex> lock(mtx_);
|
||||
if (timeout.count() == 0) {
|
||||
cv_.wait(lock, [this] { return done_; });
|
||||
return true;
|
||||
}
|
||||
return cv_.wait_for(lock, timeout, [this] { return done_; });
|
||||
}
|
||||
|
||||
int StorageResponse::status() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return status_;
|
||||
}
|
||||
|
||||
std::string StorageResponse::data() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return result_;
|
||||
}
|
||||
|
||||
size_t StorageResponse::progressCount() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return progressCount_;
|
||||
}
|
||||
|
||||
std::string StorageResponse::lastProgress() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return lastProgress_;
|
||||
}
|
||||
|
||||
StorageClient::StorageClient(std::string configJson, std::chrono::milliseconds timeout)
|
||||
: timeout_(timeout) {
|
||||
ensureNimRuntime();
|
||||
|
||||
StorageResponse resp;
|
||||
ctx_ = storage_new(configJson.c_str(), callback, &resp);
|
||||
if (!ctx_) {
|
||||
throw std::runtime_error("storage_new returned null context");
|
||||
}
|
||||
|
||||
if (!resp.wait(timeout_)) {
|
||||
storage_destroy(ctx_);
|
||||
ctx_ = nullptr;
|
||||
throw std::runtime_error("storage_new timed out");
|
||||
}
|
||||
|
||||
if (!isOk(resp.status())) {
|
||||
std::string error = resp.data();
|
||||
storage_destroy(ctx_);
|
||||
ctx_ = nullptr;
|
||||
throw std::runtime_error("storage_new failed: " + error);
|
||||
}
|
||||
}
|
||||
|
||||
StorageClient::~StorageClient() {
|
||||
if (!ctx_) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
if (started_) {
|
||||
stop();
|
||||
}
|
||||
if (!closed_) {
|
||||
close();
|
||||
}
|
||||
} catch (...) {
|
||||
// Destructors must not throw; callers should use explicit stop/close for errors.
|
||||
}
|
||||
|
||||
storage_destroy(ctx_);
|
||||
}
|
||||
|
||||
void StorageClient::start() {
|
||||
call("storage_start", [this](StorageCallback cb, void* userData) {
|
||||
return storage_start(ctx_, cb, userData);
|
||||
});
|
||||
started_ = true;
|
||||
}
|
||||
|
||||
void StorageClient::stop() {
|
||||
call("storage_stop", [this](StorageCallback cb, void* userData) {
|
||||
return storage_stop(ctx_, cb, userData);
|
||||
});
|
||||
started_ = false;
|
||||
}
|
||||
|
||||
void StorageClient::close() {
|
||||
call("storage_close", [this](StorageCallback cb, void* userData) {
|
||||
return storage_close(ctx_, cb, userData);
|
||||
});
|
||||
closed_ = true;
|
||||
}
|
||||
|
||||
std::string StorageClient::version() const {
|
||||
return takeCString(storage_version(ctx_));
|
||||
}
|
||||
|
||||
std::string StorageClient::revision() const {
|
||||
return takeCString(storage_revision(ctx_));
|
||||
}
|
||||
|
||||
std::string StorageClient::repo() {
|
||||
return call("storage_repo", [this](StorageCallback cb, void* userData) {
|
||||
return storage_repo(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::peerId() {
|
||||
return call("storage_peer_id", [this](StorageCallback cb, void* userData) {
|
||||
return storage_peer_id(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::metrics() {
|
||||
return call("storage_get_metrics", [this](StorageCallback cb, void* userData) {
|
||||
return storage_get_metrics(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::debug() {
|
||||
return call("storage_debug", [this](StorageCallback cb, void* userData) {
|
||||
return storage_debug(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::spr() {
|
||||
return call("storage_spr", [this](StorageCallback cb, void* userData) {
|
||||
return storage_spr(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::list() {
|
||||
return call("storage_list", [this](StorageCallback cb, void* userData) {
|
||||
return storage_list(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::space() {
|
||||
return call("storage_space", [this](StorageCallback cb, void* userData) {
|
||||
return storage_space(ctx_, cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::manifest(const std::string& cid) {
|
||||
return call("storage_download_manifest", [this, &cid](StorageCallback cb, void* userData) {
|
||||
return storage_download_manifest(ctx_, cid.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::exists(const std::string& cid) {
|
||||
return call("storage_exists", [this, &cid](StorageCallback cb, void* userData) {
|
||||
return storage_exists(ctx_, cid.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::deleteContent(const std::string& cid) {
|
||||
return call("storage_delete", [this, &cid](StorageCallback cb, void* userData) {
|
||||
return storage_delete(ctx_, cid.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::fetch(const std::string& cid) {
|
||||
return call("storage_fetch", [this, &cid](StorageCallback cb, void* userData) {
|
||||
return storage_fetch(ctx_, cid.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::connect(
|
||||
const std::string& peerId,
|
||||
const std::vector<std::string>& peerAddresses) {
|
||||
std::vector<const char*> addresses;
|
||||
addresses.reserve(peerAddresses.size());
|
||||
for (const auto& address : peerAddresses) {
|
||||
addresses.push_back(address.c_str());
|
||||
}
|
||||
|
||||
return call("storage_connect", [this, &peerId, &addresses](StorageCallback cb, void* userData) {
|
||||
return storage_connect(ctx_, peerId.c_str(), addresses.data(), addresses.size(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::uploadFile(const std::string& filepath, size_t chunkSize) {
|
||||
const std::string sessionId = call("storage_upload_init", [this, &filepath, chunkSize](
|
||||
StorageCallback cb,
|
||||
void* userData) {
|
||||
return storage_upload_init(ctx_, filepath.c_str(), chunkSize, cb, userData);
|
||||
});
|
||||
|
||||
return call("storage_upload_file", [this, &sessionId](StorageCallback cb, void* userData) {
|
||||
return storage_upload_file(ctx_, sessionId.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::downloadFile(
|
||||
const std::string& cid,
|
||||
const std::string& outputPath,
|
||||
size_t chunkSize,
|
||||
bool local) {
|
||||
call("storage_download_init", [this, &cid, chunkSize, local](StorageCallback cb, void* userData) {
|
||||
return storage_download_init(ctx_, cid.c_str(), chunkSize, local, cb, userData);
|
||||
});
|
||||
|
||||
return call("storage_download_stream", [this, &cid, &outputPath, chunkSize, local](
|
||||
StorageCallback cb,
|
||||
void* userData) {
|
||||
return storage_download_stream(
|
||||
ctx_, cid.c_str(), chunkSize, local, outputPath.c_str(), cb, userData);
|
||||
});
|
||||
}
|
||||
|
||||
std::string StorageClient::call(const char* name, const AsyncCall& fn) {
|
||||
StorageResponse resp;
|
||||
const int dispatchRet = fn(callback, &resp);
|
||||
if (!isOk(dispatchRet)) {
|
||||
throw std::runtime_error(std::string(name) + " dispatch failed");
|
||||
}
|
||||
|
||||
if (!resp.wait(timeout_)) {
|
||||
throw std::runtime_error(std::string(name) + " timed out");
|
||||
}
|
||||
|
||||
if (!isOk(resp.status())) {
|
||||
throw std::runtime_error(std::string(name) + " failed: " + resp.data());
|
||||
}
|
||||
|
||||
return resp.data();
|
||||
}
|
||||
|
||||
void StorageClient::callback(int callerRet, const char* msg, size_t len, void* userData) {
|
||||
if (auto* resp = static_cast<StorageResponse*>(userData)) {
|
||||
resp->setResult(callerRet, msg, len);
|
||||
}
|
||||
}
|
||||
82
tools/libstorage-cpp/storage_client.hpp
Normal file
82
tools/libstorage-cpp/storage_client.hpp
Normal file
@ -0,0 +1,82 @@
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
extern "C" {
|
||||
#include "libstorage.h"
|
||||
}
|
||||
|
||||
class StorageResponse {
|
||||
public:
|
||||
StorageResponse() = default;
|
||||
StorageResponse(const StorageResponse&) = delete;
|
||||
StorageResponse& operator=(const StorageResponse&) = delete;
|
||||
|
||||
void setResult(int callerRet, const char* msg, size_t len);
|
||||
bool wait(std::chrono::milliseconds timeout = std::chrono::seconds(60));
|
||||
|
||||
int status() const;
|
||||
std::string data() const;
|
||||
size_t progressCount() const;
|
||||
std::string lastProgress() const;
|
||||
|
||||
private:
|
||||
mutable std::mutex mtx_;
|
||||
std::condition_variable cv_;
|
||||
bool done_ = false;
|
||||
int status_ = -1;
|
||||
std::string result_;
|
||||
size_t progressCount_ = 0;
|
||||
std::string lastProgress_;
|
||||
};
|
||||
|
||||
class StorageClient {
|
||||
public:
|
||||
explicit StorageClient(
|
||||
std::string configJson,
|
||||
std::chrono::milliseconds timeout = std::chrono::seconds(120));
|
||||
StorageClient(const StorageClient&) = delete;
|
||||
StorageClient& operator=(const StorageClient&) = delete;
|
||||
~StorageClient();
|
||||
|
||||
void start();
|
||||
void stop();
|
||||
void close();
|
||||
|
||||
std::string version() const;
|
||||
std::string revision() const;
|
||||
std::string repo();
|
||||
std::string peerId();
|
||||
std::string metrics();
|
||||
std::string debug();
|
||||
std::string spr();
|
||||
std::string list();
|
||||
std::string space();
|
||||
std::string manifest(const std::string& cid);
|
||||
std::string exists(const std::string& cid);
|
||||
std::string deleteContent(const std::string& cid);
|
||||
std::string fetch(const std::string& cid);
|
||||
std::string connect(const std::string& peerId, const std::vector<std::string>& peerAddresses);
|
||||
std::string uploadFile(const std::string& filepath, size_t chunkSize);
|
||||
std::string downloadFile(
|
||||
const std::string& cid,
|
||||
const std::string& outputPath,
|
||||
size_t chunkSize,
|
||||
bool local);
|
||||
|
||||
private:
|
||||
using AsyncCall = std::function<int(StorageCallback, void*)>;
|
||||
|
||||
std::string call(const char* name, const AsyncCall& fn);
|
||||
static void callback(int callerRet, const char* msg, size_t len, void* userData);
|
||||
|
||||
void* ctx_ = nullptr;
|
||||
std::chrono::milliseconds timeout_;
|
||||
bool started_ = false;
|
||||
bool closed_ = false;
|
||||
};
|
||||
338
tools/libstorage-cpp/storage_lib.cpp
Normal file
338
tools/libstorage-cpp/storage_lib.cpp
Normal file
@ -0,0 +1,338 @@
|
||||
#include "storage_client.hpp"
|
||||
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <csignal>
|
||||
#include <cstddef>
|
||||
#include <cstring>
|
||||
#include <exception>
|
||||
#include <filesystem>
|
||||
#include <iostream>
|
||||
#include <sstream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
constexpr size_t DefaultChunkSize = 64 * 1024;
|
||||
|
||||
struct Options {
|
||||
std::string socketPath;
|
||||
std::string dataDir;
|
||||
std::string logLevel = "WARN";
|
||||
std::string network = "logos.test";
|
||||
uint16_t listenPort = 8071;
|
||||
uint16_t discPort = 8091;
|
||||
size_t chunkSize = DefaultChunkSize;
|
||||
std::chrono::milliseconds timeout = std::chrono::seconds(120);
|
||||
};
|
||||
|
||||
std::atomic<bool> g_stop{false};
|
||||
int g_serverFd = -1;
|
||||
|
||||
std::string homeDir() {
|
||||
const char* home = std::getenv("HOME");
|
||||
if (!home || std::strlen(home) == 0) {
|
||||
throw std::runtime_error("HOME is not set");
|
||||
}
|
||||
return home;
|
||||
}
|
||||
|
||||
std::string defaultSocketPath() {
|
||||
return homeDir() + "/.logos/storage/libstorage/storage_lib.sock";
|
||||
}
|
||||
|
||||
std::string defaultDataDir() {
|
||||
return homeDir() + "/.logos/storage/libstorage/node";
|
||||
}
|
||||
|
||||
std::string jsonEscape(const std::string& value) {
|
||||
std::string out;
|
||||
out.reserve(value.size() + 8);
|
||||
for (char ch : value) {
|
||||
switch (ch) {
|
||||
case '\\': out += "\\\\"; break;
|
||||
case '"': out += "\\\""; break;
|
||||
case '\n': out += "\\n"; break;
|
||||
case '\r': out += "\\r"; break;
|
||||
case '\t': out += "\\t"; break;
|
||||
default: out += ch; break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string okResponse(const std::string& result) {
|
||||
return "{\"ok\":true,\"result\":\"" + jsonEscape(result) + "\"}\n";
|
||||
}
|
||||
|
||||
std::string errorResponse(const std::string& error) {
|
||||
return "{\"ok\":false,\"error\":\"" + jsonEscape(error) + "\"}\n";
|
||||
}
|
||||
|
||||
std::string configJson(const Options& options) {
|
||||
return "{\"log-level\":\"" + jsonEscape(options.logLevel) + "\"," +
|
||||
"\"data-dir\":\"" + jsonEscape(options.dataDir) + "\"," +
|
||||
"\"network\":\"" + jsonEscape(options.network) + "\"," +
|
||||
"\"listen-port\":" + std::to_string(options.listenPort) + "," +
|
||||
"\"disc-port\":" + std::to_string(options.discPort) + "," +
|
||||
"\"metrics\":false}";
|
||||
}
|
||||
|
||||
size_t parseSize(const std::string& value) {
|
||||
size_t pos = 0;
|
||||
const auto parsed = std::stoull(value, &pos);
|
||||
if (pos != value.size()) {
|
||||
throw std::runtime_error("invalid size: " + value);
|
||||
}
|
||||
return static_cast<size_t>(parsed);
|
||||
}
|
||||
|
||||
uint16_t parsePort(const std::string& value) {
|
||||
const size_t parsed = parseSize(value);
|
||||
if (parsed == 0 || parsed > 65535) {
|
||||
throw std::runtime_error("invalid port: " + value);
|
||||
}
|
||||
return static_cast<uint16_t>(parsed);
|
||||
}
|
||||
|
||||
bool parseBool(const std::string& value) {
|
||||
if (value == "true" || value == "1" || value == "yes") {
|
||||
return true;
|
||||
}
|
||||
if (value == "false" || value == "0" || value == "no") {
|
||||
return false;
|
||||
}
|
||||
throw std::runtime_error("invalid boolean: " + value);
|
||||
}
|
||||
|
||||
std::vector<std::string> splitLine(const std::string& line) {
|
||||
std::istringstream input(line);
|
||||
std::vector<std::string> parts;
|
||||
std::string part;
|
||||
while (input >> part) {
|
||||
parts.push_back(part);
|
||||
}
|
||||
return parts;
|
||||
}
|
||||
|
||||
void printUsage() {
|
||||
std::cout <<
|
||||
"Usage:\n"
|
||||
" storage_lib [options]\n\n"
|
||||
"Options:\n"
|
||||
" --socket <path> Unix socket path\n"
|
||||
" --data-dir <path> Node data directory\n"
|
||||
" --log-level <level> TRACE, DEBUG, INFO, NOTICE, WARN, ERROR, FATAL\n"
|
||||
" --listen-port <port> Local libp2p TCP listen port (default: 8071)\n"
|
||||
" --disc-port <port> Local discovery UDP port (default: 8091)\n"
|
||||
" --network <name> Network preset (default: logos.test)\n"
|
||||
" --chunk-size <bytes> Upload/download chunk size (default: 65536)\n"
|
||||
" --timeout-ms <ms> Async operation timeout, 0 waits forever (default: 120000)\n"
|
||||
" -h, --help Show this help\n";
|
||||
}
|
||||
|
||||
Options parseArgs(int argc, char** argv) {
|
||||
Options options;
|
||||
options.socketPath = defaultSocketPath();
|
||||
options.dataDir = defaultDataDir();
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string arg = argv[i];
|
||||
if (arg == "--help" || arg == "-h") {
|
||||
printUsage();
|
||||
std::exit(0);
|
||||
} else if (arg == "--socket" && i + 1 < argc) {
|
||||
options.socketPath = argv[++i];
|
||||
} else if (arg == "--data-dir" && i + 1 < argc) {
|
||||
options.dataDir = argv[++i];
|
||||
} else if (arg == "--log-level" && i + 1 < argc) {
|
||||
options.logLevel = argv[++i];
|
||||
} else if (arg == "--listen-port" && i + 1 < argc) {
|
||||
options.listenPort = parsePort(argv[++i]);
|
||||
} else if (arg == "--disc-port" && i + 1 < argc) {
|
||||
options.discPort = parsePort(argv[++i]);
|
||||
} else if (arg == "--network" && i + 1 < argc) {
|
||||
options.network = argv[++i];
|
||||
} else if (arg == "--chunk-size" && i + 1 < argc) {
|
||||
options.chunkSize = parseSize(argv[++i]);
|
||||
} else if (arg == "--timeout-ms" && i + 1 < argc) {
|
||||
options.timeout = std::chrono::milliseconds(parseSize(argv[++i]));
|
||||
} else {
|
||||
throw std::runtime_error("unknown or incomplete option: " + arg);
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
std::string infoResult(StorageClient& client) {
|
||||
return "{\"version\":\"" + jsonEscape(client.version()) + "\"," +
|
||||
"\"revision\":\"" + jsonEscape(client.revision()) + "\"," +
|
||||
"\"repo\":\"" + jsonEscape(client.repo()) + "\"," +
|
||||
"\"peer_id\":\"" + jsonEscape(client.peerId()) + "\"}";
|
||||
}
|
||||
|
||||
std::string dispatch(StorageClient& client, const Options& options, const std::string& line) {
|
||||
const auto parts = splitLine(line);
|
||||
if (parts.empty()) {
|
||||
throw std::runtime_error("empty command");
|
||||
}
|
||||
|
||||
const auto& cmd = parts[0];
|
||||
if (cmd == "info") return infoResult(client);
|
||||
if (cmd == "version") return client.version();
|
||||
if (cmd == "revision") return client.revision();
|
||||
if (cmd == "repo") return client.repo();
|
||||
if (cmd == "peer-id" || cmd == "peerid") return client.peerId();
|
||||
if (cmd == "spr") return client.spr();
|
||||
if (cmd == "debug") return client.debug();
|
||||
if (cmd == "metrics") return client.metrics();
|
||||
if (cmd == "list") return client.list();
|
||||
if (cmd == "space") return client.space();
|
||||
|
||||
if (cmd == "upload") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: upload <file>");
|
||||
return client.uploadFile(parts[1], options.chunkSize);
|
||||
}
|
||||
if (cmd == "download") {
|
||||
if (parts.size() < 3 || parts.size() > 4) {
|
||||
throw std::runtime_error("usage: download <cid> <file> [local]");
|
||||
}
|
||||
const bool local = parts.size() == 4 ? parseBool(parts[3]) : false;
|
||||
return client.downloadFile(parts[1], parts[2], options.chunkSize, local);
|
||||
}
|
||||
if (cmd == "exists") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: exists <cid>");
|
||||
return client.exists(parts[1]);
|
||||
}
|
||||
if (cmd == "delete") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: delete <cid>");
|
||||
return client.deleteContent(parts[1]);
|
||||
}
|
||||
if (cmd == "fetch") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: fetch <cid>");
|
||||
return client.fetch(parts[1]);
|
||||
}
|
||||
if (cmd == "manifest") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: manifest <cid>");
|
||||
return client.manifest(parts[1]);
|
||||
}
|
||||
if (cmd == "connect") {
|
||||
if (parts.size() < 2) throw std::runtime_error("usage: connect <peer-id> [addr...]");
|
||||
std::vector<std::string> addresses(parts.begin() + 2, parts.end());
|
||||
return client.connect(parts[1], addresses);
|
||||
}
|
||||
if (cmd == "shutdown") {
|
||||
g_stop = true;
|
||||
return "shutting down";
|
||||
}
|
||||
|
||||
throw std::runtime_error("unknown command: " + cmd);
|
||||
}
|
||||
|
||||
std::string readLine(int fd) {
|
||||
std::string line;
|
||||
char ch = '\0';
|
||||
while (true) {
|
||||
ssize_t n = ::read(fd, &ch, 1);
|
||||
if (n == 0) break;
|
||||
if (n < 0) throw std::runtime_error(std::string("read failed: ") + std::strerror(errno));
|
||||
if (ch == '\n') break;
|
||||
line += ch;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
|
||||
void writeAll(int fd, const std::string& data) {
|
||||
const char* ptr = data.data();
|
||||
size_t left = data.size();
|
||||
while (left > 0) {
|
||||
ssize_t n = ::write(fd, ptr, left);
|
||||
if (n < 0) throw std::runtime_error(std::string("write failed: ") + std::strerror(errno));
|
||||
ptr += n;
|
||||
left -= static_cast<size_t>(n);
|
||||
}
|
||||
}
|
||||
|
||||
int createServerSocket(const std::string& socketPath) {
|
||||
if (socketPath.size() >= sizeof(sockaddr_un::sun_path)) {
|
||||
throw std::runtime_error("socket path is too long: " + socketPath);
|
||||
}
|
||||
|
||||
std::filesystem::create_directories(std::filesystem::path(socketPath).parent_path());
|
||||
::unlink(socketPath.c_str());
|
||||
|
||||
int fd = ::socket(AF_UNIX, SOCK_STREAM, 0);
|
||||
if (fd < 0) throw std::runtime_error(std::string("socket failed: ") + std::strerror(errno));
|
||||
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
std::strncpy(addr.sun_path, socketPath.c_str(), sizeof(addr.sun_path) - 1);
|
||||
|
||||
if (::bind(fd, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) {
|
||||
::close(fd);
|
||||
throw std::runtime_error(std::string("bind failed: ") + std::strerror(errno));
|
||||
}
|
||||
if (::listen(fd, 16) < 0) {
|
||||
::close(fd);
|
||||
throw std::runtime_error(std::string("listen failed: ") + std::strerror(errno));
|
||||
}
|
||||
|
||||
return fd;
|
||||
}
|
||||
|
||||
void handleSignal(int) {
|
||||
g_stop = true;
|
||||
if (g_serverFd >= 0) {
|
||||
::close(g_serverFd);
|
||||
g_serverFd = -1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
const Options options = parseArgs(argc, argv);
|
||||
|
||||
std::signal(SIGINT, handleSignal);
|
||||
std::signal(SIGTERM, handleSignal);
|
||||
|
||||
StorageClient client(configJson(options), options.timeout);
|
||||
client.start();
|
||||
|
||||
g_serverFd = createServerSocket(options.socketPath);
|
||||
std::cout << "storage_lib listening on " << options.socketPath << "\n";
|
||||
std::cout.flush();
|
||||
|
||||
while (!g_stop) {
|
||||
int clientFd = ::accept(g_serverFd, nullptr, nullptr);
|
||||
if (clientFd < 0) {
|
||||
if (g_stop) break;
|
||||
if (errno == EINTR) continue;
|
||||
throw std::runtime_error(std::string("accept failed: ") + std::strerror(errno));
|
||||
}
|
||||
|
||||
try {
|
||||
const std::string line = readLine(clientFd);
|
||||
writeAll(clientFd, okResponse(dispatch(client, options, line)));
|
||||
} catch (const std::exception& err) {
|
||||
writeAll(clientFd, errorResponse(err.what()));
|
||||
}
|
||||
::close(clientFd);
|
||||
}
|
||||
|
||||
if (g_serverFd >= 0) {
|
||||
::close(g_serverFd);
|
||||
g_serverFd = -1;
|
||||
}
|
||||
::unlink(options.socketPath.c_str());
|
||||
return 0;
|
||||
} catch (const std::exception& err) {
|
||||
std::cerr << "error: " << err.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
315
tools/libstorage-cpp/storage_lib_cli.cpp
Normal file
315
tools/libstorage-cpp/storage_lib_cli.cpp
Normal file
@ -0,0 +1,315 @@
|
||||
#include "storage_client.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <chrono>
|
||||
#include <cstddef>
|
||||
#include <exception>
|
||||
#include <fstream>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
constexpr size_t DefaultChunkSize = 64 * 1024;
|
||||
|
||||
struct Options {
|
||||
std::string dataDir = "./storage-lib-cli-data";
|
||||
std::string logLevel = "WARN";
|
||||
size_t chunkSize = DefaultChunkSize;
|
||||
std::chrono::milliseconds timeout = std::chrono::seconds(120);
|
||||
bool local = false;
|
||||
bool noStart = false;
|
||||
std::string command;
|
||||
std::vector<std::string> args;
|
||||
};
|
||||
|
||||
std::string jsonEscape(const std::string& value) {
|
||||
std::string out;
|
||||
out.reserve(value.size() + 8);
|
||||
for (char ch : value) {
|
||||
switch (ch) {
|
||||
case '\\':
|
||||
out += "\\\\";
|
||||
break;
|
||||
case '"':
|
||||
out += "\\\"";
|
||||
break;
|
||||
case '\n':
|
||||
out += "\\n";
|
||||
break;
|
||||
case '\r':
|
||||
out += "\\r";
|
||||
break;
|
||||
case '\t':
|
||||
out += "\\t";
|
||||
break;
|
||||
default:
|
||||
out += ch;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string configJson(const Options& options) {
|
||||
return "{\"log-level\":\"" + jsonEscape(options.logLevel) + "\"," +
|
||||
"\"data-dir\":\"" + jsonEscape(options.dataDir) + "\"," +
|
||||
"\"metrics\":false}";
|
||||
}
|
||||
|
||||
void printUsage() {
|
||||
std::cout <<
|
||||
"Usage:\n"
|
||||
" storage_lib_cli [options] <command> [args]\n\n"
|
||||
"Options:\n"
|
||||
" --data-dir <path> Node data directory (default: ./storage-lib-cli-data)\n"
|
||||
" --log-level <level> TRACE, DEBUG, INFO, NOTICE, WARN, ERROR, FATAL\n"
|
||||
" --chunk-size <bytes> Upload/download chunk size (default: 65536)\n"
|
||||
" --timeout-ms <ms> Async operation timeout (default: 120000)\n"
|
||||
" --local Download from local store only\n"
|
||||
" --no-start Create context but do not start node before command\n\n"
|
||||
"Commands:\n"
|
||||
" info Print version, revision, repo, and peer id\n"
|
||||
" version Print storage version\n"
|
||||
" revision Print storage revision\n"
|
||||
" repo Print configured repo/data-dir\n"
|
||||
" peer-id Print node peer id\n"
|
||||
" spr Print node signed peer record\n"
|
||||
" debug Print node debug JSON\n"
|
||||
" connect <peer-id> [addr...]\n"
|
||||
" Connect to peer using optional multiaddresses\n"
|
||||
" metrics Print metrics JSON\n"
|
||||
" list Print local manifest list JSON\n"
|
||||
" space Print storage space JSON\n"
|
||||
" manifest <cid> Print manifest JSON\n"
|
||||
" exists <cid> Check whether cid exists locally\n"
|
||||
" delete <cid> Delete locally stored content\n"
|
||||
" fetch <cid> Fetch content into the local store\n"
|
||||
" upload <file> Upload a file and print its cid\n"
|
||||
" download <cid> <file> Download cid into file\n"
|
||||
" roundtrip <in> <out> Upload, download, compare, and print cid\n"
|
||||
" repeat-roundtrip <in> <out-prefix> <count>\n"
|
||||
" Repeat roundtrip in one node session\n"
|
||||
" upload-many <file> <count>\n"
|
||||
" Upload same file repeatedly in one node session\n"
|
||||
" download-many <cid> <out-prefix> <count>\n"
|
||||
" Download same cid repeatedly in one node session\n";
|
||||
}
|
||||
|
||||
size_t parseSize(const std::string& value) {
|
||||
size_t pos = 0;
|
||||
const auto parsed = std::stoull(value, &pos);
|
||||
if (pos != value.size()) {
|
||||
throw std::runtime_error("invalid size: " + value);
|
||||
}
|
||||
return static_cast<size_t>(parsed);
|
||||
}
|
||||
|
||||
int parseCount(const std::string& value) {
|
||||
const size_t parsed = parseSize(value);
|
||||
if (parsed == 0) {
|
||||
throw std::runtime_error("count must be greater than zero");
|
||||
}
|
||||
return static_cast<int>(parsed);
|
||||
}
|
||||
|
||||
Options parseArgs(int argc, char** argv) {
|
||||
Options options;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string arg = argv[i];
|
||||
if (arg == "--help" || arg == "-h") {
|
||||
options.command = "help";
|
||||
return options;
|
||||
}
|
||||
if (arg == "--data-dir" && i + 1 < argc) {
|
||||
options.dataDir = argv[++i];
|
||||
} else if (arg == "--log-level" && i + 1 < argc) {
|
||||
options.logLevel = argv[++i];
|
||||
} else if (arg == "--chunk-size" && i + 1 < argc) {
|
||||
options.chunkSize = parseSize(argv[++i]);
|
||||
} else if (arg == "--timeout-ms" && i + 1 < argc) {
|
||||
options.timeout = std::chrono::milliseconds(parseSize(argv[++i]));
|
||||
} else if (arg == "--local") {
|
||||
options.local = true;
|
||||
} else if (arg == "--no-start") {
|
||||
options.noStart = true;
|
||||
} else if (!arg.empty() && arg[0] == '-') {
|
||||
throw std::runtime_error("unknown option: " + arg);
|
||||
} else {
|
||||
options.command = arg;
|
||||
for (++i; i < argc; ++i) {
|
||||
options.args.emplace_back(argv[i]);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
options.command = "help";
|
||||
return options;
|
||||
}
|
||||
|
||||
void requireArgCount(const Options& options, size_t count) {
|
||||
if (options.args.size() != count) {
|
||||
throw std::runtime_error("command '" + options.command + "' expects " +
|
||||
std::to_string(count) + " argument(s)");
|
||||
}
|
||||
}
|
||||
|
||||
bool filesEqual(const std::string& leftPath, const std::string& rightPath) {
|
||||
std::ifstream left(leftPath, std::ios::binary);
|
||||
std::ifstream right(rightPath, std::ios::binary);
|
||||
if (!left || !right) {
|
||||
return false;
|
||||
}
|
||||
|
||||
constexpr size_t BufferSize = 64 * 1024;
|
||||
std::vector<char> leftBuffer(BufferSize);
|
||||
std::vector<char> rightBuffer(BufferSize);
|
||||
while (left && right) {
|
||||
left.read(leftBuffer.data(), leftBuffer.size());
|
||||
right.read(rightBuffer.data(), rightBuffer.size());
|
||||
if (left.gcount() != right.gcount()) {
|
||||
return false;
|
||||
}
|
||||
if (!std::equal(
|
||||
leftBuffer.begin(),
|
||||
leftBuffer.begin() + left.gcount(),
|
||||
rightBuffer.begin())) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return left.eof() && right.eof();
|
||||
}
|
||||
|
||||
std::string indexedPath(const std::string& prefix, int index) {
|
||||
return prefix + "." + std::to_string(index);
|
||||
}
|
||||
|
||||
std::string roundtrip(
|
||||
StorageClient& client,
|
||||
const Options& options,
|
||||
const std::string& inputPath,
|
||||
const std::string& outputPath) {
|
||||
const std::string cid = client.uploadFile(inputPath, options.chunkSize);
|
||||
client.downloadFile(cid, outputPath, options.chunkSize, options.local);
|
||||
if (!filesEqual(inputPath, outputPath)) {
|
||||
throw std::runtime_error("roundtrip byte comparison failed for cid: " + cid);
|
||||
}
|
||||
return cid;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
const Options options = parseArgs(argc, argv);
|
||||
if (options.command == "help") {
|
||||
printUsage();
|
||||
return 0;
|
||||
}
|
||||
|
||||
StorageClient client(configJson(options), options.timeout);
|
||||
if (!options.noStart) {
|
||||
client.start();
|
||||
}
|
||||
|
||||
if (options.command == "info") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << "version: " << client.version() << "\n";
|
||||
std::cout << "revision: " << client.revision() << "\n";
|
||||
std::cout << "repo: " << client.repo() << "\n";
|
||||
std::cout << "peer_id: " << client.peerId() << "\n";
|
||||
} else if (options.command == "version") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.version() << "\n";
|
||||
} else if (options.command == "revision") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.revision() << "\n";
|
||||
} else if (options.command == "repo") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.repo() << "\n";
|
||||
} else if (options.command == "peer-id") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.peerId() << "\n";
|
||||
} else if (options.command == "spr") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.spr() << "\n";
|
||||
} else if (options.command == "debug") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.debug() << "\n";
|
||||
} else if (options.command == "connect") {
|
||||
if (options.args.empty()) {
|
||||
throw std::runtime_error("command 'connect' expects at least 1 argument");
|
||||
}
|
||||
std::vector<std::string> addresses(options.args.begin() + 1, options.args.end());
|
||||
std::cout << client.connect(options.args[0], addresses) << "\n";
|
||||
} else if (options.command == "metrics") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.metrics() << "\n";
|
||||
} else if (options.command == "list") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.list() << "\n";
|
||||
} else if (options.command == "space") {
|
||||
requireArgCount(options, 0);
|
||||
std::cout << client.space() << "\n";
|
||||
} else if (options.command == "manifest") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.manifest(options.args[0]) << "\n";
|
||||
} else if (options.command == "exists") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.exists(options.args[0]) << "\n";
|
||||
} else if (options.command == "delete") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.deleteContent(options.args[0]) << "\n";
|
||||
} else if (options.command == "fetch") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.fetch(options.args[0]) << "\n";
|
||||
} else if (options.command == "upload") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.uploadFile(options.args[0], options.chunkSize) << "\n";
|
||||
} else if (options.command == "download") {
|
||||
requireArgCount(options, 2);
|
||||
std::cout << client.downloadFile(
|
||||
options.args[0], options.args[1], options.chunkSize, options.local)
|
||||
<< "\n";
|
||||
} else if (options.command == "roundtrip") {
|
||||
requireArgCount(options, 2);
|
||||
std::cout << roundtrip(client, options, options.args[0], options.args[1]) << "\n";
|
||||
} else if (options.command == "repeat-roundtrip") {
|
||||
requireArgCount(options, 3);
|
||||
const int count = parseCount(options.args[2]);
|
||||
std::string lastCid;
|
||||
for (int i = 1; i <= count; ++i) {
|
||||
lastCid = roundtrip(client, options, options.args[0], indexedPath(options.args[1], i));
|
||||
std::cout << i << " " << lastCid << "\n";
|
||||
}
|
||||
std::cout << "completed repeat-roundtrip count=" << count << " last_cid=" << lastCid
|
||||
<< "\n";
|
||||
} else if (options.command == "upload-many") {
|
||||
requireArgCount(options, 2);
|
||||
const int count = parseCount(options.args[1]);
|
||||
std::string lastCid;
|
||||
for (int i = 1; i <= count; ++i) {
|
||||
lastCid = client.uploadFile(options.args[0], options.chunkSize);
|
||||
std::cout << i << " " << lastCid << "\n";
|
||||
}
|
||||
std::cout << "completed upload-many count=" << count << " last_cid=" << lastCid << "\n";
|
||||
} else if (options.command == "download-many") {
|
||||
requireArgCount(options, 3);
|
||||
const int count = parseCount(options.args[2]);
|
||||
for (int i = 1; i <= count; ++i) {
|
||||
const std::string outputPath = indexedPath(options.args[1], i);
|
||||
client.downloadFile(options.args[0], outputPath, options.chunkSize, options.local);
|
||||
std::cout << i << " " << outputPath << "\n";
|
||||
}
|
||||
std::cout << "completed download-many count=" << count << "\n";
|
||||
} else {
|
||||
throw std::runtime_error("unknown command: " + options.command);
|
||||
}
|
||||
|
||||
return 0;
|
||||
} catch (const std::exception& err) {
|
||||
std::cerr << "error: " << err.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
141
tools/libstorage-cpp/storage_lib_ctl.cpp
Normal file
141
tools/libstorage-cpp/storage_lib_ctl.cpp
Normal file
@ -0,0 +1,141 @@
|
||||
#include <cerrno>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <exception>
|
||||
#include <iostream>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <sys/socket.h>
|
||||
#include <sys/un.h>
|
||||
#include <unistd.h>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
std::string homeDir() {
|
||||
const char* home = std::getenv("HOME");
|
||||
if (!home || std::strlen(home) == 0) {
|
||||
throw std::runtime_error("HOME is not set");
|
||||
}
|
||||
return home;
|
||||
}
|
||||
|
||||
std::string defaultSocketPath() {
|
||||
const char* fromEnv = std::getenv("STORAGE_LIB_SOCKET");
|
||||
if (fromEnv && std::strlen(fromEnv) > 0) {
|
||||
return fromEnv;
|
||||
}
|
||||
return homeDir() + "/.logos/storage/libstorage/storage_lib.sock";
|
||||
}
|
||||
|
||||
void printUsage() {
|
||||
std::cout <<
|
||||
"Usage:\n"
|
||||
" storage_lib_ctl [--socket <path>] <command> [args]\n\n"
|
||||
"Commands are sent as one line to storage_lib. Examples:\n"
|
||||
" storage_lib_ctl info\n"
|
||||
" storage_lib_ctl upload README.md\n"
|
||||
" storage_lib_ctl download <cid> ./out true\n"
|
||||
" storage_lib_ctl shutdown\n";
|
||||
}
|
||||
|
||||
struct Options {
|
||||
std::string socketPath;
|
||||
std::vector<std::string> command;
|
||||
};
|
||||
|
||||
Options parseArgs(int argc, char** argv) {
|
||||
Options options;
|
||||
options.socketPath = defaultSocketPath();
|
||||
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string arg = argv[i];
|
||||
if (arg == "--help" || arg == "-h") {
|
||||
printUsage();
|
||||
std::exit(0);
|
||||
} else if (arg == "--socket" && i + 1 < argc) {
|
||||
options.socketPath = argv[++i];
|
||||
} else if (!arg.empty() && arg[0] == '-') {
|
||||
throw std::runtime_error("unknown or incomplete option: " + arg);
|
||||
} else {
|
||||
for (; i < argc; ++i) {
|
||||
options.command.emplace_back(argv[i]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (options.command.empty()) {
|
||||
throw std::runtime_error("missing command");
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
std::string commandLine(const std::vector<std::string>& command) {
|
||||
std::string line;
|
||||
for (size_t i = 0; i < command.size(); ++i) {
|
||||
if (i > 0) line += ' ';
|
||||
line += command[i];
|
||||
}
|
||||
line += '\n';
|
||||
return line;
|
||||
}
|
||||
|
||||
int connectSocket(const std::string& socketPath) {
|
||||
if (socketPath.size() >= sizeof(sockaddr_un::sun_path)) {
|
||||
throw std::runtime_error("socket path is too long: " + socketPath);
|
||||
}
|
||||
|
||||
int fd = ::socket(AF_UNIX, SOCK_STREAM, 0);
|
||||
if (fd < 0) throw std::runtime_error(std::string("socket failed: ") + std::strerror(errno));
|
||||
|
||||
sockaddr_un addr{};
|
||||
addr.sun_family = AF_UNIX;
|
||||
std::strncpy(addr.sun_path, socketPath.c_str(), sizeof(addr.sun_path) - 1);
|
||||
|
||||
if (::connect(fd, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) {
|
||||
::close(fd);
|
||||
throw std::runtime_error(std::string("connect failed: ") + std::strerror(errno));
|
||||
}
|
||||
return fd;
|
||||
}
|
||||
|
||||
void writeAll(int fd, const std::string& data) {
|
||||
const char* ptr = data.data();
|
||||
size_t left = data.size();
|
||||
while (left > 0) {
|
||||
ssize_t n = ::write(fd, ptr, left);
|
||||
if (n < 0) throw std::runtime_error(std::string("write failed: ") + std::strerror(errno));
|
||||
ptr += n;
|
||||
left -= static_cast<size_t>(n);
|
||||
}
|
||||
}
|
||||
|
||||
std::string readAll(int fd) {
|
||||
std::string out;
|
||||
char buffer[4096];
|
||||
while (true) {
|
||||
ssize_t n = ::read(fd, buffer, sizeof(buffer));
|
||||
if (n == 0) break;
|
||||
if (n < 0) throw std::runtime_error(std::string("read failed: ") + std::strerror(errno));
|
||||
out.append(buffer, static_cast<size_t>(n));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
try {
|
||||
const Options options = parseArgs(argc, argv);
|
||||
int fd = connectSocket(options.socketPath);
|
||||
writeAll(fd, commandLine(options.command));
|
||||
::shutdown(fd, SHUT_WR);
|
||||
const std::string response = readAll(fd);
|
||||
::close(fd);
|
||||
|
||||
std::cout << response;
|
||||
return response.find("\"ok\":false") == std::string::npos ? 0 : 1;
|
||||
} catch (const std::exception& err) {
|
||||
std::cerr << "error: " << err.what() << "\n";
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
@ -33,7 +33,13 @@ make -j1 NIMFLAGS="-d:disableMarchNative"
|
||||
Start a local node in the foreground:
|
||||
|
||||
```bash
|
||||
scripts/start-local-node.sh
|
||||
tools/storage-test/start-local-node.sh
|
||||
```
|
||||
|
||||
Start a libstorage-based local node in the foreground:
|
||||
|
||||
```bash
|
||||
tools/storage-test/start-local-node.sh --client lib
|
||||
```
|
||||
|
||||
Default local settings:
|
||||
@ -41,11 +47,12 @@ Default local settings:
|
||||
| Setting | Default |
|
||||
|---|---|
|
||||
| Binary | `./build/storage` |
|
||||
| Data dir | `~/.logos/storage/local-node` |
|
||||
| Data dir | `~/.logos/storage/local-node` for `storage`, `~/.logos/storage/libstorage/node` for `lib` |
|
||||
| Log level | `info` |
|
||||
| P2P TCP | `8071` |
|
||||
| Discovery UDP | `8091` |
|
||||
| REST API | `127.0.0.1:8080` |
|
||||
| Lib IPC socket | `~/.logos/storage/libstorage/storage_lib.sock` |
|
||||
| Network | `logos.test` |
|
||||
|
||||
`info` is a good default log level: it shows startup, networking, and high-level node events without the volume of `debug` or `trace`.
|
||||
@ -53,19 +60,19 @@ Default local settings:
|
||||
Use `debug` when diagnosing behavior:
|
||||
|
||||
```bash
|
||||
scripts/start-local-node.sh --log-level debug
|
||||
tools/storage-test/start-local-node.sh --log-level debug
|
||||
```
|
||||
|
||||
Use `trace` only for detailed protocol/debug investigation because it can be noisy:
|
||||
|
||||
```bash
|
||||
scripts/start-local-node.sh --log-level trace
|
||||
tools/storage-test/start-local-node.sh --log-level trace
|
||||
```
|
||||
|
||||
Show all local-node options:
|
||||
|
||||
```bash
|
||||
scripts/start-local-node.sh --help
|
||||
tools/storage-test/start-local-node.sh --help
|
||||
```
|
||||
|
||||
The local node runs in the foreground. Press `Ctrl-C` to stop it.
|
||||
@ -75,7 +82,7 @@ The local node runs in the foreground. Press `Ctrl-C` to stop it.
|
||||
Show commands:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh --help
|
||||
tools/storage-test/storage-test.sh --help
|
||||
```
|
||||
|
||||
Defaults:
|
||||
@ -85,6 +92,7 @@ Defaults:
|
||||
| Remote SSH | `storage@172.235.163.25` |
|
||||
| Remote API tunnel | `127.0.0.1:18080` |
|
||||
| Local API | `127.0.0.1:8080` |
|
||||
| Libstorage socket | `~/.logos/storage/libstorage/storage_lib.sock` |
|
||||
| CID state file | `~/.logos/storage/test/cids.log` |
|
||||
| Generated test files | `~/.logos/storage/test/files/` |
|
||||
|
||||
@ -93,8 +101,8 @@ Defaults:
|
||||
Recover the latest CID from the upload history:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh last-cid
|
||||
scripts/storage-test.sh last-cid remote
|
||||
tools/storage-test/storage-test.sh last-cid
|
||||
tools/storage-test/storage-test.sh last-cid remote
|
||||
```
|
||||
|
||||
## SSH Tunnel
|
||||
@ -102,19 +110,19 @@ scripts/storage-test.sh last-cid remote
|
||||
Start the tunnel:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh tunnel start
|
||||
tools/storage-test/storage-test.sh tunnel start
|
||||
```
|
||||
|
||||
Check it:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh tunnel status
|
||||
tools/storage-test/storage-test.sh tunnel status
|
||||
```
|
||||
|
||||
Stop it:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh tunnel stop
|
||||
tools/storage-test/storage-test.sh tunnel stop
|
||||
```
|
||||
|
||||
You do not have to stop the tunnel after each test. It is safe to leave it running while you are actively testing. Stop it when you are done, when you want to free local port `18080`, or before changing tunnel settings.
|
||||
@ -126,72 +134,112 @@ Commands that target `remote` start the tunnel automatically if it is not alread
|
||||
Terminal 1: start local node and watch logs.
|
||||
|
||||
```bash
|
||||
scripts/start-local-node.sh --log-level info
|
||||
tools/storage-test/start-local-node.sh --log-level info
|
||||
```
|
||||
|
||||
Terminal 2: upload random content to the Linode node.
|
||||
|
||||
```bash
|
||||
CID="$(scripts/storage-test.sh upload-random remote 10M)"
|
||||
CID="$(tools/storage-test/storage-test.sh remote upload-random 10M)"
|
||||
printf '%s\n' "$CID"
|
||||
```
|
||||
|
||||
If you prefer copy/paste, you can also run the upload command directly and copy the printed CID:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh upload-random remote 10M
|
||||
tools/storage-test/storage-test.sh remote upload-random 10M
|
||||
```
|
||||
|
||||
Recover it later from the upload history:
|
||||
|
||||
```bash
|
||||
CID="$(scripts/storage-test.sh last-cid remote)"
|
||||
CID="$(tools/storage-test/storage-test.sh last-cid remote)"
|
||||
```
|
||||
|
||||
Ask the local node to fetch and store the content from the network:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh fetch-local "$CID" --wait
|
||||
tools/storage-test/storage-test.sh local fetch "$CID" --wait
|
||||
```
|
||||
|
||||
Or stream the content through the local node without explicitly storing it first:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh stream-local "$CID" /tmp/logos-download.bin
|
||||
tools/storage-test/storage-test.sh local download "$CID" /tmp/logos-download.bin
|
||||
```
|
||||
|
||||
Check local presence:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh exists local "$CID"
|
||||
tools/storage-test/storage-test.sh local exists "$CID"
|
||||
```
|
||||
|
||||
List local and remote CIDs:
|
||||
List local, remote, and lib CIDs:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh list local
|
||||
scripts/storage-test.sh list remote
|
||||
tools/storage-test/storage-test.sh local list
|
||||
tools/storage-test/storage-test.sh remote list
|
||||
tools/storage-test/storage-test.sh lib list
|
||||
```
|
||||
|
||||
Delete by CID:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh delete local "$CID"
|
||||
scripts/storage-test.sh delete remote "$CID"
|
||||
tools/storage-test/storage-test.sh local delete "$CID"
|
||||
tools/storage-test/storage-test.sh remote delete "$CID"
|
||||
```
|
||||
|
||||
Delete all local CIDs:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh delete-all local --yes
|
||||
tools/storage-test/storage-test.sh local delete-all --yes
|
||||
```
|
||||
|
||||
Delete all remote CIDs:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh delete-all remote --yes
|
||||
tools/storage-test/storage-test.sh remote delete-all --yes
|
||||
```
|
||||
|
||||
## Libstorage Daemon Target
|
||||
|
||||
Build and start the libstorage daemon from `tools/libstorage-cpp`:
|
||||
|
||||
```bash
|
||||
cd tools/libstorage-cpp
|
||||
make
|
||||
cd ../..
|
||||
tools/storage-test/start-local-node.sh --client lib
|
||||
```
|
||||
|
||||
Then use the `lib` target from the repository root:
|
||||
|
||||
```bash
|
||||
tools/storage-test/storage-test.sh lib peerid
|
||||
tools/storage-test/storage-test.sh lib upload README.md
|
||||
tools/storage-test/storage-test.sh lib download <CID> /tmp/logos-lib-download.bin
|
||||
tools/storage-test/storage-test.sh lib spr
|
||||
tools/storage-test/storage-test.sh lib debug
|
||||
```
|
||||
|
||||
Override the socket with `STORAGE_LIB_SOCKET` if the daemon was started with a non-default socket path.
|
||||
|
||||
## Test Scenario
|
||||
|
||||
Run the first remote-to-local scenario against a standard local REST node:
|
||||
|
||||
```bash
|
||||
tools/storage-test/storage-test.sh local test
|
||||
```
|
||||
|
||||
Run the same scenario against the libstorage daemon:
|
||||
|
||||
```bash
|
||||
tools/storage-test/storage-test.sh lib test
|
||||
```
|
||||
|
||||
The scenario uploads random files to the remote Linode node, downloads them using the selected local target, validates SHA-256 hashes, and deletes involved CIDs from both sides. The default file sizes are `4K 1M 10M`; override with `TEST_FILE_SIZES`.
|
||||
|
||||
## Useful API Endpoints
|
||||
|
||||
| Operation | Endpoint |
|
||||
@ -212,7 +260,7 @@ Stop the local node with `Ctrl-C`.
|
||||
Stop the SSH tunnel when finished:
|
||||
|
||||
```bash
|
||||
scripts/storage-test.sh tunnel stop
|
||||
tools/storage-test/storage-test.sh tunnel stop
|
||||
```
|
||||
|
||||
Remove local test data if desired:
|
||||
@ -343,7 +343,7 @@ Then use:
|
||||
curl http://127.0.0.1:18080/api/storage/v1/peerid
|
||||
```
|
||||
|
||||
In this repository, `scripts/storage-test.sh` manages this tunnel automatically for commands targeting `remote`.
|
||||
In this repository, `tools/storage-test/storage-test.sh` manages this tunnel automatically for commands targeting `remote`.
|
||||
|
||||
## Operational Commands
|
||||
|
||||
196
tools/storage-test/start-local-node.sh
Executable file
196
tools/storage-test/start-local-node.sh
Executable file
@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
CLIENT="${STORAGE_CLIENT:-storage}"
|
||||
BINARY="${STORAGE_BINARY:-${ROOT_DIR}/build/storage}"
|
||||
LIB_BINARY="${STORAGE_LIB_BINARY:-${ROOT_DIR}/tools/libstorage-cpp/storage_lib}"
|
||||
STORAGE_DATA_DIR_DEFAULT="${HOME}/.logos/storage/local-node"
|
||||
LIB_DATA_DIR_DEFAULT="${HOME}/.logos/storage/libstorage/node"
|
||||
DATA_DIR="${STORAGE_DATA_DIR:-}"
|
||||
LOG_LEVEL="${STORAGE_LOG_LEVEL:-info}"
|
||||
LISTEN_PORT="${STORAGE_LISTEN_PORT:-8071}"
|
||||
DISC_PORT="${STORAGE_DISC_PORT:-8091}"
|
||||
API_BINDADDR="${STORAGE_API_BINDADDR:-127.0.0.1}"
|
||||
API_PORT="${STORAGE_API_PORT:-8080}"
|
||||
NETWORK="${STORAGE_NETWORK:-logos.test}"
|
||||
LIB_SOCKET="${STORAGE_LIB_SOCKET:-${HOME}/.logos/storage/libstorage/storage_lib.sock}"
|
||||
LIB_TIMEOUT_MS="${STORAGE_LIB_TIMEOUT_MS:-0}"
|
||||
LIB_CHUNK_SIZE="${STORAGE_LIB_CHUNK_SIZE:-65536}"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 [options] [-- extra args]
|
||||
|
||||
Start a local Logos Storage node in the foreground so logs are visible.
|
||||
|
||||
Options:
|
||||
--client <name> Client to launch: storage or lib [$CLIENT]
|
||||
--binary <path> Storage binary [$BINARY]
|
||||
--lib-binary <path> Libstorage daemon binary [$LIB_BINARY]
|
||||
--data-dir <path> Data directory [storage: $STORAGE_DATA_DIR_DEFAULT, lib: $LIB_DATA_DIR_DEFAULT]
|
||||
--log-level <level> Log level: trace, debug, info, notice, warn, error [$LOG_LEVEL]
|
||||
--listen-port <port> Local libp2p TCP listen port [$LISTEN_PORT]
|
||||
--disc-port <port> Local discovery UDP port [$DISC_PORT]
|
||||
--api-bindaddr <ip> REST API bind address [$API_BINDADDR]
|
||||
--api-port <port> REST API port [$API_PORT]
|
||||
--network <name> Network preset [$NETWORK]
|
||||
--socket <path> Libstorage Unix socket [$LIB_SOCKET]
|
||||
--timeout-ms <ms> Libstorage async timeout, 0 waits forever [$LIB_TIMEOUT_MS]
|
||||
--chunk-size <bytes> Libstorage upload/download chunk size [$LIB_CHUNK_SIZE]
|
||||
-h, --help Show this help.
|
||||
|
||||
Environment overrides:
|
||||
STORAGE_CLIENT, STORAGE_BINARY, STORAGE_LIB_BINARY, STORAGE_DATA_DIR,
|
||||
STORAGE_LOG_LEVEL, STORAGE_LISTEN_PORT, STORAGE_DISC_PORT, STORAGE_API_BINDADDR,
|
||||
STORAGE_API_PORT, STORAGE_NETWORK, STORAGE_LIB_SOCKET, STORAGE_LIB_TIMEOUT_MS,
|
||||
STORAGE_LIB_CHUNK_SIZE
|
||||
|
||||
Examples:
|
||||
$0
|
||||
$0 --client lib
|
||||
$0 --log-level debug
|
||||
$0 --data-dir /tmp/logos-storage-local --listen-port 8072 --disc-port 8092
|
||||
$0 --log-level trace -- --metrics --metrics-address=127.0.0.1
|
||||
|
||||
The node runs in the foreground. Press Ctrl-C to stop it.
|
||||
EOF
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--client)
|
||||
CLIENT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--binary)
|
||||
BINARY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--lib-binary)
|
||||
LIB_BINARY="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--data-dir)
|
||||
DATA_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--log-level)
|
||||
LOG_LEVEL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--listen-port)
|
||||
LISTEN_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--disc-port)
|
||||
DISC_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--api-bindaddr)
|
||||
API_BINDADDR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--api-port)
|
||||
API_PORT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--network)
|
||||
NETWORK="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--socket)
|
||||
LIB_SOCKET="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--timeout-ms)
|
||||
LIB_TIMEOUT_MS="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--chunk-size)
|
||||
LIB_CHUNK_SIZE="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
--)
|
||||
shift
|
||||
break
|
||||
;;
|
||||
*)
|
||||
printf 'error: unknown option: %s\n\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
case "$CLIENT" in
|
||||
storage|lib) ;;
|
||||
*) printf 'error: --client must be storage or lib\n' >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
if [[ -z "$DATA_DIR" ]]; then
|
||||
if [[ "$CLIENT" == 'lib' ]]; then
|
||||
DATA_DIR="$LIB_DATA_DIR_DEFAULT"
|
||||
else
|
||||
DATA_DIR="$STORAGE_DATA_DIR_DEFAULT"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ "$CLIENT" == 'storage' ]]; then
|
||||
[[ -n "$BINARY" ]] || { printf 'error: --binary cannot be empty\n' >&2; exit 1; }
|
||||
[[ -x "$BINARY" ]] || { printf 'error: storage binary not executable: %s\n' "$BINARY" >&2; exit 1; }
|
||||
else
|
||||
[[ -n "$LIB_BINARY" ]] || { printf 'error: --lib-binary cannot be empty\n' >&2; exit 1; }
|
||||
[[ -x "$LIB_BINARY" ]] || { printf 'error: libstorage daemon binary not executable: %s\n' "$LIB_BINARY" >&2; exit 1; }
|
||||
fi
|
||||
|
||||
mkdir -p "$DATA_DIR"
|
||||
|
||||
printf 'Starting local Logos Storage node\n'
|
||||
printf ' client: %s\n' "$CLIENT"
|
||||
if [[ "$CLIENT" == 'storage' ]]; then
|
||||
printf ' binary: %s\n' "$BINARY"
|
||||
else
|
||||
printf ' binary: %s\n' "$LIB_BINARY"
|
||||
fi
|
||||
printf ' data dir: %s\n' "$DATA_DIR"
|
||||
printf ' log level: %s\n' "$LOG_LEVEL"
|
||||
printf ' listen TCP: %s\n' "$LISTEN_PORT"
|
||||
printf ' discovery: %s/udp\n' "$DISC_PORT"
|
||||
printf ' network: %s\n' "$NETWORK"
|
||||
if [[ "$CLIENT" == 'storage' ]]; then
|
||||
printf ' REST API: %s:%s\n' "$API_BINDADDR" "$API_PORT"
|
||||
else
|
||||
printf ' socket: %s\n' "$LIB_SOCKET"
|
||||
printf ' timeout ms: %s\n' "$LIB_TIMEOUT_MS"
|
||||
printf ' chunk size: %s\n' "$LIB_CHUNK_SIZE"
|
||||
fi
|
||||
printf '\n'
|
||||
|
||||
if [[ "$CLIENT" == 'storage' ]]; then
|
||||
exec "$BINARY" \
|
||||
--data-dir="$DATA_DIR" \
|
||||
--log-level="$LOG_LEVEL" \
|
||||
--listen-port="$LISTEN_PORT" \
|
||||
--disc-port="$DISC_PORT" \
|
||||
--api-bindaddr="$API_BINDADDR" \
|
||||
--api-port="$API_PORT" \
|
||||
--network="$NETWORK" \
|
||||
"$@"
|
||||
fi
|
||||
|
||||
exec "$LIB_BINARY" \
|
||||
--data-dir "$DATA_DIR" \
|
||||
--log-level "$LOG_LEVEL" \
|
||||
--listen-port "$LISTEN_PORT" \
|
||||
--disc-port "$DISC_PORT" \
|
||||
--network "$NETWORK" \
|
||||
--socket "$LIB_SOCKET" \
|
||||
--timeout-ms "$LIB_TIMEOUT_MS" \
|
||||
--chunk-size "$LIB_CHUNK_SIZE" \
|
||||
"$@"
|
||||
613
tools/storage-test/storage-test.sh
Executable file
613
tools/storage-test/storage-test.sh
Executable file
@ -0,0 +1,613 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
|
||||
REMOTE_SSH_HOST="${REMOTE_SSH_HOST:-storage@172.235.163.25}"
|
||||
REMOTE_API_PORT="${REMOTE_API_PORT:-18080}"
|
||||
LOCAL_API_PORT="${LOCAL_API_PORT:-8080}"
|
||||
TUNNEL_CONTROL_PATH="${TUNNEL_CONTROL_PATH:-/tmp/logos-storage-tunnel-${USER:-user}.ctl}"
|
||||
CID_STATE_FILE="${CID_STATE_FILE:-${HOME}/.logos/storage/test/cids.log}"
|
||||
TEST_FILES_DIR="${TEST_FILES_DIR:-${HOME}/.logos/storage/test/files}"
|
||||
TEST_FILE_SIZES="${TEST_FILE_SIZES:-4K 1M 10M}"
|
||||
TEST_KEEP_FILES="${TEST_KEEP_FILES:-0}"
|
||||
STORAGE_LIB_CTL="${STORAGE_LIB_CTL:-${ROOT_DIR}/tools/libstorage-cpp/storage_lib_ctl}"
|
||||
STORAGE_LIB_SOCKET="${STORAGE_LIB_SOCKET:-${HOME}/.logos/storage/libstorage/storage_lib.sock}"
|
||||
|
||||
LOCAL_API="http://127.0.0.1:${LOCAL_API_PORT}/api/storage/v1"
|
||||
REMOTE_API="http://127.0.0.1:${REMOTE_API_PORT}/api/storage/v1"
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
Usage: $0 <target> <command> [args]
|
||||
$0 <global-command> [args]
|
||||
|
||||
Targets:
|
||||
local Local REST node at $LOCAL_API
|
||||
remote Remote REST node through SSH tunnel at $REMOTE_API
|
||||
lib Local storage_lib daemon via Unix socket
|
||||
|
||||
Target commands:
|
||||
<target> upload <file>
|
||||
Upload a file and print the returned CID.
|
||||
|
||||
<target> upload-random <size> [--keep]
|
||||
Create random content, upload it, print CID.
|
||||
|
||||
<target> download <cid> <output-file> [--local]
|
||||
Download CID into output-file. For lib, --local means local store only.
|
||||
|
||||
<target> fetch <cid> [--wait]
|
||||
Fetch CID from network into target local store. --wait is REST-only.
|
||||
|
||||
<target> list
|
||||
List manifest CIDs stored locally by target.
|
||||
|
||||
<target> delete <cid>
|
||||
Delete CID from target local storage.
|
||||
|
||||
<target> delete-all --yes
|
||||
Delete every CID returned by list from target local storage.
|
||||
|
||||
<target> exists <cid>
|
||||
Check whether target has CID locally.
|
||||
|
||||
<target> space
|
||||
Show target storage space information.
|
||||
|
||||
<target> peerid
|
||||
Show target peer ID.
|
||||
|
||||
<target> test
|
||||
Upload random files to remote, download via local/lib target, validate hashes,
|
||||
and clean up involved CIDs. Supported targets: local, lib.
|
||||
|
||||
Lib-only target commands:
|
||||
lib spr
|
||||
lib debug
|
||||
lib manifest <cid>
|
||||
lib connect <peer-id> [addr...]
|
||||
|
||||
Global commands:
|
||||
help
|
||||
Show this help.
|
||||
|
||||
tunnel start|stop|status
|
||||
Manage SSH tunnel to the remote REST API.
|
||||
|
||||
make-file <size> [output-file]
|
||||
Create random content with dd. Example: make-file 10M /tmp/logos-10M.bin
|
||||
|
||||
last-cid [target]
|
||||
Print the most recent CID from CID_STATE_FILE, optionally filtered by target.
|
||||
|
||||
Environment:
|
||||
REMOTE_SSH_HOST SSH host for the remote node [$REMOTE_SSH_HOST]
|
||||
REMOTE_API_PORT Local tunnel port for remote API [$REMOTE_API_PORT]
|
||||
LOCAL_API_PORT Local node API port [$LOCAL_API_PORT]
|
||||
STORAGE_LIB_CTL Path to storage_lib_ctl [$STORAGE_LIB_CTL]
|
||||
STORAGE_LIB_SOCKET Unix socket for storage_lib [$STORAGE_LIB_SOCKET]
|
||||
CID_STATE_FILE Upload history log [$CID_STATE_FILE]
|
||||
TEST_FILES_DIR Generated test file directory [$TEST_FILES_DIR]
|
||||
TEST_FILE_SIZES Sizes used by '<target> test' [$TEST_FILE_SIZES]
|
||||
TEST_KEEP_FILES Keep test workspace when set to 1 [$TEST_KEEP_FILES]
|
||||
|
||||
Examples:
|
||||
$0 tunnel start
|
||||
$0 remote upload-random 10M
|
||||
$0 local fetch <CID> --wait
|
||||
$0 local download <CID> /tmp/downloaded.bin
|
||||
$0 lib download <CID> /tmp/downloaded.bin
|
||||
$0 lib test
|
||||
$0 local test
|
||||
EOF
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'error: %s\n' "$*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
need() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
|
||||
}
|
||||
|
||||
check_common_deps() {
|
||||
need curl
|
||||
need jq
|
||||
}
|
||||
|
||||
is_target() {
|
||||
case "${1:-}" in
|
||||
local|remote|lib) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
target_api() {
|
||||
case "${1:-}" in
|
||||
local)
|
||||
printf '%s\n' "$LOCAL_API"
|
||||
;;
|
||||
remote)
|
||||
tunnel_start >/dev/null
|
||||
printf '%s\n' "$REMOTE_API"
|
||||
;;
|
||||
*)
|
||||
die "REST target must be 'local' or 'remote'"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
tunnel_status() {
|
||||
ssh -S "$TUNNEL_CONTROL_PATH" -O check "$REMOTE_SSH_HOST" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
tunnel_start() {
|
||||
need ssh
|
||||
if tunnel_status; then
|
||||
printf 'tunnel already running: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \
|
||||
"$REMOTE_API_PORT" "$REMOTE_SSH_HOST"
|
||||
return 0
|
||||
fi
|
||||
|
||||
ssh \
|
||||
-M \
|
||||
-S "$TUNNEL_CONTROL_PATH" \
|
||||
-fN \
|
||||
-L "127.0.0.1:${REMOTE_API_PORT}:127.0.0.1:8080" \
|
||||
"$REMOTE_SSH_HOST"
|
||||
|
||||
printf 'started tunnel: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \
|
||||
"$REMOTE_API_PORT" "$REMOTE_SSH_HOST"
|
||||
}
|
||||
|
||||
tunnel_stop() {
|
||||
need ssh
|
||||
if tunnel_status; then
|
||||
ssh -S "$TUNNEL_CONTROL_PATH" -O exit "$REMOTE_SSH_HOST" >/dev/null
|
||||
printf 'stopped tunnel\n'
|
||||
else
|
||||
printf 'tunnel not running\n'
|
||||
fi
|
||||
}
|
||||
|
||||
make_file() {
|
||||
local size="${1:-}"
|
||||
local out="${2:-}"
|
||||
[[ -n "$size" ]] || die 'make-file requires <size>'
|
||||
if [[ -z "$out" ]]; then
|
||||
mkdir -p "$TEST_FILES_DIR"
|
||||
out="${TEST_FILES_DIR}/logos-test-${size}-$(date -u +%Y%m%dT%H%M%SZ).bin"
|
||||
fi
|
||||
dd if=/dev/urandom of="$out" bs="$size" count=1 status=progress
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
record_cid() {
|
||||
local target="$1"
|
||||
local cid="$2"
|
||||
local file="$3"
|
||||
mkdir -p "$(dirname "$CID_STATE_FILE")"
|
||||
printf '%s %s %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$target" "$cid" "$file" >> "$CID_STATE_FILE"
|
||||
}
|
||||
|
||||
last_cid() {
|
||||
local target=""
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
local|remote|lib)
|
||||
[[ -z "$target" ]] || die 'target specified more than once'
|
||||
target="$1"
|
||||
;;
|
||||
*)
|
||||
die "unknown last-cid option: $1"
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
[[ -f "$CID_STATE_FILE" ]] || die "CID state file not found: $CID_STATE_FILE"
|
||||
|
||||
local cid
|
||||
if [[ -n "$target" ]]; then
|
||||
cid="$(awk -v target="$target" '$2 == target { cid = $3 } END { print cid }' "$CID_STATE_FILE")"
|
||||
else
|
||||
cid="$(awk 'NF >= 3 { cid = $3 } END { print cid }' "$CID_STATE_FILE")"
|
||||
fi
|
||||
|
||||
[[ -n "$cid" ]] || die 'no matching CID found'
|
||||
printf '%s\n' "$cid"
|
||||
}
|
||||
|
||||
lib_ctl_raw() {
|
||||
[[ -x "$STORAGE_LIB_CTL" ]] || die "storage_lib_ctl not executable: $STORAGE_LIB_CTL"
|
||||
"$STORAGE_LIB_CTL" --socket "$STORAGE_LIB_SOCKET" "$@"
|
||||
}
|
||||
|
||||
lib_result() {
|
||||
check_common_deps
|
||||
local response
|
||||
response="$(lib_ctl_raw "$@")"
|
||||
if ! printf '%s\n' "$response" | jq -e '.ok == true' >/dev/null; then
|
||||
printf '%s\n' "$response" >&2
|
||||
return 1
|
||||
fi
|
||||
printf '%s\n' "$response" | jq -r '.result'
|
||||
}
|
||||
|
||||
abs_existing_path() {
|
||||
local path="$1"
|
||||
[[ -e "$path" ]] || die "path not found: $path"
|
||||
local dir base
|
||||
dir="$(cd "$(dirname "$path")" && pwd)"
|
||||
base="$(basename "$path")"
|
||||
printf '%s/%s\n' "$dir" "$base"
|
||||
}
|
||||
|
||||
abs_output_path() {
|
||||
local path="$1"
|
||||
local dir base
|
||||
dir="$(dirname "$path")"
|
||||
base="$(basename "$path")"
|
||||
mkdir -p "$dir"
|
||||
dir="$(cd "$dir" && pwd)"
|
||||
printf '%s/%s\n' "$dir" "$base"
|
||||
}
|
||||
|
||||
rest_upload_file() {
|
||||
check_common_deps
|
||||
local target="$1"
|
||||
local file="$2"
|
||||
[[ -f "$file" ]] || die "file not found: $file"
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS \
|
||||
-H 'Content-Type: application/octet-stream' \
|
||||
--data-binary "@${file}" \
|
||||
"${api}/data"
|
||||
}
|
||||
|
||||
target_upload_file() {
|
||||
local target="$1"
|
||||
local file="$2"
|
||||
[[ -n "$file" ]] || die 'upload requires <file>'
|
||||
local cid
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
cid="$(lib_result upload "$(abs_existing_path "$file")")"
|
||||
else
|
||||
cid="$(rest_upload_file "$target" "$file")"
|
||||
fi
|
||||
printf '%s\n' "$cid"
|
||||
record_cid "$target" "$cid" "$file"
|
||||
}
|
||||
|
||||
target_upload_random() {
|
||||
local target="$1"
|
||||
local size="${2:-}"
|
||||
local keep=false
|
||||
[[ -n "$size" ]] || die 'upload-random requires <size> [--keep]'
|
||||
|
||||
shift 2
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--keep) keep=true ;;
|
||||
*) die "unknown upload-random option: $1" ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
local tmp cid
|
||||
tmp="$(mktemp "${TMPDIR:-/tmp}/logos-storage-test.XXXXXX")"
|
||||
dd if=/dev/urandom of="$tmp" bs="$size" count=1 status=progress >&2
|
||||
cid="$(target_upload_file "$target" "$tmp")"
|
||||
printf '%s\n' "$cid"
|
||||
|
||||
if [[ "$keep" == true ]]; then
|
||||
printf 'kept file: %s\n' "$tmp" >&2
|
||||
else
|
||||
rm -f "$tmp"
|
||||
fi
|
||||
}
|
||||
|
||||
target_list_cids() {
|
||||
local target="$1"
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result list | jq -r '.[]?.cid'
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS "${api}/data" | jq -r '.content[]?.cid'
|
||||
}
|
||||
|
||||
target_delete_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'delete requires <cid>'
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result delete "$cid" >/dev/null
|
||||
else
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS -X DELETE "${api}/data/${cid}" >/dev/null
|
||||
fi
|
||||
printf 'deleted %s from %s\n' "$cid" "$target"
|
||||
}
|
||||
|
||||
target_delete_all() {
|
||||
local target="$1"
|
||||
local yes="${2:-}"
|
||||
[[ "$yes" == '--yes' ]] || die 'delete-all requires --yes'
|
||||
|
||||
local cid count=0
|
||||
while IFS= read -r cid; do
|
||||
[[ -n "$cid" ]] || continue
|
||||
target_delete_cid "$target" "$cid"
|
||||
count=$((count + 1))
|
||||
done < <(target_list_cids "$target")
|
||||
printf 'deleted %d CID(s) from %s\n' "$count" "$target"
|
||||
}
|
||||
|
||||
target_simple_get() {
|
||||
local target="$1"
|
||||
local path="$2"
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
case "$path" in
|
||||
space) lib_result space ;;
|
||||
peerid) lib_result peer-id ;;
|
||||
*) die "unsupported lib get path: $path" ;;
|
||||
esac
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS "${api}/${path}"
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
target_exists_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'exists requires <cid>'
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result exists "$cid"
|
||||
else
|
||||
target_simple_get "$target" "data/${cid}/exists"
|
||||
fi
|
||||
}
|
||||
|
||||
target_fetch_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
local wait="${3:-}"
|
||||
[[ -n "$cid" ]] || die 'fetch requires <cid> [--wait]'
|
||||
[[ -z "$wait" || "$wait" == '--wait' ]] || die 'only supported option is --wait'
|
||||
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
[[ -z "$wait" ]] || printf 'warning: --wait is ignored for lib fetch\n' >&2
|
||||
lib_result fetch "$cid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api response download_id
|
||||
api="$(target_api "$target")"
|
||||
response="$(curl -fsS -X POST "${api}/data/${cid}/network")"
|
||||
printf '%s\n' "$response" | jq .
|
||||
|
||||
if [[ "$wait" != '--wait' ]]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
download_id="$(printf '%s\n' "$response" | jq -r '.downloadId // empty')"
|
||||
[[ -n "$download_id" ]] || die 'response did not contain downloadId'
|
||||
|
||||
while true; do
|
||||
local progress active received total
|
||||
progress="$(curl -fsS "${api}/data/${cid}/network/progress/${download_id}")"
|
||||
printf '%s\n' "$progress" | jq .
|
||||
active="$(printf '%s\n' "$progress" | jq -r '.active')"
|
||||
received="$(printf '%s\n' "$progress" | jq -r '.received // 0')"
|
||||
total="$(printf '%s\n' "$progress" | jq -r '.total // 0')"
|
||||
if [[ "$active" != 'true' || ( "$total" != '0' && "$received" == "$total" ) ]]; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
target_download_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
local out="${3:-}"
|
||||
local local_only=false
|
||||
[[ -n "$cid" && -n "$out" ]] || die 'download requires <cid> <output-file> [--local]'
|
||||
if [[ "${4:-}" == '--local' ]]; then
|
||||
local_only=true
|
||||
elif [[ -n "${4:-}" ]]; then
|
||||
die 'download only supports optional --local'
|
||||
fi
|
||||
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result download "$cid" "$(abs_output_path "$out")" "$local_only" >/dev/null
|
||||
else
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fL "${api}/data/${cid}/network/stream" -o "$out"
|
||||
fi
|
||||
printf '%s\n' "$out"
|
||||
}
|
||||
|
||||
lib_only_command() {
|
||||
local command="$1"
|
||||
shift
|
||||
case "$command" in
|
||||
spr|debug)
|
||||
[[ $# -eq 0 ]] || die "$command does not accept arguments"
|
||||
lib_result "$command"
|
||||
;;
|
||||
manifest)
|
||||
[[ $# -eq 1 ]] || die 'manifest requires <cid>'
|
||||
lib_result manifest "$1"
|
||||
;;
|
||||
connect)
|
||||
[[ $# -ge 1 ]] || die 'connect requires <peer-id> [addr...]'
|
||||
lib_result connect "$@"
|
||||
;;
|
||||
*)
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
size_to_bytes() {
|
||||
local value="$1"
|
||||
local number suffix
|
||||
number="${value%[KkMmGg]}"
|
||||
suffix="${value:${#number}}"
|
||||
[[ "$number" =~ ^[0-9]+$ ]] || die "invalid size: $value"
|
||||
case "$suffix" in
|
||||
K|k) printf '%s\n' $((number * 1024)) ;;
|
||||
M|m) printf '%s\n' $((number * 1024 * 1024)) ;;
|
||||
G|g) printf '%s\n' $((number * 1024 * 1024 * 1024)) ;;
|
||||
'') printf '%s\n' "$number" ;;
|
||||
*) die "invalid size suffix: $value" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
target_test() {
|
||||
local target="$1"
|
||||
[[ "$target" == 'local' || "$target" == 'lib' ]] || die 'test is supported only for local and lib targets'
|
||||
check_common_deps
|
||||
need sha256sum
|
||||
|
||||
local max_bytes=$((10 * 1024 * 1024))
|
||||
local workspace
|
||||
workspace="$(mktemp -d "${TMPDIR:-/tmp}/logos-storage-test.XXXXXX")"
|
||||
local -a cids=()
|
||||
local -a local_cids=()
|
||||
|
||||
cleanup() {
|
||||
local cid
|
||||
for cid in "${local_cids[@]:-}"; do
|
||||
target_delete_cid "$target" "$cid" >/dev/null 2>&1 || true
|
||||
done
|
||||
for cid in "${cids[@]:-}"; do
|
||||
target_delete_cid remote "$cid" >/dev/null 2>&1 || true
|
||||
done
|
||||
if [[ "$TEST_KEEP_FILES" != '1' ]]; then
|
||||
rm -rf "$workspace"
|
||||
else
|
||||
printf 'kept test workspace: %s\n' "$workspace" >&2
|
||||
fi
|
||||
}
|
||||
trap cleanup RETURN
|
||||
|
||||
printf 'test workspace: %s\n' "$workspace" >&2
|
||||
|
||||
local size index=0
|
||||
for size in $TEST_FILE_SIZES; do
|
||||
local bytes src out cid src_hash out_hash
|
||||
bytes="$(size_to_bytes "$size")"
|
||||
(( bytes <= max_bytes )) || die "test file size exceeds 10MB limit: $size"
|
||||
|
||||
index=$((index + 1))
|
||||
src="${workspace}/source-${index}-${size}.bin"
|
||||
out="${workspace}/download-${index}-${size}.bin"
|
||||
dd if=/dev/urandom of="$src" bs="$size" count=1 status=none
|
||||
src_hash="$(sha256sum "$src" | awk '{ print $1 }')"
|
||||
|
||||
printf '[%d] upload remote %s\n' "$index" "$size" >&2
|
||||
cid="$(target_upload_file remote "$src")"
|
||||
cids+=("$cid")
|
||||
|
||||
printf '[%d] download via %s cid=%s\n' "$index" "$target" "$cid" >&2
|
||||
if [[ "$target" == 'local' ]]; then
|
||||
target_fetch_cid local "$cid" --wait >/dev/null
|
||||
local_cids+=("$cid")
|
||||
target_download_cid local "$cid" "$out" >/dev/null
|
||||
else
|
||||
target_download_cid lib "$cid" "$out" >/dev/null
|
||||
local_cids+=("$cid")
|
||||
fi
|
||||
|
||||
out_hash="$(sha256sum "$out" | awk '{ print $1 }')"
|
||||
[[ "$src_hash" == "$out_hash" ]] || die "hash mismatch for cid $cid"
|
||||
printf '[%d] ok cid=%s sha256=%s\n' "$index" "$cid" "$src_hash"
|
||||
done
|
||||
|
||||
printf 'test passed target=%s files=%d\n' "$target" "$index"
|
||||
}
|
||||
|
||||
target_command() {
|
||||
local target="$1"
|
||||
local command="${2:-help}"
|
||||
shift 2 || true
|
||||
|
||||
case "$command" in
|
||||
upload) target_upload_file "$target" "${1:-}" ;;
|
||||
upload-random) target_upload_random "$target" "$@" ;;
|
||||
download) target_download_cid "$target" "$@" ;;
|
||||
fetch) target_fetch_cid "$target" "$@" ;;
|
||||
list) [[ $# -eq 0 ]] || die 'list does not accept arguments'; target_list_cids "$target" ;;
|
||||
delete) target_delete_cid "$target" "$@" ;;
|
||||
delete-all) target_delete_all "$target" "$@" ;;
|
||||
exists) target_exists_cid "$target" "$@" ;;
|
||||
space) [[ $# -eq 0 ]] || die 'space does not accept arguments'; target_simple_get "$target" 'space' ;;
|
||||
peerid) [[ $# -eq 0 ]] || die 'peerid does not accept arguments'; target_simple_get "$target" 'peerid' ;;
|
||||
test) [[ $# -eq 0 ]] || die 'test does not accept arguments yet'; target_test "$target" ;;
|
||||
spr|debug|manifest|connect)
|
||||
[[ "$target" == 'lib' ]] || die "$command is currently supported only for lib target"
|
||||
lib_only_command "$command" "$@"
|
||||
;;
|
||||
help|--help|-h)
|
||||
usage
|
||||
;;
|
||||
*)
|
||||
die "unknown command for target '$target': $command"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
cmd="${1:-help}"
|
||||
shift || true
|
||||
|
||||
if is_target "$cmd"; then
|
||||
target_command "$cmd" "$@"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
case "$cmd" in
|
||||
help|--help|-h)
|
||||
usage
|
||||
;;
|
||||
tunnel)
|
||||
case "${1:-}" in
|
||||
start) tunnel_start ;;
|
||||
stop) tunnel_stop ;;
|
||||
status)
|
||||
if tunnel_status; then
|
||||
printf 'tunnel running\n'
|
||||
else
|
||||
printf 'tunnel stopped\n'
|
||||
exit 1
|
||||
fi
|
||||
;;
|
||||
*) die 'usage: tunnel start|stop|status' ;;
|
||||
esac
|
||||
;;
|
||||
make-file) make_file "$@" ;;
|
||||
last-cid) last_cid "$@" ;;
|
||||
*)
|
||||
usage >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Loading…
x
Reference in New Issue
Block a user