From 5dd0846015f7b883f2bb953bad851ac0b2402262 Mon Sep 17 00:00:00 2001 From: Marcin Czenko Date: Thu, 25 Jun 2026 14:03:10 +0200 Subject: [PATCH] test: adds more metrics to the testing scenario align with the storage_module --- tools/libstorage-cpp/AGENTS.md | 3 + tools/libstorage-cpp/README.md | 2 + tools/libstorage-cpp/storage_client.cpp | 29 ++++ tools/libstorage-cpp/storage_client.hpp | 3 + tools/libstorage-cpp/storage_lib.cpp | 7 + tools/libstorage-cpp/storage_lib_cli.cpp | 4 + tools/storage-test/AGENTS.md | 17 ++- tools/storage-test/README.md | 5 +- tools/storage-test/storage-test.sh | 160 +++++++++++++++++++++-- 9 files changed, 209 insertions(+), 21 deletions(-) diff --git a/tools/libstorage-cpp/AGENTS.md b/tools/libstorage-cpp/AGENTS.md index 1ce40386..b79230ac 100644 --- a/tools/libstorage-cpp/AGENTS.md +++ b/tools/libstorage-cpp/AGENTS.md @@ -73,6 +73,7 @@ Keep daemon commands low-level: - `exists` - `delete` - `fetch` +- `stream-sink` - `manifest` - `connect` - `shutdown` @@ -101,6 +102,8 @@ The protocol currently does not support paths or arguments containing spaces. If `storage_lib_ctl` should return nonzero when the daemon responds with `"ok":false`. +`stream-sink [local]` uses released libstorage download primitives: `storage_download_init` followed by `storage_download_stream` with an empty output path. It discards progress bytes while counting transferred byte lengths and waits for terminal completion. + File paths sent over IPC are resolved by the daemon process. Callers that run from a different working directory, such as `tools/storage-test/storage-test.sh`, should send absolute paths. ## Daemon Options diff --git a/tools/libstorage-cpp/README.md b/tools/libstorage-cpp/README.md index 6c0e05dd..5d0c11b2 100644 --- a/tools/libstorage-cpp/README.md +++ b/tools/libstorage-cpp/README.md @@ -73,6 +73,7 @@ Send commands with `storage_lib_ctl`: ./storage_lib_ctl --socket ./storage_lib.sock manifest ./storage_lib_ctl --socket ./storage_lib.sock exists ./storage_lib_ctl --socket ./storage_lib.sock download ./downloaded.md true +./storage_lib_ctl --socket ./storage_lib.sock stream-sink false ./storage_lib_ctl --socket ./storage_lib.sock delete ./storage_lib_ctl --socket ./storage_lib.sock shutdown ``` @@ -102,6 +103,7 @@ or: - `manifest `: prints manifest JSON. - `delete `: deletes locally stored content. - `fetch `: fetches content into the local store. +- `stream-sink [local]`: streams content and discards data, returning transferred bytes. - `roundtrip `: uploads one file, downloads it, compares bytes, and prints the cid. - `repeat-roundtrip `: repeats roundtrip in one node session. - `upload-many `: uploads the same file repeatedly in one node session. diff --git a/tools/libstorage-cpp/storage_client.cpp b/tools/libstorage-cpp/storage_client.cpp index 483a904c..1ed811f6 100644 --- a/tools/libstorage-cpp/storage_client.cpp +++ b/tools/libstorage-cpp/storage_client.cpp @@ -35,6 +35,7 @@ void StorageResponse::setResult(int callerRet, const char* msg, size_t len) { if (callerRet == RET_PROGRESS) { progressCount_ += 1; + progressBytes_ += len; if (msg && len > 0) { lastProgress_.assign(msg, len); } else { @@ -78,6 +79,11 @@ size_t StorageResponse::progressCount() const { return progressCount_; } +size_t StorageResponse::progressBytes() const { + std::lock_guard lock(mtx_); + return progressBytes_; +} + std::string StorageResponse::lastProgress() const { std::lock_guard lock(mtx_); return lastProgress_; @@ -264,6 +270,29 @@ std::string StorageClient::downloadFile( }); } +size_t StorageClient::streamSink(const std::string& cid, size_t chunkSize, bool local) { + call("storage_download_init", [this, &cid, chunkSize, local](StorageCallback cb, void* userData) { + return storage_download_init(ctx_, cid.c_str(), chunkSize, local, cb, userData); + }); + + StorageResponse resp; + const int dispatchRet = storage_download_stream( + ctx_, cid.c_str(), chunkSize, local, "", callback, &resp); + if (!isOk(dispatchRet)) { + throw std::runtime_error("storage_download_stream dispatch failed"); + } + + if (!resp.wait(timeout_)) { + throw std::runtime_error("storage_download_stream timed out"); + } + + if (!isOk(resp.status())) { + throw std::runtime_error("storage_download_stream failed: " + resp.data()); + } + + return resp.progressBytes(); +} + std::string StorageClient::call(const char* name, const AsyncCall& fn) { StorageResponse resp; const int dispatchRet = fn(callback, &resp); diff --git a/tools/libstorage-cpp/storage_client.hpp b/tools/libstorage-cpp/storage_client.hpp index 956d46a6..e569112d 100644 --- a/tools/libstorage-cpp/storage_client.hpp +++ b/tools/libstorage-cpp/storage_client.hpp @@ -23,6 +23,7 @@ public: int status() const; std::string data() const; size_t progressCount() const; + size_t progressBytes() const; std::string lastProgress() const; private: @@ -32,6 +33,7 @@ private: int status_ = -1; std::string result_; size_t progressCount_ = 0; + size_t progressBytes_ = 0; std::string lastProgress_; }; @@ -68,6 +70,7 @@ public: const std::string& outputPath, size_t chunkSize, bool local); + size_t streamSink(const std::string& cid, size_t chunkSize, bool local); private: using AsyncCall = std::function; diff --git a/tools/libstorage-cpp/storage_lib.cpp b/tools/libstorage-cpp/storage_lib.cpp index fa00ef55..36108619 100644 --- a/tools/libstorage-cpp/storage_lib.cpp +++ b/tools/libstorage-cpp/storage_lib.cpp @@ -205,6 +205,13 @@ std::string dispatch(StorageClient& client, const Options& options, const std::s const bool local = parts.size() == 4 ? parseBool(parts[3]) : false; return client.downloadFile(parts[1], parts[2], options.chunkSize, local); } + if (cmd == "stream-sink") { + if (parts.size() < 2 || parts.size() > 3) { + throw std::runtime_error("usage: stream-sink [local]"); + } + const bool local = parts.size() == 3 ? parseBool(parts[2]) : false; + return std::to_string(client.streamSink(parts[1], options.chunkSize, local)); + } if (cmd == "exists") { if (parts.size() != 2) throw std::runtime_error("usage: exists "); return client.exists(parts[1]); diff --git a/tools/libstorage-cpp/storage_lib_cli.cpp b/tools/libstorage-cpp/storage_lib_cli.cpp index b07155cc..774af6ca 100644 --- a/tools/libstorage-cpp/storage_lib_cli.cpp +++ b/tools/libstorage-cpp/storage_lib_cli.cpp @@ -87,6 +87,7 @@ void printUsage() { " fetch Fetch content into the local store\n" " upload Upload a file and print its cid\n" " download Download cid into file\n" + " stream-sink Stream cid and discard data; prints bytes\n" " roundtrip Upload, download, compare, and print cid\n" " repeat-roundtrip \n" " Repeat roundtrip in one node session\n" @@ -272,6 +273,9 @@ int main(int argc, char** argv) { std::cout << client.downloadFile( options.args[0], options.args[1], options.chunkSize, options.local) << "\n"; + } else if (options.command == "stream-sink") { + requireArgCount(options, 1); + std::cout << client.streamSink(options.args[0], options.chunkSize, options.local) << "\n"; } else if (options.command == "roundtrip") { requireArgCount(options, 2); std::cout << roundtrip(client, options, options.args[0], options.args[1]) << "\n"; diff --git a/tools/storage-test/AGENTS.md b/tools/storage-test/AGENTS.md index b9290650..852628c5 100644 --- a/tools/storage-test/AGENTS.md +++ b/tools/storage-test/AGENTS.md @@ -98,6 +98,7 @@ Common target commands: - `upload-random [--keep]` - `download [--local]` - `fetch [--wait]` +- `stream-sink [--local]` - `list` - `delete ` - `delete-all --yes` @@ -139,11 +140,13 @@ Important: lib daemon file paths are resolved in the daemon process working dire 1. Generate random files with sizes from `TEST_FILE_SIZES`. 2. Upload each file to the `remote` Linode REST node. -3. Download through selected target: `local` REST or `lib` daemon. -4. Validate SHA-256 of source and downloaded file. -5. Delete involved CIDs from the selected local target and `remote`. -6. Remove temp workspace unless `TEST_KEEP_FILES=1`. -7. Write a Markdown report in the caller's current directory: `./report-YYYY-MM-DD_HH-MM-SS.md`. +3. Measure manifest resolution through selected target. +4. Measure network stream-to-sink through selected target. +5. Measure local-only write through selected target. +6. Validate SHA-256 of source and downloaded file. +7. Delete involved CIDs from the selected local target and `remote`. +8. Remove temp workspace unless `TEST_KEEP_FILES=1`. +9. Write a Markdown report in the caller's current directory: `./report-YYYY-MM-DD_HH-MM-SS.md`. Defaults: @@ -151,7 +154,9 @@ Defaults: - `TEST_KEEP_FILES=0` - Max per-file test size is 10 MB. -The report includes timestamps, duration, target, file size list, workspace path, per-file CID/hash/path details, and cleanup results. `report-*.md` is ignored by the root `.gitignore`. +The report includes timestamps, duration, target, file size list, workspace path, per-file CID/hash/path details, manifest time, network stream time/speed, local write time/speed, total time/speed, and cleanup results. `report-*.md` is ignored by the root `.gitignore`. + +The metrics flow intentionally mirrors storage-module/UI primitives while splitting work for more detail. REST local uses `/network/manifest`, `/network/stream` to `/dev/null`, then local-only `/data/{cid}`. Lib uses daemon `manifest`, `stream-sink`, then local-only `download`. ## Verification Commands diff --git a/tools/storage-test/README.md b/tools/storage-test/README.md index cd8dd3d3..6aed8d83 100644 --- a/tools/storage-test/README.md +++ b/tools/storage-test/README.md @@ -238,7 +238,9 @@ Run the same scenario against the libstorage daemon: tools/storage-test/storage-test.sh lib test ``` -The scenario uploads random files to the remote Linode node, downloads them using the selected local target, validates SHA-256 hashes, and deletes involved CIDs from both sides. The default file sizes are `4K 1M 10M`; override with `TEST_FILE_SIZES`. +The scenario uploads random files to the remote Linode node, measures manifest resolution, network stream-to-sink, and local-only write through the selected local target, validates SHA-256 hashes, and deletes involved CIDs from both sides. The default file sizes are `4K 1M 10M`; override with `TEST_FILE_SIZES`. + +The per-file metrics are `Manifest Time`, `Network Stream Time`, `Network Stream Speed`, `Local Write Time`, `Local Write Speed`, `Total Time`, and `Total Speed`. The local REST target uses `/network/manifest`, `/network/stream` to `/dev/null`, then local-only `/data/{cid}`. The lib target uses `manifest`, `stream-sink`, then local-only `download` through the daemon. The scenario prints a detailed progress summary and writes a Markdown report in the current directory: @@ -256,6 +258,7 @@ The scenario prints a detailed progress summary and writes a Markdown report in | Fetch from network into node | `POST /api/storage/v1/data/{cid}/network` | | Fetch progress | `GET /api/storage/v1/data/{cid}/network/progress/{downloadId}` | | Stream from network | `GET /api/storage/v1/data/{cid}/network/stream` | +| Stream local-only | `GET /api/storage/v1/data/{cid}` | | Local existence check | `GET /api/storage/v1/data/{cid}/exists` | | Storage space | `GET /api/storage/v1/space` | diff --git a/tools/storage-test/storage-test.sh b/tools/storage-test/storage-test.sh index 1d45c9ec..d7ecd161 100755 --- a/tools/storage-test/storage-test.sh +++ b/tools/storage-test/storage-test.sh @@ -40,6 +40,9 @@ Target commands: fetch [--wait] Fetch CID from network into target local store. --wait is REST-only. + stream-sink [--local] + Stream CID and discard data. Used by test metrics. + list List manifest CIDs stored locally by target. @@ -59,8 +62,8 @@ Target commands: Show target peer ID. test - Upload random files to remote, download via local/lib target, validate hashes, - and clean up involved CIDs. Supported targets: local, lib. + Upload random files to remote, measure manifest/network-stream/local-write, + validate hashes, and clean up involved CIDs. Supported targets: local, lib. Lib-only target commands: lib spr @@ -483,6 +486,93 @@ size_to_bytes() { esac } +now_ns() { + date +%s%N +} + +elapsed_seconds() { + local start_ns="$1" + local end_ns="$2" + awk -v start="$start_ns" -v end="$end_ns" 'BEGIN { printf "%.3f", (end - start) / 1000000000 }' +} + +sum_seconds() { + awk -v a="$1" -v b="$2" -v c="$3" 'BEGIN { printf "%.3f", a + b + c }' +} + +format_speed() { + local bytes="$1" + local seconds="$2" + awk -v bytes="$bytes" -v seconds="$seconds" ' + BEGIN { + if (seconds <= 0) { + printf "n/a" + exit + } + speed = bytes / seconds + unit = "B/s" + if (speed >= 1024) { speed /= 1024; unit = "KiB/s" } + if (speed >= 1024) { speed /= 1024; unit = "MiB/s" } + if (speed >= 1024) { speed /= 1024; unit = "GiB/s" } + printf "%.2f %s", speed, unit + } + ' +} + +target_manifest_cid() { + local target="$1" + local cid="${2:-}" + [[ -n "$cid" ]] || die 'manifest requires ' + if [[ "$target" == 'lib' ]]; then + lib_result manifest "$cid" >/dev/null + return 0 + fi + + check_common_deps + local api + api="$(target_api "$target")" + curl -fsS "${api}/data/${cid}/network/manifest" >/dev/null +} + +target_stream_sink() { + local target="$1" + local cid="${2:-}" + local local_flag="${3:-}" + [[ -n "$cid" ]] || die 'stream-sink requires [--local]' + [[ -z "$local_flag" || "$local_flag" == '--local' ]] || die 'stream-sink only supports optional --local' + + if [[ "$target" == 'lib' ]]; then + lib_result stream-sink "$cid" "$([[ "$local_flag" == '--local' ]] && printf true || printf false)" >/dev/null + return 0 + fi + + check_common_deps + local api path + api="$(target_api "$target")" + if [[ "$local_flag" == '--local' ]]; then + path="${api}/data/${cid}" + else + path="${api}/data/${cid}/network/stream" + fi + curl -fLsS "$path" -o /dev/null +} + +target_download_local_cid() { + local target="$1" + local cid="${2:-}" + local out="${3:-}" + [[ -n "$cid" && -n "$out" ]] || die 'local download requires ' + if [[ "$target" == 'lib' ]]; then + target_download_cid lib "$cid" "$out" --local >/dev/null + return 0 + fi + + check_common_deps + local api + api="$(target_api "$target")" + curl -fLsS "${api}/data/${cid}" -o "$out" +} + target_test() { local target="$1" [[ "$target" == 'local' || "$target" == 'lib' ]] || die 'test is supported only for local and lib targets' @@ -504,6 +594,13 @@ target_test() { local -a result_downloads=() local -a result_cids=() local -a result_hashes=() + local -a result_manifest_times=() + local -a result_network_times=() + local -a result_network_speeds=() + local -a result_write_times=() + local -a result_write_speeds=() + local -a result_total_times=() + local -a result_total_speeds=() local -a cleanup_messages=() cleanup() { @@ -557,13 +654,20 @@ target_test() { printf -- "- **Cleanup failures:** \`%s\`\n\n" "$cleanup_failures" printf '## Files\n\n' - printf '| # | Size | Bytes | CID | SHA-256 | Source | Download |\n' - printf '|---:|---:|---:|---|---|---|---|\n' + printf '| # | Size | Bytes | Manifest Time | Network Stream Time | Network Stream Speed | Local Write Time | Local Write Speed | Total Time | Total Speed | CID | SHA-256 | Source | Download |\n' + printf '|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|\n' for i in "${!result_cids[@]}"; do - printf "| %d | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` |\n" \ + printf "| %d | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` |\n" \ "$((i + 1))" \ "${result_sizes[$i]}" \ "${result_bytes[$i]}" \ + "${result_manifest_times[$i]}" \ + "${result_network_times[$i]}" \ + "${result_network_speeds[$i]}" \ + "${result_write_times[$i]}" \ + "${result_write_speeds[$i]}" \ + "${result_total_times[$i]}" \ + "${result_total_speeds[$i]}" \ "${result_cids[$i]}" \ "${result_hashes[$i]}" \ "${result_sources[$i]}" \ @@ -589,6 +693,9 @@ target_test() { local size index=0 for size in $TEST_FILE_SIZES; do local bytes src out cid src_hash out_hash + local manifest_start manifest_end network_start network_end write_start write_end + local manifest_seconds network_seconds write_seconds total_seconds + local network_speed write_speed total_speed bytes="$(size_to_bytes "$size")" (( bytes <= max_bytes )) || die "test file size exceeds 10MB limit: $size" @@ -603,15 +710,28 @@ target_test() { cid="$(target_upload_file remote "$src")" cids+=("$cid") - printf '[%d] Download via %s: %s\n' "$index" "$target" "$cid" - if [[ "$target" == 'local' ]]; then - target_fetch_cid local "$cid" --wait >/dev/null - local_cids+=("$cid") - target_download_cid local "$cid" "$out" >/dev/null - else - target_download_cid lib "$cid" "$out" >/dev/null - local_cids+=("$cid") - fi + printf '[%d] Resolve manifest via %s: %s\n' "$index" "$target" "$cid" + manifest_start="$(now_ns)" + target_manifest_cid "$target" "$cid" + manifest_end="$(now_ns)" + manifest_seconds="$(elapsed_seconds "$manifest_start" "$manifest_end")" + + printf '[%d] Network stream via %s\n' "$index" "$target" + network_start="$(now_ns)" + target_stream_sink "$target" "$cid" + network_end="$(now_ns)" + network_seconds="$(elapsed_seconds "$network_start" "$network_end")" + network_speed="$(format_speed "$bytes" "$network_seconds")" + local_cids+=("$cid") + + printf '[%d] Local write via %s\n' "$index" "$target" + write_start="$(now_ns)" + target_download_local_cid "$target" "$cid" "$out" + write_end="$(now_ns)" + write_seconds="$(elapsed_seconds "$write_start" "$write_end")" + write_speed="$(format_speed "$bytes" "$write_seconds")" + total_seconds="$(sum_seconds "$manifest_seconds" "$network_seconds" "$write_seconds")" + total_speed="$(format_speed "$bytes" "$total_seconds")" out_hash="$(sha256sum "$out" | awk '{ print $1 }')" [[ "$src_hash" == "$out_hash" ]] || die "hash mismatch for cid $cid" @@ -621,6 +741,17 @@ target_test() { result_downloads+=("$out") result_cids+=("$cid") result_hashes+=("$src_hash") + result_manifest_times+=("${manifest_seconds}s") + result_network_times+=("${network_seconds}s") + result_network_speeds+=("$network_speed") + result_write_times+=("${write_seconds}s") + result_write_speeds+=("$write_speed") + result_total_times+=("${total_seconds}s") + result_total_speeds+=("$total_speed") + printf '[%d] Manifest: %ss\n' "$index" "$manifest_seconds" + printf '[%d] Network stream: %ss, %s\n' "$index" "$network_seconds" "$network_speed" + printf '[%d] Local write: %ss, %s\n' "$index" "$write_seconds" "$write_speed" + printf '[%d] Total: %ss, %s\n' "$index" "$total_seconds" "$total_speed" printf '[%d] OK sha256=%s\n' "$index" "$src_hash" done @@ -646,6 +777,7 @@ target_command() { upload-random) target_upload_random "$target" "$@" ;; download) target_download_cid "$target" "$@" ;; fetch) target_fetch_cid "$target" "$@" ;; + stream-sink) target_stream_sink "$target" "$@" ;; list) [[ $# -eq 0 ]] || die 'list does not accept arguments'; target_list_cids "$target" ;; delete) target_delete_cid "$target" "$@" ;; delete-all) target_delete_all "$target" "$@" ;;