mirror of
https://github.com/logos-storage/logos-storage-nim.git
synced 2026-07-27 12:03:20 +00:00
test: adds more metrics to the testing scenario align with the storage_module
This commit is contained in:
Marcin Czenko
parent
362bbac743
commit
5dd0846015
@ -73,6 +73,7 @@ Keep daemon commands low-level:
|
||||
- `exists`
|
||||
- `delete`
|
||||
- `fetch`
|
||||
- `stream-sink`
|
||||
- `manifest`
|
||||
- `connect`
|
||||
- `shutdown`
|
||||
@ -101,6 +102,8 @@ The protocol currently does not support paths or arguments containing spaces. If
|
||||
|
||||
`storage_lib_ctl` should return nonzero when the daemon responds with `"ok":false`.
|
||||
|
||||
`stream-sink <cid> [local]` uses released libstorage download primitives: `storage_download_init` followed by `storage_download_stream` with an empty output path. It discards progress bytes while counting transferred byte lengths and waits for terminal completion.
|
||||
|
||||
File paths sent over IPC are resolved by the daemon process. Callers that run from a different working directory, such as `tools/storage-test/storage-test.sh`, should send absolute paths.
|
||||
|
||||
## Daemon Options
|
||||
|
||||
@ -73,6 +73,7 @@ Send commands with `storage_lib_ctl`:
|
||||
./storage_lib_ctl --socket ./storage_lib.sock manifest <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock exists <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock download <cid> ./downloaded.md true
|
||||
./storage_lib_ctl --socket ./storage_lib.sock stream-sink <cid> false
|
||||
./storage_lib_ctl --socket ./storage_lib.sock delete <cid>
|
||||
./storage_lib_ctl --socket ./storage_lib.sock shutdown
|
||||
```
|
||||
@ -102,6 +103,7 @@ or:
|
||||
- `manifest <cid>`: prints manifest JSON.
|
||||
- `delete <cid>`: deletes locally stored content.
|
||||
- `fetch <cid>`: fetches content into the local store.
|
||||
- `stream-sink <cid> [local]`: streams content and discards data, returning transferred bytes.
|
||||
- `roundtrip <in> <out>`: uploads one file, downloads it, compares bytes, and prints the cid.
|
||||
- `repeat-roundtrip <in> <out-prefix> <count>`: repeats roundtrip in one node session.
|
||||
- `upload-many <file> <count>`: uploads the same file repeatedly in one node session.
|
||||
|
||||
@ -35,6 +35,7 @@ void StorageResponse::setResult(int callerRet, const char* msg, size_t len) {
|
||||
|
||||
if (callerRet == RET_PROGRESS) {
|
||||
progressCount_ += 1;
|
||||
progressBytes_ += len;
|
||||
if (msg && len > 0) {
|
||||
lastProgress_.assign(msg, len);
|
||||
} else {
|
||||
@ -78,6 +79,11 @@ size_t StorageResponse::progressCount() const {
|
||||
return progressCount_;
|
||||
}
|
||||
|
||||
size_t StorageResponse::progressBytes() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return progressBytes_;
|
||||
}
|
||||
|
||||
std::string StorageResponse::lastProgress() const {
|
||||
std::lock_guard<std::mutex> lock(mtx_);
|
||||
return lastProgress_;
|
||||
@ -264,6 +270,29 @@ std::string StorageClient::downloadFile(
|
||||
});
|
||||
}
|
||||
|
||||
size_t StorageClient::streamSink(const std::string& cid, size_t chunkSize, bool local) {
|
||||
call("storage_download_init", [this, &cid, chunkSize, local](StorageCallback cb, void* userData) {
|
||||
return storage_download_init(ctx_, cid.c_str(), chunkSize, local, cb, userData);
|
||||
});
|
||||
|
||||
StorageResponse resp;
|
||||
const int dispatchRet = storage_download_stream(
|
||||
ctx_, cid.c_str(), chunkSize, local, "", callback, &resp);
|
||||
if (!isOk(dispatchRet)) {
|
||||
throw std::runtime_error("storage_download_stream dispatch failed");
|
||||
}
|
||||
|
||||
if (!resp.wait(timeout_)) {
|
||||
throw std::runtime_error("storage_download_stream timed out");
|
||||
}
|
||||
|
||||
if (!isOk(resp.status())) {
|
||||
throw std::runtime_error("storage_download_stream failed: " + resp.data());
|
||||
}
|
||||
|
||||
return resp.progressBytes();
|
||||
}
|
||||
|
||||
std::string StorageClient::call(const char* name, const AsyncCall& fn) {
|
||||
StorageResponse resp;
|
||||
const int dispatchRet = fn(callback, &resp);
|
||||
|
||||
@ -23,6 +23,7 @@ public:
|
||||
int status() const;
|
||||
std::string data() const;
|
||||
size_t progressCount() const;
|
||||
size_t progressBytes() const;
|
||||
std::string lastProgress() const;
|
||||
|
||||
private:
|
||||
@ -32,6 +33,7 @@ private:
|
||||
int status_ = -1;
|
||||
std::string result_;
|
||||
size_t progressCount_ = 0;
|
||||
size_t progressBytes_ = 0;
|
||||
std::string lastProgress_;
|
||||
};
|
||||
|
||||
@ -68,6 +70,7 @@ public:
|
||||
const std::string& outputPath,
|
||||
size_t chunkSize,
|
||||
bool local);
|
||||
size_t streamSink(const std::string& cid, size_t chunkSize, bool local);
|
||||
|
||||
private:
|
||||
using AsyncCall = std::function<int(StorageCallback, void*)>;
|
||||
|
||||
@ -205,6 +205,13 @@ std::string dispatch(StorageClient& client, const Options& options, const std::s
|
||||
const bool local = parts.size() == 4 ? parseBool(parts[3]) : false;
|
||||
return client.downloadFile(parts[1], parts[2], options.chunkSize, local);
|
||||
}
|
||||
if (cmd == "stream-sink") {
|
||||
if (parts.size() < 2 || parts.size() > 3) {
|
||||
throw std::runtime_error("usage: stream-sink <cid> [local]");
|
||||
}
|
||||
const bool local = parts.size() == 3 ? parseBool(parts[2]) : false;
|
||||
return std::to_string(client.streamSink(parts[1], options.chunkSize, local));
|
||||
}
|
||||
if (cmd == "exists") {
|
||||
if (parts.size() != 2) throw std::runtime_error("usage: exists <cid>");
|
||||
return client.exists(parts[1]);
|
||||
|
||||
@ -87,6 +87,7 @@ void printUsage() {
|
||||
" fetch <cid> Fetch content into the local store\n"
|
||||
" upload <file> Upload a file and print its cid\n"
|
||||
" download <cid> <file> Download cid into file\n"
|
||||
" stream-sink <cid> Stream cid and discard data; prints bytes\n"
|
||||
" roundtrip <in> <out> Upload, download, compare, and print cid\n"
|
||||
" repeat-roundtrip <in> <out-prefix> <count>\n"
|
||||
" Repeat roundtrip in one node session\n"
|
||||
@ -272,6 +273,9 @@ int main(int argc, char** argv) {
|
||||
std::cout << client.downloadFile(
|
||||
options.args[0], options.args[1], options.chunkSize, options.local)
|
||||
<< "\n";
|
||||
} else if (options.command == "stream-sink") {
|
||||
requireArgCount(options, 1);
|
||||
std::cout << client.streamSink(options.args[0], options.chunkSize, options.local) << "\n";
|
||||
} else if (options.command == "roundtrip") {
|
||||
requireArgCount(options, 2);
|
||||
std::cout << roundtrip(client, options, options.args[0], options.args[1]) << "\n";
|
||||
|
||||
@ -98,6 +98,7 @@ Common target commands:
|
||||
- `upload-random <size> [--keep]`
|
||||
- `download <cid> <output-file> [--local]`
|
||||
- `fetch <cid> [--wait]`
|
||||
- `stream-sink <cid> [--local]`
|
||||
- `list`
|
||||
- `delete <cid>`
|
||||
- `delete-all --yes`
|
||||
@ -139,11 +140,13 @@ Important: lib daemon file paths are resolved in the daemon process working dire
|
||||
|
||||
1. Generate random files with sizes from `TEST_FILE_SIZES`.
|
||||
2. Upload each file to the `remote` Linode REST node.
|
||||
3. Download through selected target: `local` REST or `lib` daemon.
|
||||
4. Validate SHA-256 of source and downloaded file.
|
||||
5. Delete involved CIDs from the selected local target and `remote`.
|
||||
6. Remove temp workspace unless `TEST_KEEP_FILES=1`.
|
||||
7. Write a Markdown report in the caller's current directory: `./report-YYYY-MM-DD_HH-MM-SS.md`.
|
||||
3. Measure manifest resolution through selected target.
|
||||
4. Measure network stream-to-sink through selected target.
|
||||
5. Measure local-only write through selected target.
|
||||
6. Validate SHA-256 of source and downloaded file.
|
||||
7. Delete involved CIDs from the selected local target and `remote`.
|
||||
8. Remove temp workspace unless `TEST_KEEP_FILES=1`.
|
||||
9. Write a Markdown report in the caller's current directory: `./report-YYYY-MM-DD_HH-MM-SS.md`.
|
||||
|
||||
Defaults:
|
||||
|
||||
@ -151,7 +154,9 @@ Defaults:
|
||||
- `TEST_KEEP_FILES=0`
|
||||
- Max per-file test size is 10 MB.
|
||||
|
||||
The report includes timestamps, duration, target, file size list, workspace path, per-file CID/hash/path details, and cleanup results. `report-*.md` is ignored by the root `.gitignore`.
|
||||
The report includes timestamps, duration, target, file size list, workspace path, per-file CID/hash/path details, manifest time, network stream time/speed, local write time/speed, total time/speed, and cleanup results. `report-*.md` is ignored by the root `.gitignore`.
|
||||
|
||||
The metrics flow intentionally mirrors storage-module/UI primitives while splitting work for more detail. REST local uses `/network/manifest`, `/network/stream` to `/dev/null`, then local-only `/data/{cid}`. Lib uses daemon `manifest`, `stream-sink`, then local-only `download`.
|
||||
|
||||
## Verification Commands
|
||||
|
||||
|
||||
@ -238,7 +238,9 @@ Run the same scenario against the libstorage daemon:
|
||||
tools/storage-test/storage-test.sh lib test
|
||||
```
|
||||
|
||||
The scenario uploads random files to the remote Linode node, downloads them using the selected local target, validates SHA-256 hashes, and deletes involved CIDs from both sides. The default file sizes are `4K 1M 10M`; override with `TEST_FILE_SIZES`.
|
||||
The scenario uploads random files to the remote Linode node, measures manifest resolution, network stream-to-sink, and local-only write through the selected local target, validates SHA-256 hashes, and deletes involved CIDs from both sides. The default file sizes are `4K 1M 10M`; override with `TEST_FILE_SIZES`.
|
||||
|
||||
The per-file metrics are `Manifest Time`, `Network Stream Time`, `Network Stream Speed`, `Local Write Time`, `Local Write Speed`, `Total Time`, and `Total Speed`. The local REST target uses `/network/manifest`, `/network/stream` to `/dev/null`, then local-only `/data/{cid}`. The lib target uses `manifest`, `stream-sink`, then local-only `download` through the daemon.
|
||||
|
||||
The scenario prints a detailed progress summary and writes a Markdown report in the current directory:
|
||||
|
||||
@ -256,6 +258,7 @@ The scenario prints a detailed progress summary and writes a Markdown report in
|
||||
| Fetch from network into node | `POST /api/storage/v1/data/{cid}/network` |
|
||||
| Fetch progress | `GET /api/storage/v1/data/{cid}/network/progress/{downloadId}` |
|
||||
| Stream from network | `GET /api/storage/v1/data/{cid}/network/stream` |
|
||||
| Stream local-only | `GET /api/storage/v1/data/{cid}` |
|
||||
| Local existence check | `GET /api/storage/v1/data/{cid}/exists` |
|
||||
| Storage space | `GET /api/storage/v1/space` |
|
||||
|
||||
|
||||
@ -40,6 +40,9 @@ Target commands:
|
||||
<target> fetch <cid> [--wait]
|
||||
Fetch CID from network into target local store. --wait is REST-only.
|
||||
|
||||
<target> stream-sink <cid> [--local]
|
||||
Stream CID and discard data. Used by test metrics.
|
||||
|
||||
<target> list
|
||||
List manifest CIDs stored locally by target.
|
||||
|
||||
@ -59,8 +62,8 @@ Target commands:
|
||||
Show target peer ID.
|
||||
|
||||
<target> test
|
||||
Upload random files to remote, download via local/lib target, validate hashes,
|
||||
and clean up involved CIDs. Supported targets: local, lib.
|
||||
Upload random files to remote, measure manifest/network-stream/local-write,
|
||||
validate hashes, and clean up involved CIDs. Supported targets: local, lib.
|
||||
|
||||
Lib-only target commands:
|
||||
lib spr
|
||||
@ -483,6 +486,93 @@ size_to_bytes() {
|
||||
esac
|
||||
}
|
||||
|
||||
now_ns() {
|
||||
date +%s%N
|
||||
}
|
||||
|
||||
elapsed_seconds() {
|
||||
local start_ns="$1"
|
||||
local end_ns="$2"
|
||||
awk -v start="$start_ns" -v end="$end_ns" 'BEGIN { printf "%.3f", (end - start) / 1000000000 }'
|
||||
}
|
||||
|
||||
sum_seconds() {
|
||||
awk -v a="$1" -v b="$2" -v c="$3" 'BEGIN { printf "%.3f", a + b + c }'
|
||||
}
|
||||
|
||||
format_speed() {
|
||||
local bytes="$1"
|
||||
local seconds="$2"
|
||||
awk -v bytes="$bytes" -v seconds="$seconds" '
|
||||
BEGIN {
|
||||
if (seconds <= 0) {
|
||||
printf "n/a"
|
||||
exit
|
||||
}
|
||||
speed = bytes / seconds
|
||||
unit = "B/s"
|
||||
if (speed >= 1024) { speed /= 1024; unit = "KiB/s" }
|
||||
if (speed >= 1024) { speed /= 1024; unit = "MiB/s" }
|
||||
if (speed >= 1024) { speed /= 1024; unit = "GiB/s" }
|
||||
printf "%.2f %s", speed, unit
|
||||
}
|
||||
'
|
||||
}
|
||||
|
||||
target_manifest_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
[[ -n "$cid" ]] || die 'manifest requires <cid>'
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result manifest "$cid" >/dev/null
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fsS "${api}/data/${cid}/network/manifest" >/dev/null
|
||||
}
|
||||
|
||||
target_stream_sink() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
local local_flag="${3:-}"
|
||||
[[ -n "$cid" ]] || die 'stream-sink requires <cid> [--local]'
|
||||
[[ -z "$local_flag" || "$local_flag" == '--local' ]] || die 'stream-sink only supports optional --local'
|
||||
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
lib_result stream-sink "$cid" "$([[ "$local_flag" == '--local' ]] && printf true || printf false)" >/dev/null
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api path
|
||||
api="$(target_api "$target")"
|
||||
if [[ "$local_flag" == '--local' ]]; then
|
||||
path="${api}/data/${cid}"
|
||||
else
|
||||
path="${api}/data/${cid}/network/stream"
|
||||
fi
|
||||
curl -fLsS "$path" -o /dev/null
|
||||
}
|
||||
|
||||
target_download_local_cid() {
|
||||
local target="$1"
|
||||
local cid="${2:-}"
|
||||
local out="${3:-}"
|
||||
[[ -n "$cid" && -n "$out" ]] || die 'local download requires <cid> <output-file>'
|
||||
if [[ "$target" == 'lib' ]]; then
|
||||
target_download_cid lib "$cid" "$out" --local >/dev/null
|
||||
return 0
|
||||
fi
|
||||
|
||||
check_common_deps
|
||||
local api
|
||||
api="$(target_api "$target")"
|
||||
curl -fLsS "${api}/data/${cid}" -o "$out"
|
||||
}
|
||||
|
||||
target_test() {
|
||||
local target="$1"
|
||||
[[ "$target" == 'local' || "$target" == 'lib' ]] || die 'test is supported only for local and lib targets'
|
||||
@ -504,6 +594,13 @@ target_test() {
|
||||
local -a result_downloads=()
|
||||
local -a result_cids=()
|
||||
local -a result_hashes=()
|
||||
local -a result_manifest_times=()
|
||||
local -a result_network_times=()
|
||||
local -a result_network_speeds=()
|
||||
local -a result_write_times=()
|
||||
local -a result_write_speeds=()
|
||||
local -a result_total_times=()
|
||||
local -a result_total_speeds=()
|
||||
local -a cleanup_messages=()
|
||||
|
||||
cleanup() {
|
||||
@ -557,13 +654,20 @@ target_test() {
|
||||
printf -- "- **Cleanup failures:** \`%s\`\n\n" "$cleanup_failures"
|
||||
|
||||
printf '## Files\n\n'
|
||||
printf '| # | Size | Bytes | CID | SHA-256 | Source | Download |\n'
|
||||
printf '|---:|---:|---:|---|---|---|---|\n'
|
||||
printf '| # | Size | Bytes | Manifest Time | Network Stream Time | Network Stream Speed | Local Write Time | Local Write Speed | Total Time | Total Speed | CID | SHA-256 | Source | Download |\n'
|
||||
printf '|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|---|---|---|\n'
|
||||
for i in "${!result_cids[@]}"; do
|
||||
printf "| %d | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` |\n" \
|
||||
printf "| %d | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` | \`%s\` |\n" \
|
||||
"$((i + 1))" \
|
||||
"${result_sizes[$i]}" \
|
||||
"${result_bytes[$i]}" \
|
||||
"${result_manifest_times[$i]}" \
|
||||
"${result_network_times[$i]}" \
|
||||
"${result_network_speeds[$i]}" \
|
||||
"${result_write_times[$i]}" \
|
||||
"${result_write_speeds[$i]}" \
|
||||
"${result_total_times[$i]}" \
|
||||
"${result_total_speeds[$i]}" \
|
||||
"${result_cids[$i]}" \
|
||||
"${result_hashes[$i]}" \
|
||||
"${result_sources[$i]}" \
|
||||
@ -589,6 +693,9 @@ target_test() {
|
||||
local size index=0
|
||||
for size in $TEST_FILE_SIZES; do
|
||||
local bytes src out cid src_hash out_hash
|
||||
local manifest_start manifest_end network_start network_end write_start write_end
|
||||
local manifest_seconds network_seconds write_seconds total_seconds
|
||||
local network_speed write_speed total_speed
|
||||
bytes="$(size_to_bytes "$size")"
|
||||
(( bytes <= max_bytes )) || die "test file size exceeds 10MB limit: $size"
|
||||
|
||||
@ -603,15 +710,28 @@ target_test() {
|
||||
cid="$(target_upload_file remote "$src")"
|
||||
cids+=("$cid")
|
||||
|
||||
printf '[%d] Download via %s: %s\n' "$index" "$target" "$cid"
|
||||
if [[ "$target" == 'local' ]]; then
|
||||
target_fetch_cid local "$cid" --wait >/dev/null
|
||||
local_cids+=("$cid")
|
||||
target_download_cid local "$cid" "$out" >/dev/null
|
||||
else
|
||||
target_download_cid lib "$cid" "$out" >/dev/null
|
||||
local_cids+=("$cid")
|
||||
fi
|
||||
printf '[%d] Resolve manifest via %s: %s\n' "$index" "$target" "$cid"
|
||||
manifest_start="$(now_ns)"
|
||||
target_manifest_cid "$target" "$cid"
|
||||
manifest_end="$(now_ns)"
|
||||
manifest_seconds="$(elapsed_seconds "$manifest_start" "$manifest_end")"
|
||||
|
||||
printf '[%d] Network stream via %s\n' "$index" "$target"
|
||||
network_start="$(now_ns)"
|
||||
target_stream_sink "$target" "$cid"
|
||||
network_end="$(now_ns)"
|
||||
network_seconds="$(elapsed_seconds "$network_start" "$network_end")"
|
||||
network_speed="$(format_speed "$bytes" "$network_seconds")"
|
||||
local_cids+=("$cid")
|
||||
|
||||
printf '[%d] Local write via %s\n' "$index" "$target"
|
||||
write_start="$(now_ns)"
|
||||
target_download_local_cid "$target" "$cid" "$out"
|
||||
write_end="$(now_ns)"
|
||||
write_seconds="$(elapsed_seconds "$write_start" "$write_end")"
|
||||
write_speed="$(format_speed "$bytes" "$write_seconds")"
|
||||
total_seconds="$(sum_seconds "$manifest_seconds" "$network_seconds" "$write_seconds")"
|
||||
total_speed="$(format_speed "$bytes" "$total_seconds")"
|
||||
|
||||
out_hash="$(sha256sum "$out" | awk '{ print $1 }')"
|
||||
[[ "$src_hash" == "$out_hash" ]] || die "hash mismatch for cid $cid"
|
||||
@ -621,6 +741,17 @@ target_test() {
|
||||
result_downloads+=("$out")
|
||||
result_cids+=("$cid")
|
||||
result_hashes+=("$src_hash")
|
||||
result_manifest_times+=("${manifest_seconds}s")
|
||||
result_network_times+=("${network_seconds}s")
|
||||
result_network_speeds+=("$network_speed")
|
||||
result_write_times+=("${write_seconds}s")
|
||||
result_write_speeds+=("$write_speed")
|
||||
result_total_times+=("${total_seconds}s")
|
||||
result_total_speeds+=("$total_speed")
|
||||
printf '[%d] Manifest: %ss\n' "$index" "$manifest_seconds"
|
||||
printf '[%d] Network stream: %ss, %s\n' "$index" "$network_seconds" "$network_speed"
|
||||
printf '[%d] Local write: %ss, %s\n' "$index" "$write_seconds" "$write_speed"
|
||||
printf '[%d] Total: %ss, %s\n' "$index" "$total_seconds" "$total_speed"
|
||||
printf '[%d] OK sha256=%s\n' "$index" "$src_hash"
|
||||
done
|
||||
|
||||
@ -646,6 +777,7 @@ target_command() {
|
||||
upload-random) target_upload_random "$target" "$@" ;;
|
||||
download) target_download_cid "$target" "$@" ;;
|
||||
fetch) target_fetch_cid "$target" "$@" ;;
|
||||
stream-sink) target_stream_sink "$target" "$@" ;;
|
||||
list) [[ $# -eq 0 ]] || die 'list does not accept arguments'; target_list_cids "$target" ;;
|
||||
delete) target_delete_cid "$target" "$@" ;;
|
||||
delete-all) target_delete_all "$target" "$@" ;;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user