mirror of
https://github.com/logos-storage/logos-storage-local-harness.git
synced 2026-07-22 14:59:31 +00:00
Merge 6b6df2016d731da4a8c3a2aedba42e26c1745034 into f86b1d79a9916f102a53c645f2f988708c5b8096
This commit is contained in:
Chrysostomos Nanakos
GitHub
commit
fdb74cff59
@ -6,11 +6,24 @@
|
||||
#
|
||||
# Outputs download times to a log file.
|
||||
#
|
||||
# Usage: k-node.sh <node_count> <repetitions> <output_log> <file_sizes...>
|
||||
# Usage: k-node.sh <node_count> <seeder_count> <repetitions> <output_log> [stagger_delay] [mix_enabled] [relay_backend] <file_sizes...>
|
||||
#
|
||||
# Example: k-node.sh 5 10 ./k-node-5-10.csv 100 200 500
|
||||
# Arguments:
|
||||
# stagger_delay - Optional delay in seconds between starting each leecher (default: 0 = simultaneous)
|
||||
# mix_enabled - Optional true|false. When true, DHT lookups are routed
|
||||
# through the Mix protocol via dht-proxy.
|
||||
# relay_backend - Optional standalone|storage|mix_relay_dht (default: storage).
|
||||
# Only meaningful when mix_enabled=true.
|
||||
# standalone -> mix_relay_dht binary with --no-dht-proxy
|
||||
# (pure Mix relay, no DHT)
|
||||
# storage -> use the storage binary as a Mix relay
|
||||
# mix_relay_dht -> mix_relay_dht binary
|
||||
# (relay + DHT proxy)
|
||||
set -e -o pipefail
|
||||
|
||||
# Default block size used by storage when chunking uploaded files (KB).
|
||||
BLOCK_SIZE_KB=64
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
# shellcheck source=./src/clh
|
||||
@ -20,9 +33,23 @@ node_count="${1:-2}"
|
||||
seeder_count="${2:-1}"
|
||||
repetitions="${3:-1}"
|
||||
output_log="${4:-"${OUTPUTS}/k-node-$(date +%s)-${RANDOM}.csv"}"
|
||||
stagger_delay="${5:-0}"
|
||||
mix_enabled="${6:-false}"
|
||||
relay_backend="${7:-storage}"
|
||||
|
||||
if [ "$#" -gt 4 ]; then
|
||||
shift 4
|
||||
case "${relay_backend}" in
|
||||
standalone|storage|mix_relay_dht) ;;
|
||||
*)
|
||||
echoerr "Error: invalid relay_backend='${relay_backend}'" \
|
||||
"(use 'standalone', 'storage', or 'mix_relay_dht')"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
export CDX_MIX_RELAY_BACKEND="${relay_backend}"
|
||||
|
||||
if [ "$#" -gt 7 ]; then
|
||||
shift 7
|
||||
file_sizes=("$@")
|
||||
else
|
||||
echoerr "No file sizes specified, using default (100)."
|
||||
@ -35,17 +62,51 @@ echoerr "* Nodes: ${node_count}"
|
||||
echoerr "* Seeders: ${seeder_count}"
|
||||
echoerr "* Repetitions: ${repetitions}"
|
||||
echoerr "* File Sizes: ${file_sizes[*]}"
|
||||
echoerr "* Stagger Delay: ${stagger_delay}s"
|
||||
echoerr "* Timing log: ${output_log}"
|
||||
if [ "$mix_enabled" = "true" ]; then
|
||||
echoerr "* Relay backend: ${relay_backend}"
|
||||
fi
|
||||
|
||||
relay_count="${_cdx_mix_min_pool}"
|
||||
|
||||
if [ "$mix_enabled" = "true" ]; then
|
||||
cdx_require_mix_pool_binary || exit 1
|
||||
|
||||
if [ "${relay_backend}" = "standalone" ]; then
|
||||
cdx_require_mix_relay_dht_binary || exit 1
|
||||
fi
|
||||
|
||||
mix_pool_dir="${_experiment_output}/mix-pool"
|
||||
cdx_generate_mix_pool "${relay_count}" "${mix_pool_dir}" || exit 1
|
||||
|
||||
export CDX_MIX_POOL_DIR="${mix_pool_dir}"
|
||||
export CDX_MIX_ENABLED=true
|
||||
|
||||
echoerr "* Mix: enabled, ${relay_count} relays via ${relay_backend} backend," \
|
||||
"pool=${mix_pool_dir}/pool.json"
|
||||
else
|
||||
echoerr "* Mix: disabled"
|
||||
fi
|
||||
|
||||
# TODO: procmon management should be moved into
|
||||
# experiment lifecycle management.
|
||||
# TODO: we should register this process with procmon
|
||||
# so its also killed if something fails.
|
||||
trap pm_stop EXIT INT TERM
|
||||
trap "cdx_stop_relays; cdx_stop_bootstrap; pm_stop" EXIT INT TERM
|
||||
pm_start
|
||||
|
||||
cdx_set_log_level "INFO;trace:blockexcnetwork,blockexcengine,discoveryengine"
|
||||
cdx_launch_network "${node_count}"
|
||||
cdx_set_log_level "INFO;info:blockexcnetwork,blockexcengine,discoveryengine"
|
||||
cdx_set_relay_log_level "INFO"
|
||||
|
||||
cdx_launch_bootstrap || exit 1
|
||||
bootstrap_spr=$(cdx_get_bootstrap_spr) || exit 1
|
||||
|
||||
if [ "$mix_enabled" = "true" ]; then
|
||||
cdx_launch_relays "${relay_count}" "${bootstrap_spr}" || exit 1
|
||||
fi
|
||||
|
||||
cdx_launch_network "${node_count}" "${bootstrap_spr}"
|
||||
|
||||
for file_size in "${file_sizes[@]}"; do
|
||||
for i in $(seq 1 "${repetitions}"); do
|
||||
@ -58,6 +119,12 @@ for file_size in "${file_sizes[@]}"; do
|
||||
|
||||
handles=()
|
||||
for j in $(seq "${seeder_count}" "$((node_count - 1))"); do
|
||||
if [ "$stagger_delay" -gt 0 ] && [ "$j" -gt "${seeder_count}" ]; then
|
||||
echoerr "Staggering: waiting ${stagger_delay}s before starting leecher $j..."
|
||||
sleep "$stagger_delay"
|
||||
fi
|
||||
|
||||
echoerr "Starting leecher $j download..."
|
||||
cdx_download_file_async "$j" "$cid"
|
||||
# shellcheck disable=SC2128
|
||||
handles+=("$result")
|
||||
@ -66,5 +133,70 @@ for file_size in "${file_sizes[@]}"; do
|
||||
await_all "${handles[@]}" "Inf"
|
||||
|
||||
cdx_log_timings_end
|
||||
|
||||
echoerr "=== Memory usage ==="
|
||||
for j in $(seq 0 "$((node_count - 1))"); do
|
||||
storage_pid=$(pgrep -f "storage.*--data-dir.*storage-$j" | head -1)
|
||||
if [ -z "$storage_pid" ]; then
|
||||
echoerr "Node $j: storage process not found"
|
||||
continue
|
||||
fi
|
||||
mem=$(ps -p "$storage_pid" -o rss= 2>/dev/null | awk '{print int($1/1024)}')
|
||||
[ -n "$mem" ] && echoerr "Node $j (PID $storage_pid): ${mem} MB"
|
||||
done
|
||||
|
||||
echoerr ""
|
||||
echoerr "=== Blocks served per node (peer-to-peer analysis) ==="
|
||||
total_blocks_served=0
|
||||
seeder_blocks_served=0
|
||||
leecher_blocks_served=0
|
||||
|
||||
for j in $(seq 0 "$((node_count - 1))"); do
|
||||
node_type="leecher"
|
||||
if [ "$j" -lt "$seeder_count" ]; then
|
||||
node_type="seeder"
|
||||
fi
|
||||
|
||||
# Query Prometheus metrics endpoint for storage_block_exchange_blocks_sent
|
||||
blocks_served=$(curl -s "http://localhost:$(_cdx_metrics_port "$j")/metrics" 2>/dev/null | \
|
||||
grep '^storage_block_exchange_blocks_sent_total' | \
|
||||
awk '{printf "%.0f", $2}')
|
||||
|
||||
if [ -z "$blocks_served" ]; then
|
||||
echoerr "Node $j ($node_type): Unable to query metrics"
|
||||
else
|
||||
echoerr "Node $j ($node_type): $blocks_served blocks sent"
|
||||
total_blocks_served=$((total_blocks_served + blocks_served))
|
||||
if [ "$node_type" = "seeder" ]; then
|
||||
seeder_blocks_served=$((seeder_blocks_served + blocks_served))
|
||||
else
|
||||
leecher_blocks_served=$((leecher_blocks_served + blocks_served))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
expected_blocks=$((file_size * 1024 / BLOCK_SIZE_KB))
|
||||
total_downloads=$((node_count - seeder_count))
|
||||
seeder_expected=$((expected_blocks * total_downloads))
|
||||
|
||||
echoerr "---"
|
||||
echoerr "Total blocks served by all nodes: $total_blocks_served"
|
||||
echoerr " Seeder(s) served: $seeder_blocks_served blocks"
|
||||
echoerr " Leechers served: $leecher_blocks_served blocks"
|
||||
echoerr ""
|
||||
echoerr "Expected seeder load (no peer-to-peer): $seeder_expected blocks"
|
||||
|
||||
if [ "$seeder_blocks_served" -lt "$seeder_expected" ]; then
|
||||
seeder_savings=$((seeder_expected - seeder_blocks_served))
|
||||
seeder_percent=$((seeder_savings * 100 / seeder_expected))
|
||||
echoerr "Seeder bandwidth savings: $seeder_savings blocks ($seeder_percent%)"
|
||||
if [ "$total_blocks_served" -gt 0 ]; then
|
||||
p2p_ratio=$((leecher_blocks_served * 100 / total_blocks_served))
|
||||
echoerr "Peer-to-peer ratio: $p2p_ratio% of blocks served by leechers"
|
||||
fi
|
||||
else
|
||||
echoerr "No peer-to-peer detected (seeder served all or more blocks)"
|
||||
fi
|
||||
echoerr "=== End blocks served analysis ==="
|
||||
done
|
||||
done
|
||||
|
||||
@ -31,7 +31,7 @@ detect_platform() {
|
||||
}
|
||||
|
||||
fetch_latest_version() {
|
||||
curl -sL "${API_BASE_URL}/releases/latest" |
|
||||
curl -sL "${API_BASE_URL}/releases" |
|
||||
grep -oE '"tag_name": "[^"]+"' |
|
||||
head -1 |
|
||||
sed 's/"tag_name": "//;s/"$//'
|
||||
@ -44,7 +44,7 @@ main() {
|
||||
echo "Detected platform: ${platform}"
|
||||
|
||||
echo "Fetching latest release info..."
|
||||
version=$(fetch_latest_version)
|
||||
version="${STORAGE_VERSION:-$(fetch_latest_version)}"
|
||||
|
||||
if [[ -z "$version" ]]; then
|
||||
echo "Error: Could not determine latest version" >&2
|
||||
@ -54,7 +54,7 @@ main() {
|
||||
echo "Latest version: ${version}"
|
||||
|
||||
executable_name="logos-storage-${platform}-${version}"
|
||||
download_url="${DOWNLOAD_BASE_URL}/${version}/${executable_name}.tar.gz"
|
||||
download_url="${DOWNLOAD_BASE_URL}/${version}/${executable_name}.zip"
|
||||
|
||||
echo "Downloading from: ${download_url}"
|
||||
|
||||
@ -62,11 +62,13 @@ main() {
|
||||
# shellcheck disable=SC2064
|
||||
trap "rm -rf '${tmp_dir}'" EXIT
|
||||
|
||||
curl -sL "$download_url" -o "${tmp_dir}/archive.tar.gz"
|
||||
if ! curl -sL --fail "$download_url" -o "${tmp_dir}/archive.zip"; then
|
||||
echo "Error: failed to download release asset from ${download_url}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Extracting..."
|
||||
echo "tar -xzf ${tmp_dir}/archive.tar.gz -C ${tmp_dir}"
|
||||
tar -xzf "${tmp_dir}/archive.tar.gz" -C "$tmp_dir"
|
||||
unzip -q "${tmp_dir}/archive.zip" -d "$tmp_dir"
|
||||
|
||||
mkdir -p "$output_dir"
|
||||
|
||||
|
||||
@ -52,5 +52,5 @@ prom_remove() {
|
||||
node="$4"\
|
||||
node_type="$5"
|
||||
|
||||
rm "${_prom_output}/${metrics_port}-${experiment_type}-${experiment_id}-${node}-${node_type}.json" || true
|
||||
rm -f "${_prom_output}/${metrics_port}-${experiment_type}-${experiment_id}-${node}-${node_type}.json"
|
||||
}
|
||||
|
||||
261
src/storage.bash
261
src/storage.bash
@ -23,6 +23,141 @@ fi
|
||||
|
||||
echoerr "[storage] Using binary at ${_cdx_binary}"
|
||||
|
||||
# Mix pool tool binary (only required when an experiment turns Mix on)
|
||||
if [ -n "${MIX_POOL_BINARY:-}" ]; then
|
||||
_cdx_mix_pool_binary="${MIX_POOL_BINARY}"
|
||||
fi
|
||||
|
||||
if [ -n "${MIX_RELAY_DHT_BINARY:-}" ]; then
|
||||
_cdx_mix_relay_dht_binary="${MIX_RELAY_DHT_BINARY}"
|
||||
fi
|
||||
|
||||
_cdx_mix_min_pool=4
|
||||
|
||||
# Tracked PIDs for relay processes (separate from storage node PIDs)
|
||||
declare -A _cdx_mix_relay_pids
|
||||
|
||||
cdx_require_binary() {
|
||||
local path="$1" label="$2" env_name="$3"
|
||||
if [ -z "$path" ] || [ ! -x "$path" ]; then
|
||||
echoerr "Error: no valid ${label} binary found." \
|
||||
"Set ${env_name} to point to a compiled ${label}."
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
cdx_require_mix_pool_binary() {
|
||||
cdx_require_binary \
|
||||
"${_cdx_mix_pool_binary:-}" "mix_pool" "MIX_POOL_BINARY"
|
||||
}
|
||||
|
||||
cdx_require_mix_relay_dht_binary() {
|
||||
cdx_require_binary \
|
||||
"${_cdx_mix_relay_dht_binary:-}" "mix_relay_dht" "MIX_RELAY_DHT_BINARY"
|
||||
}
|
||||
|
||||
cdx_generate_mix_pool() {
|
||||
local pool_size="$1" pool_dir="$2"
|
||||
cdx_require_mix_pool_binary || return 1
|
||||
rm -rf "${pool_dir}"
|
||||
mkdir -p "${pool_dir}"
|
||||
"${_cdx_mix_pool_binary}" init \
|
||||
--pool="${pool_dir}/pool.json" \
|
||||
--count="${pool_size}" \
|
||||
--outdir="${pool_dir}/relays" >&2 || return 1
|
||||
}
|
||||
|
||||
cdx_launch_relay() {
|
||||
local relay_index="$1" bootstrap_spr="${2:-}" base_port log_file data_dir cmd backend
|
||||
backend="${CDX_MIX_RELAY_BACKEND:-standalone}"
|
||||
|
||||
if [ -z "${CDX_MIX_POOL_DIR:-}" ]; then
|
||||
echoerr "Error: cdx_launch_relay requires CDX_MIX_POOL_DIR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
_cdx_init_global_outputs || return 1
|
||||
|
||||
base_port="${CDX_RELAY_BASE_PORT:-4242}"
|
||||
data_dir="${CDX_MIX_POOL_DIR}/relays/relay_${relay_index}"
|
||||
|
||||
case "${backend}" in
|
||||
standalone)
|
||||
cdx_require_mix_relay_dht_binary || return 1
|
||||
log_file="${_cdx_logs}/relay-${relay_index}.log"
|
||||
cmd="${_cdx_mix_relay_dht_binary} \
|
||||
--data-dir=${data_dir} \
|
||||
--listen-ip=127.0.0.1 \
|
||||
--listen-port=$((base_port + relay_index)) \
|
||||
--no-dht-proxy \
|
||||
'--log-level=${_cdx_relay_log_level}'"
|
||||
;;
|
||||
storage)
|
||||
log_file="${_cdx_logs}/storage-relay-${relay_index}.log"
|
||||
local relay_api_port=$((9080 + relay_index))
|
||||
local relay_disc_port=$((9190 + relay_index))
|
||||
local relay_metrics_port=$((9290 + relay_index))
|
||||
cmd="${_cdx_binary} --nat:none \
|
||||
--listen-ip=127.0.0.1 \
|
||||
--listen-port=$((base_port + relay_index)) \
|
||||
--data-dir=${data_dir} \
|
||||
--api-port=${relay_api_port} \
|
||||
--disc-port=${relay_disc_port} \
|
||||
--metrics-port=${relay_metrics_port} \
|
||||
--no-bootstrap-node \
|
||||
--mix-enabled \
|
||||
--mix-pool=${CDX_MIX_POOL_DIR}/pool.json \
|
||||
'--log-level=${_cdx_relay_log_level}'"
|
||||
;;
|
||||
mix_relay_dht)
|
||||
cdx_require_mix_relay_dht_binary || return 1
|
||||
log_file="${_cdx_logs}/relay-dht-${relay_index}.log"
|
||||
local relay_disc_port=$((9190 + relay_index))
|
||||
cmd="${_cdx_mix_relay_dht_binary} \
|
||||
--data-dir=${data_dir} \
|
||||
--listen-ip=127.0.0.1 \
|
||||
--listen-port=$((base_port + relay_index)) \
|
||||
--disc-port=${relay_disc_port} \
|
||||
'--log-level=${_cdx_relay_log_level}'"
|
||||
if [[ -n "${bootstrap_spr}" ]]; then
|
||||
cmd="${cmd} --bootstrap-node=${bootstrap_spr}"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echoerr "Error: invalid CDX_MIX_RELAY_BACKEND='${backend}'" \
|
||||
"(use 'standalone', 'storage', or 'mix_relay_dht')"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
pm_async "bash" "-c" "exec ${cmd} &> ${log_file}" \
|
||||
-%- "mix-relay (${backend})" "${relay_index}"
|
||||
_cdx_mix_relay_pids[$relay_index]=$!
|
||||
|
||||
# Relay has no HTTP endpoint to poll; brief sleep to let it bind the port.
|
||||
sleep 0.5
|
||||
}
|
||||
|
||||
cdx_launch_relays() {
|
||||
local count="$1" bootstrap_spr="${2:-}" i backend
|
||||
backend="${CDX_MIX_RELAY_BACKEND:-standalone}"
|
||||
echoerr "Launching ${count} Mix relays (backend: ${backend})..."
|
||||
for i in $(seq 0 "$((count - 1))"); do
|
||||
cdx_launch_relay "$i" "${bootstrap_spr}" || return 1
|
||||
done
|
||||
}
|
||||
|
||||
cdx_stop_relays() {
|
||||
local idx pid
|
||||
for idx in "${!_cdx_mix_relay_pids[@]}"; do
|
||||
pid="${_cdx_mix_relay_pids[$idx]}"
|
||||
if kill -0 "$pid" 2>/dev/null; then
|
||||
kill -TERM "$pid" 2>/dev/null
|
||||
fi
|
||||
done
|
||||
_cdx_mix_relay_pids=()
|
||||
}
|
||||
|
||||
# Custom prefix for timing logs
|
||||
_cdx_timing_prefix=""
|
||||
# Log file where timings are aggregated
|
||||
@ -32,18 +167,22 @@ _cdx_base_api_port=8080
|
||||
_cdx_base_disc_port=8190
|
||||
_cdx_base_metrics_port=8290
|
||||
_cdx_node_start_timeout=30
|
||||
# Default options set for Logos Storage nodes
|
||||
_cdx_defaultopts=()
|
||||
# Log level for Logos Storage nodes.
|
||||
_cdx_log_level="INFO"
|
||||
_cdx_relay_log_level="INFO"
|
||||
|
||||
echoerr "[storage] Node log level is ${_cdx_log_level}"
|
||||
echoerr "[storage] Relay log level is ${_cdx_relay_log_level}"
|
||||
|
||||
# PID array for known Logos Storage node processes
|
||||
# FIXME: right now only processes destroyed with cdx_destroy_node are removed from
|
||||
# this array.
|
||||
declare -A _cdx_pids
|
||||
|
||||
_cdx_bootstrap_pid=""
|
||||
_cdx_bootstrap_api_port=7080
|
||||
_cdx_bootstrap_disc_port=7190
|
||||
|
||||
cdx_set_outputs() {
|
||||
# Output folders
|
||||
_cdx_output="$1"
|
||||
@ -95,8 +234,13 @@ cdx_set_log_level() {
|
||||
_cdx_log_level="$1"
|
||||
}
|
||||
|
||||
cdx_set_relay_log_level() {
|
||||
_cdx_relay_log_level="$1"
|
||||
}
|
||||
|
||||
cdx_cmdline() {
|
||||
local node_index spr cdx_cmd="${_cdx_binary} --nat:none" opts=("$@")
|
||||
local node_index spr proxy_spr \
|
||||
cdx_cmd="${_cdx_binary} --nat:none --listen-ip=127.0.0.1" opts=("$@")
|
||||
|
||||
opts+=("${_cdx_defaultopts[@]}")
|
||||
|
||||
@ -111,9 +255,17 @@ cdx_cmdline() {
|
||||
spr="${opts[0]}"
|
||||
cdx_cmd="${cdx_cmd} --bootstrap-node=$spr"
|
||||
;;
|
||||
--dht-mix-proxy)
|
||||
shift_arr opts
|
||||
proxy_spr="${opts[0]}"
|
||||
cdx_cmd="${cdx_cmd} --dht-mix-proxy=$proxy_spr"
|
||||
;;
|
||||
--metrics)
|
||||
cdx_cmd="${cdx_cmd} --metrics --metrics-port=$(_cdx_metrics_port "$node_index") --metrics-address=0.0.0.0"
|
||||
;;
|
||||
--no-bootstrap-node)
|
||||
cdx_cmd="${cdx_cmd} --no-bootstrap-node"
|
||||
;;
|
||||
*)
|
||||
echoerr "Error: unknown option $opt"
|
||||
return 1
|
||||
@ -127,6 +279,14 @@ cdx_cmdline() {
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ "${CDX_MIX_ENABLED:-}" == "true" ]]; then
|
||||
if [[ -z "${CDX_MIX_POOL_DIR:-}" ]]; then
|
||||
echoerr "Error: CDX_MIX_ENABLED requires CDX_MIX_POOL_DIR"
|
||||
return 1
|
||||
fi
|
||||
cdx_cmd="${cdx_cmd} --mix-enabled --mix-pool=${CDX_MIX_POOL_DIR}/pool.json"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2140
|
||||
echo "${cdx_cmd}"\
|
||||
"--data-dir=${_cdx_data}/storage-${node_index} --api-port=$(_cdx_api_port "$node_index")"\
|
||||
@ -134,11 +294,12 @@ cdx_cmdline() {
|
||||
}
|
||||
|
||||
cdx_get_spr() {
|
||||
local node_index="$1" spr
|
||||
local node_index="$1" field="${2:-spr}" spr
|
||||
|
||||
spr=$(curl --silent --fail "http://localhost:$(_cdx_api_port "$node_index")/api/storage/v1/debug/info" | grep -oe 'spr:[^"]\+')
|
||||
spr=$(curl --silent --fail "http://localhost:$(_cdx_api_port "$node_index")/api/storage/v1/debug/info" \
|
||||
| grep -oP '"'"$field"'"\s*:\s*"\K[^"]+')
|
||||
if [[ -z "$spr" ]]; then
|
||||
echoerr "Error: unable to get SPR for node $node_index"
|
||||
echoerr "Error: unable to get $field for node $node_index"
|
||||
return 1
|
||||
fi
|
||||
|
||||
@ -163,17 +324,95 @@ cdx_launch_node() {
|
||||
cdx_ensure_ready "$node_index"
|
||||
}
|
||||
|
||||
cdx_launch_bootstrap() {
|
||||
_cdx_init_global_outputs || return 1
|
||||
local data_dir="${_cdx_data}/bootstrap"
|
||||
mkdir -p "${data_dir}" || return 1
|
||||
|
||||
local cmd
|
||||
cmd="${_cdx_binary} --nat:none --listen-ip=127.0.0.1 \
|
||||
--no-bootstrap-node \
|
||||
--data-dir=${data_dir} \
|
||||
--api-port=${_cdx_bootstrap_api_port} \
|
||||
--disc-port=${_cdx_bootstrap_disc_port} \
|
||||
'--log-level=${_cdx_log_level}'"
|
||||
|
||||
pm_async "bash" "-c" "exec ${cmd} &> ${_cdx_logs}/bootstrap.log" \
|
||||
-%- "storage" "bootstrap"
|
||||
_cdx_bootstrap_pid=$!
|
||||
|
||||
local start="${SECONDS}"
|
||||
while true; do
|
||||
if cdx_get_bootstrap_spr 2> /dev/null > /dev/null; then
|
||||
echoerr "Bootstrap node is ready."
|
||||
return 0
|
||||
fi
|
||||
if (( SECONDS - start > _cdx_node_start_timeout )); then
|
||||
echoerr "Bootstrap node did not start within ${_cdx_node_start_timeout} seconds."
|
||||
return 1
|
||||
fi
|
||||
sleep 0.2
|
||||
done
|
||||
}
|
||||
|
||||
cdx_get_bootstrap_spr() {
|
||||
local spr
|
||||
spr=$(curl --silent --fail \
|
||||
"http://localhost:${_cdx_bootstrap_api_port}/api/storage/v1/debug/info" \
|
||||
| grep -oP '"spr"\s*:\s*"\K[^"]+')
|
||||
if [[ -z "$spr" ]]; then
|
||||
echoerr "Error: unable to get spr for bootstrap node"
|
||||
return 1
|
||||
fi
|
||||
echo "${spr}"
|
||||
}
|
||||
|
||||
cdx_stop_bootstrap() {
|
||||
if [[ -n "${_cdx_bootstrap_pid}" ]] && kill -0 "${_cdx_bootstrap_pid}" 2>/dev/null; then
|
||||
kill -TERM "${_cdx_bootstrap_pid}" 2>/dev/null
|
||||
fi
|
||||
_cdx_bootstrap_pid=""
|
||||
}
|
||||
|
||||
cdx_launch_network() {
|
||||
local node_count="$1" bootstrap_spr
|
||||
local node_count="$1" bootstrap_spr="${2:-}"
|
||||
local mix_node_spr extra_args=()
|
||||
if [[ "$node_count" -lt 2 ]]; then
|
||||
echoerr "Error: a Logos Storage network needs at least 2 nodes"
|
||||
return 1
|
||||
fi
|
||||
if [[ -z "${bootstrap_spr}" ]]; then
|
||||
echoerr "Error: cdx_launch_network requires a bootstrap SPR (2nd arg)"
|
||||
return 1
|
||||
fi
|
||||
|
||||
cdx_launch_node 0 --bootstrap-node "${bootstrap_spr}" || return 1
|
||||
|
||||
if [[ "${CDX_MIX_ENABLED:-}" == "true" ]]; then
|
||||
if [[ "${CDX_MIX_RELAY_BACKEND:-standalone}" == "mix_relay_dht" ]]; then
|
||||
local relay_count="${#_cdx_mix_relay_pids[@]}" i mix_node_spr_file tries
|
||||
for i in $(seq 0 "$((relay_count - 1))"); do
|
||||
mix_node_spr_file="${CDX_MIX_POOL_DIR}/relays/relay_${i}/mix_node.spr"
|
||||
tries=0
|
||||
while [[ ! -s "${mix_node_spr_file}" && $tries -lt 60 ]]; do
|
||||
sleep 0.2
|
||||
tries=$((tries + 1))
|
||||
done
|
||||
if [[ ! -s "${mix_node_spr_file}" ]]; then
|
||||
echoerr "Error: mix_relay_dht ${i} did not write ${mix_node_spr_file}"
|
||||
return 1
|
||||
fi
|
||||
mix_node_spr=$(cat "${mix_node_spr_file}")
|
||||
extra_args+=("--dht-mix-proxy" "$mix_node_spr")
|
||||
done
|
||||
else
|
||||
mix_node_spr=$(cdx_get_spr 0 providerRecord) || return 1
|
||||
extra_args+=("--dht-mix-proxy" "$mix_node_spr")
|
||||
fi
|
||||
fi
|
||||
|
||||
cdx_launch_node 0 || return 1
|
||||
bootstrap_spr=$(cdx_get_spr 0) || return 1
|
||||
for i in $(seq 1 "$((node_count - 1))"); do
|
||||
cdx_launch_node "$i" "--bootstrap-node" "$bootstrap_spr" || return 1
|
||||
cdx_launch_node "$i" "--bootstrap-node" "$bootstrap_spr" "${extra_args[@]}" || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
@ -341,4 +580,4 @@ cdx_log_timings_end() {
|
||||
|
||||
_cdx_timing_log="/dev/null"
|
||||
_cdx_timing_prefix=""
|
||||
}
|
||||
}
|
||||
|
||||
@ -13,14 +13,14 @@ setup() {
|
||||
|
||||
@test "should generate the correct Logos Storage command line for node 0" {
|
||||
# shellcheck disable=SC2140
|
||||
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none"\
|
||||
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none --listen-ip=127.0.0.1"\
|
||||
" --data-dir=${_cdx_output}/data/storage-0"\
|
||||
" --api-port=8080 --disc-port=8190 '--log-level=INFO'"
|
||||
}
|
||||
|
||||
@test "should generate the correct Logos Storage command line for node 1" {
|
||||
# shellcheck disable=SC2140
|
||||
assert_equal "$(cdx_cmdline 1 '--bootstrap-node' 'node-spr')" "${_cdx_binary} --nat:none"\
|
||||
assert_equal "$(cdx_cmdline 1 '--bootstrap-node' 'node-spr')" "${_cdx_binary} --nat:none --listen-ip=127.0.0.1"\
|
||||
" --bootstrap-node=node-spr"\
|
||||
" --data-dir=${_cdx_output}/data/storage-1"\
|
||||
" --api-port=8081 --disc-port=8191 '--log-level=INFO'"
|
||||
@ -33,7 +33,7 @@ setup() {
|
||||
|
||||
@test "should generate metrics options when metrics enabled for node" {
|
||||
# shellcheck disable=SC2140
|
||||
assert_equal "$(cdx_cmdline 0 --metrics)" "${_cdx_binary} --nat:none"\
|
||||
assert_equal "$(cdx_cmdline 0 --metrics)" "${_cdx_binary} --nat:none --listen-ip=127.0.0.1"\
|
||||
" --metrics --metrics-port=8290 --metrics-address=0.0.0.0"\
|
||||
" --data-dir=${_cdx_output}/data/storage-0"\
|
||||
" --api-port=8080 --disc-port=8190 '--log-level=INFO'"
|
||||
@ -43,7 +43,7 @@ setup() {
|
||||
cdx_set_log_level "DEBUG"
|
||||
|
||||
# shellcheck disable=SC2140
|
||||
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none"\
|
||||
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none --listen-ip=127.0.0.1"\
|
||||
" --data-dir=${_cdx_output}/data/storage-0"\
|
||||
" --api-port=8080 --disc-port=8190 '--log-level=DEBUG'"
|
||||
}
|
||||
@ -157,7 +157,10 @@ setup() {
|
||||
@test "should launch a Logos Storage network and allow uploading and downloading" {
|
||||
pm_start
|
||||
|
||||
assert cdx_launch_network 5
|
||||
assert cdx_launch_bootstrap
|
||||
bootstrap_spr=$(cdx_get_bootstrap_spr)
|
||||
|
||||
assert cdx_launch_network 5 "$bootstrap_spr"
|
||||
|
||||
filename=$(cdx_generate_file 10)
|
||||
cid=$(cdx_upload_file 0 "$filename")
|
||||
@ -182,7 +185,10 @@ setup() {
|
||||
|
||||
cdx_log_timings_start "${_cdx_output}/experiment-0.csv" "experiment-0,100MB"
|
||||
|
||||
assert cdx_launch_network 5
|
||||
assert cdx_launch_bootstrap
|
||||
bootstrap_spr=$(cdx_get_bootstrap_spr)
|
||||
|
||||
assert cdx_launch_network 5 "$bootstrap_spr"
|
||||
|
||||
filename=$(cdx_generate_file 10)
|
||||
cid=$(cdx_upload_file 0 "$filename")
|
||||
@ -223,5 +229,6 @@ setup() {
|
||||
}
|
||||
|
||||
teardown() {
|
||||
cdx_stop_bootstrap
|
||||
clean_outputs
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user