#!/usr/bin/env bash set -euo pipefail REMOTE_SSH_HOST="${REMOTE_SSH_HOST:-storage@172.235.163.25}" REMOTE_API_PORT="${REMOTE_API_PORT:-18080}" LOCAL_API_PORT="${LOCAL_API_PORT:-8080}" TUNNEL_CONTROL_PATH="${TUNNEL_CONTROL_PATH:-/tmp/logos-storage-tunnel-${USER:-user}.ctl}" CID_STATE_FILE="${CID_STATE_FILE:-${HOME}/.logos/storage/test/cids.log}" TEST_FILES_DIR="${TEST_FILES_DIR:-${HOME}/.logos/storage/test/files}" LOCAL_API="http://127.0.0.1:${LOCAL_API_PORT}/api/storage/v1" REMOTE_API="http://127.0.0.1:${REMOTE_API_PORT}/api/storage/v1" usage() { cat < [options] Environment: REMOTE_SSH_HOST SSH host for the remote node [$REMOTE_SSH_HOST] REMOTE_API_PORT Local tunnel port for remote API [$REMOTE_API_PORT] LOCAL_API_PORT Local node API port [$LOCAL_API_PORT] TUNNEL_CONTROL_PATH SSH control socket path [$TUNNEL_CONTROL_PATH] CID_STATE_FILE Upload history log [$CID_STATE_FILE] TEST_FILES_DIR Default generated test file directory [$TEST_FILES_DIR] Targets: local API at $LOCAL_API remote API through SSH tunnel at $REMOTE_API Commands: help Show this help. tunnel start|stop|status Manage SSH tunnel to the remote node API. make-file [output-file] Create random content with dd. Example: make-file 10M /tmp/logos-10M.bin upload Upload a file to target node and print the returned CID. Appends CID to CID_STATE_FILE. Example: upload remote /tmp/logos-10M.bin upload-random [--keep] Create a temporary random file, upload it, print CID. Deletes temp file unless --keep is passed. Example: upload-random remote 10M last-cid [target] Print the most recent CID from CID_STATE_FILE, optionally filtered by target. list List manifest CIDs stored locally by target node. delete Delete CID from target node local storage. delete-all --yes Delete every CID returned by list from target node local storage. exists Check whether target node has CID locally. space Show target node storage space information. peerid Show target node peer ID. fetch-local [--wait] Ask local node to fetch CID from the network. With --wait, poll progress until the background download is inactive or complete. stream-local Stream CID from the network through local node into output-file. Examples: $0 tunnel start $0 upload-random remote 10M $0 fetch-local --wait $0 stream-local /tmp/downloaded.bin $0 delete remote $0 delete-all local --yes EOF } die() { printf 'error: %s\n' "$*" >&2 exit 1 } need() { command -v "$1" >/dev/null 2>&1 || die "missing required command: $1" } check_common_deps() { need curl need jq } tunnel_status() { ssh -S "$TUNNEL_CONTROL_PATH" -O check "$REMOTE_SSH_HOST" >/dev/null 2>&1 } tunnel_start() { need ssh if tunnel_status; then printf 'tunnel already running: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \ "$REMOTE_API_PORT" "$REMOTE_SSH_HOST" return 0 fi ssh \ -M \ -S "$TUNNEL_CONTROL_PATH" \ -fN \ -L "127.0.0.1:${REMOTE_API_PORT}:127.0.0.1:8080" \ "$REMOTE_SSH_HOST" printf 'started tunnel: 127.0.0.1:%s -> %s:127.0.0.1:8080\n' \ "$REMOTE_API_PORT" "$REMOTE_SSH_HOST" } tunnel_stop() { need ssh if tunnel_status; then ssh -S "$TUNNEL_CONTROL_PATH" -O exit "$REMOTE_SSH_HOST" >/dev/null printf 'stopped tunnel\n' else printf 'tunnel not running\n' fi } target_api() { case "${1:-}" in local) printf '%s\n' "$LOCAL_API" ;; remote) tunnel_start >/dev/null printf '%s\n' "$REMOTE_API" ;; *) die "target must be 'local' or 'remote'" ;; esac } make_file() { local size="${1:-}" local out="${2:-}" [[ -n "$size" ]] || die 'make-file requires ' if [[ -z "$out" ]]; then mkdir -p "$TEST_FILES_DIR" out="${TEST_FILES_DIR}/logos-test-${size}-$(date -u +%Y%m%dT%H%M%SZ).bin" fi dd if=/dev/urandom of="$out" bs="$size" count=1 status=progress printf '%s\n' "$out" } upload_file() { check_common_deps local target="${1:-}" local file="${2:-}" [[ -n "$file" ]] || die 'upload requires ' [[ -f "$file" ]] || die "file not found: $file" [[ -z "${3:-}" ]] || die 'upload does not accept extra options' local api cid api="$(target_api "$target")" cid="$(curl -fsS \ -H 'Content-Type: application/octet-stream' \ --data-binary "@${file}" \ "${api}/data")" printf '%s\n' "$cid" mkdir -p "$(dirname "$CID_STATE_FILE")" printf '%s %s %s %s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$target" "$cid" "$file" >> "$CID_STATE_FILE" } upload_random() { local target="${1:-}" local size="${2:-}" local keep=false [[ -n "$target" && -n "$size" ]] || die 'upload-random requires [--keep]' shift 2 while [[ $# -gt 0 ]]; do case "$1" in --keep) keep=true ;; *) die "unknown upload-random option: $1" ;; esac shift done local tmp cid tmp="$(mktemp "${TMPDIR:-/tmp}/logos-storage-test.XXXXXX")" dd if=/dev/urandom of="$tmp" bs="$size" count=1 status=progress >&2 cid="$(upload_file "$target" "$tmp")" printf '%s\n' "$cid" if [[ "$keep" == true ]]; then printf 'kept file: %s\n' "$tmp" >&2 else rm -f "$tmp" fi } last_cid() { local target="" while [[ $# -gt 0 ]]; do case "$1" in local|remote) [[ -z "$target" ]] || die 'target specified more than once' target="$1" ;; *) die "unknown last-cid option: $1" ;; esac shift done [[ -f "$CID_STATE_FILE" ]] || die "CID state file not found: $CID_STATE_FILE" local cid if [[ -n "$target" ]]; then cid="$(awk -v target="$target" '$2 == target { cid = $3 } END { print cid }' "$CID_STATE_FILE")" else cid="$(awk 'NF >= 3 { cid = $3 } END { print cid }' "$CID_STATE_FILE")" fi [[ -n "$cid" ]] || die 'no matching CID found' printf '%s\n' "$cid" } list_cids() { check_common_deps local api api="$(target_api "${1:-}")" curl -fsS "${api}/data" | jq -r '.content[]?.cid' } delete_cid() { check_common_deps local target="${1:-}" local cid="${2:-}" [[ -n "$cid" ]] || die 'delete requires ' local api api="$(target_api "$target")" curl -fsS -X DELETE "${api}/data/${cid}" >/dev/null printf 'deleted %s from %s\n' "$cid" "$target" } delete_all() { local target="${1:-}" local yes="${2:-}" [[ "$yes" == '--yes' ]] || die 'delete-all requires --yes' local cid count=0 while IFS= read -r cid; do [[ -n "$cid" ]] || continue delete_cid "$target" "$cid" count=$((count + 1)) done < <(list_cids "$target") printf 'deleted %d CID(s) from %s\n' "$count" "$target" } simple_get() { check_common_deps local target="${1:-}" local path="${2:-}" local api api="$(target_api "$target")" curl -fsS "${api}/${path}" printf '\n' } exists_cid() { local target="${1:-}" local cid="${2:-}" [[ -n "$cid" ]] || die 'exists requires ' simple_get "$target" "data/${cid}/exists" } fetch_local() { check_common_deps local cid="${1:-}" local wait="${2:-}" [[ -n "$cid" ]] || die 'fetch-local requires [--wait]' [[ -z "$wait" || "$wait" == '--wait' ]] || die 'only supported option is --wait' local response download_id response="$(curl -fsS -X POST "${LOCAL_API}/data/${cid}/network")" printf '%s\n' "$response" | jq . if [[ "$wait" != '--wait' ]]; then return 0 fi download_id="$(printf '%s\n' "$response" | jq -r '.downloadId // empty')" [[ -n "$download_id" ]] || die 'response did not contain downloadId' while true; do local progress active received total progress="$(curl -fsS "${LOCAL_API}/data/${cid}/network/progress/${download_id}")" printf '%s\n' "$progress" | jq . active="$(printf '%s\n' "$progress" | jq -r '.active')" received="$(printf '%s\n' "$progress" | jq -r '.received // 0')" total="$(printf '%s\n' "$progress" | jq -r '.total // 0')" if [[ "$active" != 'true' || ( "$total" != '0' && "$received" == "$total" ) ]]; then break fi sleep 2 done } stream_local() { local cid="${1:-}" local out="${2:-}" [[ -n "$cid" && -n "$out" ]] || die 'stream-local requires ' curl -fL "${LOCAL_API}/data/${cid}/network/stream" -o "$out" printf '%s\n' "$out" } cmd="${1:-help}" shift || true case "$cmd" in help|--help|-h) usage ;; tunnel) case "${1:-}" in start) tunnel_start ;; stop) tunnel_stop ;; status) if tunnel_status; then printf 'tunnel running\n' else printf 'tunnel stopped\n' exit 1 fi ;; *) die 'usage: tunnel start|stop|status' ;; esac ;; make-file) make_file "$@" ;; upload) upload_file "$@" ;; upload-random) upload_random "$@" ;; last-cid) last_cid "$@" ;; list) list_cids "$@" ;; delete) delete_cid "$@" ;; delete-all) delete_all "$@" ;; exists) exists_cid "$@" ;; space) simple_get "${1:-}" 'space' ;; peerid) simple_get "${1:-}" 'peerid' ;; fetch-local) fetch_local "$@" ;; stream-local) stream_local "$@" ;; *) usage >&2 exit 1 ;; esac