From 2781f2b9ecbdc34d8fe010abb65e4d4cffce2a41 Mon Sep 17 00:00:00 2001 From: gmega Date: Thu, 19 Jun 2025 18:46:50 -0300 Subject: [PATCH] feat: add synchronous verifiable upload/download --- src/codex.bash | 102 +++++++++++++++++++++++++++++++++---------- src/procmon.bash | 2 + src/utils.bash | 13 ++++++ test/test_codex.bats | 27 +++++++++--- 4 files changed, 116 insertions(+), 28 deletions(-) diff --git a/src/codex.bash b/src/codex.bash index f444bb2..43b14b1 100644 --- a/src/codex.bash +++ b/src/codex.bash @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -o pipefail + LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} # shellcheck source=./src/config.bash @@ -8,13 +10,9 @@ source "${LIB_SRC}/utils.bash" # shellcheck source=./src/procmon.bash source "${LIB_SRC}/procmon.bash" -_cdx_output=$(clh_output_folder "codex") -_cdx_logs="${_cdx_output}/logs" -_cdx_data="${_cdx_output}/data" - +# Codex binary if [ -z "${CODEX_BINARY}" ]; then - run command -v codex - _cdx_binary="${output}" + _cdx_binary="$(command -v codex)" else _cdx_binary="${CODEX_BINARY}" fi @@ -24,29 +22,44 @@ if [ ! -f "${_cdx_binary}" ]; then exit 1 fi +# Output folders +_cdx_output=$(clh_output_folder "codex") +_cdx_genfiles="${_cdx_output}/genfiles" +_cdx_downloads="${_cdx_output}/downloads" +_cdx_uploads="${_cdx_output}/uploads" +_cdx_logs="${_cdx_output}/logs" +_cdx_data="${_cdx_output}/data" + +# Base ports and timeouts _cdx_base_api_port=8080 _cdx_base_disc_port=8190 _cdx_base_metrics_port=8290 - _cdx_node_start_timeout=30 +# PID array for known Codex node processes declare -A _cdx_pids +_cdx_api_port() { + local node_index="$1" + echo $((_cdx_base_api_port + node_index)) +} + +_cdx_disc_port() { + local node_index="$1" + echo $((_cdx_base_disc_port + node_index)) +} + +_cdx_metrics_port() { + local node_index="$1" + echo $((_cdx_base_metrics_port + node_index)) +} + cdx_cmdline() { - local api_port\ - disc_port\ - metrics_port\ - spr\ - node_index\ - cdx_cmd="${_cdx_binary} --nat:none" + local node_index spr cdx_cmd="${_cdx_binary} --nat:none" node_index="$1" shift - api_port=$((_cdx_base_api_port + node_index)) - disc_port=$((_cdx_base_disc_port + node_index)) - metrics_port=$((_cdx_base_metrics_port + node_index)) - while [[ "$#" -gt 0 ]]; do case "$1" in --bootstrap-node) @@ -55,7 +68,7 @@ cdx_cmdline() { cdx_cmd="${cdx_cmd} --bootstrap-node=$spr" ;; --metrics) - cdx_cmd="${cdx_cmd} --metrics --metrics-port=${metrics_port} --metrics-address=0.0.0.0" + cdx_cmd="${cdx_cmd} --metrics --metrics-port=$(_cdx_metrics_port "$node_index") --metrics-address=0.0.0.0" ;; *) echoerr "Error: unknown option $1" @@ -73,14 +86,13 @@ cdx_cmdline() { # shellcheck disable=SC2140 echo "${cdx_cmd}"\ " --log-file=${_cdx_logs}/codex-${node_index}.log --data-dir=${_cdx_data}/codex-${node_index}"\ -" --api-port=${api_port} --disc-port=${disc_port} --log-level=INFO" +" --api-port=$(_cdx_api_port "$node_index") --disc-port=$(_cdx_disc_port "$node_index") --log-level=INFO" } cdx_get_spr() { - local node_index="$1" api_port spr - api_port=$((_cdx_base_api_port + node_index)) + local node_index="$1" spr - spr=$(curl --silent --fail "http://localhost:${api_port}/api/codex/v1/debug/info" | grep -oe 'spr:[^"]\+') + spr=$(curl --silent --fail "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/debug/info" | grep -oe 'spr:[^"]\+') if [[ -z "$spr" ]]; then echoerr "Error: unable to get SPR for node $node_index" return 1 @@ -147,4 +159,50 @@ _cdx_ensure_outputs() { local node_index="$1" mkdir -p "${_cdx_logs}" || return 1 mkdir -p "${_cdx_data}/codex-${node_index}" || return 1 + mkdir -p "${_cdx_genfiles}" || return 1 + mkdir -p "${_cdx_downloads}/codex-${node_index}" || return 1 + mkdir -p "${_cdx_uploads}/codex-${node_index}" || return 1 +} + +cdx_generate_file() { + local size_mb="${1}" filename + filename="${_cdx_genfiles}/file-$(date +%s).bin" + + echoerr "Generating file ${filename} of size ${size_mb}MB" + dd if=/dev/urandom of="${filename}" bs=1M count="${size_mb}" || return 1 + echo "${filename}" +} + +cdx_upload_file() { + local node_index="$1" filename="$2" content_sha1 cid + + content_sha1=$(sha1 "$filename") || return 1 + + echoerr "Uploading file ${filename} to node ${node_index}" + + cid=$(curl --silent --fail\ + -XPOST "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/data"\ + -T "${filename}") || return 1 + + echoerr "Upload SHA-1 is ${content_sha1}" + + echo "${content_sha1}" > "${_cdx_uploads}/codex-${node_index}/${cid}.sha1" + echo "${cid}" +} + +cdx_download_file() { + local node_index="$1" cid="$2" + curl --silent --fail\ + -XGET "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/data/$cid/network/stream"\ + -o "${_cdx_downloads}/codex-${node_index}/$cid" || return 1 +} + +cdx_upload_sha1() { + local node_index="$1" cid="$2" + cat "${_cdx_uploads}/codex-${node_index}/${cid}.sha1" || return 1 +} + +cdx_download_sha1() { + local node_index="$1" cid="$2" + sha1 "${_cdx_downloads}/codex-${node_index}/$cid" || return 1 } \ No newline at end of file diff --git a/src/procmon.bash b/src/procmon.bash index 300eb55..7893f88 100644 --- a/src/procmon.bash +++ b/src/procmon.bash @@ -1,4 +1,6 @@ #!/usr/bin/env bash +set -o pipefail + LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} # shellcheck source=./src/config.bash diff --git a/src/utils.bash b/src/utils.bash index d9c9fb9..94fd659 100644 --- a/src/utils.bash +++ b/src/utils.bash @@ -1,10 +1,19 @@ #!/usr/bin/env bash +set -o pipefail LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} # shellcheck source=./src/config.bash source "${LIB_SRC}/config.bash" +if ! command -v sha1sum > /dev/null; then + echoerr "Error: sha1sum is required for computing file hashes" + exit 1 +fi + +OUTPUTS=${OUTPUTS:-$(mktemp -d)} || exit 1 +OUTPUTS=$(realpath "$OUTPUTS") || exit 1 + clh_output_folder() { echo "${OUTPUTS}/$1" } @@ -23,4 +32,8 @@ await() { sleep 0.1 done return 0 +} + +sha1() { + sha1sum "$1" | cut -d ' ' -f 1 || return 1 } \ No newline at end of file diff --git a/test/test_codex.bats b/test/test_codex.bats index e692949..8aa5563 100755 --- a/test/test_codex.bats +++ b/test/test_codex.bats @@ -8,17 +8,17 @@ setup() { } @test "should generate the correct Codex command line for node 0" { - assert_equal "$(cdx_cmdline 0)" "codex --nat:none"\ + assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none"\ " --log-file=${_cdx_output}/logs/codex-0.log"\ " --data-dir=${_cdx_output}/data/codex-0"\ -" --api-port=8080 --disc-port=8190 --loglevel=INFO" +" --api-port=8080 --disc-port=8190 --log-level=INFO" } @test "should generate the correct Codex command line for node 1" { - assert_equal "$(cdx_cmdline 1 '--bootstrap-node' 'node-spr')" "codex --nat:none"\ + assert_equal "$(cdx_cmdline 1 '--bootstrap-node' 'node-spr')" "${_cdx_binary} --nat:none"\ " --bootstrap-node=node-spr --log-file=${_cdx_output}/logs/codex-1.log"\ " --data-dir=${_cdx_output}/data/codex-1"\ -" --api-port=8081 --disc-port=8191 --loglevel=INFO" +" --api-port=8081 --disc-port=8191 --log-level=INFO" } @test "should refuse to generate the command line for node > 0 if no SPR is provided" { @@ -27,11 +27,11 @@ setup() { } @test "should generate metrics options when metrics enabled for node" { - assert_equal "$(cdx_cmdline 0 --metrics)" "codex --nat:none"\ + assert_equal "$(cdx_cmdline 0 --metrics)" "${_cdx_binary} --nat:none"\ " --metrics --metrics-port=8290 --metrics-address=0.0.0.0"\ " --log-file=${_cdx_output}/logs/codex-0.log"\ " --data-dir=${_cdx_output}/data/codex-0"\ -" --api-port=8080 --disc-port=8190 --loglevel=INFO" +" --api-port=8080 --disc-port=8190 --log-level=INFO" } @test "should fail readiness check if node is not running" { @@ -73,3 +73,18 @@ setup() { pm_stop } + +@test "should upload and synchronously download file to Codex node" { + pm_start + + assert cdx_launch_node 0 + assert cdx_ensure_ready 0 3 + + filename=$(cdx_generate_file 10) + cid=$(cdx_upload_file 0 "$filename") + + cdx_download_file 0 "$cid" + assert_equal $(sha1 "${filename}") $(cdx_download_sha1 0 "$cid") + + pm_stop +} \ No newline at end of file