From 6be2b6ff510d7241828a07f48ca250480d6dcfa7 Mon Sep 17 00:00:00 2001 From: gmega Date: Mon, 23 Jun 2025 18:55:17 -0300 Subject: [PATCH] chore: allow output to be customized per-module --- src/clh | 9 ++++++ src/codex.bash | 48 +++++++++++++++++++----------- src/experiment.bash | 28 ++++++++++++----- src/procmon.bash | 20 +++++++++++-- src/prometheus.bash | 17 +++++++++-- src/utils.bash | 15 ---------- test/test_codex.bats | 13 ++++---- test/test_experiment.bats | 23 +++++++++----- test/test_helper/common_setup.bash | 10 +++++-- test/test_procmon.bats | 5 +++- test/test_prometheus.bats | 2 ++ test/test_utils.bats | 17 +++++++++++ 12 files changed, 145 insertions(+), 62 deletions(-) create mode 100644 test/test_utils.bats diff --git a/src/clh b/src/clh index 2a367d5..3a68b48 100644 --- a/src/clh +++ b/src/clh @@ -2,5 +2,14 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} +PROM_TARGETS_DIR=${PROM_TARGETS_DIR:-"${LIB_SRC}/../dashboard/targets/"} +PROM_TARGETS_DIR=$(realpath "${PROM_TARGETS_DIR}") + +OUTPUTS=${OUTPUTS:-"${LIB_SRC}/../outputs"} +OUTPUTS=$(realpath "${OUTPUTS}") + # shellcheck source=./src/experiment.bash source "${LIB_SRC}/experiment.bash" + +exp_set_outputs "${OUTPUTS}" +prom_set_outputs "${PROM_TARGETS_DIR}" \ No newline at end of file diff --git a/src/codex.bash b/src/codex.bash index d7bbc98..f97b17a 100644 --- a/src/codex.bash +++ b/src/codex.bash @@ -20,37 +20,45 @@ if [ ! -f "${_cdx_binary}" ]; then exit 1 fi -# Output folders -_cdx_output=$(clh_output_folder "codex") -# generated files -_cdx_genfiles="${_cdx_output}/genfiles" -# downloaded files, per node. File names are CIDs -_cdx_downloads="${_cdx_output}/downloads" -# SHA1 of uploaded files, per node. File names are CIDs -_cdx_uploads="${_cdx_output}/uploads" -# Codex node logs, per node -_cdx_logs="${_cdx_output}/logs" -# Codex data directories, per node -_cdx_data="${_cdx_output}/data" - -# Partial timings, per operation per node -_cdx_timing_partials="${_cdx_output}/timing" # Custom prefix for timing logs _cdx_timing_prefix="" # Log file where timings are aggregated _cdx_timing_log="/dev/null" - -_cdx_defaultopts=() - # Base ports and timeouts _cdx_base_api_port=8080 _cdx_base_disc_port=8190 _cdx_base_metrics_port=8290 _cdx_node_start_timeout=30 +# Default options set for Codex nodes +_cdx_defaultopts=() # PID array for known Codex node processes declare -A _cdx_pids +cdx_set_outputs() { + # Output folders + _cdx_output="$1" + # generated files + _cdx_genfiles="${_cdx_output}/genfiles" + # downloaded files, per node. File names are CIDs + _cdx_downloads="${_cdx_output}/downloads" + # SHA1 of uploaded files, per node. File names are CIDs + _cdx_uploads="${_cdx_output}/uploads" + # Codex node logs, per node + _cdx_logs="${_cdx_output}/logs" + # Codex data directories, per node + _cdx_data="${_cdx_output}/data" + # Partial timings, per operation per node + _cdx_timing_partials="${_cdx_output}/timing" +} + +_ensure_outputs_set() { + if [ -z "${_cdx_output}" ]; then + echoerr "Error: outputs not set" + return 1 + fi +} + _cdx_api_port() { local node_index="$1" echo $((_cdx_base_api_port + node_index)) @@ -197,6 +205,8 @@ cdx_ensure_ready() { _cdx_init_node_outputs() { local node_index="$1" + _ensure_outputs_set || return 1 + mkdir -p "${_cdx_data}/codex-${node_index}" || return 1 mkdir -p "${_cdx_downloads}/codex-${node_index}" || return 1 mkdir -p "${_cdx_uploads}/codex-${node_index}" || return 1 @@ -206,6 +216,8 @@ _cdx_init_node_outputs() { # being piggybacked on cdx_launch_node and cdx_log_timings_start # so we don't have to add extra initialization calls. _cdx_init_global_outputs() { + _ensure_outputs_set || return 1 + mkdir -p "${_cdx_logs}" || return 1 mkdir -p "${_cdx_genfiles}" || return 1 mkdir -p "${_cdx_timing_partials}" || return 1 diff --git a/src/experiment.bash b/src/experiment.bash index 71c42f3..94fdf75 100644 --- a/src/experiment.bash +++ b/src/experiment.bash @@ -13,22 +13,34 @@ source "${LIB_SRC}/prometheus.bash" _experiment_type="" _experiment_id="" +exp_set_outputs() { + _exp_outputs="${1}" + + mkdir -p "${_exp_outputs}" || return 1 +} + +_ensure_outputs_set() { + if [ -z "${_exp_outputs}" ]; then + echoerr "experiments output not set" + return 1 + fi +} + exp_start() { local experiment_id experiment_type="$1" - experiment_id="$(date +%s)-${RANDOM}" || return 1 + _ensure_outputs_set || return 1 - # FIXME: this is pretty clumsy/confusing. We're "initing" the - # harness just so it sets the base output folder, and then - # "initing" it again. - if [ -z "${_clh_output}" ]; then - clh_init - fi + experiment_id="$(date +%s)-${RANDOM}" || return 1 _experiment_id="${experiment_id}" _experiment_type="${experiment_type}" + _experiment_output="${_exp_outputs}/${experiment_type}-${experiment_id}" + + mkdir -p "${_experiment_output}" || return 1 + pm_set_outputs "${_experiment_output}/pm" + cdx_set_outputs "${_experiment_output}/codex" - clh_init "${_clh_output}/${_experiment_id}" || return 1 cdx_add_defaultopts "--metrics" pm_register_callback "codex" _codex_target_changed diff --git a/src/procmon.bash b/src/procmon.bash index 4beca39..37e0e04 100644 --- a/src/procmon.bash +++ b/src/procmon.bash @@ -12,12 +12,20 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} # shellcheck source=./src/utils.bash source "${LIB_SRC}/utils.bash" -_pm_output=$(clh_output_folder "pm") _pm_pid="" declare -g -A _pm_callbacks +pm_set_outputs() { + _pm_output="$1" +} + _pm_init_output() { + if [ -z "${_pm_output}" ]; then + echoerr "Error: outputs not set" + return 1 + fi + rm -rf "${_pm_output}" || true mkdir -p "${_pm_output}" } @@ -28,16 +36,16 @@ _pm_init_output() { # 0 otherwise pm_start() { _pm_assert_state_not "running" || return 1 - _pm_init_output + _pm_init_output || return 1 echoerr "[procmon] starting process monitor" export _pm_output ( _pm_pid=${BASHPID} + echoerr "[procmon] started with PID $_pm_pid" while true; do pm_known_pids - echoerr "Known PIDs:" "${result[@]}" for pid in "${result[@]}"; do if kill -0 "${pid}" 2> /dev/null; then continue @@ -136,6 +144,12 @@ pm_state() { return 1 } +pm_is_running() { + local state + state=$(pm_state) + [ "$state" = "running" ] +} + _pm_halt() { _pm_assert_state "running" || return 1 diff --git a/src/prometheus.bash b/src/prometheus.bash index 4a10307..1e06ae1 100644 --- a/src/prometheus.bash +++ b/src/prometheus.bash @@ -6,7 +6,20 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)} # shellcheck source=./src/utils.bash source "${LIB_SRC}/utils.bash" -_prom_output=${PROM_TARGETS_DIR:-$(clh_output_folder "prometheus")} +_prom_output="" + +prom_set_outputs() { + _prom_output="$1" +} + +_prom_init_output() { + if [ -z "${_prom_output}" ]; then + echoerr "Error: outputs not set" + return 1 + fi + + mkdir -p "${_prom_output}" +} prom_add() { local metrics_port="$1"\ @@ -15,7 +28,7 @@ prom_add() { node="$4"\ node_type="$5" - mkdir -p "${_prom_output}" + _prom_init_output || return 1 cat > "${_prom_output}/${metrics_port}-${experiment_type}-${experiment_id}-${node}-${node_type}.json" < /dev/null; then exit 1 fi -clh_init() { - _clh_output=${1:-$(mktemp -d)} || exit 1 - _clh_output=$(realpath "$_clh_output") || exit 1 - mkdir -p "${_clh_output}" || exit 1 - export _clh_output -} - -clh_output_folder() { - echo "${_clh_output}/$1" -} - -clh_destroy() { - rm -rf "${_clh_output}" || true -} - echoerr() { echo "$@" >&2 } diff --git a/test/test_codex.bats b/test/test_codex.bats index 1a25933..2d27e4d 100755 --- a/test/test_codex.bats +++ b/test/test_codex.bats @@ -1,11 +1,14 @@ #!/usr/bin/env bats -# shellcheck disable=SC2128 +# shellcheck disable=SC2128,SC2076 setup() { load test_helper/common_setup common_setup # shellcheck source=./src/codex.bash source "${LIB_SRC}/codex.bash" + + pm_set_outputs "${TEST_OUTPUTS}/pm" + cdx_set_outputs "${TEST_OUTPUTS}/codex" } @test "should generate the correct Codex command line for node 0" { @@ -39,7 +42,7 @@ setup() { } @test "should allow setting of global default options" { - ! [[ "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]] + [[ ! "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]] cdx_add_defaultopts "--metrics" @@ -47,7 +50,7 @@ setup() { cdx_clear_defaultopts - ! [[ "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]] + [[ ! "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]] } @test "should fail readiness check if node is not running" { @@ -55,7 +58,7 @@ setup() { } @test "should pass readiness check if node is running" { - data_dir=$(clh_output_folder "codex-temp") + data_dir="${TEST_OUTPUTS}/codex-temp" "${_cdx_binary}" --nat:none --data-dir="$data_dir" &> /dev/null & pid=$! @@ -209,5 +212,5 @@ setup() { } teardown() { - clh_destroy + clean_outputs } diff --git a/test/test_experiment.bats b/test/test_experiment.bats index 09e05cd..fc71041 100644 --- a/test/test_experiment.bats +++ b/test/test_experiment.bats @@ -1,22 +1,29 @@ #!/usr/bin/env bats -# shellcheck disable=SC2128 +# shellcheck disable=SC2128,SC2076 setup() { load test_helper/common_setup common_setup # shellcheck source=./src/experiment.bash source "${LIB_SRC}/experiment.bash" + exp_set_outputs "${TEST_OUTPUTS}" + prom_set_outputs "${TEST_OUTPUTS}/prometheus" } @test "should create experiment folder and set it as the global harness output" { - output_base="${_clh_output}" - exp_start "experiment-type" - experiment_output="${output_base}/experiment-type-[0-9]+-[0-9]+" - [[ "${_clh_output}" =~ ${experiment_output} ]] + experiment_output="${TEST_OUTPUTS}/experiment-type-[0-9]+-[0-9]+" + found=false + + for output in "${TEST_OUTPUTS}"/*; do + if [[ "$output" =~ ${experiment_output} ]]; then + found=true + fi + done + + assert [ "$found" = true ] - assert [ -d "${_clh_output}" ] } @test "should launch Codex nodes with metrics enabled when there is an experiment in scope" { @@ -39,5 +46,7 @@ setup() { } teardown() { - pm_stop + if pm_is_running; then + pm_stop + fi } diff --git a/test/test_helper/common_setup.bash b/test/test_helper/common_setup.bash index bdf994b..5215fbd 100644 --- a/test/test_helper/common_setup.bash +++ b/test/test_helper/common_setup.bash @@ -3,10 +3,14 @@ common_setup() { load test_helper/bats-support/load load test_helper/bats-assert/load - export LIB_SRC="${BATS_TEST_DIRNAME}/../src" + LIB_SRC="$(realpath "${BATS_TEST_DIRNAME}/../src")" + TEST_OUTPUTS="$(realpath "${BATS_TEST_DIRNAME}/../test_outputs")" + export LIB_SRC TEST_OUTPUTS # shellcheck source=./src/utils.bash source "${LIB_SRC}/utils.bash" - - clh_init "${LIB_SRC}/../test_outputs" } + +clean_outputs() { + rm -rf "${TEST_OUTPUTS}" +} \ No newline at end of file diff --git a/test/test_procmon.bats b/test/test_procmon.bats index fad3d66..8e6f3f9 100644 --- a/test/test_procmon.bats +++ b/test/test_procmon.bats @@ -1,3 +1,4 @@ +#!/usr/bin/env bats # shellcheck disable=SC2128 setup() { load test_helper/common_setup @@ -5,6 +6,8 @@ setup() { # shellcheck source=./src/procmon.bash source "${LIB_SRC}/procmon.bash" + + pm_set_outputs "${TEST_OUTPUTS}/pm" } @test "should kill processes recursively" { @@ -234,4 +237,4 @@ callback() { assert_equal "$(cat "${_pm_output}/${pid}-sleepy-start-args")" "arg1 arg2" assert_equal "$(cat "${_pm_output}/${pid}-sleepy-exit-args")" "arg1 arg2" -} \ No newline at end of file +} diff --git a/test/test_prometheus.bats b/test/test_prometheus.bats index a9ce171..b6f4bac 100644 --- a/test/test_prometheus.bats +++ b/test/test_prometheus.bats @@ -1,9 +1,11 @@ +#!/usr/bin/env bats setup() { load test_helper/common_setup common_setup # shellcheck source=./src/prometheus.bash source "${LIB_SRC}/prometheus.bash" + prom_set_outputs "${TEST_OUTPUTS}/prometheus" } contains() { diff --git a/test/test_utils.bats b/test/test_utils.bats new file mode 100644 index 0000000..81d4c1f --- /dev/null +++ b/test/test_utils.bats @@ -0,0 +1,17 @@ +#!/usr/bin/env bats +setup() { + load test_helper/common_setup + common_setup +} + +@test "should shift an array" { + local arr=(1 2 3) + shift_arr arr + assert_equal "${arr[*]}" "2 3" +} + +@test "should shift and array by n places" { + local arr=(1 2 3 4) + shift_arr arr 2 + assert_equal "${arr[*]}" "3 4" +} \ No newline at end of file