chore: allow output to be customized per-module

src/clh

@@ -2,5 +2,14 @@
 
 LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
 
+PROM_TARGETS_DIR=${PROM_TARGETS_DIR:-"${LIB_SRC}/../dashboard/targets/"}
+PROM_TARGETS_DIR=$(realpath "${PROM_TARGETS_DIR}")
+
+OUTPUTS=${OUTPUTS:-"${LIB_SRC}/../outputs"}
+OUTPUTS=$(realpath "${OUTPUTS}")
+
 # shellcheck source=./src/experiment.bash
 source "${LIB_SRC}/experiment.bash"
+
+exp_set_outputs "${OUTPUTS}"
+prom_set_outputs "${PROM_TARGETS_DIR}"

src/codex.bash

@@ -20,37 +20,45 @@ if [ ! -f "${_cdx_binary}" ]; then
   exit 1
 fi
 
-# Output folders
-_cdx_output=$(clh_output_folder "codex")
-# generated files
-_cdx_genfiles="${_cdx_output}/genfiles"
-# downloaded files, per node. File names are CIDs
-_cdx_downloads="${_cdx_output}/downloads"
-# SHA1 of uploaded files, per node. File names are CIDs
-_cdx_uploads="${_cdx_output}/uploads"
-# Codex node logs, per node
-_cdx_logs="${_cdx_output}/logs"
-# Codex data directories, per node
-_cdx_data="${_cdx_output}/data"
-
-# Partial timings, per operation per node
-_cdx_timing_partials="${_cdx_output}/timing"
 # Custom prefix for timing logs
 _cdx_timing_prefix=""
 # Log file where timings are aggregated
 _cdx_timing_log="/dev/null"
-
-_cdx_defaultopts=()
-
 # Base ports and timeouts
 _cdx_base_api_port=8080
 _cdx_base_disc_port=8190
 _cdx_base_metrics_port=8290
 _cdx_node_start_timeout=30
+# Default options set for Codex nodes
+_cdx_defaultopts=()
 
 # PID array for known Codex node processes
 declare -A _cdx_pids
 
+cdx_set_outputs() {
+  # Output folders
+  _cdx_output="$1"
+  # generated files
+  _cdx_genfiles="${_cdx_output}/genfiles"
+  # downloaded files, per node. File names are CIDs
+  _cdx_downloads="${_cdx_output}/downloads"
+  # SHA1 of uploaded files, per node. File names are CIDs
+  _cdx_uploads="${_cdx_output}/uploads"
+  # Codex node logs, per node
+  _cdx_logs="${_cdx_output}/logs"
+  # Codex data directories, per node
+  _cdx_data="${_cdx_output}/data"
+  # Partial timings, per operation per node
+  _cdx_timing_partials="${_cdx_output}/timing"
+}
+
+_ensure_outputs_set() {
+  if [ -z "${_cdx_output}" ]; then
+    echoerr "Error: outputs not set"
+    return 1
+  fi
+}
+
 _cdx_api_port() {
   local node_index="$1"
   echo $((_cdx_base_api_port + node_index))
@@ -197,6 +205,8 @@ cdx_ensure_ready() {
 
 _cdx_init_node_outputs() {
   local node_index="$1"
+  _ensure_outputs_set || return 1
+
   mkdir -p "${_cdx_data}/codex-${node_index}" || return 1
   mkdir -p "${_cdx_downloads}/codex-${node_index}" || return 1
   mkdir -p "${_cdx_uploads}/codex-${node_index}" || return 1
@@ -206,6 +216,8 @@ _cdx_init_node_outputs() {
 #   being piggybacked on cdx_launch_node and cdx_log_timings_start
 #   so we don't have to add extra initialization calls.
 _cdx_init_global_outputs() {
+  _ensure_outputs_set || return 1
+
   mkdir -p "${_cdx_logs}" || return 1
   mkdir -p "${_cdx_genfiles}" || return 1
   mkdir -p "${_cdx_timing_partials}" || return 1

src/experiment.bash

@@ -13,22 +13,34 @@ source "${LIB_SRC}/prometheus.bash"
 _experiment_type=""
 _experiment_id=""
 
+exp_set_outputs() {
+  _exp_outputs="${1}"
+
+  mkdir -p "${_exp_outputs}" || return 1
+}
+
+_ensure_outputs_set() {
+  if [ -z "${_exp_outputs}" ]; then
+    echoerr "experiments output not set"
+    return 1
+  fi
+}
+
 exp_start() {
   local experiment_id experiment_type="$1"
 
-  experiment_id="$(date +%s)-${RANDOM}" || return 1
+  _ensure_outputs_set || return 1
 
-  # FIXME: this is pretty clumsy/confusing. We're "initing" the
-  #   harness just so it sets the base output folder, and then
-  #   "initing" it again.
-  if [ -z "${_clh_output}" ]; then
-    clh_init
-  fi
+  experiment_id="$(date +%s)-${RANDOM}" || return 1
 
   _experiment_id="${experiment_id}"
   _experiment_type="${experiment_type}"
+  _experiment_output="${_exp_outputs}/${experiment_type}-${experiment_id}"
+
+  mkdir -p "${_experiment_output}" || return 1
+  pm_set_outputs "${_experiment_output}/pm"
+  cdx_set_outputs "${_experiment_output}/codex"
 
-  clh_init "${_clh_output}/${_experiment_id}" || return 1
   cdx_add_defaultopts "--metrics"
 
   pm_register_callback "codex" _codex_target_changed

src/procmon.bash

@@ -12,12 +12,20 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
 # shellcheck source=./src/utils.bash
 source "${LIB_SRC}/utils.bash"
 
-_pm_output=$(clh_output_folder "pm")
 _pm_pid=""
 
 declare -g -A _pm_callbacks
 
+pm_set_outputs() {
+  _pm_output="$1"
+}
+
 _pm_init_output() {
+  if [ -z "${_pm_output}" ]; then
+    echoerr "Error: outputs not set"
+    return 1
+  fi
+
   rm -rf "${_pm_output}" || true
   mkdir -p "${_pm_output}"
 }
@@ -28,16 +36,16 @@ _pm_init_output() {
 #   0 otherwise
 pm_start() {
   _pm_assert_state_not "running" || return 1
-  _pm_init_output
+  _pm_init_output || return 1
 
   echoerr "[procmon] starting process monitor"
 
   export _pm_output
   (
     _pm_pid=${BASHPID}
+    echoerr "[procmon] started with PID $_pm_pid"
     while true; do
       pm_known_pids
-      echoerr "Known PIDs:" "${result[@]}"
       for pid in "${result[@]}"; do
         if kill -0 "${pid}" 2> /dev/null; then
           continue
@@ -136,6 +144,12 @@ pm_state() {
   return 1
 }
 
+pm_is_running() {
+  local state
+  state=$(pm_state)
+  [ "$state" = "running" ]
+}
+
 _pm_halt() {
   _pm_assert_state "running" || return 1
 

src/prometheus.bash

@@ -6,7 +6,20 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
 # shellcheck source=./src/utils.bash
 source "${LIB_SRC}/utils.bash"
 
-_prom_output=${PROM_TARGETS_DIR:-$(clh_output_folder "prometheus")}
+_prom_output=""
+
+prom_set_outputs() {
+  _prom_output="$1"
+}
+
+_prom_init_output() {
+  if [ -z "${_prom_output}" ]; then
+    echoerr "Error: outputs not set"
+    return 1
+  fi
+
+  mkdir -p "${_prom_output}"
+}
 
 prom_add() {
   local metrics_port="$1"\
@@ -15,7 +28,7 @@ prom_add() {
     node="$4"\
     node_type="$5"
 
-  mkdir -p "${_prom_output}"
+  _prom_init_output || return 1
 
   cat > "${_prom_output}/${metrics_port}-${experiment_type}-${experiment_id}-${node}-${node_type}.json" <<EOF
 {

src/utils.bash

@@ -6,21 +6,6 @@ if ! command -v sha1sum > /dev/null; then
   exit 1
 fi
 
-clh_init() {
-  _clh_output=${1:-$(mktemp -d)} || exit 1
-  _clh_output=$(realpath "$_clh_output") || exit 1
-  mkdir -p "${_clh_output}" || exit 1
-  export _clh_output
-}
-
-clh_output_folder() {
-  echo "${_clh_output}/$1"
-}
-
-clh_destroy() {
-  rm -rf "${_clh_output}" || true
-}
-
 echoerr() {
   echo "$@" >&2
 }

test/test_codex.bats

@@ -1,11 +1,14 @@
 #!/usr/bin/env bats
-# shellcheck disable=SC2128
+# shellcheck disable=SC2128,SC2076
 setup() {
   load test_helper/common_setup
   common_setup
 
   # shellcheck source=./src/codex.bash
   source "${LIB_SRC}/codex.bash"
+
+  pm_set_outputs "${TEST_OUTPUTS}/pm"
+  cdx_set_outputs "${TEST_OUTPUTS}/codex"
 }
 
 @test "should generate the correct Codex command line for node 0" {
@@ -39,7 +42,7 @@ setup() {
 }
 
 @test "should allow setting of global default options" {
-  ! [[ "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]]
+  [[ ! "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]]
 
   cdx_add_defaultopts "--metrics"
 
@@ -47,7 +50,7 @@ setup() {
 
   cdx_clear_defaultopts
 
-  ! [[ "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]]
+  [[ ! "$(cdx_cmdline 0)" =~ "--metrics --metrics-port=8290 --metrics-address=0.0.0.0" ]]
 }
 
 @test "should fail readiness check if node is not running" {
@@ -55,7 +58,7 @@ setup() {
 }
 
 @test "should pass readiness check if node is running" {
-  data_dir=$(clh_output_folder "codex-temp")
+  data_dir="${TEST_OUTPUTS}/codex-temp"
   "${_cdx_binary}" --nat:none --data-dir="$data_dir" &> /dev/null &
   pid=$!
 
@@ -209,5 +212,5 @@ setup() {
 }
 
 teardown() {
-  clh_destroy
+  clean_outputs
 }

test/test_experiment.bats

@@ -1,22 +1,29 @@
 #!/usr/bin/env bats
-# shellcheck disable=SC2128
+# shellcheck disable=SC2128,SC2076
 setup() {
   load test_helper/common_setup
   common_setup
 
   # shellcheck source=./src/experiment.bash
   source "${LIB_SRC}/experiment.bash"
+  exp_set_outputs "${TEST_OUTPUTS}"
+  prom_set_outputs "${TEST_OUTPUTS}/prometheus"
 }
 
 @test "should create experiment folder and set it as the global harness output" {
-  output_base="${_clh_output}"
-
   exp_start "experiment-type"
-  experiment_output="${output_base}/experiment-type-[0-9]+-[0-9]+"
 
-  [[ "${_clh_output}" =~ ${experiment_output} ]]
+  experiment_output="${TEST_OUTPUTS}/experiment-type-[0-9]+-[0-9]+"
+  found=false
+
+  for output in "${TEST_OUTPUTS}"/*; do
+    if [[ "$output" =~ ${experiment_output} ]]; then
+      found=true
+    fi
+  done
+
+  assert [ "$found" = true ]
 
-  assert [ -d "${_clh_output}" ]
 }
 
 @test "should launch Codex nodes with metrics enabled when there is an experiment in scope" {
@@ -39,5 +46,7 @@ setup() {
 }
 
 teardown() {
-  pm_stop
+  if pm_is_running; then
+    pm_stop
+  fi
 }

test/test_helper/common_setup.bash

@@ -3,10 +3,14 @@ common_setup() {
   load test_helper/bats-support/load
   load test_helper/bats-assert/load
 
-  export LIB_SRC="${BATS_TEST_DIRNAME}/../src"
+  LIB_SRC="$(realpath "${BATS_TEST_DIRNAME}/../src")"
+  TEST_OUTPUTS="$(realpath "${BATS_TEST_DIRNAME}/../test_outputs")"
+  export LIB_SRC TEST_OUTPUTS
 
   # shellcheck source=./src/utils.bash
   source "${LIB_SRC}/utils.bash"
-
-  clh_init "${LIB_SRC}/../test_outputs"
 }
+
+clean_outputs() {
+  rm -rf "${TEST_OUTPUTS}"
+}

test/test_procmon.bats

@@ -1,3 +1,4 @@
+#!/usr/bin/env bats
 # shellcheck disable=SC2128
 setup() {
   load test_helper/common_setup
@@ -5,6 +6,8 @@ setup() {
 
   # shellcheck source=./src/procmon.bash
   source "${LIB_SRC}/procmon.bash"
+
+  pm_set_outputs "${TEST_OUTPUTS}/pm"
 }
 
 @test "should kill processes recursively" {
@@ -234,4 +237,4 @@ callback() {
 
   assert_equal "$(cat "${_pm_output}/${pid}-sleepy-start-args")" "arg1 arg2"
   assert_equal "$(cat "${_pm_output}/${pid}-sleepy-exit-args")" "arg1 arg2"
-}
+}

test/test_prometheus.bats

@@ -1,9 +1,11 @@
+#!/usr/bin/env bats
 setup() {
   load test_helper/common_setup
   common_setup
 
   # shellcheck source=./src/prometheus.bash
   source "${LIB_SRC}/prometheus.bash"
+  prom_set_outputs "${TEST_OUTPUTS}/prometheus"
 }
 
 contains() {

test/test_utils.bats

@@ -0,0 +1,17 @@
+#!/usr/bin/env bats
+setup() {
+  load test_helper/common_setup
+  common_setup
+}
+
+@test "should shift an array" {
+  local arr=(1 2 3)
+  shift_arr arr
+  assert_equal "${arr[*]}" "2 3"
+}
+
+@test "should shift and array by n places" {
+  local arr=(1 2 3 4)
+  shift_arr arr 2
+  assert_equal "${arr[*]}" "3 4"
+}