Merge pull request #1 from logos-storage/chore/rename

chore: rename Codex -> Logos Storage and add script to get binary; improve docs
This commit is contained in:
Giuliano Mega 2026-01-28 16:03:10 -03:00 committed by GitHub
commit fee91f88b9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 189 additions and 88 deletions

View File

@ -1 +0,0 @@
test/codex

View File

@ -1,4 +1,4 @@
name: Lint and Test the Codex Local Harness
name: Lint and Test the Logos Storage Local Harness
on:
push:
@ -6,6 +6,7 @@ on:
- main
tags:
- 'v*.*.*'
pull_request:
workflow_dispatch:
jobs:
@ -18,9 +19,8 @@ jobs:
with:
submodules: recursive
- name: Get Codex
run: |
curl -s https://get.codex.storage/install.sh | bash
- name: Get Logos Storage
run: ./scripts/get-logos-storage.sh /usr/local/bin
- name: Lint
uses: ludeeus/action-shellcheck@master

View File

@ -1,3 +1,24 @@
# codex-local-harness
# logos-storage-local-harness
A local, 100% bash-based implementation of a Logos Storage harness that can be used to launch
a cluster and run experiments against it.
Some features:
**Containerless.** Compile your binary and you are ready to go. No need to build
Docker images or fiddle with container networking. This makes this particularly suitable for
quick experimentation: edit the code, compile, launch the cluster.
**Monitoring.** The harness includes a Grafana/Prometheus setup which runs on containers and
can be started with docker compose. The harness will automatically add Logos Storage
processes to Prometheus monitoring with proper experiment and node labels.
**Primitives.** The harness is built as a collection of primitives which allow:
1. setting up and tearing down Logos Storage nodes;
2. scripting nodes.
It has been designed with experimentation in mind: setting up scenarios in which nodes are
started/stopped and execute a concerted set of actions over which we then do measurements.
The harness supports concurrent actions and exposes simple future-like synchronization
primitives.
A local, 100% bash-based implementation of a Codex harness that can be used to launch a Codex cluster and run experiments against it.

View File

@ -3,7 +3,7 @@ apiVersion: 1
providers:
- name: 'default'
orgId: 1
folder: 'Codex'
folder: 'Logos Storage'
type: file
disableDeletion: true
updateIntervalSeconds: 10

View File

@ -9,7 +9,7 @@ scrape_configs:
static_configs:
- targets: ['host.docker.internal:9090']
- job_name: 'codex-metrics'
- job_name: 'storage-metrics'
file_sd_configs:
- files:
- '/etc/prometheus/targets/*.json'

View File

@ -1,7 +1,8 @@
#!/usr/bin/env bash
#
# k-nodes runs a Codex network with k nodes in which a file is uploaded to
# node zero and then the remainder k - 1 nodes download it concurrently.
# k-nodes runs a Logos Storage network with k nodes in which a file is
# uploaded to node zero and then the remainder k - 1 nodes download it
# concurrently.
#
# Outputs download times to a log file.
#

80
scripts/get-logos-storage.sh Executable file
View File

@ -0,0 +1,80 @@
#!/usr/bin/env bash
set -euo pipefail
# Fetches the latest Logos Storage binary from GitHub releases.
#
# Usage: ./scripts/get0-logos-storage.sh [output_dir]
# output_dir: Directory to place the binary (default: ./bin)
readonly API_BASE_URL="https://api.github.com/repos/logos-storage/logos-storage-nim"
readonly DOWNLOAD_BASE_URL="https://github.com/logos-storage/logos-storage-nim/releases/download"
readonly BINARY_NAME="storage"
output_dir="${1:-./}"
detect_platform() {
local os arch
case "$(uname -s)" in
Linux*) os="linux" ;;
Darwin*) os="darwin" ;;
*) echo "Unsupported OS: $(uname -s)" >&2; exit 1 ;;
esac
case "$(uname -m)" in
x86_64|amd64) arch="amd64" ;;
arm64|aarch64) arch="arm64" ;;
*) echo "Unsupported architecture: $(uname -m)" >&2; exit 1 ;;
esac
echo "${os}-${arch}"
}
fetch_latest_version() {
curl -sL "${API_BASE_URL}/releases/latest" |
grep -oE '"tag_name": "[^"]+"' |
head -1 |
sed 's/"tag_name": "//;s/"$//'
}
main() {
local platform version download_url tmp_dir executable_name
platform=$(detect_platform)
echo "Detected platform: ${platform}"
echo "Fetching latest release info..."
version=$(fetch_latest_version)
if [[ -z "$version" ]]; then
echo "Error: Could not determine latest version" >&2
exit 1
fi
echo "Latest version: ${version}"
executable_name="logos-storage-${platform}-${version}"
download_url="${DOWNLOAD_BASE_URL}/${version}/${executable_name}.tar.gz"
echo "Downloading from: ${download_url}"
tmp_dir=$(mktemp -d)
# shellcheck disable=SC2064
trap "rm -rf '${tmp_dir}'" EXIT
curl -sL "$download_url" -o "${tmp_dir}/archive.tar.gz"
echo "Extracting..."
echo "tar -xzf ${tmp_dir}/archive.tar.gz -C ${tmp_dir}"
tar -xzf "${tmp_dir}/archive.tar.gz" -C "$tmp_dir"
mkdir -p "$output_dir"
mv "${tmp_dir}/${executable_name}" "${output_dir}/${BINARY_NAME}"
chmod +x "${output_dir}/${BINARY_NAME}"
echo "Logos Storage binary installed to: ${output_dir}/${BINARY_NAME}"
"${output_dir}/${BINARY_NAME}" --version || true
}
main "$@"

View File

@ -5,8 +5,8 @@ LIB_SRC=${LIB_SRC:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}
# shellcheck source=./src/utils.bash
source "${LIB_SRC}/utils.bash"
# shellcheck source=./src/codex.bash
source "${LIB_SRC}/codex.bash"
# shellcheck source=./src/storage.bash
source "${LIB_SRC}/storage.bash"
# shellcheck source=./src/prometheus.bash
source "${LIB_SRC}/prometheus.bash"
@ -39,16 +39,16 @@ exp_start() {
mkdir -p "${_experiment_output}" || return 1
pm_set_outputs "${_experiment_output}/pm"
cdx_set_outputs "${_experiment_output}/codex"
cdx_set_outputs "${_experiment_output}/storage"
cdx_add_defaultopts "--metrics"
pm_register_callback "codex" _codex_target_changed
pm_register_callback "storage" _storage_target_changed
echoerr "[exp] Experiment ID is ${experiment_id}"
}
_codex_target_changed() {
_storage_target_changed() {
local event="$1"
if [ "$event" = "start" ]; then
shift 3
@ -64,7 +64,7 @@ _add_target() {
metrics_port=$(_cdx_metrics_port "$node_index") || return 1
prom_add "${metrics_port}" "${_experiment_type}" "${_experiment_id}"\
"${node_index}" "codex"
"${node_index}" "storage"
}
_remove_target() {
@ -72,5 +72,5 @@ _remove_target() {
metrics_port=$(_cdx_metrics_port "$node_index") || return 1
prom_remove "${metrics_port}" "${_experiment_type}" "${_experiment_id}"\
"${node_index}" "codex"
"${node_index}" "storage"
}

View File

@ -8,20 +8,20 @@ source "${LIB_SRC}/utils.bash"
# shellcheck source=./src/procmon.bash
source "${LIB_SRC}/procmon.bash"
# Codex binary
if [ -z "${CODEX_BINARY}" ]; then
_cdx_binary="$(command -v codex)" || true
# Logos Storage binary
if [ -z "${STORAGE_BINARY}" ]; then
_cdx_binary="$(command -v storage)" || true
else
_cdx_binary="${CODEX_BINARY}"
_cdx_binary="${STORAGE_BINARY}"
fi
if [ ! -f "${_cdx_binary}" ]; then
echoerr "Error: no valid Codex binary found."\
"Set CODEX_BINARY to point to a valid Codex binary."
echoerr "Error: no valid Logos Storage binary found."\
"Set STORAGE_BINARY to point to a valid Logos Storage binary."
exit 1
fi
echoerr "[codex] Using binary at ${_cdx_binary}"
echoerr "[storage] Using binary at ${_cdx_binary}"
# Custom prefix for timing logs
_cdx_timing_prefix=""
@ -32,14 +32,14 @@ _cdx_base_api_port=8080
_cdx_base_disc_port=8190
_cdx_base_metrics_port=8290
_cdx_node_start_timeout=30
# Default options set for Codex nodes
# Default options set for Logos Storage nodes
_cdx_defaultopts=()
# Log level for Codex nodes.
# Log level for Logos Storage nodes.
_cdx_log_level="INFO"
echoerr "[codex] Node log level is ${_cdx_log_level}"
echoerr "[storage] Node log level is ${_cdx_log_level}"
# PID array for known Codex node processes
# PID array for known Logos Storage node processes
# FIXME: right now only processes destroyed with cdx_destroy_node are removed from
# this array.
declare -A _cdx_pids
@ -53,9 +53,9 @@ cdx_set_outputs() {
_cdx_downloads="${_cdx_output}/downloads"
# SHA1 of uploaded files, per node. File names are CIDs
_cdx_uploads="${_cdx_output}/uploads"
# Codex node logs, per node
# Logos Storage node logs, per node
_cdx_logs="${_cdx_output}/logs"
# Codex data directories, per node
# Logos Storage data directories, per node
_cdx_data="${_cdx_output}/data"
# Partial timings, per operation per node
_cdx_timing_partials="${_cdx_output}/timing"
@ -129,14 +129,14 @@ cdx_cmdline() {
# shellcheck disable=SC2140
echo "${cdx_cmd}"\
"--data-dir=${_cdx_data}/codex-${node_index} --api-port=$(_cdx_api_port "$node_index")"\
"--data-dir=${_cdx_data}/storage-${node_index} --api-port=$(_cdx_api_port "$node_index")"\
"--disc-port=$(_cdx_disc_port "$node_index") '--log-level=${_cdx_log_level}'"
}
cdx_get_spr() {
local node_index="$1" spr
spr=$(curl --silent --fail "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/debug/info" | grep -oe 'spr:[^"]\+')
spr=$(curl --silent --fail "http://localhost:$(_cdx_api_port "$node_index")/api/storage/v1/debug/info" | grep -oe 'spr:[^"]\+')
if [[ -z "$spr" ]]; then
echoerr "Error: unable to get SPR for node $node_index"
return 1
@ -151,13 +151,13 @@ cdx_launch_node() {
_cdx_init_global_outputs || return 1
_cdx_init_node_outputs "${node_index}" || return 1
local codex_cmd
codex_cmd=$(cdx_cmdline "$@") || return 1
local storage_cmd
storage_cmd=$(cdx_cmdline "$@") || return 1
cmd_array=()
IFS=' ' read -r -a cmd_array <<<"$codex_cmd"
IFS=' ' read -r -a cmd_array <<<"$storage_cmd"
pm_async "bash" "-c" "exec ${cmd_array[*]} &> ${_cdx_logs}/codex-${node_index}.log" -%- "codex" "${node_index}"
pm_async "bash" "-c" "exec ${cmd_array[*]} &> ${_cdx_logs}/storage-${node_index}.log" -%- "storage" "${node_index}"
_cdx_pids[$node_index]=$!
cdx_ensure_ready "$node_index"
@ -166,7 +166,7 @@ cdx_launch_node() {
cdx_launch_network() {
local node_count="$1" bootstrap_spr
if [[ "$node_count" -lt 2 ]]; then
echoerr "Error: a Codex network needs at least 2 nodes"
echoerr "Error: a Logos Storage network needs at least 2 nodes"
return 1
fi
@ -199,8 +199,8 @@ cdx_destroy_node() {
unset "_cdx_pids[$node_index]"
if [ "$wipe_data" = true ]; then
rm -rf "${_cdx_data}/codex-${node_index}"
rm -rf "${_cdx_logs}/codex-${node_index}.log"
rm -rf "${_cdx_data}/storage-${node_index}"
rm -rf "${_cdx_logs}/storage-${node_index}.log"
fi
}
@ -209,12 +209,12 @@ cdx_ensure_ready() {
echoerr "Waiting ${timeout} seconds for node ${node_index} to be ready."
while true; do
if cdx_get_spr "$node_index" 2> /dev/null; then
echoerr "Codex node $node_index is ready."
echoerr "Logos Storage node $node_index is ready."
return 0
fi
if (( SECONDS - start > timeout )); then
echoerr "Codex node $node_index did not start within ${timeout} seconds."
echoerr "Logos Storage node $node_index did not start within ${timeout} seconds."
return 1
fi
@ -226,9 +226,9 @@ _cdx_init_node_outputs() {
local node_index="$1"
_ensure_outputs_set || return 1
mkdir -p "${_cdx_data}/codex-${node_index}" || return 1
mkdir -p "${_cdx_downloads}/codex-${node_index}" || return 1
mkdir -p "${_cdx_uploads}/codex-${node_index}" || return 1
mkdir -p "${_cdx_data}/storage-${node_index}" || return 1
mkdir -p "${_cdx_downloads}/storage-${node_index}" || return 1
mkdir -p "${_cdx_uploads}/storage-${node_index}" || return 1
}
# XXX: output initialization is a bit of a pain. Right now it's
@ -259,12 +259,12 @@ cdx_upload_file() {
echoerr "Uploading file ${filename} to node ${node_index}"
cid=$(curl --silent --fail\
-XPOST "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/data"\
-XPOST "http://localhost:$(_cdx_api_port "$node_index")/api/storage/v1/data"\
-T "${filename}") || return 1
echoerr "Upload SHA-1 is ${content_sha1}"
echo "${content_sha1}" > "${_cdx_uploads}/codex-${node_index}/${cid}.sha1"
echo "${content_sha1}" > "${_cdx_uploads}/storage-${node_index}/${cid}.sha1"
echo "${cid}"
}
@ -277,9 +277,9 @@ cdx_download_file() {
# puts the most recent entries first, while at the same time breaking ties arbitrarily
# for entries that happen within the same second.
{ time curl --silent --fail\
-XGET "http://localhost:$(_cdx_api_port "$node_index")/api/codex/v1/data/$cid/network/stream"\
-o "${_cdx_downloads}/codex-${node_index}/$cid" ; } 2> \
"${_cdx_timing_partials}/codex-${node_index}-${timestamp}-${RANDOM}.csv"
-XGET "http://localhost:$(_cdx_api_port "$node_index")/api/storage/v1/data/$cid/network/stream"\
-o "${_cdx_downloads}/storage-${node_index}/$cid" ; } 2> \
"${_cdx_timing_partials}/storage-${node_index}-${timestamp}-${RANDOM}.csv"
}
cdx_download_file_async() {
@ -288,12 +288,12 @@ cdx_download_file_async() {
cdx_upload_sha1() {
local node_index="$1" cid="$2"
cat "${_cdx_uploads}/codex-${node_index}/${cid}.sha1" || return 1
cat "${_cdx_uploads}/storage-${node_index}/${cid}.sha1" || return 1
}
cdx_download_sha1() {
local node_index="$1" cid="$2"
sha1 "${_cdx_downloads}/codex-${node_index}/$cid" || return 1
sha1 "${_cdx_downloads}/storage-${node_index}/$cid" || return 1
}
cdx_check_download() {

@ -1 +1 @@
Subproject commit 941bf35348987999cb510b17f4c59b67486aa818
Subproject commit 5f12b3172105a0f26bce629ff8ae0ac76c4bd61e

View File

@ -25,19 +25,19 @@ setup() {
assert [ "$found" = true ]
}
@test "should launch Codex nodes with metrics enabled when there is an experiment in scope" {
@test "should launch Logos Storage nodes with metrics enabled when there is an experiment in scope" {
exp_start "experiment-type"
[[ "$(cdx_cmdline 0)" =~ "--metrics-port=8290 --metrics-address=0.0.0.0" ]]
}
@test "should add a prometheus target for each Codex node when there is an experiment in scope" {
@test "should add a prometheus target for each Logos Storage node when there is an experiment in scope" {
exp_start "k-node"
pm_start
cdx_launch_node 0
config_file="${_prom_output}/8290-k-node-${_experiment_id}-0-codex.json"
config_file="${_prom_output}/8290-k-node-${_experiment_id}-0-storage.json"
assert [ -f "$config_file" ]
cdx_destroy_node 0

@ -1 +1 @@
Subproject commit 035e99c84759427bf160a6613904e9d57f838ea9
Subproject commit 697471b7a89d3ab38571f38c6c7c4b460d1f5e35

@ -1 +1 @@
Subproject commit d007fc1f451abbad55204fa9c9eb3e6ed5dc5f61
Subproject commit 0954abb9925cad550424cebca2b99255d4eabe96

View File

@ -14,9 +14,9 @@ contains() {
}
@test "should create prometheus configurations on start callback" {
prom_add "8290" "experiment" "84858" "node-1" "codex"
prom_add "8290" "experiment" "84858" "node-1" "storage"
config_file="${_prom_output}/8290-experiment-84858-node-1-codex.json"
config_file="${_prom_output}/8290-experiment-84858-node-1-storage.json"
assert [ -f "${config_file}" ]
@ -24,16 +24,16 @@ contains() {
assert contains "${config_file}" '"job": "experiment"'
assert contains "${config_file}" '"experiment_id": "84858"'
assert contains "${config_file}" '"node": "node-1"'
assert contains "${config_file}" '"node_type": "codex"'
assert contains "${config_file}" '"node_type": "storage"'
}
@test "should remove prometheus configurations on stop callback" {
prom_add "8290" "experiment" "84858" "node-1" "codex"
prom_add "8290" "experiment" "84858" "node-1" "storage"
config_file="${_prom_output}/8290-experiment-84858-node-1-codex.json"
config_file="${_prom_output}/8290-experiment-84858-node-1-storage.json"
assert [ -f "${config_file}" ]
prom_remove "8290" "experiment" "84858" "node-1" "codex"
prom_remove "8290" "experiment" "84858" "node-1" "storage"
assert [ ! -f "${config_file}" ]
}

View File

@ -4,25 +4,25 @@ setup() {
load test_helper/common_setup
common_setup
# shellcheck source=./src/codex.bash
source "${LIB_SRC}/codex.bash"
# shellcheck source=./src/storage.bash
source "${LIB_SRC}/storage.bash"
pm_set_outputs "${TEST_OUTPUTS}/pm"
cdx_set_outputs "${TEST_OUTPUTS}/codex"
cdx_set_outputs "${TEST_OUTPUTS}/storage"
}
@test "should generate the correct Codex command line for node 0" {
@test "should generate the correct Logos Storage command line for node 0" {
# shellcheck disable=SC2140
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none"\
" --data-dir=${_cdx_output}/data/codex-0"\
" --data-dir=${_cdx_output}/data/storage-0"\
" --api-port=8080 --disc-port=8190 '--log-level=INFO'"
}
@test "should generate the correct Codex command line for node 1" {
@test "should generate the correct Logos Storage command line for node 1" {
# shellcheck disable=SC2140
assert_equal "$(cdx_cmdline 1 '--bootstrap-node' 'node-spr')" "${_cdx_binary} --nat:none"\
" --bootstrap-node=node-spr"\
" --data-dir=${_cdx_output}/data/codex-1"\
" --data-dir=${_cdx_output}/data/storage-1"\
" --api-port=8081 --disc-port=8191 '--log-level=INFO'"
}
@ -35,16 +35,16 @@ setup() {
# shellcheck disable=SC2140
assert_equal "$(cdx_cmdline 0 --metrics)" "${_cdx_binary} --nat:none"\
" --metrics --metrics-port=8290 --metrics-address=0.0.0.0"\
" --data-dir=${_cdx_output}/data/codex-0"\
" --data-dir=${_cdx_output}/data/storage-0"\
" --api-port=8080 --disc-port=8190 '--log-level=INFO'"
}
@test "should modify the Codex log-level when specified" {
@test "should modify the Logos Storage log-level when specified" {
cdx_set_log_level "DEBUG"
# shellcheck disable=SC2140
assert_equal "$(cdx_cmdline 0)" "${_cdx_binary} --nat:none"\
" --data-dir=${_cdx_output}/data/codex-0"\
" --data-dir=${_cdx_output}/data/storage-0"\
" --api-port=8080 --disc-port=8190 '--log-level=DEBUG'"
}
@ -65,7 +65,7 @@ setup() {
}
@test "should pass readiness check if node is running" {
data_dir="${TEST_OUTPUTS}/codex-temp"
data_dir="${TEST_OUTPUTS}/storage-temp"
"${_cdx_binary}" --nat:none --data-dir="$data_dir" &> /dev/null &
pid=$!
@ -76,23 +76,23 @@ setup() {
rm -rf "$data_dir"
}
@test "should launch a Codex node" {
@test "should launch a Logos Storage node" {
pm_start
assert cdx_launch_node 0
assert cdx_ensure_ready 0 3
# We should see a log file and a data directory.
assert [ -f "${_cdx_output}/logs/codex-0.log" ]
assert [ -d "${_cdx_output}/data/codex-0" ]
assert [ -f "${_cdx_output}/logs/storage-0.log" ]
assert [ -d "${_cdx_output}/data/storage-0" ]
pid="${_cdx_pids[0]}"
assert [ -n "$pid" ]
cdx_destroy_node 0 true
refute [ -d "${_cdx_output}/data/codex-0" ]
refute [ -f "${_cdx_output}/logs/codex-0.log" ]
refute [ -d "${_cdx_output}/data/storage-0" ]
refute [ -f "${_cdx_output}/logs/storage-0.log" ]
assert [ -z "${_cdx_pids[0]}" ]
# Node should already be dead.
@ -103,19 +103,19 @@ setup() {
@test "should check downloaded content" {
mkdir -p "${_cdx_genfiles}"
mkdir -p "${_cdx_uploads}/codex-0"
mkdir -p "${_cdx_downloads}/codex-1"
mkdir -p "${_cdx_uploads}/storage-0"
mkdir -p "${_cdx_downloads}/storage-1"
filename=$(cdx_generate_file 10)
sha1 "$filename" > "${_cdx_uploads}/codex-0/fakecid.sha1"
cp "$filename" "${_cdx_downloads}/codex-1/fakecid"
sha1 "$filename" > "${_cdx_uploads}/storage-0/fakecid.sha1"
cp "$filename" "${_cdx_downloads}/storage-1/fakecid"
# Checks that the file uploaded at 0 matches the file downloaded at 1.
assert cdx_check_download 0 1 "fakecid"
}
@test "should upload and synchronously download file from Codex node" {
@test "should upload and synchronously download file from Logos Storage node" {
pm_start
assert cdx_launch_node 0
@ -131,7 +131,7 @@ setup() {
pm_stop
}
@test "should upload and asynchronously download file from Codex node" {
@test "should upload and asynchronously download file from Logos Storage node" {
pm_start
assert cdx_launch_node 0
@ -150,11 +150,11 @@ setup() {
pm_stop
}
@test "should refuse to launch a Codex network with less than 2 nodes" {
@test "should refuse to launch a Logos Storage network with less than 2 nodes" {
refute cdx_launch_network 1
}
@test "should launch a Codex network and allow uploading and downloading" {
@test "should launch a Logos Storage network and allow uploading and downloading" {
pm_start
assert cdx_launch_network 5