diff --git a/scripts/run-examples.sh b/scripts/run-examples.sh index 6aebca0..208071e 100755 --- a/scripts/run-examples.sh +++ b/scripts/run-examples.sh @@ -189,6 +189,14 @@ select_image() { export IMAGE_TAG="${IMAGE}" export NOMOS_TESTNET_IMAGE="${IMAGE}" + + if [ "${MODE}" = "k8s" ]; then + if [ "${selection}" = "ecr" ]; then + export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-inImage}" + else + export NOMOS_KZG_MODE="${NOMOS_KZG_MODE:-hostPath}" + fi + fi } select_image @@ -450,7 +458,11 @@ fi echo "==> Running ${BIN} for ${RUN_SECS}s" cd "${ROOT_DIR}" if [ "$MODE" = "compose" ] || [ "$MODE" = "k8s" ]; then - KZG_PATH="${KZG_CONTAINER_PATH}" + if [ "$MODE" = "k8s" ] && [ "${NOMOS_KZG_MODE:-hostPath}" = "inImage" ]; then + KZG_PATH="${NOMOS_KZG_IN_IMAGE_PARAMS_PATH:-/opt/nomos/kzg-params/kzgrs_test_params}" + else + KZG_PATH="${KZG_CONTAINER_PATH}" + fi else KZG_PATH="${KZG_HOST_PATH}" fi diff --git a/testing-framework/assets/stack/Dockerfile b/testing-framework/assets/stack/Dockerfile index 9acf884..8de713c 100644 --- a/testing-framework/assets/stack/Dockerfile +++ b/testing-framework/assets/stack/Dockerfile @@ -167,6 +167,19 @@ RUN apt-get update && apt-get install -yq \ COPY --from=builder /opt/circuits /opt/circuits +# Provide a stable in-image location for the KZG test parameters so EKS runs do +# not rely on hostPath volumes. +RUN set -eu; \ + mkdir -p /opt/nomos/kzg-params; \ + if [ -f "/opt/circuits/kzgrs_test_params/kzgrs_test_params" ]; then \ + cp "/opt/circuits/kzgrs_test_params/kzgrs_test_params" "/opt/nomos/kzg-params/kzgrs_test_params"; \ + elif [ -f "/opt/circuits/kzgrs_test_params" ]; then \ + cp "/opt/circuits/kzgrs_test_params" "/opt/nomos/kzg-params/kzgrs_test_params"; \ + else \ + echo "ERROR: kzgrs_test_params not found under /opt/circuits; cannot seed /opt/nomos/kzg-params" >&2; \ + exit 1; \ + fi + COPY --from=builder /workspace/artifacts/nomos-node /usr/bin/nomos-node COPY --from=builder /workspace/artifacts/nomos-executor /usr/bin/nomos-executor COPY --from=builder /workspace/artifacts/nomos-cli /usr/bin/nomos-cli diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/executor-deployments.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/executor-deployments.yaml index 160d0b9..baaa33b 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/executor-deployments.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/executor-deployments.yaml @@ -31,6 +31,8 @@ spec: env: - name: CFG_SERVER_ADDR value: http://{{ include "nomos-runner.fullname" $root }}-cfgsync:{{ $root.Values.cfgsync.port }} + - name: NOMOS_KZGRS_PARAMS_PATH + value: '{{ if eq $root.Values.kzg.mode "inImage" }}{{ $root.Values.kzg.inImageParamsPath }}{{ else }}{{ $root.Values.kzg.hostPathParamsPath }}{{ end }}' {{- range $key, $value := $node.env }} - name: {{ $key }} value: "{{ $value }}" @@ -39,9 +41,11 @@ spec: - name: assets mountPath: /etc/nomos readOnly: true + {{- if eq $root.Values.kzg.mode "hostPath" }} - name: kzg-params mountPath: /kzgrs_test_params readOnly: true + {{- end }} volumes: - name: assets configMap: @@ -58,8 +62,10 @@ spec: path: scripts/run_nomos_executor.sh - key: run_nomos_node.sh path: scripts/run_nomos_node.sh + {{- if eq $root.Values.kzg.mode "hostPath" }} - name: kzg-params persistentVolumeClaim: claimName: {{ include "nomos-runner.fullname" $root }}-kzg readOnly: true + {{- end }} {{- end }} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/pv.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/pv.yaml index 3af3b2e..6372cde 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/pv.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/pv.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.kzg.mode "hostPath" }} apiVersion: v1 kind: PersistentVolume metadata: @@ -14,3 +15,4 @@ spec: hostPath: path: {{ .Values.kzg.hostPath }} type: {{ .Values.kzg.hostPathType }} +{{- end }} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/pvc.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/pvc.yaml index 52248fe..93f4081 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/pvc.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/pvc.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.kzg.mode "hostPath" }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -12,3 +13,4 @@ spec: resources: requests: storage: {{ .Values.kzg.storageSize }} +{{- end }} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/templates/validator-deployments.yaml b/testing-framework/runners/k8s/helm/nomos-runner/templates/validator-deployments.yaml index 4c12d34..ed0f4e0 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/templates/validator-deployments.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/templates/validator-deployments.yaml @@ -31,6 +31,8 @@ spec: env: - name: CFG_SERVER_ADDR value: http://{{ include "nomos-runner.fullname" $root }}-cfgsync:{{ $root.Values.cfgsync.port }} + - name: NOMOS_KZGRS_PARAMS_PATH + value: '{{ if eq $root.Values.kzg.mode "inImage" }}{{ $root.Values.kzg.inImageParamsPath }}{{ else }}{{ $root.Values.kzg.hostPathParamsPath }}{{ end }}' {{- range $key, $value := $node.env }} - name: {{ $key }} value: "{{ $value }}" @@ -39,9 +41,11 @@ spec: - name: assets mountPath: /etc/nomos readOnly: true + {{- if eq $root.Values.kzg.mode "hostPath" }} - name: kzg-params mountPath: /kzgrs_test_params readOnly: true + {{- end }} volumes: - name: assets configMap: @@ -56,8 +60,10 @@ spec: path: scripts/run_nomos.sh - key: run_nomos_node.sh path: scripts/run_nomos_node.sh + {{- if eq $root.Values.kzg.mode "hostPath" }} - name: kzg-params persistentVolumeClaim: claimName: {{ include "nomos-runner.fullname" $root }}-kzg readOnly: true + {{- end }} {{- end }} diff --git a/testing-framework/runners/k8s/helm/nomos-runner/values.yaml b/testing-framework/runners/k8s/helm/nomos-runner/values.yaml index b3b3c35..8d25abf 100644 --- a/testing-framework/runners/k8s/helm/nomos-runner/values.yaml +++ b/testing-framework/runners/k8s/helm/nomos-runner/values.yaml @@ -20,6 +20,9 @@ executors: nodes: [] kzg: + mode: "hostPath" + hostPathParamsPath: "/kzgrs_test_params/kzgrs_test_params" + inImageParamsPath: "/opt/nomos/kzg-params/kzgrs_test_params" hostPath: "/var/lib/nomos/kzgrs_test_params" hostPathType: "Directory" storageSize: "1Gi" diff --git a/testing-framework/runners/k8s/src/infrastructure/assets.rs b/testing-framework/runners/k8s/src/infrastructure/assets.rs index e7b6be7..7809eeb 100644 --- a/testing-framework/runners/k8s/src/infrastructure/assets.rs +++ b/testing-framework/runners/k8s/src/infrastructure/assets.rs @@ -18,7 +18,8 @@ use tracing::{debug, info}; /// Paths and image metadata required to deploy the Helm chart. pub struct RunnerAssets { pub image: String, - pub kzg_path: PathBuf, + pub kzg_mode: KzgMode, + pub kzg_path: Option, pub chart_path: PathBuf, pub cfgsync_file: PathBuf, pub run_cfgsync_script: PathBuf, @@ -70,6 +71,19 @@ pub enum AssetsError { }, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum KzgMode { + HostPath, + InImage, +} + +fn kzg_mode() -> KzgMode { + match env::var("NOMOS_KZG_MODE").ok().as_deref() { + Some("inImage") => KzgMode::InImage, + _ => KzgMode::HostPath, + } +} + /// Render cfgsync config, Helm values, and locate scripts/KZG assets for a /// topology. pub fn prepare_assets(topology: &GeneratedTopology) -> Result { @@ -89,24 +103,34 @@ pub fn prepare_assets(topology: &GeneratedTopology) -> Result Some(validate_kzg_params(&root)?), + KzgMode::InImage => None, + }; let chart_path = helm_chart_path()?; let values_yaml = render_values_yaml(topology)?; let values_file = write_temp_file(tempdir.path(), "values.yaml", values_yaml)?; let image = env::var("NOMOS_TESTNET_IMAGE") .unwrap_or_else(|_| String::from("logos-blockchain-testing:test")); + let kzg_display = kzg_path + .as_ref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "".to_string()); debug!( cfgsync = %cfgsync_file.display(), values = %values_file.display(), image, - kzg = %kzg_path.display(), + kzg_mode = ?kzg_mode, + kzg = %kzg_display, chart = %chart_path.display(), "k8s runner assets prepared" ); Ok(RunnerAssets { image, + kzg_mode, kzg_path, chart_path, cfgsync_file, diff --git a/testing-framework/runners/k8s/src/infrastructure/helm.rs b/testing-framework/runners/k8s/src/infrastructure/helm.rs index 0c6fa41..6862a7a 100644 --- a/testing-framework/runners/k8s/src/infrastructure/helm.rs +++ b/testing-framework/runners/k8s/src/infrastructure/helm.rs @@ -4,7 +4,7 @@ use thiserror::Error; use tokio::process::Command; use tracing::{debug, info}; -use crate::infrastructure::assets::{RunnerAssets, cfgsync_port_value, workspace_root}; +use crate::infrastructure::assets::{KzgMode, RunnerAssets, cfgsync_port_value, workspace_root}; /// Errors returned from Helm invocations. #[derive(Debug, Error)] @@ -32,10 +32,20 @@ pub async fn install_release( validators: usize, executors: usize, ) -> Result<(), HelmError> { - let host_path_type = if assets.kzg_path.is_dir() { - "Directory" - } else { - "File" + let (host_path_type, host_path) = match assets.kzg_mode { + KzgMode::HostPath => { + let host_path = assets + .kzg_path + .as_ref() + .expect("kzg_path must be present for HostPath mode"); + let host_path_type = if host_path.is_dir() { + "Directory" + } else { + "File" + }; + (Some(host_path_type), Some(host_path)) + } + KzgMode::InImage => (None, None), }; info!( release, @@ -44,7 +54,11 @@ pub async fn install_release( executors, image = %assets.image, cfgsync_port = cfgsync_port_value(), - kzg = %assets.kzg_path.display(), + kzg_mode = ?assets.kzg_mode, + kzg = %host_path + .as_ref() + .map(|p| p.display().to_string()) + .unwrap_or_else(|| "".to_string()), values = %assets.values_file.display(), "installing helm release" ); @@ -67,12 +81,13 @@ pub async fn install_release( .arg(format!("executors.count={executors}")) .arg("--set") .arg(format!("cfgsync.port={}", cfgsync_port_value())) - .arg("--set") - .arg(format!("kzg.hostPath={}", assets.kzg_path.display())) - .arg("--set") - .arg(format!("kzg.hostPathType={host_path_type}")) .arg("-f") .arg(&assets.values_file) + .arg("--set") + .arg(match assets.kzg_mode { + KzgMode::HostPath => "kzg.mode=hostPath", + KzgMode::InImage => "kzg.mode=inImage", + }) .arg("--set-file") .arg(format!("cfgsync.config={}", assets.cfgsync_file.display())) .arg("--set-file") @@ -98,6 +113,13 @@ pub async fn install_release( .stdout(Stdio::piped()) .stderr(Stdio::piped()); + if let (Some(host_path), Some(host_path_type)) = (host_path, host_path_type) { + cmd.arg("--set") + .arg(format!("kzg.hostPath={}", host_path.display())) + .arg("--set") + .arg(format!("kzg.hostPathType={host_path_type}")); + } + if let Ok(root) = workspace_root() { cmd.current_dir(root); }