diff --git a/testing-framework/deployers/local/src/binary/mod.rs b/testing-framework/deployers/local/src/binary/mod.rs index 24f326a..d8111f6 100644 --- a/testing-framework/deployers/local/src/binary/mod.rs +++ b/testing-framework/deployers/local/src/binary/mod.rs @@ -17,8 +17,9 @@ use cache::BinaryCache; pub(super) use types::optional_path_display; pub use types::{ BinaryProviderError, BinaryProviderRef, BuildBinaryProvider, BuildCommand, - DownloadBinaryProvider, DownloadChecksum, DownloadUrl, EnvBinaryProvider, - FallbackBinaryProvider, PathBinaryProvider, + DownloadBinaryProvider, DownloadChecksum, DownloadProcessor, DownloadProcessorError, + DownloadProcessorFn, DownloadUrl, EnvBinaryProvider, FallbackBinaryProvider, + PathBinaryProvider, }; /// Resolves an executable path for a local process. diff --git a/testing-framework/deployers/local/src/binary/providers/download.rs b/testing-framework/deployers/local/src/binary/providers/download.rs index 24a4882..c24825c 100644 --- a/testing-framework/deployers/local/src/binary/providers/download.rs +++ b/testing-framework/deployers/local/src/binary/providers/download.rs @@ -17,7 +17,7 @@ use sha2::{Digest as _, Sha256}; use tracing::info; use crate::binary::{ - BinaryProvider, BinaryProviderError, DownloadBinaryProvider, DownloadUrl, + BinaryProvider, BinaryProviderError, DownloadBinaryProvider, DownloadChecksum, DownloadUrl, lock::BinaryProviderLock, optional_path_display, }; @@ -33,7 +33,7 @@ impl BinaryProvider for DownloadBinaryProvider { let bytes = self.download_bytes(&url)?; self.verify_checksum(&path, &bytes)?; - self.write_binary(&path, &bytes)?; + self.prepare_binary(&path, &bytes)?; Ok(Some(path)) } @@ -44,8 +44,14 @@ impl BinaryProvider for DownloadBinaryProvider { fn cache_key(&self) -> String { format!( - "download:{}:{}", + "download:{}:{}:{}:{}", self.url.cache_key(), + self.sha256 + .as_ref() + .map_or_else(String::new, DownloadChecksum::cache_key), + self.processor + .as_ref() + .map_or("", |processor| processor.cache_key()), optional_path_display(&self.cache_dir) ) } @@ -60,6 +66,15 @@ impl DownloadUrl { } } +impl DownloadChecksum { + fn cache_key(&self) -> String { + match self { + Self::Fixed(checksum) => checksum.to_ascii_lowercase(), + Self::Env(env_var) => format!("env:{env_var}"), + } + } +} + impl DownloadBinaryProvider { fn cached_binary_path(&self, url: &str) -> Result { let cache_dir = self.cache_dir(); @@ -92,13 +107,76 @@ impl DownloadBinaryProvider { }) } - fn write_binary(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> { - fs::write(path, bytes).map_err(|source| BinaryProviderError::Io { - path: path.to_owned(), + fn prepare_binary(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> { + let artifact = path.with_extension("download"); + let output = path.with_extension("part"); + self.remove_temporary_file(&artifact); + self.remove_temporary_file(&output); + + let result = self + .materialize_output(&artifact, &output, bytes) + .and_then(|()| { + self.ensure_processed_output(&output)?; + self.make_executable(&output)?; + fs::rename(&output, path).map_err(|source| BinaryProviderError::Io { + path: path.to_owned(), + source, + }) + }); + + self.remove_temporary_file(&artifact); + if result.is_err() { + self.remove_temporary_file(&output); + } + + result + } + + fn materialize_output( + &self, + artifact: &Path, + output: &Path, + bytes: &[u8], + ) -> Result<(), BinaryProviderError> { + let Some(processor) = &self.processor else { + return fs::write(output, bytes).map_err(|source| BinaryProviderError::Io { + path: output.to_owned(), + source, + }); + }; + + fs::write(artifact, bytes).map_err(|source| BinaryProviderError::Io { + path: artifact.to_owned(), source, })?; + processor.process(artifact, output).map_err(|source| { + BinaryProviderError::DownloadProcessing { + processor: processor.cache_key().to_owned(), + source, + } + }) + } - self.make_executable(path) + fn ensure_processed_output(&self, output: &Path) -> Result<(), BinaryProviderError> { + if output.is_file() { + return Ok(()); + } + + Err(BinaryProviderError::MissingProcessedOutput { + processor: self.processor.as_ref().map_or_else( + || "identity".to_owned(), + |processor| processor.cache_key().to_owned(), + ), + path: output.to_owned(), + }) + } + + fn remove_temporary_file(&self, path: &Path) { + if let Err(error) = fs::remove_file(path) + && error.kind() != std::io::ErrorKind::NotFound + { + tracing::warn!(path = %path.display(), %error, "failed to remove temporary download file"); + } } fn verify_checksum(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> { @@ -163,6 +241,14 @@ impl DownloadBinaryProvider { fn download_file_name(&self, url: &str) -> String { let mut hasher = DefaultHasher::new(); url.hash(&mut hasher); + self.sha256 + .as_ref() + .and_then(DownloadChecksum::resolve) + .hash(&mut hasher); + self.processor + .as_ref() + .map(|processor| processor.cache_key()) + .hash(&mut hasher); format!("binary-{:x}", hasher.finish()) } diff --git a/testing-framework/deployers/local/src/binary/tests.rs b/testing-framework/deployers/local/src/binary/tests.rs index cb29dd4..15c1a26 100644 --- a/testing-framework/deployers/local/src/binary/tests.rs +++ b/testing-framework/deployers/local/src/binary/tests.rs @@ -158,6 +158,7 @@ fn downloads_binary_from_minimal_http_server() { url: DownloadUrl::Fixed(server.url()), sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))), cache_dir: Some(temp.path().join("cache")), + processor: None, }; let path = provider.resolve().expect("download provider resolves"); @@ -172,6 +173,7 @@ fn rejects_download_checksum_mismatch() { url: DownloadUrl::Fixed(server.url()), sha256: Some(DownloadChecksum::Fixed("00".repeat(32))), cache_dir: Some(temp.path().join("cache")), + processor: None, }; let error = provider @@ -184,6 +186,61 @@ fn rejects_download_checksum_mismatch() { )); } +#[test] +fn processes_downloaded_artifact_before_publishing_binary() { + let temp = TempDir::new().expect("temp dir"); + let body = b"archive:downloaded-node"; + let server = SingleResponseServer::start(body); + let process_count = Arc::new(AtomicUsize::new(0)); + let callback_count = Arc::clone(&process_count); + let provider = DownloadBinaryProvider { + url: DownloadUrl::Fixed(server.url()), + sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))), + cache_dir: Some(temp.path().join("cache")), + processor: None, + } + .with_processor_fn("strip-test-archive-v1", move |artifact, output| { + callback_count.fetch_add(1, Ordering::SeqCst); + let contents = fs::read(artifact)?; + fs::write( + output, + contents.strip_prefix(b"archive:").unwrap_or(&contents), + )?; + Ok(()) + }); + + let path = provider.resolve().expect("processed download resolves"); + + assert_eq!( + fs::read(path).expect("processed binary"), + b"downloaded-node" + ); + assert_eq!(process_count.load(Ordering::SeqCst), 1); +} + +#[test] +fn rejects_processor_that_does_not_create_output() { + let temp = TempDir::new().expect("temp dir"); + let body = b"archive"; + let server = SingleResponseServer::start(body); + let provider = DownloadBinaryProvider { + url: DownloadUrl::Fixed(server.url()), + sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))), + cache_dir: Some(temp.path().join("cache")), + processor: None, + } + .with_processor_fn("empty-v1", |_artifact, _output| Ok(())); + + let error = provider + .resolve() + .expect_err("missing processed output is rejected"); + + assert!(matches!( + error, + BinaryProviderError::MissingProcessedOutput { .. } + )); +} + fn write_file(path: &Path, contents: &[u8]) { fs::write(path, contents).expect("write file"); } diff --git a/testing-framework/deployers/local/src/binary/types.rs b/testing-framework/deployers/local/src/binary/types.rs index 50c41c5..5fcd591 100644 --- a/testing-framework/deployers/local/src/binary/types.rs +++ b/testing-framework/deployers/local/src/binary/types.rs @@ -1,4 +1,8 @@ -use std::{env, fmt, iter, path::PathBuf, sync::Arc}; +use std::{ + env, fmt, iter, + path::{Path, PathBuf}, + sync::Arc, +}; use thiserror::Error; @@ -132,7 +136,64 @@ impl BuildBinaryProvider { } } -#[derive(Clone, Debug)] +/// Error returned by integration-specific download processors. +pub type DownloadProcessorError = Box; + +/// Post-download preparation step for artifacts that are not directly +/// executable. +/// +/// Implementations receive the checksum-verified downloaded artifact and must +/// materialize the executable at `output`. The stable cache key ensures that a +/// change in preparation logic invalidates the previously prepared binary. +pub trait DownloadProcessor: Send + Sync { + fn process(&self, artifact: &Path, output: &Path) -> Result<(), DownloadProcessorError>; + + fn cache_key(&self) -> &str; +} + +type DownloadProcessorCallback = + dyn Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync; + +/// Named callback adapter for lightweight, integration-specific processing. +#[derive(Clone)] +pub struct DownloadProcessorFn { + cache_key: String, + callback: Arc, +} + +impl DownloadProcessorFn { + #[must_use] + pub fn new( + cache_key: impl Into, + callback: impl Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync + 'static, + ) -> Self { + Self { + cache_key: cache_key.into(), + callback: Arc::new(callback), + } + } +} + +impl DownloadProcessor for DownloadProcessorFn { + fn process(&self, artifact: &Path, output: &Path) -> Result<(), DownloadProcessorError> { + (self.callback)(artifact, output) + } + + fn cache_key(&self) -> &str { + &self.cache_key + } +} + +impl fmt::Debug for DownloadProcessorFn { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DownloadProcessorFn") + .field("cache_key", &self.cache_key) + .finish_non_exhaustive() + } +} + +#[derive(Clone)] pub struct DownloadBinaryProvider { /// Download source used to fetch the executable. pub url: DownloadUrl, @@ -143,6 +204,8 @@ pub struct DownloadBinaryProvider { /// If unset, downloads are cached under `target/.tf-binaries` in the /// current process directory. pub cache_dir: Option, + /// Optional preparation step for archives or other packaged artifacts. + pub processor: Option>, } impl DownloadBinaryProvider { @@ -152,8 +215,42 @@ impl DownloadBinaryProvider { url: DownloadUrl::Fixed(url.into()), sha256: None, cache_dir: None, + processor: None, } } + + #[must_use] + pub fn with_processor(mut self, processor: impl DownloadProcessor + 'static) -> Self { + self.processor = Some(Arc::new(processor)); + self + } + + #[must_use] + pub fn with_processor_fn( + self, + cache_key: impl Into, + callback: impl Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync + 'static, + ) -> Self { + self.with_processor(DownloadProcessorFn::new(cache_key, callback)) + } +} + +impl fmt::Debug for DownloadBinaryProvider { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DownloadBinaryProvider") + .field("url", &self.url) + .field("sha256", &self.sha256) + .field("cache_dir", &self.cache_dir) + .field( + "processor", + &self + .processor + .as_ref() + .map(|processor| processor.cache_key()), + ) + .finish() + } } #[derive(Clone, Debug)] @@ -253,6 +350,23 @@ pub enum BinaryProviderError { /// Actual lowercase SHA-256 checksum of the downloaded bytes. actual: String, }, + /// A download processor completed without creating its configured output. + #[error("download processor {processor} did not produce binary output {path:?}")] + MissingProcessedOutput { + /// Stable identifier of the processor that failed to produce output. + processor: String, + /// Expected temporary executable output path. + path: PathBuf, + }, + /// A configured download processor failed while preparing the executable. + #[error("download processor {processor} failed: {source}")] + DownloadProcessing { + /// Stable identifier of the processor that failed. + processor: String, + #[source] + /// Processor-specific error. + source: DownloadProcessorError, + }, /// Filesystem operation failed while preparing or resolving a binary. #[error("failed to prepare binary path {path:?}: {source}")] Io { diff --git a/testing-framework/deployers/local/src/lib.rs b/testing-framework/deployers/local/src/lib.rs index 14eda23..fd4751a 100644 --- a/testing-framework/deployers/local/src/lib.rs +++ b/testing-framework/deployers/local/src/lib.rs @@ -8,8 +8,9 @@ pub mod process; pub use binary::{ BinaryProvider, BinaryProviderError, BinaryProviderRef, BuildBinaryProvider, BuildCommand, - DownloadBinaryProvider, DownloadChecksum, DownloadUrl, EnvBinaryProvider, - FallbackBinaryProvider, PathBinaryProvider, + DownloadBinaryProvider, DownloadChecksum, DownloadProcessor, DownloadProcessorError, + DownloadProcessorFn, DownloadUrl, EnvBinaryProvider, FallbackBinaryProvider, + PathBinaryProvider, }; pub use deployer::{ProcessDeployer, ProcessDeployerError}; pub use env::{