logos-blockchain/logos-blockchain-testing
1
0
Fork 0
mirror of https://github.com/logos-blockchain/logos-blockchain-testing.git synced 2026-07-18 21:49:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(tf): process downloaded binary artifacts

Browse Source
This commit is contained in:
andrussal 2026-07-14 02:45:35 +02:00
parent a364088627
commit c3281cf59c
5 changed files with 272 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
testing-framework/deployers/local/src/binary/mod.rs
View File

@ -17,8 +17,9 @@ use cache::BinaryCache;
pub(super) use types::optional_path_display;
pub use types::{
BinaryProviderError, BinaryProviderRef, BuildBinaryProvider, BuildCommand,
DownloadBinaryProvider, DownloadChecksum, DownloadUrl, EnvBinaryProvider,
FallbackBinaryProvider, PathBinaryProvider,
DownloadBinaryProvider, DownloadChecksum, DownloadProcessor, DownloadProcessorError,
DownloadProcessorFn, DownloadUrl, EnvBinaryProvider, FallbackBinaryProvider,
PathBinaryProvider,
};
/// Resolves an executable path for a local process.

100
testing-framework/deployers/local/src/binary/providers/download.rs
View File

@ -17,7 +17,7 @@ use sha2::{Digest as _, Sha256};
use tracing::info;
use crate::binary::{
BinaryProvider, BinaryProviderError, DownloadBinaryProvider, DownloadUrl,
BinaryProvider, BinaryProviderError, DownloadBinaryProvider, DownloadChecksum, DownloadUrl,
lock::BinaryProviderLock, optional_path_display,
};
@ -33,7 +33,7 @@ impl BinaryProvider for DownloadBinaryProvider {
let bytes = self.download_bytes(&url)?;
self.verify_checksum(&path, &bytes)?;
self.write_binary(&path, &bytes)?;
self.prepare_binary(&path, &bytes)?;
Ok(Some(path))
}
@ -44,8 +44,14 @@ impl BinaryProvider for DownloadBinaryProvider {
fn cache_key(&self) -> String {
format!(
"download:{}:{}",
"download:{}:{}:{}:{}",
self.url.cache_key(),
self.sha256
.as_ref()
.map_or_else(String::new, DownloadChecksum::cache_key),
self.processor
.as_ref()
.map_or("", |processor| processor.cache_key()),
optional_path_display(&self.cache_dir)
)
}
@ -60,6 +66,15 @@ impl DownloadUrl {
}
}
impl DownloadChecksum {
fn cache_key(&self) -> String {
match self {
Self::Fixed(checksum) => checksum.to_ascii_lowercase(),
Self::Env(env_var) => format!("env:{env_var}"),
}
}
}
impl DownloadBinaryProvider {
fn cached_binary_path(&self, url: &str) -> Result<PathBuf, BinaryProviderError> {
let cache_dir = self.cache_dir();
@ -92,13 +107,76 @@ impl DownloadBinaryProvider {
})
}
fn write_binary(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> {
fs::write(path, bytes).map_err(|source| BinaryProviderError::Io {
path: path.to_owned(),
fn prepare_binary(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> {
let artifact = path.with_extension("download");
let output = path.with_extension("part");
self.remove_temporary_file(&artifact);
self.remove_temporary_file(&output);
let result = self
.materialize_output(&artifact, &output, bytes)
.and_then(|()| {
self.ensure_processed_output(&output)?;
self.make_executable(&output)?;
fs::rename(&output, path).map_err(|source| BinaryProviderError::Io {
path: path.to_owned(),
source,
})
});
self.remove_temporary_file(&artifact);
if result.is_err() {
self.remove_temporary_file(&output);
}
result
}
fn materialize_output(
&self,
artifact: &Path,
output: &Path,
bytes: &[u8],
) -> Result<(), BinaryProviderError> {
let Some(processor) = &self.processor else {
return fs::write(output, bytes).map_err(|source| BinaryProviderError::Io {
path: output.to_owned(),
source,
});
};
fs::write(artifact, bytes).map_err(|source| BinaryProviderError::Io {
path: artifact.to_owned(),
source,
})?;
processor.process(artifact, output).map_err(|source| {
BinaryProviderError::DownloadProcessing {
processor: processor.cache_key().to_owned(),
source,
}
})
}
self.make_executable(path)
fn ensure_processed_output(&self, output: &Path) -> Result<(), BinaryProviderError> {
if output.is_file() {
return Ok(());
}
Err(BinaryProviderError::MissingProcessedOutput {
processor: self.processor.as_ref().map_or_else(
|| "identity".to_owned(),
|processor| processor.cache_key().to_owned(),
),
path: output.to_owned(),
})
}
fn remove_temporary_file(&self, path: &Path) {
if let Err(error) = fs::remove_file(path)
&& error.kind() != std::io::ErrorKind::NotFound
{
tracing::warn!(path = %path.display(), %error, "failed to remove temporary download file");
}
}
fn verify_checksum(&self, path: &Path, bytes: &[u8]) -> Result<(), BinaryProviderError> {
@ -163,6 +241,14 @@ impl DownloadBinaryProvider {
fn download_file_name(&self, url: &str) -> String {
let mut hasher = DefaultHasher::new();
url.hash(&mut hasher);
self.sha256
.as_ref()
.and_then(DownloadChecksum::resolve)
.hash(&mut hasher);
self.processor
.as_ref()
.map(|processor| processor.cache_key())
.hash(&mut hasher);
format!("binary-{:x}", hasher.finish())
}

57
testing-framework/deployers/local/src/binary/tests.rs
View File

@ -158,6 +158,7 @@ fn downloads_binary_from_minimal_http_server() {
url: DownloadUrl::Fixed(server.url()),
sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))),
cache_dir: Some(temp.path().join("cache")),
processor: None,
};
let path = provider.resolve().expect("download provider resolves");
@ -172,6 +173,7 @@ fn rejects_download_checksum_mismatch() {
url: DownloadUrl::Fixed(server.url()),
sha256: Some(DownloadChecksum::Fixed("00".repeat(32))),
cache_dir: Some(temp.path().join("cache")),
processor: None,
};
let error = provider
@ -184,6 +186,61 @@ fn rejects_download_checksum_mismatch() {
));
}
#[test]
fn processes_downloaded_artifact_before_publishing_binary() {
let temp = TempDir::new().expect("temp dir");
let body = b"archive:downloaded-node";
let server = SingleResponseServer::start(body);
let process_count = Arc::new(AtomicUsize::new(0));
let callback_count = Arc::clone(&process_count);
let provider = DownloadBinaryProvider {
url: DownloadUrl::Fixed(server.url()),
sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))),
cache_dir: Some(temp.path().join("cache")),
processor: None,
}
.with_processor_fn("strip-test-archive-v1", move |artifact, output| {
callback_count.fetch_add(1, Ordering::SeqCst);
let contents = fs::read(artifact)?;
fs::write(
output,
contents.strip_prefix(b"archive:").unwrap_or(&contents),
)?;
Ok(())
});
let path = provider.resolve().expect("processed download resolves");
assert_eq!(
fs::read(path).expect("processed binary"),
b"downloaded-node"
);
assert_eq!(process_count.load(Ordering::SeqCst), 1);
}
#[test]
fn rejects_processor_that_does_not_create_output() {
let temp = TempDir::new().expect("temp dir");
let body = b"archive";
let server = SingleResponseServer::start(body);
let provider = DownloadBinaryProvider {
url: DownloadUrl::Fixed(server.url()),
sha256: Some(DownloadChecksum::Fixed(sha256_hex(body))),
cache_dir: Some(temp.path().join("cache")),
processor: None,
}
.with_processor_fn("empty-v1", |_artifact, _output| Ok(()));
let error = provider
.resolve()
.expect_err("missing processed output is rejected");
assert!(matches!(
error,
BinaryProviderError::MissingProcessedOutput { .. }
));
}
fn write_file(path: &Path, contents: &[u8]) {
fs::write(path, contents).expect("write file");
}

118
testing-framework/deployers/local/src/binary/types.rs
View File

@ -1,4 +1,8 @@
use std::{env, fmt, iter, path::PathBuf, sync::Arc};
use std::{
env, fmt, iter,
path::{Path, PathBuf},
sync::Arc,
};
use thiserror::Error;
@ -132,7 +136,64 @@ impl BuildBinaryProvider {
}
}
#[derive(Clone, Debug)]
/// Error returned by integration-specific download processors.
pub type DownloadProcessorError = Box<dyn std::error::Error + Send + Sync + 'static>;
/// Post-download preparation step for artifacts that are not directly
/// executable.
///
/// Implementations receive the checksum-verified downloaded artifact and must
/// materialize the executable at `output`. The stable cache key ensures that a
/// change in preparation logic invalidates the previously prepared binary.
pub trait DownloadProcessor: Send + Sync {
fn process(&self, artifact: &Path, output: &Path) -> Result<(), DownloadProcessorError>;
fn cache_key(&self) -> &str;
}
type DownloadProcessorCallback =
dyn Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync;
/// Named callback adapter for lightweight, integration-specific processing.
#[derive(Clone)]
pub struct DownloadProcessorFn {
cache_key: String,
callback: Arc<DownloadProcessorCallback>,
}
impl DownloadProcessorFn {
#[must_use]
pub fn new(
cache_key: impl Into<String>,
callback: impl Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync + 'static,
) -> Self {
Self {
cache_key: cache_key.into(),
callback: Arc::new(callback),
}
}
}
impl DownloadProcessor for DownloadProcessorFn {
fn process(&self, artifact: &Path, output: &Path) -> Result<(), DownloadProcessorError> {
(self.callback)(artifact, output)
}
fn cache_key(&self) -> &str {
&self.cache_key
}
}
impl fmt::Debug for DownloadProcessorFn {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DownloadProcessorFn")
.field("cache_key", &self.cache_key)
.finish_non_exhaustive()
}
}
#[derive(Clone)]
pub struct DownloadBinaryProvider {
/// Download source used to fetch the executable.
pub url: DownloadUrl,
@ -143,6 +204,8 @@ pub struct DownloadBinaryProvider {
/// If unset, downloads are cached under `target/.tf-binaries` in the
/// current process directory.
pub cache_dir: Option<PathBuf>,
/// Optional preparation step for archives or other packaged artifacts.
pub processor: Option<Arc<dyn DownloadProcessor>>,
}
impl DownloadBinaryProvider {
@ -152,8 +215,42 @@ impl DownloadBinaryProvider {
url: DownloadUrl::Fixed(url.into()),
sha256: None,
cache_dir: None,
processor: None,
}
}
#[must_use]
pub fn with_processor(mut self, processor: impl DownloadProcessor + 'static) -> Self {
self.processor = Some(Arc::new(processor));
self
}
#[must_use]
pub fn with_processor_fn(
self,
cache_key: impl Into<String>,
callback: impl Fn(&Path, &Path) -> Result<(), DownloadProcessorError> + Send + Sync + 'static,
) -> Self {
self.with_processor(DownloadProcessorFn::new(cache_key, callback))
}
}
impl fmt::Debug for DownloadBinaryProvider {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("DownloadBinaryProvider")
.field("url", &self.url)
.field("sha256", &self.sha256)
.field("cache_dir", &self.cache_dir)
.field(
"processor",
&self
.processor
.as_ref()
.map(|processor| processor.cache_key()),
)
.finish()
}
}
#[derive(Clone, Debug)]
@ -253,6 +350,23 @@ pub enum BinaryProviderError {
/// Actual lowercase SHA-256 checksum of the downloaded bytes.
actual: String,
},
/// A download processor completed without creating its configured output.
#[error("download processor {processor} did not produce binary output {path:?}")]
MissingProcessedOutput {
/// Stable identifier of the processor that failed to produce output.
processor: String,
/// Expected temporary executable output path.
path: PathBuf,
},
/// A configured download processor failed while preparing the executable.
#[error("download processor {processor} failed: {source}")]
DownloadProcessing {
/// Stable identifier of the processor that failed.
processor: String,
#[source]
/// Processor-specific error.
source: DownloadProcessorError,
},
/// Filesystem operation failed while preparing or resolving a binary.
#[error("failed to prepare binary path {path:?}: {source}")]
Io {

5
testing-framework/deployers/local/src/lib.rs
View File

@ -8,8 +8,9 @@ pub mod process;
pub use binary::{
BinaryProvider, BinaryProviderError, BinaryProviderRef, BuildBinaryProvider, BuildCommand,
DownloadBinaryProvider, DownloadChecksum, DownloadUrl, EnvBinaryProvider,
FallbackBinaryProvider, PathBinaryProvider,
DownloadBinaryProvider, DownloadChecksum, DownloadProcessor, DownloadProcessorError,
DownloadProcessorFn, DownloadUrl, EnvBinaryProvider, FallbackBinaryProvider,
PathBinaryProvider,
};
pub use deployer::{ProcessDeployer, ProcessDeployerError};
pub use env::{