codex-storage/cs-codex-dist-tests
1
0
Fork 0
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git synced 2025-02-10 19:16:46 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cannot assume retry uploads will require as much space as file is large

Browse Source
This commit is contained in:
Ben 2024-06-05 09:59:03 +02:00
parent b888999b61
commit bf75c25152
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
1 changed files with 15 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -62,18 +62,22 @@ namespace CodexPlugin
public string UploadFile(FileStream fileStream) public string UploadFile(FileStream fileStream)
{ {
CheckSpaceIsAvailable(fileStream); LogSpaceStatistics("Before upload");
return OnCodex( var str = OnCodex(
api => api.UploadAsync(fileStream), api => api.UploadAsync(fileStream),
CreateRetryConfig(nameof(UploadFile), f => CheckSpaceIsAvailable(fileStream))); CreateRetryConfig(nameof(UploadFile)));
LogSpaceStatistics("After upload");
return str;
} }
public Stream DownloadFile(string contentId) public Stream DownloadFile(string contentId)
{ {
var fileResponse = OnCodex( var fileResponse = OnCodex(
api => api.DownloadNetworkAsync(contentId), api => api.DownloadNetworkAsync(contentId),
CreateRetryConfig(nameof(DownloadFile), f => { })); CreateRetryConfig(nameof(DownloadFile)));
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
return fileResponse.Stream; return fileResponse.Stream;
@ -160,18 +164,6 @@ namespace CodexPlugin
return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); return Container.Containers.Single().GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag);
} }
private void CheckSpaceIsAvailable(FileStream fileStream)
{
var space = Space();
if (space.FreeBytes < fileStream.Length)
{
var msg = $"Not enough space available. File: {fileStream.Length} Free: {space.FreeBytes}";
tools.GetLog().Error(msg);
throw new Exception(msg);
}
}
private void CheckContainerCrashed(HttpClient client) private void CheckContainerCrashed(HttpClient client)
{ {
if (hasContainerCrashed) throw new Exception($"Container {GetName()} has crashed."); if (hasContainerCrashed) throw new Exception($"Container {GetName()} has crashed.");
@ -196,14 +188,13 @@ namespace CodexPlugin
hasContainerCrashed = true; hasContainerCrashed = true;
} }
private Retry CreateRetryConfig(string description, Action<Failure> onFailure) private Retry CreateRetryConfig(string description)
{ {
var timeSet = tools.TimeSet; var timeSet = tools.TimeSet;
var log = tools.GetLog(); var log = tools.GetLog();
return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure => return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
{ {
onFailure(failure);
if (failure.Duration.TotalSeconds < timeSet.HttpCallTimeout().TotalSeconds) if (failure.Duration.TotalSeconds < timeSet.HttpCallTimeout().TotalSeconds)
{ {
Investigate(log, failure, timeSet); Investigate(log, failure, timeSet);
@ -242,15 +233,7 @@ namespace CodexPlugin
{ {
try try
{ {
var space = Space(); LogSpaceStatistics();
log.Log($"Got space statistics: {space}");
log.Log($"Free space: {space.FreeBytes}");
if (space.FreeBytes < 1.MB().SizeInBytes)
{
log.Log("There's less than 1MB free. Stopping...");
Throw(failure);
}
} }
catch (Exception e) catch (Exception e)
{ {
@ -260,6 +243,11 @@ namespace CodexPlugin
} }
} }
private void LogSpaceStatistics(string prefix = "")
{
tools.GetLog().Log($"{prefix} Space statistics: {Space()}");
}
private void Throw(Failure failure) private void Throw(Failure failure)
{ {
throw failure.Exception; throw failure.Exception;