logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-01-08 16:33:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Improves crash detection

Browse Source
This commit is contained in:
benbierens 2024-06-14 09:05:56 +02:00
parent 7b5e802efb
commit 54f053cfcc
No known key found for this signature in database
GPG Key ID: 877D2C2E09A22F3A
4 changed files with 48 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Framework/KubernetesWorkflow/CrashWatcher.cs
View File

@ -50,7 +50,9 @@ namespace KubernetesWorkflow
public bool HasContainerCrashed()
{
using var client = new Kubernetes(config);
return HasContainerBeenRestarted(client);
var result = HasContainerBeenRestarted(client);
if (result) DownloadCrashedContainerLogs(client);
return result;
}
private void Worker()

1
Framework/KubernetesWorkflow/K8sCluster.cs
View File

@ -16,6 +16,7 @@ namespace KubernetesWorkflow
{
var config = GetConfig();
UpdateHostAddress(config);
config.SkipTlsVerify = true; // Required for operation on Wings cluster.
return config;
}

64
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -37,20 +37,23 @@ namespace CodexPlugin
public DebugPeer GetDebugPeer(string peerId)
{
// Cannot use openAPI: debug/peer endpoint is not specified there.
var endpoint = GetEndpoint();
var str = endpoint.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
return CrashCheck(() =>
{
return new DebugPeer
{
IsPeerFound = false
};
}
var endpoint = GetEndpoint();
var str = endpoint.HttpGetString($"debug/peer/{peerId}");
var result = endpoint.Deserialize<DebugPeer>(str);
result.IsPeerFound = true;
return result;
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new DebugPeer
{
IsPeerFound = false
};
}
var result = endpoint.Deserialize<DebugPeer>(str);
result.IsPeerFound = true;
return result;
});
}
public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
@ -105,13 +108,16 @@ namespace CodexPlugin
public StoragePurchase GetPurchaseStatus(string purchaseId)
{
var endpoint = GetEndpoint();
return Time.Retry(() =>
return CrashCheck(() =>
{
var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
}, nameof(GetPurchaseStatus));
var endpoint = GetEndpoint();
return Time.Retry(() =>
{
var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
}, nameof(GetPurchaseStatus));
});
// TODO: current getpurchase api does not line up with its openapi spec.
// return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
@ -174,7 +180,19 @@ namespace CodexPlugin
var address = GetAddress();
var api = new CodexApi(client);
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
return Time.Wait(action(api));
return CrashCheck(() => Time.Wait(action(api)));
}
private T CrashCheck<T>(Func<T> action)
{
try
{
return action();
}
finally
{
CrashWatcher.HasContainerCrashed();
}
}
private IEndpoint GetEndpoint()
@ -237,7 +255,6 @@ namespace CodexPlugin
if (string.IsNullOrEmpty(debugInfo.Spr))
{
Log("Did not get value debug/info response.");
DownloadLog();
Throw(failure);
}
else
@ -248,14 +265,12 @@ namespace CodexPlugin
catch (Exception ex)
{
Log("Got exception from debug/info call: " + ex);
DownloadLog();
Throw(failure);
}
if (failure.Duration < timeSet.HttpCallTimeout())
{
Log("Retry failed within HTTP timeout duration.");
DownloadLog();
Throw(failure);
}
}
@ -269,10 +284,5 @@ namespace CodexPlugin
{
log.Log($"{GetName()} {msg}");
}
private void DownloadLog()
{
tools.CreateWorkflow().DownloadContainerLog(Container.Containers.Single(), this);
}
}
}

15
Tests/CodexTests/BasicTests/MarketplaceTests.cs
View File

@ -112,18 +112,17 @@ namespace CodexTests.BasicTests
};
var purchaseContract = client.Marketplace.RequestStorage(purchase);
var contractCid = purchaseContract.ContentId;
Assert.That(uploadCid.Id, Is.Not.EqualTo(contractCid.Id));
var downloader = StartCodex(s => s.WithName("Downloader"));
var uploadedFile = downloader.DownloadContent(uploadCid);
testFile.AssertIsEqual(uploadedFile);
// Download both from client.
testFile.AssertIsEqual(client.DownloadContent(uploadCid));
testFile.AssertIsEqual(client.DownloadContent(contractCid));
var contractFile = downloader.DownloadContent(contractCid);
testFile.AssertIsEqual(contractFile);
// Download both from another node.
var downloader = StartCodex(s => s.WithName("Downloader"));
testFile.AssertIsEqual(downloader.DownloadContent(uploadCid));
testFile.AssertIsEqual(downloader.DownloadContent(contractCid));
}
private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth)