2023-04-13 07:33:10 +00:00
|
|
|
|
using DistTestCore.Codex;
|
2023-04-14 07:54:07 +00:00
|
|
|
|
using DistTestCore.Logs;
|
2023-04-14 08:51:35 +00:00
|
|
|
|
using DistTestCore.Marketplace;
|
2023-04-13 12:36:17 +00:00
|
|
|
|
using DistTestCore.Metrics;
|
2023-06-07 06:30:10 +00:00
|
|
|
|
using Logging;
|
2023-04-13 07:33:10 +00:00
|
|
|
|
using NUnit.Framework;
|
2023-08-02 13:11:27 +00:00
|
|
|
|
using Utils;
|
2023-04-13 07:33:10 +00:00
|
|
|
|
|
|
|
|
|
namespace DistTestCore
|
|
|
|
|
{
|
|
|
|
|
public interface IOnlineCodexNode
|
|
|
|
|
{
|
2023-04-19 07:19:06 +00:00
|
|
|
|
string GetName();
|
2023-04-13 07:33:10 +00:00
|
|
|
|
CodexDebugResponse GetDebugInfo();
|
2023-05-11 10:44:53 +00:00
|
|
|
|
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
2023-04-13 07:33:10 +00:00
|
|
|
|
ContentId UploadFile(TestFile file);
|
2023-06-04 06:59:51 +00:00
|
|
|
|
TestFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
2023-04-13 07:33:10 +00:00
|
|
|
|
void ConnectToPeer(IOnlineCodexNode node);
|
2023-08-16 14:13:29 +00:00
|
|
|
|
IDownloadedLog DownloadLog(int? tailLines = null);
|
2023-04-13 12:36:17 +00:00
|
|
|
|
IMetricsAccess Metrics { get; }
|
2023-04-14 08:51:35 +00:00
|
|
|
|
IMarketplaceAccess Marketplace { get; }
|
2023-07-31 09:51:29 +00:00
|
|
|
|
CodexDebugVersionResponse Version { get; }
|
2023-04-25 10:52:11 +00:00
|
|
|
|
ICodexSetup BringOffline();
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public class OnlineCodexNode : IOnlineCodexNode
|
|
|
|
|
{
|
|
|
|
|
private const string SuccessfullyConnectedMessage = "Successfully connected to peer";
|
|
|
|
|
private const string UploadFailedMessage = "Unable to store block";
|
|
|
|
|
private readonly TestLifecycle lifecycle;
|
|
|
|
|
|
2023-04-14 08:51:35 +00:00
|
|
|
|
public OnlineCodexNode(TestLifecycle lifecycle, CodexAccess codexAccess, CodexNodeGroup group, IMetricsAccess metricsAccess, IMarketplaceAccess marketplaceAccess)
|
2023-04-13 07:33:10 +00:00
|
|
|
|
{
|
|
|
|
|
this.lifecycle = lifecycle;
|
|
|
|
|
CodexAccess = codexAccess;
|
|
|
|
|
Group = group;
|
2023-04-13 12:36:17 +00:00
|
|
|
|
Metrics = metricsAccess;
|
2023-04-14 08:51:35 +00:00
|
|
|
|
Marketplace = marketplaceAccess;
|
2023-07-31 09:51:29 +00:00
|
|
|
|
Version = new CodexDebugVersionResponse();
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public CodexAccess CodexAccess { get; }
|
|
|
|
|
public CodexNodeGroup Group { get; }
|
2023-04-13 12:36:17 +00:00
|
|
|
|
public IMetricsAccess Metrics { get; }
|
2023-04-14 08:51:35 +00:00
|
|
|
|
public IMarketplaceAccess Marketplace { get; }
|
2023-07-31 09:51:29 +00:00
|
|
|
|
public CodexDebugVersionResponse Version { get; private set; }
|
2023-04-13 07:33:10 +00:00
|
|
|
|
|
|
|
|
|
public string GetName()
|
|
|
|
|
{
|
2023-04-30 08:56:19 +00:00
|
|
|
|
return CodexAccess.Container.Name;
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public CodexDebugResponse GetDebugInfo()
|
|
|
|
|
{
|
2023-06-29 14:07:49 +00:00
|
|
|
|
var debugInfo = CodexAccess.GetDebugInfo();
|
2023-05-18 08:42:04 +00:00
|
|
|
|
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
|
|
|
|
|
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
|
2023-04-19 12:57:00 +00:00
|
|
|
|
return debugInfo;
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-05-11 10:44:53 +00:00
|
|
|
|
public CodexDebugPeerResponse GetDebugPeer(string peerId)
|
|
|
|
|
{
|
2023-06-29 14:07:49 +00:00
|
|
|
|
return CodexAccess.GetDebugPeer(peerId);
|
2023-05-11 10:44:53 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 07:33:10 +00:00
|
|
|
|
public ContentId UploadFile(TestFile file)
|
|
|
|
|
{
|
|
|
|
|
using var fileStream = File.OpenRead(file.Filename);
|
2023-06-07 06:30:10 +00:00
|
|
|
|
|
2023-08-08 07:44:38 +00:00
|
|
|
|
var logMessage = $"Uploading file {file.Describe()}...";
|
2023-08-02 13:11:27 +00:00
|
|
|
|
Log(logMessage);
|
2023-06-07 06:30:10 +00:00
|
|
|
|
var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
|
|
|
|
|
{
|
2023-06-29 14:07:49 +00:00
|
|
|
|
return CodexAccess.UploadFile(fileStream);
|
2023-06-07 06:30:10 +00:00
|
|
|
|
});
|
|
|
|
|
|
2023-07-11 06:19:14 +00:00
|
|
|
|
if (string.IsNullOrEmpty(response)) Assert.Fail("Received empty response.");
|
|
|
|
|
if (response.StartsWith(UploadFailedMessage)) Assert.Fail("Node failed to store block.");
|
|
|
|
|
|
2023-04-13 07:33:10 +00:00
|
|
|
|
Log($"Uploaded file. Received contentId: '{response}'.");
|
|
|
|
|
return new ContentId(response);
|
|
|
|
|
}
|
|
|
|
|
|
2023-06-04 06:59:51 +00:00
|
|
|
|
public TestFile? DownloadContent(ContentId contentId, string fileLabel = "")
|
2023-04-13 07:33:10 +00:00
|
|
|
|
{
|
2023-08-08 07:44:38 +00:00
|
|
|
|
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
|
2023-08-02 13:11:27 +00:00
|
|
|
|
Log(logMessage);
|
2023-06-04 06:59:51 +00:00
|
|
|
|
var file = lifecycle.FileManager.CreateEmptyTestFile(fileLabel);
|
2023-06-07 06:30:10 +00:00
|
|
|
|
Stopwatch.Measure(lifecycle.Log, logMessage, () => DownloadToFile(contentId.Id, file));
|
2023-06-07 07:32:56 +00:00
|
|
|
|
Log($"Downloaded file {file.Describe()} to '{file.Filename}'.");
|
2023-04-13 07:33:10 +00:00
|
|
|
|
return file;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public void ConnectToPeer(IOnlineCodexNode node)
|
|
|
|
|
{
|
|
|
|
|
var peer = (OnlineCodexNode)node;
|
|
|
|
|
|
|
|
|
|
Log($"Connecting to peer {peer.GetName()}...");
|
|
|
|
|
var peerInfo = node.GetDebugInfo();
|
2023-06-29 14:07:49 +00:00
|
|
|
|
var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
|
2023-04-13 07:33:10 +00:00
|
|
|
|
|
|
|
|
|
Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
|
|
|
|
|
Log($"Successfully connected to peer {peer.GetName()}.");
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-16 14:13:29 +00:00
|
|
|
|
public IDownloadedLog DownloadLog(int? tailLines = null)
|
2023-04-13 09:30:19 +00:00
|
|
|
|
{
|
2023-08-16 14:13:29 +00:00
|
|
|
|
return lifecycle.DownloadLog(CodexAccess.Container, tailLines);
|
2023-04-13 09:30:19 +00:00
|
|
|
|
}
|
2023-04-13 07:33:10 +00:00
|
|
|
|
|
2023-04-25 10:52:11 +00:00
|
|
|
|
public ICodexSetup BringOffline()
|
|
|
|
|
{
|
2023-05-04 12:55:39 +00:00
|
|
|
|
if (Group.Count() > 1) throw new InvalidOperationException("Codex-nodes that are part of a group cannot be " +
|
|
|
|
|
"individually shut down. Use 'BringOffline()' on the group object to stop the group. This method is only " +
|
|
|
|
|
"available for codex-nodes in groups of 1.");
|
|
|
|
|
|
2023-04-25 10:52:11 +00:00
|
|
|
|
return Group.BringOffline();
|
|
|
|
|
}
|
|
|
|
|
|
2023-07-31 09:51:29 +00:00
|
|
|
|
public void EnsureOnlineGetVersionResponse()
|
|
|
|
|
{
|
2023-08-02 13:11:27 +00:00
|
|
|
|
var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online");
|
2023-07-31 09:51:29 +00:00
|
|
|
|
var nodePeerId = debugInfo.id;
|
|
|
|
|
var nodeName = CodexAccess.Container.Name;
|
|
|
|
|
|
|
|
|
|
if (!debugInfo.codex.IsValid())
|
|
|
|
|
{
|
|
|
|
|
throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
|
|
|
|
|
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
|
|
|
|
|
Version = debugInfo.codex;
|
|
|
|
|
}
|
|
|
|
|
|
2023-04-13 07:33:10 +00:00
|
|
|
|
private string GetPeerMultiAddress(OnlineCodexNode peer, CodexDebugResponse peerInfo)
|
|
|
|
|
{
|
|
|
|
|
var multiAddress = peerInfo.addrs.First();
|
|
|
|
|
// Todo: Is there a case where First address in list is not the way?
|
|
|
|
|
|
|
|
|
|
// The peer we want to connect is in a different pod.
|
|
|
|
|
// We must replace the default IP with the pod IP in the multiAddress.
|
2023-08-07 08:44:48 +00:00
|
|
|
|
return multiAddress.Replace("0.0.0.0", peer.CodexAccess.Container.Pod.PodInfo.Ip);
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void DownloadToFile(string contentId, TestFile file)
|
|
|
|
|
{
|
|
|
|
|
using var fileStream = File.OpenWrite(file.Filename);
|
2023-04-19 12:57:00 +00:00
|
|
|
|
try
|
|
|
|
|
{
|
2023-06-29 14:07:49 +00:00
|
|
|
|
using var downloadStream = CodexAccess.DownloadFile(contentId);
|
2023-04-19 12:57:00 +00:00
|
|
|
|
downloadStream.CopyTo(fileStream);
|
|
|
|
|
}
|
|
|
|
|
catch
|
|
|
|
|
{
|
|
|
|
|
Log($"Failed to download file '{contentId}'.");
|
|
|
|
|
throw;
|
|
|
|
|
}
|
2023-04-13 07:33:10 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void Log(string msg)
|
|
|
|
|
{
|
|
|
|
|
lifecycle.Log.Log($"{GetName()}: {msg}");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public class ContentId
|
|
|
|
|
{
|
|
|
|
|
public ContentId(string id)
|
|
|
|
|
{
|
|
|
|
|
Id = id;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public string Id { get; }
|
|
|
|
|
}
|
|
|
|
|
}
|