status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Flattens CodexNode into CodexAccess

This commit is contained in:
benbierens 2023-06-29 16:07:49 +02:00
parent d985e3191a
commit 66e6cdc027
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
13 changed files with 112 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
CodexNetDeployer/CodexNodeStarter.cs
View File

@ -34,15 +34,14 @@ namespace CodexNetDeployer
var containers = workflow.Start(1, Location.Unspecified, new CodexContainerRecipe(), workflowStartup);
var container = containers.Containers.First();
var codexAccess = new CodexAccess(lifecycle, container);
var codexAccess = new CodexAccess(lifecycle.Log, container, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(container));
var account = gethResult.MarketplaceNetwork.Bootstrap.AllAccounts.Accounts[i];
var tokenAddress = gethResult.MarketplaceNetwork.Marketplace.TokenAddress;
var marketAccess = new MarketplaceAccess(lifecycle, gethResult.MarketplaceNetwork, account, codexAccess);
try
{
var debugInfo = codexAccess.Node.GetDebugInfo();
var debugInfo = codexAccess.GetDebugInfo();
if (!string.IsNullOrWhiteSpace(debugInfo.spr))
{
Console.Write("Online\t");

6
ContinuousTests/CodexNodeFactory.cs → ContinuousTests/CodexAccessFactory.cs
View File

@ -5,14 +5,14 @@ using Logging;
namespace ContinuousTests
{
public class CodexNodeFactory
public class CodexAccessFactory
{
public CodexNode[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
public CodexAccess[] Create(RunningContainer[] containers, BaseLog log, ITimeSet timeSet)
{
return containers.Select(container =>
{
var address = container.ClusterExternalAddress;
return new CodexNode(log, timeSet, address);
return new CodexAccess(log, container, timeSet, address);
}).ToArray();
}
}

10
ContinuousTests/ContinuousTest.cs
View File

@ -21,7 +21,7 @@ namespace ContinuousTests
private const string UploadFailedMessage = "Unable to store block";
public void Initialize(CodexNode[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
public void Initialize(CodexAccess[] nodes, BaseLog log, FileManager fileManager, Configuration configuration, CancellationToken cancelToken)
{
Nodes = nodes;
Log = log;
@ -39,7 +39,7 @@ namespace ContinuousTests
}
}
public CodexNode[] Nodes { get; private set; } = null!;
public CodexAccess[] Nodes { get; private set; } = null!;
public BaseLog Log { get; private set; } = null!;
public IFileManager FileManager { get; private set; } = null!;
public Configuration Configuration { get; private set; } = null!;
@ -61,7 +61,7 @@ namespace ContinuousTests
}
}
public ContentId? UploadFile(CodexNode node, TestFile file)
public ContentId? UploadFile(CodexAccess node, TestFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -79,7 +79,7 @@ namespace ContinuousTests
return new ContentId(response);
}
public TestFile DownloadFile(CodexNode node, ContentId contentId, string fileLabel = "")
public TestFile DownloadFile(CodexAccess node, ContentId contentId, string fileLabel = "")
{
var logMessage = $"Downloading for contentId: '{contentId.Id}'...";
var file = FileManager.CreateEmptyTestFile(fileLabel);
@ -88,7 +88,7 @@ namespace ContinuousTests
return file;
}
private void DownloadToFile(CodexNode node, string contentId, TestFile file)
private void DownloadToFile(CodexAccess node, string contentId, TestFile file)
{
using var fileStream = File.OpenWrite(file.Filename);
try

11
ContinuousTests/NodeRunner.cs
View File

@ -11,14 +11,14 @@ namespace ContinuousTests
public class NodeRunner
{
private readonly K8sFactory k8SFactory = new K8sFactory();
private readonly CodexNode[] nodes;
private readonly CodexAccess[] nodes;
private readonly Configuration config;
private readonly ITimeSet timeSet;
private readonly BaseLog log;
private readonly string customNamespace;
private readonly int ethereumAccountIndex;
public NodeRunner(CodexNode[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
public NodeRunner(CodexAccess[] nodes, Configuration config, ITimeSet timeSet, BaseLog log, string customNamespace, int ethereumAccountIndex)
{
this.nodes = nodes;
this.config = config;
@ -33,12 +33,12 @@ namespace ContinuousTests
RunNode(nodes.ToList().PickOneRandom(), operation, 0.TestTokens());
}
public void RunNode(CodexNode bootstrapNode, Action<CodexAccess, MarketplaceAccess> operation)
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess> operation)
{
RunNode(bootstrapNode, operation, 0.TestTokens());
}
public void RunNode(CodexNode bootstrapNode, Action<CodexAccess, MarketplaceAccess> operation, TestToken mintTestTokens)
public void RunNode(CodexAccess bootstrapNode, Action<CodexAccess, MarketplaceAccess> operation, TestToken mintTestTokens)
{
var (workflowCreator, lifecycle) = CreateFacilities();
var flow = workflowCreator.CreateWorkflow();
@ -68,7 +68,8 @@ namespace ContinuousTests
}
var container = rc.Containers[0];
var codexAccess = new CodexAccess(lifecycle, container);
var address = lifecycle.Configuration.GetAddress(container);
var codexAccess = new CodexAccess(log, container, lifecycle.TimeSet, address);
var marketAccess = new MarketplaceAccess(lifecycle, marketplaceNetwork, account, codexAccess);
try

6
ContinuousTests/SingleTestRun.cs
View File

@ -10,14 +10,14 @@ namespace ContinuousTests
{
public class SingleTestRun
{
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
private readonly List<Exception> exceptions = new List<Exception>();
private readonly TaskFactory taskFactory;
private readonly Configuration config;
private readonly BaseLog overviewLog;
private readonly TestHandle handle;
private readonly CancellationToken cancelToken;
private readonly CodexNode[] nodes;
private readonly CodexAccess[] nodes;
private readonly FileManager fileManager;
private readonly FixtureLog fixtureLog;
private readonly string testName;
@ -171,7 +171,7 @@ namespace ContinuousTests
overviewLog.Log( testName + ": " + msg);
}
private CodexNode[] CreateRandomNodes(int number)
private CodexAccess[] CreateRandomNodes(int number)
{
var containers = SelectRandomContainers(number);
fixtureLog.Log("Selected nodes: " + string.Join(",", containers.Select(c => c.Name)));

4
ContinuousTests/StartupChecker.cs
View File

@ -8,7 +8,7 @@ namespace ContinuousTests
public class StartupChecker
{
private readonly TestFactory testFactory = new TestFactory();
private readonly CodexNodeFactory codexNodeFactory = new CodexNodeFactory();
private readonly CodexAccessFactory codexNodeFactory = new CodexAccessFactory();
private readonly Configuration config;
public StartupChecker(Configuration config)
@ -77,7 +77,7 @@ namespace ContinuousTests
}
}
private bool EnsureOnline(CodexNode n)
private bool EnsureOnline(CodexAccess n)
{
try
{

4
ContinuousTests/Tests/PerformanceTests.cs
View File

@ -42,7 +42,7 @@ namespace ContinuousTests.Tests
public override TimeSpan RunTestEvery => TimeSpan.FromHours(1);
public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments;
public void UploadTest(int megabytes, CodexNode uploadNode)
public void UploadTest(int megabytes, CodexAccess uploadNode)
{
var file = FileManager.GenerateTestFile(megabytes.MB());
@ -56,7 +56,7 @@ namespace ContinuousTests.Tests
Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached.");
}
public void DownloadTest(int megabytes, CodexNode uploadNode, CodexNode downloadNode)
public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode)
{
var file = FileManager.GenerateTestFile(megabytes.MB());

10
ContinuousTests/Tests/TransientNodeTest.cs
View File

@ -15,9 +15,9 @@ namespace ContinuousTests.Tests
private TestFile file = null!;
private ContentId cid = null!;
private CodexNode UploadBootstapNode { get { return Nodes[0]; } }
private CodexNode DownloadBootstapNode { get { return Nodes[1]; } }
private CodexNode IntermediateNode { get { return Nodes[2]; } }
private CodexAccess UploadBootstapNode { get { return Nodes[0]; } }
private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } }
private CodexAccess IntermediateNode { get { return Nodes[2]; } }
[TestMoment(t: 0)]
public void UploadWithTransientNode()
@ -26,7 +26,7 @@ namespace ContinuousTests.Tests
NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess) =>
{
cid = UploadFile(codexAccess.Node, file)!;
cid = UploadFile(codexAccess, file)!;
Assert.That(cid, Is.Not.Null);
var resultFile = DownloadFile(IntermediateNode, cid);
@ -39,7 +39,7 @@ namespace ContinuousTests.Tests
{
NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess) =>
{
var resultFile = DownloadFile(codexAccess.Node, cid);
var resultFile = DownloadFile(codexAccess, cid);
file.AssertIsEqual(resultFile);
});
}

88
DistTestCore/Codex/CodexAccess.cs
View File

@ -1,40 +1,86 @@
using KubernetesWorkflow;
using Logging;
using Utils;
namespace DistTestCore.Codex
{
public class CodexAccess
{
private readonly TestLifecycle lifecycle;
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexAccess(TestLifecycle lifecycle, RunningContainer runningContainer)
public CodexAccess(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
{
this.lifecycle = lifecycle;
Container = runningContainer;
var address = lifecycle.Configuration.GetAddress(Container);
Node = new CodexNode(lifecycle.Log, lifecycle.TimeSet, address);
this.log = log;
Container = container;
this.timeSet = timeSet;
Address = address;
}
public RunningContainer Container { get; }
public CodexNode Node { get; }
public Address Address { get; }
public void EnsureOnline()
public CodexDebugResponse GetDebugInfo()
{
try
{
var debugInfo = Node.GetDebugInfo();
if (debugInfo == null || string.IsNullOrEmpty(debugInfo.id)) throw new InvalidOperationException("Unable to get debug-info from codex node at startup.");
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
}
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
catch (Exception e)
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
lifecycle.Log.Error($"Failed to start codex node: {e}. Test infra failure.");
throw new InvalidOperationException($"Failed to start codex node. Test infra failure.", e);
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public string UploadFile(FileStream fileStream)
{
return Http().HttpPostStream("upload", fileStream);
}
public Stream DownloadFile(string contentId)
{
return Http().HttpGetStream("download/" + contentId);
}
public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
{
return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
}
public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
{
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
{
return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http(TimeSpan? timeoutOverride = null)
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
}
}
}

80
DistTestCore/Codex/CodexNode.cs → DistTestCore/Codex/CodexApiTypes.cs
View File

@ -4,86 +4,6 @@ using Utils;
namespace DistTestCore.Codex
{
public class CodexNode
{
private readonly BaseLog log;
private readonly ITimeSet timeSet;
public CodexNode(BaseLog log, RunningContainer container, ITimeSet timeSet, Address address)
{
this.log = log;
Container = container;
this.timeSet = timeSet;
Address = address;
}
public RunningContainer Container { get; }
public Address Address { get; }
public CodexDebugResponse GetDebugInfo()
{
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public string UploadFile(FileStream fileStream)
{
return Http().HttpPostStream("upload", fileStream);
}
public Stream DownloadFile(string contentId)
{
return Http().HttpGetStream("download/" + contentId);
}
public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request)
{
return Http().HttpPostJson<CodexSalesAvailabilityRequest, CodexSalesAvailabilityResponse>("sales/availability", request);
}
public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId)
{
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public CodexStoragePurchase GetPurchaseStatus(string purchaseId)
{
return Http().HttpGetJson<CodexStoragePurchase>($"storage/purchases/{purchaseId}");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
private Http Http(TimeSpan? timeoutOverride = null)
{
return new Http(log, timeSet, Address, baseUrl: "/api/codex/v1", timeoutOverride);
}
}
public class CodexDebugResponse
{
public string id { get; set; } = string.Empty;

11
DistTestCore/CodexNodeGroup.cs
View File

@ -64,12 +64,19 @@ namespace DistTestCore
public void EnsureOnline()
{
foreach (var node in Nodes) node.CodexAccess.EnsureOnline();
foreach (var node in Nodes)
{
var debugInfo = node.CodexAccess.GetDebugInfo();
var nodePeerId = debugInfo.id;
var nodeName = node.CodexAccess.Container.Name;
lifecycle.Log.AddStringReplace(nodePeerId, nodeName);
lifecycle.Log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
}
private OnlineCodexNode CreateOnlineCodexNode(RunningContainer c, ICodexNodeFactory factory)
{
var access = new CodexAccess(lifecycle, c);
var access = new CodexAccess(lifecycle.Log, c, lifecycle.TimeSet, lifecycle.Configuration.GetAddress(c));
return factory.CreateOnlineCodexNode(access, this);
}
}

4
DistTestCore/Marketplace/MarketplaceAccess.cs
View File

@ -50,7 +50,7 @@ namespace DistTestCore.Marketplace
$"proofProbability: {proofProbability}, " +
$"duration: {Time.FormatDuration(duration)})");
var response = codexAccess.Node.RequestStorage(request, contentId.Id);
var response = codexAccess.RequestStorage(request, contentId.Id);
if (response == "Purchasing not available")
{
@ -78,7 +78,7 @@ namespace DistTestCore.Marketplace
$"maxCollateral: {maxCollateral}, " +
$"maxDuration: {Time.FormatDuration(maxDuration)})");
var response = codexAccess.Node.SalesAvailability(request);
var response = codexAccess.SalesAvailability(request);
Log($"Storage successfully made available. Id: {response.id}");

12
DistTestCore/OnlineCodexNode.cs
View File

@ -49,7 +49,7 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var debugInfo = CodexAccess.Node.GetDebugInfo();
var debugInfo = CodexAccess.GetDebugInfo();
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
return debugInfo;
@ -57,12 +57,12 @@ namespace DistTestCore
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return CodexAccess.Node.GetDebugPeer(peerId);
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.Node.GetDebugPeer(peerId, timeout);
return CodexAccess.GetDebugPeer(peerId, timeout);
}
public ContentId UploadFile(TestFile file)
@ -72,7 +72,7 @@ namespace DistTestCore
var logMessage = $"Uploading file {file.Describe()}...";
var response = Stopwatch.Measure(lifecycle.Log, logMessage, () =>
{
return CodexAccess.Node.UploadFile(fileStream);
return CodexAccess.UploadFile(fileStream);
});
if (response.StartsWith(UploadFailedMessage))
@ -101,7 +101,7 @@ namespace DistTestCore
Log($"Connecting to peer {peer.GetName()}...");
var peerInfo = node.GetDebugInfo();
var response = CodexAccess.Node.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo));
Assert.That(response, Is.EqualTo(SuccessfullyConnectedMessage), "Unable to connect codex nodes.");
Log($"Successfully connected to peer {peer.GetName()}.");
@ -141,7 +141,7 @@ namespace DistTestCore
using var fileStream = File.OpenWrite(file.Filename);
try
{
using var downloadStream = CodexAccess.Node.DownloadFile(contentId);
using var downloadStream = CodexAccess.DownloadFile(contentId);
downloadStream.CopyTo(fileStream);
}
catch