codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/peer-discovery-tests'

This commit is contained in:
benbierens 2023-05-31 14:12:22 +02:00
parent 1b490c475e 3f7e623eb6
commit 59692fafa3
No known key found for this signature in database
GPG Key ID: FE44815D96D0A1AA
27 changed files with 735 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
DistTestCore/AutoBootstrapDistTest.cs
View File

@ -1,9 +1,9 @@
namespace DistTestCore
using NUnit.Framework;
namespace DistTestCore
{
public class AutoBootstrapDistTest : DistTest
{
private IOnlineCodexNode? bootstrapNode;
public override IOnlineCodexNode SetupCodexBootstrapNode(Action<ICodexSetup> setup)
{
throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " +
@ -12,19 +12,18 @@
public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes);
var codexSetup = CreateCodexSetup(numberOfNodes);
setup(codexSetup);
codexSetup.WithBootstrapNode(EnsureBootstapNode());
codexSetup.WithBootstrapNode(BootstrapNode);
return BringOnline(codexSetup);
}
private IOnlineCodexNode EnsureBootstapNode()
[SetUp]
public void SetUpBootstrapNode()
{
if (bootstrapNode == null)
{
bootstrapNode = base.SetupCodexBootstrapNode(s => { });
}
return bootstrapNode;
BootstrapNode = BringOnline(CreateCodexSetup(1))[0];
}
protected IOnlineCodexNode BootstrapNode { get; private set; } = null!;
}
}

74
DistTestCore/Codex/CodexAccess.cs
View File

@ -19,7 +19,30 @@ namespace DistTestCore.Codex
public CodexDebugResponse GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
return Http(TimeSpan.FromSeconds(2)).HttpGetJson<CodexDebugResponse>("debug/info");
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return GetDebugPeer(peerId, TimeSpan.FromSeconds(2));
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
var http = Http(timeout);
var str = http.HttpGetString($"debug/peer/{peerId}");
if (str.ToLowerInvariant() == "unable to find peer!")
{
return new CodexDebugPeerResponse
{
IsPeerFound = false
};
}
var result = http.TryJsonDeserialize<CodexDebugPeerResponse>(str);
result.IsPeerFound = true;
return result;
}
public string UploadFile(FileStream fileStream)
@ -42,6 +65,11 @@ namespace DistTestCore.Codex
return Http().HttpPostJson($"storage/request/{contentId}", request);
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
}
public void EnsureOnline()
{
try
@ -51,7 +79,8 @@ namespace DistTestCore.Codex
var nodePeerId = debugInfo.id;
var nodeName = Container.Name;
log.AddStringReplace(nodePeerId, $"___{nodeName}___");
log.AddStringReplace(nodePeerId, nodeName);
log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName);
}
catch (Exception e)
{
@ -60,16 +89,11 @@ namespace DistTestCore.Codex
}
}
private Http Http()
private Http Http(TimeSpan? timeoutOverride = null)
{
var ip = Container.Pod.Cluster.IP;
var ip = Container.Pod.Cluster.HostAddress;
var port = Container.ServicePorts[0].Number;
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1");
}
public string ConnectToPeer(string peerId, string peerMultiAddress)
{
return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}");
return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride);
}
}
@ -82,6 +106,22 @@ namespace DistTestCore.Codex
public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
public CodexDebugVersionResponse codex { get; set; } = new();
public CodexDebugTableResponse table { get; set; } = new();
}
public class CodexDebugTableResponse
{
public CodexDebugTableNodeResponse localNode { get; set; } = new();
public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
}
public class CodexDebugTableNodeResponse
{
public string nodeId { get; set; } = string.Empty;
public string peerId { get; set; } = string.Empty;
public string record { get; set; } = string.Empty;
public string address { get; set; } = string.Empty;
public bool seen { get; set; }
}
public class EnginePeerResponse
@ -110,6 +150,20 @@ namespace DistTestCore.Codex
public string revision { get; set; } = string.Empty;
}
public class CodexDebugPeerResponse
{
public bool IsPeerFound { get; set; }
public string peerId { get; set; } = string.Empty;
public long seqNo { get; set; }
public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty<CodexDebugPeerAddressResponse>();
}
public class CodexDebugPeerAddressResponse
{
public string address { get; set; } = string.Empty;
}
public class CodexSalesAvailabilityRequest
{
public string size { get; set; } = string.Empty;

9
DistTestCore/Codex/CodexContainerRecipe.cs
View File

@ -13,6 +13,7 @@ namespace DistTestCore.Codex
public const string DockerImage = "thatbenbierens/codexlocal:latest";
#endif
public const string MetricsPortTag = "metrics_port";
public const string DiscoveryPortTag = "discovery-port";
protected override string Image => DockerImage;
@ -22,7 +23,8 @@ namespace DistTestCore.Codex
AddExposedPortAndVar("API_PORT");
AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}");
AddInternalPortAndVar("DISC_PORT");
AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag);
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
var listenPort = AddInternalPort();
AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
@ -31,11 +33,6 @@ namespace DistTestCore.Codex
{
AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr);
}
if (config.LogLevel != null)
{
AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant());
}
if (config.StorageQuota != null)
{
AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!);

7
DistTestCore/Codex/CodexStartupConfig.cs
View File

@ -5,9 +5,14 @@ namespace DistTestCore.Codex
{
public class CodexStartupConfig
{
public CodexStartupConfig(CodexLogLevel logLevel)
{
LogLevel = logLevel;
}
public string? NameOverride { get; set; }
public Location Location { get; set; }
public CodexLogLevel? LogLevel { get; set; }
public CodexLogLevel LogLevel { get; }
public ByteSize? StorageQuota { get; set; }
public bool MetricsEnabled { get; set; }
public MarketplaceInitialConfig? MarketplaceConfig { get; set; }

12
DistTestCore/CodexSetup.cs
View File

@ -8,7 +8,6 @@ namespace DistTestCore
{
ICodexSetup WithName(string name);
ICodexSetup At(Location location);
ICodexSetup WithLogLevel(CodexLogLevel level);
ICodexSetup WithBootstrapNode(IOnlineCodexNode node);
ICodexSetup WithStorageQuota(ByteSize storageQuota);
ICodexSetup EnableMetrics();
@ -20,7 +19,8 @@ namespace DistTestCore
{
public int NumberOfNodes { get; }
public CodexSetup(int numberOfNodes)
public CodexSetup(int numberOfNodes, CodexLogLevel logLevel)
: base(logLevel)
{
NumberOfNodes = numberOfNodes;
}
@ -43,12 +43,6 @@ namespace DistTestCore
return this;
}
public ICodexSetup WithLogLevel(CodexLogLevel level)
{
LogLevel = level;
return this;
}
public ICodexSetup WithStorageQuota(ByteSize storageQuota)
{
StorageQuota = storageQuota;
@ -80,7 +74,7 @@ namespace DistTestCore
private IEnumerable<string> DescribeArgs()
{
if (LogLevel != null) yield return $"LogLevel={LogLevel}";
yield return $"LogLevel={LogLevel}";
if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}";
if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}";
}

3
DistTestCore/CodexStarter.cs
View File

@ -1,6 +1,7 @@
using DistTestCore.Codex;
using DistTestCore.Marketplace;
using KubernetesWorkflow;
using Logging;
namespace DistTestCore
{
@ -74,7 +75,7 @@ namespace DistTestCore
{
var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory);
RunningGroups.Add(group);
group.EnsureOnline();
Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true);
return group;
}

8
DistTestCore/Configuration.cs
View File

@ -1,4 +1,5 @@
using KubernetesWorkflow;
using DistTestCore.Codex;
using KubernetesWorkflow;
namespace DistTestCore
{
@ -28,5 +29,10 @@ namespace DistTestCore
{
return "TestDataFiles";
}
public CodexLogLevel GetCodexLogLevel()
{
return CodexLogLevel.Trace;
}
}
}

45
DistTestCore/DistTest.cs
View File

@ -1,4 +1,5 @@
using DistTestCore.Codex;
using DistTestCore.Helpers;
using DistTestCore.Logs;
using DistTestCore.Marketplace;
using DistTestCore.Metrics;
@ -25,8 +26,14 @@ namespace DistTestCore
testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray();
fixtureLog = new FixtureLog(configuration.GetLogConfig());
PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this);
PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this);
}
public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; }
public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; }
[OneTimeSetUp]
public void GlobalSetup()
{
@ -85,6 +92,17 @@ namespace DistTestCore
return Get().FileManager.GenerateTestFile(size);
}
/// <summary>
/// Any test files generated in 'action' will be deleted after it returns.
/// This helps prevent large tests from filling up discs.
/// </summary>
public void ScopedTestFiles(Action action)
{
Get().FileManager.PushFileSet();
action();
Get().FileManager.PopFileSet();
}
public IOnlineCodexNode SetupCodexBootstrapNode()
{
return SetupCodexBootstrapNode(s => { });
@ -116,7 +134,7 @@ namespace DistTestCore
public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action<ICodexSetup> setup)
{
var codexSetup = new CodexSetup(numberOfNodes);
var codexSetup = CreateCodexSetup(numberOfNodes);
setup(codexSetup);
@ -128,16 +146,31 @@ namespace DistTestCore
return Get().CodexStarter.BringOnline((CodexSetup)codexSetup);
}
protected void Log(string msg)
public IEnumerable<IOnlineCodexNode> GetAllOnlineCodexNodes()
{
TestContext.Progress.WriteLine(msg);
Get().Log.Log(msg);
return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes);
}
protected void Debug(string msg)
public BaseLog GetTestLog()
{
return Get().Log;
}
public void Log(string msg)
{
TestContext.Progress.WriteLine(msg);
Get().Log.Debug(msg);
GetTestLog().Log(msg);
}
public void Debug(string msg)
{
TestContext.Progress.WriteLine(msg);
GetTestLog().Debug(msg);
}
protected CodexSetup CreateCodexSetup(int numberOfNodes)
{
return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel());
}
private TestLifecycle Get()

25
DistTestCore/FileManager.cs
View File

@ -9,6 +9,8 @@ namespace DistTestCore
TestFile CreateEmptyTestFile();
TestFile GenerateTestFile(ByteSize size);
void DeleteAllTestFiles();
void PushFileSet();
void PopFileSet();
}
public class FileManager : IFileManager
@ -18,6 +20,7 @@ namespace DistTestCore
private readonly Random random = new Random();
private readonly TestLog log;
private readonly string folder;
private readonly List<List<TestFile>> fileSetStack = new List<List<TestFile>>();
public FileManager(TestLog log, Configuration configuration)
{
@ -31,6 +34,7 @@ namespace DistTestCore
{
var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin"));
File.Create(result.Filename).Close();
if (fileSetStack.Any()) fileSetStack.Last().Add(result);
return result;
}
@ -47,6 +51,27 @@ namespace DistTestCore
DeleteDirectory();
}
public void PushFileSet()
{
fileSetStack.Add(new List<TestFile>());
}
public void PopFileSet()
{
if (!fileSetStack.Any()) return;
var pop = fileSetStack.Last();
fileSetStack.Remove(pop);
foreach (var file in pop)
{
try
{
File.Delete(file.Filename);
}
catch { }
}
}
private void GenerateFileBytes(TestFile result, ByteSize size)
{
long bytesLeft = size.SizeInBytes;

239
DistTestCore/Helpers/PeerConnectionTestHelpers.cs Normal file
View File

@ -0,0 +1,239 @@
using DistTestCore.Codex;
using NUnit.Framework;
using Utils;
namespace DistTestCore.Helpers
{
public class PeerConnectionTestHelpers
{
private readonly Random random = new Random();
private readonly DistTest test;
public PeerConnectionTestHelpers(DistTest test)
{
this.test = test;
}
public void AssertFullyConnected(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullyConnected(nodes.ToArray());
}
public void AssertFullyConnected(params IOnlineCodexNode[] nodes)
{
test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
var entries = CreateEntries(nodes);
var pairs = CreatePairs(entries);
RetryWhilePairs(pairs, () =>
{
CheckAndRemoveSuccessful(pairs);
});
if (pairs.Any())
{
test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage())));
}
else
{
test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
}
private static void RetryWhilePairs(List<Pair> pairs, Action action)
{
var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10);
while (pairs.Any() && timeout > DateTime.UtcNow)
{
action();
if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5));
}
}
private void CheckAndRemoveSuccessful(List<Pair> pairs)
{
var checkTasks = pairs.Select(p => Task.Run(() =>
{
ApplyRandomDelay();
p.Check();
})).ToArray();
Task.WaitAll(checkTasks);
foreach (var pair in pairs.ToArray())
{
if (pair.Success)
{
test.Log(pair.GetMessage());
pairs.Remove(pair);
}
}
}
private static Entry[] CreateEntries(IOnlineCodexNode[] nodes)
{
var entries = nodes.Select(n => new Entry(n)).ToArray();
var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray();
if (incorrectDiscoveryEndpoints.Any())
{
Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " +
string.Join(Environment.NewLine, incorrectDiscoveryEndpoints));
}
return entries;
}
private static List<Pair> CreatePairs(Entry[] entries)
{
return CreatePairsIterator(entries).ToList();
}
private static IEnumerable<Pair> CreatePairsIterator(Entry[] entries)
{
for (var x = 0; x < entries.Length; x++)
{
for (var y = x + 1; y < entries.Length; y++)
{
yield return new Pair(entries[x], entries[y]);
}
}
}
private void ApplyRandomDelay()
{
// Calling all the nodes all at the same time is not exactly nice.
Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000)));
}
public class Entry
{
public Entry(IOnlineCodexNode node)
{
Node = node;
Response = node.GetDebugInfo();
}
public IOnlineCodexNode Node { get; }
public CodexDebugResponse Response { get; }
public IEnumerable<string> GetInCorrectDiscoveryEndpoints(Entry[] allEntries)
{
foreach (var peer in Response.table.nodes)
{
var expected = GetExpectedDiscoveryEndpoint(allEntries, peer);
if (expected != peer.address)
{
yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'";
}
}
}
private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node)
{
var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId);
if (peer == null) return $"peerId: {node.peerId} is not known.";
var n = (OnlineCodexNode)peer.Node;
var ip = n.CodexAccess.Container.Pod.Ip;
var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag);
return $"{ip}:{discPort.Number}";
}
}
public enum PeerConnectionState
{
Unknown,
Connection,
NoConnection,
}
public class Pair
{
private readonly TimeSpan timeout = TimeSpan.FromSeconds(60);
private TimeSpan aToBTime = TimeSpan.FromSeconds(0);
private TimeSpan bToATime = TimeSpan.FromSeconds(0);
public Pair(Entry a, Entry b)
{
A = a;
B = b;
}
public Entry A { get; }
public Entry B { get; }
public PeerConnectionState AKnowsB { get; private set; }
public PeerConnectionState BKnowsA { get; private set; }
public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } }
public void Check()
{
aToBTime = Measure(() => AKnowsB = Knows(A, B));
bToATime = Measure(() => BKnowsA = Knows(B, A));
}
public string GetMessage()
{
return GetResultMessage() + GetTimePostfix();
}
private string GetResultMessage()
{
var aName = A.Response.id;
var bName = B.Response.id;
if (Success)
{
return $"{aName} and {bName} know each other.";
}
return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}";
}
private string GetTimePostfix()
{
var aName = A.Response.id;
var bName = B.Response.id;
return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)";
}
private static TimeSpan Measure(Action action)
{
var start = DateTime.UtcNow;
action();
return DateTime.UtcNow - start;
}
private PeerConnectionState Knows(Entry a, Entry b)
{
lock (a)
{
var peerId = b.Response.id;
try
{
var response = a.Node.GetDebugPeer(peerId, timeout);
if (!response.IsPeerFound)
{
return PeerConnectionState.NoConnection;
}
if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any())
{
return PeerConnectionState.Connection;
}
}
catch
{
}
// Didn't get a conclusive answer. Try again later.
return PeerConnectionState.Unknown;
}
}
}
}
}

52
DistTestCore/Helpers/PeerDownloadTestHelpers.cs Normal file
View File

@ -0,0 +1,52 @@
namespace DistTestCore.Helpers
{
public class PeerDownloadTestHelpers
{
private readonly DistTest test;
public PeerDownloadTestHelpers(DistTest test)
{
this.test = test;
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes)
{
AssertFullDownloadInterconnectivity(nodes, 1.MB());
}
public void AssertFullDownloadInterconnectivity(IEnumerable<IOnlineCodexNode> nodes, ByteSize testFileSize)
{
test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'...");
foreach (var node in nodes)
{
var uploader = node;
var downloaders = nodes.Where(n => n != uploader).ToArray();
test.ScopedTestFiles(() =>
{
PerformTest(uploader, downloaders, testFileSize);
});
}
test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}");
}
private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize)
{
// 1 test file per downloader.
var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray();
// Upload all the test files to the uploader.
var contentIds = files.Select(uploader.UploadFile).ToArray();
// Each downloader should retrieve its own test file.
for (var i = 0; i < downloaders.Length; i++)
{
var expectedFile = files[i];
var downloadedFile = downloaders[i].DownloadContent(contentIds[i]);
expectedFile.AssertIsEqual(downloadedFile);
}
}
}
}

76
DistTestCore/Http.cs
View File

@ -1,6 +1,5 @@
using Logging;
using Newtonsoft.Json;
using NUnit.Framework;
using System.Net.Http.Headers;
using System.Net.Http.Json;
using Utils;
@ -14,15 +13,16 @@ namespace DistTestCore
private readonly string ip;
private readonly int port;
private readonly string baseUrl;
private readonly TimeSpan? timeoutOverride;
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl)
public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl, TimeSpan? timeoutOverride = null)
{
this.log = log;
this.timeSet = timeSet;
this.ip = ip;
this.port = port;
this.baseUrl = baseUrl;
this.timeoutOverride = timeoutOverride;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
}
@ -38,7 +38,7 @@ namespace DistTestCore
var str = Time.Wait(result.Content.ReadAsStringAsync());
Log(url, str);
return str; ;
});
}, $"HTTP-GET:{route}");
}
public T HttpGetJson<T>(string route)
@ -62,10 +62,10 @@ namespace DistTestCore
using var content = JsonContent.Create(body);
Log(url, JsonConvert.SerializeObject(body));
var result = Time.Wait(client.PostAsync(url, content));
var str= Time.Wait(result.Content.ReadAsStringAsync());
var str = Time.Wait(result.Content.ReadAsStringAsync());
Log(url, str);
return str;
});
}, $"HTTP-POST-JSON: {route}");
}
public string HttpPostStream(string route, Stream stream)
@ -81,7 +81,7 @@ namespace DistTestCore
var str =Time.Wait(response.Content.ReadAsStringAsync());
Log(url, str);
return str;
});
}, $"HTTP-POST-STREAM: {route}");
}
public Stream HttpGetStream(string route)
@ -92,7 +92,20 @@ namespace DistTestCore
var url = GetUrl() + route;
Log(url, "~ STREAM ~");
return Time.Wait(client.GetStreamAsync(url));
});
}, $"HTTP-GET-STREAM: {route}");
}
public T TryJsonDeserialize<T>(string json)
{
try
{
return JsonConvert.DeserializeObject<T>(json)!;
}
catch (Exception exception)
{
var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
throw new InvalidOperationException(msg, exception);
}
}
private string GetUrl()
@ -105,47 +118,24 @@ namespace DistTestCore
log.Debug($"({url}) = '{message}'", 3);
}
private T Retry<T>(Func<T> operation)
private T Retry<T>(Func<T> operation, string description)
{
var retryCounter = 0;
while (true)
{
try
{
return operation();
}
catch (Exception exception)
{
timeSet.HttpCallRetryDelay();
retryCounter++;
if (retryCounter > timeSet.HttpCallRetryCount())
{
Assert.Fail(exception.ToString());
throw;
}
}
}
}
private static T TryJsonDeserialize<T>(string json)
{
try
{
return JsonConvert.DeserializeObject<T>(json)!;
}
catch (Exception exception)
{
var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}";
Assert.Fail(msg);
throw new InvalidOperationException(msg, exception);
}
return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description);
}
private HttpClient GetClient()
{
if (timeoutOverride.HasValue)
{
return GetClient(timeoutOverride.Value);
}
return GetClient(timeSet.HttpCallTimeout());
}
private HttpClient GetClient(TimeSpan timeout)
{
var client = new HttpClient();
client.Timeout = timeSet.HttpCallTimeout();
client.Timeout = timeout;
return client;
}
}

29
DistTestCore/Marketplace/ContainerInfoExtractor.cs
View File

@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace
return marketplaceAbi;
}
private string Retry(Func<string> fetch)
{
var result = string.Empty;
Time.WaitUntil(() =>
{
result = Catch(fetch);
return !string.IsNullOrEmpty(result);
}, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3));
return result;
}
private string Catch(Func<string> fetch)
{
try
{
return fetch();
}
catch
{
return string.Empty;
}
}
private string FetchAccountsCsv()
{
return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename);
@ -116,6 +92,11 @@ namespace DistTestCore.Marketplace
var privateKey = tokens[1];
return new GethAccount(account, privateKey);
}
private static string Retry(Func<string> fetch)
{
return Time.Retry(fetch, nameof(ContainerInfoExtractor));
}
}
public class PubKeyFinder : LogHandler, ILogHandler

2
DistTestCore/Marketplace/GethBootstrapNodeInfo.cs
View File

@ -23,7 +23,7 @@ namespace DistTestCore.Marketplace
public NethereumInteraction StartInteraction(BaseLog log)
{
var ip = RunningContainers.RunningPod.Cluster.IP;
var ip = RunningContainers.RunningPod.Cluster.HostAddress;
var port = RunningContainers.Containers[0].ServicePorts[0].Number;
var account = Account;

2
DistTestCore/Marketplace/GethCompanionNodeInfo.cs
View File

@ -17,7 +17,7 @@ namespace DistTestCore.Marketplace
public NethereumInteraction StartInteraction(BaseLog log, GethAccount account)
{
var ip = RunningContainer.Pod.Cluster.IP;
var ip = RunningContainer.Pod.Cluster.HostAddress;
var port = RunningContainer.ServicePorts[0].Number;
var privateKey = account.PrivateKey;

2
DistTestCore/Metrics/MetricsQuery.cs
View File

@ -16,7 +16,7 @@ namespace DistTestCore.Metrics
http = new Http(
log,
timeSet,
runningContainers.RunningPod.Cluster.IP,
runningContainers.RunningPod.Cluster.HostAddress,
runningContainers.Containers[0].ServicePorts[0].Number,
"api/v1");
}

15
DistTestCore/OnlineCodexNode.cs
View File

@ -10,6 +10,8 @@ namespace DistTestCore
{
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout);
ContentId UploadFile(TestFile file);
TestFile? DownloadContent(ContentId contentId);
void ConnectToPeer(IOnlineCodexNode node);
@ -47,10 +49,21 @@ namespace DistTestCore
public CodexDebugResponse GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
Log($"Got DebugInfo with id: '{debugInfo.id}'.");
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}");
return debugInfo;
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
{
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout)
{
return CodexAccess.GetDebugPeer(peerId, timeout);
}
public ContentId UploadFile(TestFile file)
{
Log($"Uploading file of size {file.GetFileSize()}...");

21
DistTestCore/Timing.cs
View File

@ -1,5 +1,4 @@
using NUnit.Framework;
using Utils;
namespace DistTestCore
{
@ -11,8 +10,8 @@ namespace DistTestCore
public interface ITimeSet
{
TimeSpan HttpCallTimeout();
int HttpCallRetryCount();
void HttpCallRetryDelay();
TimeSpan HttpCallRetryTimeout();
TimeSpan HttpCallRetryDelay();
TimeSpan WaitForK8sServiceDelay();
TimeSpan K8sOperationTimeout();
TimeSpan WaitForMetricTimeout();
@ -25,14 +24,14 @@ namespace DistTestCore
return TimeSpan.FromSeconds(10);
}
public int HttpCallRetryCount()
public TimeSpan HttpCallRetryTimeout()
{
return 5;
return TimeSpan.FromMinutes(1);
}
public void HttpCallRetryDelay()
public TimeSpan HttpCallRetryDelay()
{
Time.Sleep(TimeSpan.FromSeconds(3));
return TimeSpan.FromSeconds(3);
}
public TimeSpan WaitForK8sServiceDelay()
@ -58,14 +57,14 @@ namespace DistTestCore
return TimeSpan.FromHours(2);
}
public int HttpCallRetryCount()
public TimeSpan HttpCallRetryTimeout()
{
return 2;
return TimeSpan.FromHours(5);
}
public void HttpCallRetryDelay()
public TimeSpan HttpCallRetryDelay()
{
Time.Sleep(TimeSpan.FromMinutes(5));
return TimeSpan.FromMinutes(5);
}
public TimeSpan WaitForK8sServiceDelay()

15
KubernetesWorkflow/K8sCluster.cs
View File

@ -10,12 +10,12 @@ namespace KubernetesWorkflow
}
public Configuration Configuration { get; }
public string IP { get; private set; } = string.Empty;
public string HostAddress { get; private set; } = string.Empty;
public KubernetesClientConfiguration GetK8sClientConfig()
{
var config = GetConfig();
UpdateIp(config);
UpdateHostAddress(config);
return config;
}
@ -47,10 +47,17 @@ namespace KubernetesWorkflow
}
}
private void UpdateIp(KubernetesClientConfiguration config)
private void UpdateHostAddress(KubernetesClientConfiguration config)
{
var host = config.Host.Replace("https://", "");
IP = host.Substring(0, host.IndexOf(':'));
if (host.Contains(":"))
{
HostAddress = host.Substring(0, host.IndexOf(':'));
}
else
{
HostAddress = config.Host;
}
}
}
}

2
LongTests/BasicTests/LargeFileTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace TestsLong.BasicTests
@ -11,7 +10,6 @@ namespace TestsLong.BasicTests
public void OneClientLargeFileTest()
{
var primary = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Warn)
.WithStorageQuota(20.GB()));
var testFile = GenerateTestFile(10.GB());

2
LongTests/BasicTests/TestInfraTests.cs
View File

@ -1,5 +1,4 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace TestsLong.BasicTests
@ -32,7 +31,6 @@ namespace TestsLong.BasicTests
public void DownloadConsistencyTest()
{
var primary = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(2.MB()));
var testFile = GenerateTestFile(1.MB());

4
Tests/BasicTests/ExampleTests.cs
View File

@ -11,7 +11,7 @@ namespace Tests.BasicTests
[Test]
public void CodexLogExample()
{
var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var primary = SetupCodexNode();
primary.UploadFile(GenerateTestFile(5.MB()));
@ -47,7 +47,6 @@ namespace Tests.BasicTests
var buyerInitialBalance = 1000.TestTokens();
var seller = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithStorageQuota(11.GB())
.EnableMarketplace(sellerInitialBalance));
@ -61,7 +60,6 @@ namespace Tests.BasicTests
var testFile = GenerateTestFile(10.MB());
var buyer = SetupCodexNode(s => s
.WithLogLevel(CodexLogLevel.Trace)
.WithBootstrapNode(seller)
.EnableMarketplace(buyerInitialBalance));

85
Tests/BasicTests/PeerTests.cs
View File

@ -1,85 +0,0 @@
using DistTestCore;
using DistTestCore.Codex;
using NUnit.Framework;
namespace Tests.BasicTests
{
[TestFixture]
public class PeerTests : DistTest
{
[Test]
public void TwoNodes()
{
var primary = SetupCodexBootstrapNode();
var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary));
primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up.
// This is required for the enginePeers to show up.
//var file = GenerateTestFile(10.MB());
//var contentId = primary.UploadFile(file);
//var file2 = secondary.DownloadContent(contentId);
//file.AssertIsEqual(file2);
AssertKnowEachother(primary, secondary);
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
public void VariableNodes(int number)
{
var bootstrap = SetupCodexBootstrapNode();
var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap));
var file = GenerateTestFile(10.MB());
var contentId = nodes.First().UploadFile(file);
var file2 = nodes.Last().DownloadContent(contentId);
file.AssertIsEqual(file2);
// <TODO REMOVE THIS>
foreach (var node in nodes) bootstrap.ConnectToPeer(node);
for (var x = 0; x < number; x++)
{
for (var y = x + 1; y < number; y++)
{
nodes[x].ConnectToPeer(nodes[y]);
}
}
// </TODO REMOVE THIS>
foreach (var node in nodes) AssertKnowEachother(node, bootstrap);
for (var x = 0; x < number; x++)
{
for (var y = x + 1; y < number; y++)
{
AssertKnowEachother(nodes[x], nodes[y]);
}
}
}
private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b)
{
AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo());
}
private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b)
{
AssertKnows(a, b);
AssertKnows(b, a);
}
private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b)
{
var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId));
var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId));
Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]");
Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]");
Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found.");
Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found.");
}
}
}

6
Tests/DurabilityTests/DurabilityTests.cs
View File

@ -31,10 +31,10 @@ namespace Tests.DurabilityTests
[Test]
public void DataRetentionTest()
{
var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace));
var bootstrapNode = SetupCodexBootstrapNode();
var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode));
var startGroup = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode));
var finishGroup = SetupCodexNodes(10, s => s.WithBootstrapNode(bootstrapNode));
var file = GenerateTestFile(10.MB());

62
Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs Normal file
View File

@ -0,0 +1,62 @@
using DistTestCore;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
[TestFixture]
public class LayeredDiscoveryTests : DistTest
{
[Test]
public void TwoLayersTest()
{
var root = SetupCodexNode();
var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
AssertAllNodesConnected();
}
[Test]
public void ThreeLayersTest()
{
var root = SetupCodexNode();
var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root));
var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root));
var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node));
var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node));
AssertAllNodesConnected();
}
[TestCase(3)]
[TestCase(5)]
[TestCase(10)]
[TestCase(20)]
[TestCase(50)]
public void NodeChainTest(int chainLength)
{
var node = SetupCodexNode();
for (var i = 1; i < chainLength; i++)
{
node = SetupCodexNode(s => s.WithBootstrapNode(node));
}
AssertAllNodesConnected();
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

74
Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs Normal file
View File

@ -0,0 +1,74 @@
using DistTestCore;
using DistTestCore.Codex;
using DistTestCore.Helpers;
using NUnit.Framework;
using Utils;
namespace Tests.PeerDiscoveryTests
{
[TestFixture]
public class PeerDiscoveryTests : AutoBootstrapDistTest
{
[Test]
public void CanReportUnknownPeerId()
{
var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK";
var node = SetupCodexNode();
var result = node.GetDebugPeer(unknownId);
Assert.That(result.IsPeerFound, Is.False);
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
public void VariableNodes(int number)
{
SetupCodexNodes(number);
AssertAllNodesConnected();
}
[TestCase(2)]
[TestCase(3)]
[TestCase(10)]
[TestCase(20)]
public void VariableNodesInPods(int number)
{
for (var i = 0; i < number; i++)
{
SetupCodexNode();
}
AssertAllNodesConnected();
}
[TestCase(3, 3)]
[TestCase(3, 5)]
[TestCase(3, 10)]
[TestCase(5, 10)]
[TestCase(3, 20)]
[TestCase(5, 20)]
public void StagedVariableNodes(int numberOfNodes, int numberOfStages)
{
for (var i = 0; i < numberOfStages; i++)
{
SetupCodexNodes(numberOfNodes);
AssertAllNodesConnected();
}
for (int i = 0; i < 5; i++)
{
Time.Sleep(TimeSpan.FromSeconds(30));
AssertAllNodesConnected();
}
}
private void AssertAllNodesConnected()
{
PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes());
//PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes());
}
}
}

69
Utils/Time.cs
View File

@ -38,5 +38,74 @@
state = predicate();
}
}
public static void Retry(Action action, string description)
{
Retry(action, TimeSpan.FromMinutes(1), description);
}
public static T Retry<T>(Func<T> action, string description)
{
return Retry(action, TimeSpan.FromMinutes(1), description);
}
public static void Retry(Action action, TimeSpan timeout, string description)
{
Retry(action, timeout, TimeSpan.FromSeconds(1), description);
}
public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
{
return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
}
public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var exceptions = new List<Exception>();
while (true)
{
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
}
try
{
action();
return;
}
catch (Exception ex)
{
exceptions.Add(ex);
}
Sleep(retryTime);
}
}
public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
{
var start = DateTime.UtcNow;
var exceptions = new List<Exception>();
while (true)
{
if (DateTime.UtcNow - start > timeout)
{
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
}
try
{
return action();
}
catch (Exception ex)
{
exceptions.Add(ex);
}
Sleep(retryTime);
}
}
}
}