diff --git a/DistTestCore/AutoBootstrapDistTest.cs b/DistTestCore/AutoBootstrapDistTest.cs index 352e2ec..edf182f 100644 --- a/DistTestCore/AutoBootstrapDistTest.cs +++ b/DistTestCore/AutoBootstrapDistTest.cs @@ -1,9 +1,9 @@ -namespace DistTestCore +using NUnit.Framework; + +namespace DistTestCore { public class AutoBootstrapDistTest : DistTest { - private IOnlineCodexNode? bootstrapNode; - public override IOnlineCodexNode SetupCodexBootstrapNode(Action setup) { throw new Exception("AutoBootstrapDistTest creates and attaches a single boostrap node for you. " + @@ -12,19 +12,18 @@ public override ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) { - var codexSetup = new CodexSetup(numberOfNodes); + var codexSetup = CreateCodexSetup(numberOfNodes); setup(codexSetup); - codexSetup.WithBootstrapNode(EnsureBootstapNode()); + codexSetup.WithBootstrapNode(BootstrapNode); return BringOnline(codexSetup); } - private IOnlineCodexNode EnsureBootstapNode() + [SetUp] + public void SetUpBootstrapNode() { - if (bootstrapNode == null) - { - bootstrapNode = base.SetupCodexBootstrapNode(s => { }); - } - return bootstrapNode; + BootstrapNode = BringOnline(CreateCodexSetup(1))[0]; } + + protected IOnlineCodexNode BootstrapNode { get; private set; } = null!; } } diff --git a/DistTestCore/Codex/CodexAccess.cs b/DistTestCore/Codex/CodexAccess.cs index 8654878..587f7f0 100644 --- a/DistTestCore/Codex/CodexAccess.cs +++ b/DistTestCore/Codex/CodexAccess.cs @@ -19,7 +19,30 @@ namespace DistTestCore.Codex public CodexDebugResponse GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + return Http(TimeSpan.FromSeconds(2)).HttpGetJson("debug/info"); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return GetDebugPeer(peerId, TimeSpan.FromSeconds(2)); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + var http = Http(timeout); + var str = http.HttpGetString($"debug/peer/{peerId}"); + + if (str.ToLowerInvariant() == "unable to find peer!") + { + return new CodexDebugPeerResponse + { + IsPeerFound = false + }; + } + + var result = http.TryJsonDeserialize(str); + result.IsPeerFound = true; + return result; } public string UploadFile(FileStream fileStream) @@ -42,6 +65,11 @@ namespace DistTestCore.Codex return Http().HttpPostJson($"storage/request/{contentId}", request); } + public string ConnectToPeer(string peerId, string peerMultiAddress) + { + return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + } + public void EnsureOnline() { try @@ -51,7 +79,8 @@ namespace DistTestCore.Codex var nodePeerId = debugInfo.id; var nodeName = Container.Name; - log.AddStringReplace(nodePeerId, $"___{nodeName}___"); + log.AddStringReplace(nodePeerId, nodeName); + log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); } catch (Exception e) { @@ -60,16 +89,11 @@ namespace DistTestCore.Codex } } - private Http Http() + private Http Http(TimeSpan? timeoutOverride = null) { - var ip = Container.Pod.Cluster.IP; + var ip = Container.Pod.Cluster.HostAddress; var port = Container.ServicePorts[0].Number; - return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1"); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + return new Http(log, timeSet, ip, port, baseUrl: "/api/codex/v1", timeoutOverride); } } @@ -82,6 +106,22 @@ namespace DistTestCore.Codex public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); public CodexDebugVersionResponse codex { get; set; } = new(); + public CodexDebugTableResponse table { get; set; } = new(); + } + + public class CodexDebugTableResponse + { + public CodexDebugTableNodeResponse localNode { get; set; } = new(); + public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty(); + } + + public class CodexDebugTableNodeResponse + { + public string nodeId { get; set; } = string.Empty; + public string peerId { get; set; } = string.Empty; + public string record { get; set; } = string.Empty; + public string address { get; set; } = string.Empty; + public bool seen { get; set; } } public class EnginePeerResponse @@ -110,6 +150,20 @@ namespace DistTestCore.Codex public string revision { get; set; } = string.Empty; } + public class CodexDebugPeerResponse + { + public bool IsPeerFound { get; set; } + + public string peerId { get; set; } = string.Empty; + public long seqNo { get; set; } + public CodexDebugPeerAddressResponse[] addresses { get; set; } = Array.Empty(); + } + + public class CodexDebugPeerAddressResponse + { + public string address { get; set; } = string.Empty; + } + public class CodexSalesAvailabilityRequest { public string size { get; set; } = string.Empty; diff --git a/DistTestCore/Codex/CodexContainerRecipe.cs b/DistTestCore/Codex/CodexContainerRecipe.cs index ff263a7..834f7af 100644 --- a/DistTestCore/Codex/CodexContainerRecipe.cs +++ b/DistTestCore/Codex/CodexContainerRecipe.cs @@ -13,6 +13,7 @@ namespace DistTestCore.Codex public const string DockerImage = "thatbenbierens/codexlocal:latest"; #endif public const string MetricsPortTag = "metrics_port"; + public const string DiscoveryPortTag = "discovery-port"; protected override string Image => DockerImage; @@ -22,7 +23,8 @@ namespace DistTestCore.Codex AddExposedPortAndVar("API_PORT"); AddEnvVar("DATA_DIR", $"datadir{ContainerNumber}"); - AddInternalPortAndVar("DISC_PORT"); + AddInternalPortAndVar("DISC_PORT", DiscoveryPortTag); + AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); var listenPort = AddInternalPort(); AddEnvVar("LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}"); @@ -31,11 +33,6 @@ namespace DistTestCore.Codex { AddEnvVar("BOOTSTRAP_SPR", config.BootstrapSpr); } - - if (config.LogLevel != null) - { - AddEnvVar("LOG_LEVEL", config.LogLevel.ToString()!.ToUpperInvariant()); - } if (config.StorageQuota != null) { AddEnvVar("STORAGE_QUOTA", config.StorageQuota.SizeInBytes.ToString()!); diff --git a/DistTestCore/Codex/CodexStartupConfig.cs b/DistTestCore/Codex/CodexStartupConfig.cs index 9b3cb8a..b13512b 100644 --- a/DistTestCore/Codex/CodexStartupConfig.cs +++ b/DistTestCore/Codex/CodexStartupConfig.cs @@ -5,9 +5,14 @@ namespace DistTestCore.Codex { public class CodexStartupConfig { + public CodexStartupConfig(CodexLogLevel logLevel) + { + LogLevel = logLevel; + } + public string? NameOverride { get; set; } public Location Location { get; set; } - public CodexLogLevel? LogLevel { get; set; } + public CodexLogLevel LogLevel { get; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } diff --git a/DistTestCore/CodexSetup.cs b/DistTestCore/CodexSetup.cs index 5b5f3c0..83c5b9b 100644 --- a/DistTestCore/CodexSetup.cs +++ b/DistTestCore/CodexSetup.cs @@ -8,7 +8,6 @@ namespace DistTestCore { ICodexSetup WithName(string name); ICodexSetup At(Location location); - ICodexSetup WithLogLevel(CodexLogLevel level); ICodexSetup WithBootstrapNode(IOnlineCodexNode node); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup EnableMetrics(); @@ -20,7 +19,8 @@ namespace DistTestCore { public int NumberOfNodes { get; } - public CodexSetup(int numberOfNodes) + public CodexSetup(int numberOfNodes, CodexLogLevel logLevel) + : base(logLevel) { NumberOfNodes = numberOfNodes; } @@ -43,12 +43,6 @@ namespace DistTestCore return this; } - public ICodexSetup WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - public ICodexSetup WithStorageQuota(ByteSize storageQuota) { StorageQuota = storageQuota; @@ -80,7 +74,7 @@ namespace DistTestCore private IEnumerable DescribeArgs() { - if (LogLevel != null) yield return $"LogLevel={LogLevel}"; + yield return $"LogLevel={LogLevel}"; if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; if (StorageQuota != null) yield return $"StorageQuote={StorageQuota}"; } diff --git a/DistTestCore/CodexStarter.cs b/DistTestCore/CodexStarter.cs index e36ebca..1c49cad 100644 --- a/DistTestCore/CodexStarter.cs +++ b/DistTestCore/CodexStarter.cs @@ -1,6 +1,7 @@ using DistTestCore.Codex; using DistTestCore.Marketplace; using KubernetesWorkflow; +using Logging; namespace DistTestCore { @@ -74,7 +75,7 @@ namespace DistTestCore { var group = new CodexNodeGroup(lifecycle, codexSetup, runningContainers, codexNodeFactory); RunningGroups.Add(group); - group.EnsureOnline(); + Stopwatch.Measure(lifecycle.Log, "EnsureOnline", group.EnsureOnline, debug: true); return group; } diff --git a/DistTestCore/Configuration.cs b/DistTestCore/Configuration.cs index ee01559..ac7d470 100644 --- a/DistTestCore/Configuration.cs +++ b/DistTestCore/Configuration.cs @@ -1,4 +1,5 @@ -using KubernetesWorkflow; +using DistTestCore.Codex; +using KubernetesWorkflow; namespace DistTestCore { @@ -28,5 +29,10 @@ namespace DistTestCore { return "TestDataFiles"; } + + public CodexLogLevel GetCodexLogLevel() + { + return CodexLogLevel.Trace; + } } } diff --git a/DistTestCore/DistTest.cs b/DistTestCore/DistTest.cs index e5ee885..bf4d59f 100644 --- a/DistTestCore/DistTest.cs +++ b/DistTestCore/DistTest.cs @@ -1,4 +1,5 @@ using DistTestCore.Codex; +using DistTestCore.Helpers; using DistTestCore.Logs; using DistTestCore.Marketplace; using DistTestCore.Metrics; @@ -25,8 +26,14 @@ namespace DistTestCore testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); fixtureLog = new FixtureLog(configuration.GetLogConfig()); + + PeerConnectionTestHelpers = new PeerConnectionTestHelpers(this); + PeerDownloadTestHelpers = new PeerDownloadTestHelpers(this); } + public PeerConnectionTestHelpers PeerConnectionTestHelpers { get; } + public PeerDownloadTestHelpers PeerDownloadTestHelpers { get; } + [OneTimeSetUp] public void GlobalSetup() { @@ -85,6 +92,17 @@ namespace DistTestCore return Get().FileManager.GenerateTestFile(size); } + /// + /// Any test files generated in 'action' will be deleted after it returns. + /// This helps prevent large tests from filling up discs. + /// + public void ScopedTestFiles(Action action) + { + Get().FileManager.PushFileSet(); + action(); + Get().FileManager.PopFileSet(); + } + public IOnlineCodexNode SetupCodexBootstrapNode() { return SetupCodexBootstrapNode(s => { }); @@ -116,7 +134,7 @@ namespace DistTestCore public virtual ICodexNodeGroup SetupCodexNodes(int numberOfNodes, Action setup) { - var codexSetup = new CodexSetup(numberOfNodes); + var codexSetup = CreateCodexSetup(numberOfNodes); setup(codexSetup); @@ -128,16 +146,31 @@ namespace DistTestCore return Get().CodexStarter.BringOnline((CodexSetup)codexSetup); } - protected void Log(string msg) + public IEnumerable GetAllOnlineCodexNodes() { - TestContext.Progress.WriteLine(msg); - Get().Log.Log(msg); + return Get().CodexStarter.RunningGroups.SelectMany(g => g.Nodes); } - protected void Debug(string msg) + public BaseLog GetTestLog() + { + return Get().Log; + } + + public void Log(string msg) { TestContext.Progress.WriteLine(msg); - Get().Log.Debug(msg); + GetTestLog().Log(msg); + } + + public void Debug(string msg) + { + TestContext.Progress.WriteLine(msg); + GetTestLog().Debug(msg); + } + + protected CodexSetup CreateCodexSetup(int numberOfNodes) + { + return new CodexSetup(numberOfNodes, configuration.GetCodexLogLevel()); } private TestLifecycle Get() diff --git a/DistTestCore/FileManager.cs b/DistTestCore/FileManager.cs index ae58cd6..97cf27c 100644 --- a/DistTestCore/FileManager.cs +++ b/DistTestCore/FileManager.cs @@ -9,6 +9,8 @@ namespace DistTestCore TestFile CreateEmptyTestFile(); TestFile GenerateTestFile(ByteSize size); void DeleteAllTestFiles(); + void PushFileSet(); + void PopFileSet(); } public class FileManager : IFileManager @@ -18,6 +20,7 @@ namespace DistTestCore private readonly Random random = new Random(); private readonly TestLog log; private readonly string folder; + private readonly List> fileSetStack = new List>(); public FileManager(TestLog log, Configuration configuration) { @@ -31,6 +34,7 @@ namespace DistTestCore { var result = new TestFile(Path.Combine(folder, Guid.NewGuid().ToString() + "_test.bin")); File.Create(result.Filename).Close(); + if (fileSetStack.Any()) fileSetStack.Last().Add(result); return result; } @@ -47,6 +51,27 @@ namespace DistTestCore DeleteDirectory(); } + public void PushFileSet() + { + fileSetStack.Add(new List()); + } + + public void PopFileSet() + { + if (!fileSetStack.Any()) return; + var pop = fileSetStack.Last(); + fileSetStack.Remove(pop); + + foreach (var file in pop) + { + try + { + File.Delete(file.Filename); + } + catch { } + } + } + private void GenerateFileBytes(TestFile result, ByteSize size) { long bytesLeft = size.SizeInBytes; diff --git a/DistTestCore/Helpers/PeerConnectionTestHelpers.cs b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs new file mode 100644 index 0000000..92114bc --- /dev/null +++ b/DistTestCore/Helpers/PeerConnectionTestHelpers.cs @@ -0,0 +1,239 @@ +using DistTestCore.Codex; +using NUnit.Framework; +using Utils; + +namespace DistTestCore.Helpers +{ + public class PeerConnectionTestHelpers + { + private readonly Random random = new Random(); + private readonly DistTest test; + + public PeerConnectionTestHelpers(DistTest test) + { + this.test = test; + } + + public void AssertFullyConnected(IEnumerable nodes) + { + AssertFullyConnected(nodes.ToArray()); + } + + public void AssertFullyConnected(params IOnlineCodexNode[] nodes) + { + test.Log($"Asserting peers are fully-connected for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + var entries = CreateEntries(nodes); + var pairs = CreatePairs(entries); + + RetryWhilePairs(pairs, () => + { + CheckAndRemoveSuccessful(pairs); + }); + + if (pairs.Any()) + { + test.Log($"Unsuccessful! Peers are not fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); + Assert.Fail(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); + test.Log(string.Join(Environment.NewLine, pairs.Select(p => p.GetMessage()))); + } + else + { + test.Log($"Success! Peers are fully-connected: {string.Join(",", nodes.Select(n => n.GetName()))}"); + } + } + + private static void RetryWhilePairs(List pairs, Action action) + { + var timeout = DateTime.UtcNow + TimeSpan.FromMinutes(10); + while (pairs.Any() && timeout > DateTime.UtcNow) + { + action(); + + if (pairs.Any()) Time.Sleep(TimeSpan.FromSeconds(5)); + } + } + + private void CheckAndRemoveSuccessful(List pairs) + { + var checkTasks = pairs.Select(p => Task.Run(() => + { + ApplyRandomDelay(); + p.Check(); + })).ToArray(); + + Task.WaitAll(checkTasks); + + foreach (var pair in pairs.ToArray()) + { + if (pair.Success) + { + test.Log(pair.GetMessage()); + pairs.Remove(pair); + } + } + } + + private static Entry[] CreateEntries(IOnlineCodexNode[] nodes) + { + var entries = nodes.Select(n => new Entry(n)).ToArray(); + var incorrectDiscoveryEndpoints = entries.SelectMany(e => e.GetInCorrectDiscoveryEndpoints(entries)).ToArray(); + + if (incorrectDiscoveryEndpoints.Any()) + { + Assert.Fail("Some nodes contain peer records with incorrect discovery ip/port information: " + + string.Join(Environment.NewLine, incorrectDiscoveryEndpoints)); + } + + return entries; + } + + private static List CreatePairs(Entry[] entries) + { + return CreatePairsIterator(entries).ToList(); + } + + private static IEnumerable CreatePairsIterator(Entry[] entries) + { + for (var x = 0; x < entries.Length; x++) + { + for (var y = x + 1; y < entries.Length; y++) + { + yield return new Pair(entries[x], entries[y]); + } + } + } + + private void ApplyRandomDelay() + { + // Calling all the nodes all at the same time is not exactly nice. + Time.Sleep(TimeSpan.FromMicroseconds(random.Next(10, 1000))); + } + + public class Entry + { + public Entry(IOnlineCodexNode node) + { + Node = node; + Response = node.GetDebugInfo(); + } + + public IOnlineCodexNode Node { get; } + public CodexDebugResponse Response { get; } + + public IEnumerable GetInCorrectDiscoveryEndpoints(Entry[] allEntries) + { + foreach (var peer in Response.table.nodes) + { + var expected = GetExpectedDiscoveryEndpoint(allEntries, peer); + if (expected != peer.address) + { + yield return $"Node:{Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'"; + } + } + } + + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) + { + var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); + if (peer == null) return $"peerId: {node.peerId} is not known."; + + var n = (OnlineCodexNode)peer.Node; + var ip = n.CodexAccess.Container.Pod.Ip; + var discPort = n.CodexAccess.Container.Recipe.GetPortByTag(CodexContainerRecipe.DiscoveryPortTag); + return $"{ip}:{discPort.Number}"; + } + } + + public enum PeerConnectionState + { + Unknown, + Connection, + NoConnection, + } + + public class Pair + { + private readonly TimeSpan timeout = TimeSpan.FromSeconds(60); + private TimeSpan aToBTime = TimeSpan.FromSeconds(0); + private TimeSpan bToATime = TimeSpan.FromSeconds(0); + + public Pair(Entry a, Entry b) + { + A = a; + B = b; + } + + public Entry A { get; } + public Entry B { get; } + public PeerConnectionState AKnowsB { get; private set; } + public PeerConnectionState BKnowsA { get; private set; } + public bool Success { get { return AKnowsB == PeerConnectionState.Connection && BKnowsA == PeerConnectionState.Connection; } } + + public void Check() + { + aToBTime = Measure(() => AKnowsB = Knows(A, B)); + bToATime = Measure(() => BKnowsA = Knows(B, A)); + } + + public string GetMessage() + { + return GetResultMessage() + GetTimePostfix(); + } + + private string GetResultMessage() + { + var aName = A.Response.id; + var bName = B.Response.id; + + if (Success) + { + return $"{aName} and {bName} know each other."; + } + + return $"[{aName}-->{bName}] = {AKnowsB} AND [{aName}<--{bName}] = {BKnowsA}"; + } + + private string GetTimePostfix() + { + var aName = A.Response.id; + var bName = B.Response.id; + + return $" ({aName}->{bName}: {aToBTime.TotalMinutes} seconds, {bName}->{aName}: {bToATime.TotalSeconds} seconds)"; + } + + private static TimeSpan Measure(Action action) + { + var start = DateTime.UtcNow; + action(); + return DateTime.UtcNow - start; + } + + private PeerConnectionState Knows(Entry a, Entry b) + { + lock (a) + { + var peerId = b.Response.id; + + try + { + var response = a.Node.GetDebugPeer(peerId, timeout); + if (!response.IsPeerFound) + { + return PeerConnectionState.NoConnection; + } + if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + { + return PeerConnectionState.Connection; + } + } + catch + { + } + + // Didn't get a conclusive answer. Try again later. + return PeerConnectionState.Unknown; + } + } + } + } +} diff --git a/DistTestCore/Helpers/PeerDownloadTestHelpers.cs b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs new file mode 100644 index 0000000..6929ceb --- /dev/null +++ b/DistTestCore/Helpers/PeerDownloadTestHelpers.cs @@ -0,0 +1,52 @@ +namespace DistTestCore.Helpers +{ + public class PeerDownloadTestHelpers + { + private readonly DistTest test; + + public PeerDownloadTestHelpers(DistTest test) + { + this.test = test; + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes) + { + AssertFullDownloadInterconnectivity(nodes, 1.MB()); + } + + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + { + test.Log($"Asserting full download interconnectivity for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + foreach (var node in nodes) + { + var uploader = node; + var downloaders = nodes.Where(n => n != uploader).ToArray(); + + test.ScopedTestFiles(() => + { + PerformTest(uploader, downloaders, testFileSize); + }); + } + + test.Log($"Success! Full download interconnectivity for nodes: {string.Join(",", nodes.Select(n => n.GetName()))}"); + } + + private void PerformTest(IOnlineCodexNode uploader, IOnlineCodexNode[] downloaders, ByteSize testFileSize) + { + // 1 test file per downloader. + var files = downloaders.Select(d => test.GenerateTestFile(testFileSize)).ToArray(); + + // Upload all the test files to the uploader. + var contentIds = files.Select(uploader.UploadFile).ToArray(); + + // Each downloader should retrieve its own test file. + for (var i = 0; i < downloaders.Length; i++) + { + var expectedFile = files[i]; + var downloadedFile = downloaders[i].DownloadContent(contentIds[i]); + + expectedFile.AssertIsEqual(downloadedFile); + } + } + } +} diff --git a/DistTestCore/Http.cs b/DistTestCore/Http.cs index 3dfee9d..955e4ef 100644 --- a/DistTestCore/Http.cs +++ b/DistTestCore/Http.cs @@ -1,6 +1,5 @@ using Logging; using Newtonsoft.Json; -using NUnit.Framework; using System.Net.Http.Headers; using System.Net.Http.Json; using Utils; @@ -14,15 +13,16 @@ namespace DistTestCore private readonly string ip; private readonly int port; private readonly string baseUrl; + private readonly TimeSpan? timeoutOverride; - public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl) + public Http(BaseLog log, ITimeSet timeSet, string ip, int port, string baseUrl, TimeSpan? timeoutOverride = null) { this.log = log; this.timeSet = timeSet; this.ip = ip; this.port = port; this.baseUrl = baseUrl; - + this.timeoutOverride = timeoutOverride; if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; } @@ -38,7 +38,7 @@ namespace DistTestCore var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); return str; ; - }); + }, $"HTTP-GET:{route}"); } public T HttpGetJson(string route) @@ -62,10 +62,10 @@ namespace DistTestCore using var content = JsonContent.Create(body); Log(url, JsonConvert.SerializeObject(body)); var result = Time.Wait(client.PostAsync(url, content)); - var str= Time.Wait(result.Content.ReadAsStringAsync()); + var str = Time.Wait(result.Content.ReadAsStringAsync()); Log(url, str); return str; - }); + }, $"HTTP-POST-JSON: {route}"); } public string HttpPostStream(string route, Stream stream) @@ -81,7 +81,7 @@ namespace DistTestCore var str =Time.Wait(response.Content.ReadAsStringAsync()); Log(url, str); return str; - }); + }, $"HTTP-POST-STREAM: {route}"); } public Stream HttpGetStream(string route) @@ -92,7 +92,20 @@ namespace DistTestCore var url = GetUrl() + route; Log(url, "~ STREAM ~"); return Time.Wait(client.GetStreamAsync(url)); - }); + }, $"HTTP-GET-STREAM: {route}"); + } + + public T TryJsonDeserialize(string json) + { + try + { + return JsonConvert.DeserializeObject(json)!; + } + catch (Exception exception) + { + var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; + throw new InvalidOperationException(msg, exception); + } } private string GetUrl() @@ -105,47 +118,24 @@ namespace DistTestCore log.Debug($"({url}) = '{message}'", 3); } - private T Retry(Func operation) + private T Retry(Func operation, string description) { - var retryCounter = 0; - - while (true) - { - try - { - return operation(); - } - catch (Exception exception) - { - timeSet.HttpCallRetryDelay(); - retryCounter++; - if (retryCounter > timeSet.HttpCallRetryCount()) - { - Assert.Fail(exception.ToString()); - throw; - } - } - } - } - - private static T TryJsonDeserialize(string json) - { - try - { - return JsonConvert.DeserializeObject(json)!; - } - catch (Exception exception) - { - var msg = $"Failed to deserialize JSON: '{json}' with exception: {exception}"; - Assert.Fail(msg); - throw new InvalidOperationException(msg, exception); - } + return Time.Retry(operation, timeSet.HttpCallRetryTimeout(), timeSet.HttpCallRetryDelay(), description); } private HttpClient GetClient() + { + if (timeoutOverride.HasValue) + { + return GetClient(timeoutOverride.Value); + } + return GetClient(timeSet.HttpCallTimeout()); + } + + private HttpClient GetClient(TimeSpan timeout) { var client = new HttpClient(); - client.Timeout = timeSet.HttpCallTimeout(); + client.Timeout = timeout; return client; } } diff --git a/DistTestCore/Marketplace/ContainerInfoExtractor.cs b/DistTestCore/Marketplace/ContainerInfoExtractor.cs index f99827b..1aadc44 100644 --- a/DistTestCore/Marketplace/ContainerInfoExtractor.cs +++ b/DistTestCore/Marketplace/ContainerInfoExtractor.cs @@ -56,30 +56,6 @@ namespace DistTestCore.Marketplace return marketplaceAbi; } - private string Retry(Func fetch) - { - var result = string.Empty; - Time.WaitUntil(() => - { - result = Catch(fetch); - return !string.IsNullOrEmpty(result); - }, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(3)); - - return result; - } - - private string Catch(Func fetch) - { - try - { - return fetch(); - } - catch - { - return string.Empty; - } - } - private string FetchAccountsCsv() { return workflow.ExecuteCommand(container, "cat", GethContainerRecipe.AccountsFilename); @@ -116,6 +92,11 @@ namespace DistTestCore.Marketplace var privateKey = tokens[1]; return new GethAccount(account, privateKey); } + + private static string Retry(Func fetch) + { + return Time.Retry(fetch, nameof(ContainerInfoExtractor)); + } } public class PubKeyFinder : LogHandler, ILogHandler diff --git a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs index b59fb80..ab38cb1 100644 --- a/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs +++ b/DistTestCore/Marketplace/GethBootstrapNodeInfo.cs @@ -23,7 +23,7 @@ namespace DistTestCore.Marketplace public NethereumInteraction StartInteraction(BaseLog log) { - var ip = RunningContainers.RunningPod.Cluster.IP; + var ip = RunningContainers.RunningPod.Cluster.HostAddress; var port = RunningContainers.Containers[0].ServicePorts[0].Number; var account = Account; diff --git a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs index 5731ab3..6c4b8e8 100644 --- a/DistTestCore/Marketplace/GethCompanionNodeInfo.cs +++ b/DistTestCore/Marketplace/GethCompanionNodeInfo.cs @@ -17,7 +17,7 @@ namespace DistTestCore.Marketplace public NethereumInteraction StartInteraction(BaseLog log, GethAccount account) { - var ip = RunningContainer.Pod.Cluster.IP; + var ip = RunningContainer.Pod.Cluster.HostAddress; var port = RunningContainer.ServicePorts[0].Number; var privateKey = account.PrivateKey; diff --git a/DistTestCore/Metrics/MetricsQuery.cs b/DistTestCore/Metrics/MetricsQuery.cs index 8c5f24f..f162690 100644 --- a/DistTestCore/Metrics/MetricsQuery.cs +++ b/DistTestCore/Metrics/MetricsQuery.cs @@ -16,7 +16,7 @@ namespace DistTestCore.Metrics http = new Http( log, timeSet, - runningContainers.RunningPod.Cluster.IP, + runningContainers.RunningPod.Cluster.HostAddress, runningContainers.Containers[0].ServicePorts[0].Number, "api/v1"); } diff --git a/DistTestCore/OnlineCodexNode.cs b/DistTestCore/OnlineCodexNode.cs index 7fb30e1..322fb04 100644 --- a/DistTestCore/OnlineCodexNode.cs +++ b/DistTestCore/OnlineCodexNode.cs @@ -10,6 +10,8 @@ namespace DistTestCore { string GetName(); CodexDebugResponse GetDebugInfo(); + CodexDebugPeerResponse GetDebugPeer(string peerId); + CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout); ContentId UploadFile(TestFile file); TestFile? DownloadContent(ContentId contentId); void ConnectToPeer(IOnlineCodexNode node); @@ -47,10 +49,21 @@ namespace DistTestCore public CodexDebugResponse GetDebugInfo() { var debugInfo = CodexAccess.GetDebugInfo(); - Log($"Got DebugInfo with id: '{debugInfo.id}'."); + var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); + Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); return debugInfo; } + public CodexDebugPeerResponse GetDebugPeer(string peerId) + { + return CodexAccess.GetDebugPeer(peerId); + } + + public CodexDebugPeerResponse GetDebugPeer(string peerId, TimeSpan timeout) + { + return CodexAccess.GetDebugPeer(peerId, timeout); + } + public ContentId UploadFile(TestFile file) { Log($"Uploading file of size {file.GetFileSize()}..."); diff --git a/DistTestCore/Timing.cs b/DistTestCore/Timing.cs index 9771767..cff8b09 100644 --- a/DistTestCore/Timing.cs +++ b/DistTestCore/Timing.cs @@ -1,5 +1,4 @@ using NUnit.Framework; -using Utils; namespace DistTestCore { @@ -11,8 +10,8 @@ namespace DistTestCore public interface ITimeSet { TimeSpan HttpCallTimeout(); - int HttpCallRetryCount(); - void HttpCallRetryDelay(); + TimeSpan HttpCallRetryTimeout(); + TimeSpan HttpCallRetryDelay(); TimeSpan WaitForK8sServiceDelay(); TimeSpan K8sOperationTimeout(); TimeSpan WaitForMetricTimeout(); @@ -25,14 +24,14 @@ namespace DistTestCore return TimeSpan.FromSeconds(10); } - public int HttpCallRetryCount() + public TimeSpan HttpCallRetryTimeout() { - return 5; + return TimeSpan.FromMinutes(1); } - public void HttpCallRetryDelay() + public TimeSpan HttpCallRetryDelay() { - Time.Sleep(TimeSpan.FromSeconds(3)); + return TimeSpan.FromSeconds(3); } public TimeSpan WaitForK8sServiceDelay() @@ -58,14 +57,14 @@ namespace DistTestCore return TimeSpan.FromHours(2); } - public int HttpCallRetryCount() + public TimeSpan HttpCallRetryTimeout() { - return 2; + return TimeSpan.FromHours(5); } - public void HttpCallRetryDelay() + public TimeSpan HttpCallRetryDelay() { - Time.Sleep(TimeSpan.FromMinutes(5)); + return TimeSpan.FromMinutes(5); } public TimeSpan WaitForK8sServiceDelay() diff --git a/KubernetesWorkflow/K8sCluster.cs b/KubernetesWorkflow/K8sCluster.cs index 4d5a772..9ad7a6a 100644 --- a/KubernetesWorkflow/K8sCluster.cs +++ b/KubernetesWorkflow/K8sCluster.cs @@ -10,12 +10,12 @@ namespace KubernetesWorkflow } public Configuration Configuration { get; } - public string IP { get; private set; } = string.Empty; + public string HostAddress { get; private set; } = string.Empty; public KubernetesClientConfiguration GetK8sClientConfig() { var config = GetConfig(); - UpdateIp(config); + UpdateHostAddress(config); return config; } @@ -47,10 +47,17 @@ namespace KubernetesWorkflow } } - private void UpdateIp(KubernetesClientConfiguration config) + private void UpdateHostAddress(KubernetesClientConfiguration config) { var host = config.Host.Replace("https://", ""); - IP = host.Substring(0, host.IndexOf(':')); + if (host.Contains(":")) + { + HostAddress = host.Substring(0, host.IndexOf(':')); + } + else + { + HostAddress = config.Host; + } } } } diff --git a/LongTests/BasicTests/LargeFileTests.cs b/LongTests/BasicTests/LargeFileTests.cs index 63b33f5..3149ee3 100644 --- a/LongTests/BasicTests/LargeFileTests.cs +++ b/LongTests/BasicTests/LargeFileTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests @@ -11,7 +10,6 @@ namespace TestsLong.BasicTests public void OneClientLargeFileTest() { var primary = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Warn) .WithStorageQuota(20.GB())); var testFile = GenerateTestFile(10.GB()); diff --git a/LongTests/BasicTests/TestInfraTests.cs b/LongTests/BasicTests/TestInfraTests.cs index 0b5e640..9b3111f 100644 --- a/LongTests/BasicTests/TestInfraTests.cs +++ b/LongTests/BasicTests/TestInfraTests.cs @@ -1,5 +1,4 @@ using DistTestCore; -using DistTestCore.Codex; using NUnit.Framework; namespace TestsLong.BasicTests @@ -32,7 +31,6 @@ namespace TestsLong.BasicTests public void DownloadConsistencyTest() { var primary = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(2.MB())); var testFile = GenerateTestFile(1.MB()); diff --git a/Tests/BasicTests/ExampleTests.cs b/Tests/BasicTests/ExampleTests.cs index 6951433..3a4884c 100644 --- a/Tests/BasicTests/ExampleTests.cs +++ b/Tests/BasicTests/ExampleTests.cs @@ -11,7 +11,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = SetupCodexNode(s => s.WithLogLevel(CodexLogLevel.Trace)); + var primary = SetupCodexNode(); primary.UploadFile(GenerateTestFile(5.MB())); @@ -47,7 +47,6 @@ namespace Tests.BasicTests var buyerInitialBalance = 1000.TestTokens(); var seller = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithStorageQuota(11.GB()) .EnableMarketplace(sellerInitialBalance)); @@ -61,7 +60,6 @@ namespace Tests.BasicTests var testFile = GenerateTestFile(10.MB()); var buyer = SetupCodexNode(s => s - .WithLogLevel(CodexLogLevel.Trace) .WithBootstrapNode(seller) .EnableMarketplace(buyerInitialBalance)); diff --git a/Tests/BasicTests/PeerTests.cs b/Tests/BasicTests/PeerTests.cs deleted file mode 100644 index 09b41ac..0000000 --- a/Tests/BasicTests/PeerTests.cs +++ /dev/null @@ -1,85 +0,0 @@ -using DistTestCore; -using DistTestCore.Codex; -using NUnit.Framework; - -namespace Tests.BasicTests -{ - [TestFixture] - public class PeerTests : DistTest - { - [Test] - public void TwoNodes() - { - var primary = SetupCodexBootstrapNode(); - var secondary = SetupCodexNode(s => s.WithBootstrapNode(primary)); - - primary.ConnectToPeer(secondary); // TODO REMOVE THIS: This is required for the switchPeers to show up. - - // This is required for the enginePeers to show up. - //var file = GenerateTestFile(10.MB()); - //var contentId = primary.UploadFile(file); - //var file2 = secondary.DownloadContent(contentId); - //file.AssertIsEqual(file2); - - AssertKnowEachother(primary, secondary); - } - - [TestCase(2)] - [TestCase(3)] - [TestCase(10)] - public void VariableNodes(int number) - { - var bootstrap = SetupCodexBootstrapNode(); - var nodes = SetupCodexNodes(number, s => s.WithBootstrapNode(bootstrap)); - - var file = GenerateTestFile(10.MB()); - var contentId = nodes.First().UploadFile(file); - var file2 = nodes.Last().DownloadContent(contentId); - file.AssertIsEqual(file2); - - // - foreach (var node in nodes) bootstrap.ConnectToPeer(node); - for (var x = 0; x < number; x++) - { - for (var y = x + 1; y < number; y++) - { - nodes[x].ConnectToPeer(nodes[y]); - } - } - // - - foreach (var node in nodes) AssertKnowEachother(node, bootstrap); - - for (var x = 0; x < number; x++) - { - for (var y = x + 1; y < number; y++) - { - AssertKnowEachother(nodes[x], nodes[y]); - } - } - } - - private void AssertKnowEachother(IOnlineCodexNode a, IOnlineCodexNode b) - { - AssertKnowEachother(a.GetDebugInfo(), b.GetDebugInfo()); - } - - private void AssertKnowEachother(CodexDebugResponse a, CodexDebugResponse b) - { - AssertKnows(a, b); - AssertKnows(b, a); - } - - private void AssertKnows(CodexDebugResponse a, CodexDebugResponse b) - { - var enginePeers = string.Join(",", a.enginePeers.Select(p => p.peerId)); - var switchPeers = string.Join(",", a.switchPeers.Select(p => p.peerId)); - - Debug($"{a.id} is looking for {b.id} in engine-peers [{enginePeers}]"); - Debug($"{a.id} is looking for {b.id} in switch-peers [{switchPeers}]"); - - Assert.That(a.enginePeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in engine-peers [{enginePeers}] but it was not found."); - Assert.That(a.switchPeers.Any(p => p.peerId == b.id), $"{a.id} was looking for '{b.id}' in switch-peers [{switchPeers}] but it was not found."); - } - } -} diff --git a/Tests/DurabilityTests/DurabilityTests.cs b/Tests/DurabilityTests/DurabilityTests.cs index 008799b..1267400 100644 --- a/Tests/DurabilityTests/DurabilityTests.cs +++ b/Tests/DurabilityTests/DurabilityTests.cs @@ -31,10 +31,10 @@ namespace Tests.DurabilityTests [Test] public void DataRetentionTest() { - var bootstrapNode = SetupCodexBootstrapNode(s => s.WithLogLevel(CodexLogLevel.Trace)); + var bootstrapNode = SetupCodexBootstrapNode(); - var startGroup = SetupCodexNodes(2, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); - var finishGroup = SetupCodexNodes(10, s => s.WithLogLevel(CodexLogLevel.Trace).WithBootstrapNode(bootstrapNode)); + var startGroup = SetupCodexNodes(2, s => s.WithBootstrapNode(bootstrapNode)); + var finishGroup = SetupCodexNodes(10, s => s.WithBootstrapNode(bootstrapNode)); var file = GenerateTestFile(10.MB()); diff --git a/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs new file mode 100644 index 0000000..306d305 --- /dev/null +++ b/Tests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -0,0 +1,62 @@ +using DistTestCore; +using DistTestCore.Helpers; +using NUnit.Framework; +using Utils; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class LayeredDiscoveryTests : DistTest + { + [Test] + public void TwoLayersTest() + { + var root = SetupCodexNode(); + var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Target = SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + + AssertAllNodesConnected(); + } + + [Test] + public void ThreeLayersTest() + { + var root = SetupCodexNode(); + var l1Source = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l1Node = SetupCodexNode(s => s.WithBootstrapNode(root)); + var l2Node = SetupCodexNode(s => s.WithBootstrapNode(l1Node)); + var l3Target = SetupCodexNode(s => s.WithBootstrapNode(l2Node)); + + AssertAllNodesConnected(); + } + + [TestCase(3)] + [TestCase(5)] + [TestCase(10)] + [TestCase(20)] + [TestCase(50)] + public void NodeChainTest(int chainLength) + { + var node = SetupCodexNode(); + for (var i = 1; i < chainLength; i++) + { + node = SetupCodexNode(s => s.WithBootstrapNode(node)); + } + + AssertAllNodesConnected(); + + for (int i = 0; i < 5; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } + } + + private void AssertAllNodesConnected() + { + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); + } + } +} diff --git a/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs new file mode 100644 index 0000000..41b0a66 --- /dev/null +++ b/Tests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -0,0 +1,74 @@ +using DistTestCore; +using DistTestCore.Codex; +using DistTestCore.Helpers; +using NUnit.Framework; +using Utils; + +namespace Tests.PeerDiscoveryTests +{ + [TestFixture] + public class PeerDiscoveryTests : AutoBootstrapDistTest + { + [Test] + public void CanReportUnknownPeerId() + { + var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK"; + var node = SetupCodexNode(); + + var result = node.GetDebugPeer(unknownId); + Assert.That(result.IsPeerFound, Is.False); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + public void VariableNodes(int number) + { + SetupCodexNodes(number); + + AssertAllNodesConnected(); + } + + [TestCase(2)] + [TestCase(3)] + [TestCase(10)] + [TestCase(20)] + public void VariableNodesInPods(int number) + { + for (var i = 0; i < number; i++) + { + SetupCodexNode(); + } + + AssertAllNodesConnected(); + } + + [TestCase(3, 3)] + [TestCase(3, 5)] + [TestCase(3, 10)] + [TestCase(5, 10)] + [TestCase(3, 20)] + [TestCase(5, 20)] + public void StagedVariableNodes(int numberOfNodes, int numberOfStages) + { + for (var i = 0; i < numberOfStages; i++) + { + SetupCodexNodes(numberOfNodes); + + AssertAllNodesConnected(); + } + + for (int i = 0; i < 5; i++) + { + Time.Sleep(TimeSpan.FromSeconds(30)); + AssertAllNodesConnected(); + } + } + + private void AssertAllNodesConnected() + { + PeerConnectionTestHelpers.AssertFullyConnected(GetAllOnlineCodexNodes()); + //PeerDownloadTestHelpers.AssertFullDownloadInterconnectivity(GetAllOnlineCodexNodes()); + } + } +} diff --git a/Utils/Time.cs b/Utils/Time.cs index 0f4f71b..6ae2640 100644 --- a/Utils/Time.cs +++ b/Utils/Time.cs @@ -38,5 +38,74 @@ state = predicate(); } } + + public static void Retry(Action action, string description) + { + Retry(action, TimeSpan.FromMinutes(1), description); + } + + public static T Retry(Func action, string description) + { + return Retry(action, TimeSpan.FromMinutes(1), description); + } + + public static void Retry(Action action, TimeSpan timeout, string description) + { + Retry(action, timeout, TimeSpan.FromSeconds(1), description); + } + + public static T Retry(Func action, TimeSpan timeout, string description) + { + return Retry(action, timeout, TimeSpan.FromSeconds(1), description); + } + + public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description) + { + var start = DateTime.UtcNow; + var exceptions = new List(); + while (true) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); + } + + try + { + action(); + return; + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + Sleep(retryTime); + } + } + + public static T Retry(Func action, TimeSpan timeout, TimeSpan retryTime, string description) + { + var start = DateTime.UtcNow; + var exceptions = new List(); + while (true) + { + if (DateTime.UtcNow - start > timeout) + { + throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions)); + } + + try + { + return action(); + } + catch (Exception ex) + { + exceptions.Add(ex); + } + + Sleep(retryTime); + } + } } }