diff --git a/ProjectPlugins/CodexClient/CodexAccess.cs b/ProjectPlugins/CodexClient/CodexAccess.cs index 7f35d233..9f0c22c7 100644 --- a/ProjectPlugins/CodexClient/CodexAccess.cs +++ b/ProjectPlugins/CodexClient/CodexAccess.cs @@ -31,7 +31,9 @@ namespace CodexClient public IDownloadedLog DownloadLog(string additionalName = "") { - return processControl.DownloadLog(log.CreateSubfile(GetName() + additionalName)); + var file = log.CreateSubfile(GetName() + additionalName); + Log($"Downloading logs for '{GetName()}' to '{file.Filename}'"); + return processControl.DownloadLog(file); } public string GetImageName() diff --git a/ProjectPlugins/CodexClient/CodexNode.cs b/ProjectPlugins/CodexClient/CodexNode.cs index 5cdf7b02..df18079c 100644 --- a/ProjectPlugins/CodexClient/CodexNode.cs +++ b/ProjectPlugins/CodexClient/CodexNode.cs @@ -79,7 +79,7 @@ namespace CodexClient InitializePeerNodeId(); InitializeLogReplacements(); - hooks.OnNodeStarted(peerId, nodeId); + hooks.OnNodeStarted(this, peerId, nodeId); } public IMarketplaceAccess Marketplace { get; } @@ -350,9 +350,9 @@ namespace CodexClient cts.Cancel(); throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}"); } - catch + catch (Exception ex) { - Log($"Failed to download file '{contentId}'."); + Log($"Failed to download file '{contentId}': {ex}"); throw; } } diff --git a/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs b/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs index 83e45926..0557f138 100644 --- a/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs +++ b/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs @@ -9,11 +9,14 @@ namespace CodexClient.Hooks public class CodexHooksFactory { - public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider(); + public List Providers { get; } = new List(); public ICodexNodeHooks CreateHooks(string nodeName) { - return Provider.CreateHooks(nodeName); + if (Providers.Count == 0) return new DoNothingCodexHooks(); + + var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray(); + return new MuxingCodexNodeHooks(hooks); } } @@ -43,7 +46,7 @@ namespace CodexClient.Hooks { } - public void OnNodeStarted(string peerId, string nodeId) + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) { } diff --git a/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs b/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs index 8becef0a..926a5d1d 100644 --- a/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs +++ b/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs @@ -5,7 +5,7 @@ namespace CodexClient.Hooks public interface ICodexNodeHooks { void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount); - void OnNodeStarted(string peerId, string nodeId); + void OnNodeStarted(ICodexNode node, string peerId, string nodeId); void OnNodeStopping(); void OnFileUploading(string uid, ByteSize size); void OnFileUploaded(string uid, ByteSize size, ContentId cid); @@ -15,4 +15,64 @@ namespace CodexClient.Hooks void OnStorageContractUpdated(StoragePurchase purchaseStatus); void OnStorageAvailabilityCreated(StorageAvailability response); } + + public class MuxingCodexNodeHooks : ICodexNodeHooks + { + private readonly ICodexNodeHooks[] backingHooks; + + public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks) + { + this.backingHooks = backingHooks; + } + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + foreach (var h in backingHooks) h.OnFileDownloaded(size, cid); + } + + public void OnFileDownloading(ContentId cid) + { + foreach (var h in backingHooks) h.OnFileDownloading(cid); + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid); + } + + public void OnFileUploading(string uid, ByteSize size) + { + foreach (var h in backingHooks) h.OnFileUploading(uid, size); + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount); + } + + public void OnNodeStopping() + { + foreach (var h in backingHooks) h.OnNodeStopping(); + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response); + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract); + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + { + foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus); + } + } } diff --git a/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs b/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs index 44c843d7..63128977 100644 --- a/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs +++ b/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs @@ -44,7 +44,7 @@ namespace CodexPlugin public bool HasCrashed() { - return false; + return process.HasExited; } public void Stop(bool waitTillStopped) diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index b208347c..a68be9e6 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -68,9 +68,10 @@ namespace CodexPlugin } } - public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider) + public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider) { - hooksFactory.Provider = hooksProvider; + if (hooksFactory.Providers.Contains(hooksProvider)) return; + hooksFactory.Providers.Add(hooksProvider); } private CodexSetup GetSetup(int numberOfNodes, Action setup) diff --git a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs index 06b9246e..383ac38b 100644 --- a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs @@ -39,9 +39,9 @@ namespace CodexPlugin return ci.StartCodexNodes(number, s => { }); } - public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider) + public static void AddCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider) { - Plugin(ci).SetCodexHooksProvider(hooksProvider); + Plugin(ci).AddCodexHooksProvider(hooksProvider); } private static CodexPlugin Plugin(CoreInterface ci) diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs index 346715ca..db4702d7 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs @@ -32,7 +32,7 @@ namespace CodexPlugin.OverwatchSupport }); } - public void OnNodeStarted(string peerId, string nodeId) + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) { if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId)) { diff --git a/Tests/DistTestCore/TestLifecycle.cs b/Tests/DistTestCore/TestLifecycle.cs index 2a11cedb..3d642d20 100644 --- a/Tests/DistTestCore/TestLifecycle.cs +++ b/Tests/DistTestCore/TestLifecycle.cs @@ -115,6 +115,9 @@ namespace DistTestCore { try { + // TODO: This code is built on k8s containers. + // It should be remapped to use the project plugin's support for downloading logs (via IProcessControl). + // For now, leave this. Add support for Codex non-container logs using the codex node hooks. var result = new List(); result.AddRange(stoppedContainerLogs); foreach (var rc in runningContainers) diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index 3d83021a..99671edb 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -1,5 +1,6 @@ using BlockchainUtils; using CodexClient; +using CodexClient.Hooks; using CodexContractsPlugin; using CodexNetDeployer; using CodexPlugin; @@ -16,14 +17,85 @@ using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using OverwatchTranscript; +using Utils; namespace CodexTests { + public class CodexLogTrackerProvider : ICodexHooksProvider + { + private readonly Action addNode; + + public CodexLogTrackerProvider(Action addNode) + { + this.addNode = addNode; + } + + // See TestLifecycle.cs DownloadAllLogs() + public ICodexNodeHooks CreateHooks(string nodeName) + { + return new CodexLogTracker(addNode); + } + + public class CodexLogTracker : ICodexNodeHooks + { + private readonly Action addNode; + + public CodexLogTracker(Action addNode) + { + this.addNode = addNode; + } + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + addNode(node); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + } + + public void OnNodeStopping() + { + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + { + } + } + } + public class CodexDistTest : DistTest { private static readonly Dictionary writers = new Dictionary(); private static readonly Dictionary blockCaches = new Dictionary(); + // this entire structure is not good and needs to be destroyed at the earliest convenience: + private static readonly Dictionary> nodes = new Dictionary>(); + public CodexDistTest() { ProjectPlugin.Load(); @@ -43,12 +115,24 @@ namespace CodexTests { base.LifecycleStart(lifecycle); SetupTranscript(lifecycle); + + Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => + { + if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List()); + nodes[lifecycle].Add(n); + })); } protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) { base.LifecycleStop(lifecycle, result); TeardownTranscript(lifecycle, result); + + if (!result.Success) + { + var codexNodes = nodes[lifecycle]; + foreach (var node in codexNodes) node.DownloadLog(); + } } public ICodexNode StartCodex() @@ -178,7 +262,7 @@ namespace CodexTests var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); - Ci.SetCodexHooksProvider(writer); + Ci.AddCodexHooksProvider(writer); writers.Add(lifecycle, writer); } diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs index 6c619969..7ffc2d81 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using NUnit.Framework; using Utils; diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs index 6f007c4e..601d2cfc 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs @@ -1,7 +1,8 @@ -using NUnit.Framework; +using CodexTests; +using NUnit.Framework; using Utils; -namespace CodexTests.DownloadConnectivityTests +namespace ExperimentalTests.DownloadConnectivityTests { [TestFixture] public class DetectBlockRetransmitTest : AutoBootstrapDistTest diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 8898f49c..ec128154 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,9 +1,10 @@ using CodexClient; using CodexContractsPlugin; +using CodexTests; using NUnit.Framework; using Utils; -namespace CodexTests.DownloadConnectivityTests +namespace ExperimentalTests.DownloadConnectivityTests { [TestFixture] public class FullyConnectedDownloadTests : AutoBootstrapDistTest diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs index 8b8f9d1e..a206d475 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs @@ -1,10 +1,11 @@ using CodexClient; +using CodexTests; using FileUtils; using Logging; using NUnit.Framework; using Utils; -namespace CodexTests.DownloadConnectivityTests +namespace ExperimentalTests.DownloadConnectivityTests { [TestFixture] public class MultiswarmTests : AutoBootstrapDistTest @@ -206,7 +207,7 @@ namespace CodexTests.DownloadConnectivityTests var available = NodePlans.Where(n => n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn) ).ToArray(); - if (available.Any()) return RandomUtils.GetOneRandom(available); + if (available.Any()) return available.GetOneRandom(); var newNodePlan = new NodePlan(NodePlans.Count); NodePlans.Add(newNodePlan); @@ -218,7 +219,7 @@ namespace CodexTests.DownloadConnectivityTests var available = NodePlans.Where(n => n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn) ).ToArray(); - if (available.Any()) return RandomUtils.GetOneRandom(available); + if (available.Any()) return available.GetOneRandom(); var newNodePlan = new NodePlan(NodePlans.Count); NodePlans.Add(newNodePlan);