workaround for downloading codex logs

ProjectPlugins/CodexClient/CodexAccess.cs

@@ -31,7 +31,9 @@ namespace CodexClient
 
         public IDownloadedLog DownloadLog(string additionalName = "")
         {
-            return processControl.DownloadLog(log.CreateSubfile(GetName() + additionalName));
+            var file = log.CreateSubfile(GetName() + additionalName);
+            Log($"Downloading logs for '{GetName()}' to '{file.Filename}'");
+            return processControl.DownloadLog(file);
         }
 
         public string GetImageName()

ProjectPlugins/CodexClient/CodexNode.cs

@@ -79,7 +79,7 @@ namespace CodexClient
             InitializePeerNodeId();
             InitializeLogReplacements();
 
-            hooks.OnNodeStarted(peerId, nodeId);
+            hooks.OnNodeStarted(this, peerId, nodeId);
         }
 
         public IMarketplaceAccess Marketplace { get; }
@@ -350,9 +350,9 @@ namespace CodexClient
                 cts.Cancel();
                 throw new TimeoutException($"Download of '{contentId}' timed out after {Time.FormatDuration(timeout)}");
             }
-            catch
+            catch (Exception ex)
             {
-                Log($"Failed to download file '{contentId}'.");
+                Log($"Failed to download file '{contentId}': {ex}");
                 throw;
             }
         }

ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs

@@ -9,11 +9,14 @@ namespace CodexClient.Hooks
 
     public class CodexHooksFactory
     {
-        public ICodexHooksProvider Provider { get; set; } = new DoNothingHooksProvider();
+        public List<ICodexHooksProvider> Providers { get; } = new List<ICodexHooksProvider>();
 
         public ICodexNodeHooks CreateHooks(string nodeName)
         {
-            return Provider.CreateHooks(nodeName);
+            if (Providers.Count == 0) return new DoNothingCodexHooks();
+
+            var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray();
+            return new MuxingCodexNodeHooks(hooks);
         }
     }
 
@@ -43,7 +46,7 @@ namespace CodexClient.Hooks
         {
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
         }
 

ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs

@@ -5,7 +5,7 @@ namespace CodexClient.Hooks
     public interface ICodexNodeHooks
     {
         void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount);
-        void OnNodeStarted(string peerId, string nodeId);
+        void OnNodeStarted(ICodexNode node, string peerId, string nodeId);
         void OnNodeStopping();
         void OnFileUploading(string uid, ByteSize size);
         void OnFileUploaded(string uid, ByteSize size, ContentId cid);
@@ -15,4 +15,64 @@ namespace CodexClient.Hooks
         void OnStorageContractUpdated(StoragePurchase purchaseStatus);
         void OnStorageAvailabilityCreated(StorageAvailability response);
     }
+
+    public class MuxingCodexNodeHooks : ICodexNodeHooks
+    {
+        private readonly ICodexNodeHooks[] backingHooks;
+
+        public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks)
+        {
+            this.backingHooks = backingHooks;
+        }
+
+        public void OnFileDownloaded(ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloaded(size, cid);
+        }
+
+        public void OnFileDownloading(ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileDownloading(cid);
+        }
+
+        public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+        {
+            foreach (var h in backingHooks) h.OnFileUploaded(uid, size, cid);
+        }
+
+        public void OnFileUploading(string uid, ByteSize size)
+        {
+            foreach (var h in backingHooks) h.OnFileUploading(uid, size);
+        }
+
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId);
+        }
+
+        public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+        {
+            foreach (var h in backingHooks) h.OnNodeStarting(startUtc, image, ethAccount);
+        }
+
+        public void OnNodeStopping()
+        {
+            foreach (var h in backingHooks) h.OnNodeStopping();
+        }
+
+        public void OnStorageAvailabilityCreated(StorageAvailability response)
+        {
+            foreach (var h in backingHooks) h.OnStorageAvailabilityCreated(response);
+        }
+
+        public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractSubmitted(storagePurchaseContract);
+        }
+
+        public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+        {
+            foreach (var h in backingHooks) h.OnStorageContractUpdated(purchaseStatus);
+        }
+    }
 }

ProjectPlugins/CodexPlugin/BinaryProcessControl.cs

@@ -44,7 +44,7 @@ namespace CodexPlugin
 
         public bool HasCrashed()
         {
-            return false;
+            return process.HasExited;
         }
 
         public void Stop(bool waitTillStopped)

ProjectPlugins/CodexPlugin/CodexPlugin.cs

@@ -68,9 +68,10 @@ namespace CodexPlugin
             }
         }
 
-        public void SetCodexHooksProvider(ICodexHooksProvider hooksProvider)
+        public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider)
         {
-            hooksFactory.Provider = hooksProvider;
+            if (hooksFactory.Providers.Contains(hooksProvider)) return;
+            hooksFactory.Providers.Add(hooksProvider);
         }
 
         private CodexSetup GetSetup(int numberOfNodes, Action<ICodexSetup> setup)

ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs

@@ -39,9 +39,9 @@ namespace CodexPlugin
             return ci.StartCodexNodes(number, s => { });
         }
 
-        public static void SetCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
+        public static void AddCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider)
         {
-            Plugin(ci).SetCodexHooksProvider(hooksProvider);
+            Plugin(ci).AddCodexHooksProvider(hooksProvider);
         }
 
         private static CodexPlugin Plugin(CoreInterface ci)

ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs

@@ -32,7 +32,7 @@ namespace CodexPlugin.OverwatchSupport
             });
         }
 
-        public void OnNodeStarted(string peerId, string nodeId)
+        public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
         {
             if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId))
             {

Tests/DistTestCore/TestLifecycle.cs

@@ -115,6 +115,9 @@ namespace DistTestCore
         {
             try
             {
+                // TODO: This code is built on k8s containers.
+                // It should be remapped to use the project plugin's support for downloading logs (via IProcessControl).
+                // For now, leave this. Add support for Codex non-container logs using the codex node hooks.
                 var result = new List<IDownloadedLog>();
                 result.AddRange(stoppedContainerLogs);
                 foreach (var rc in runningContainers)

Tests/ExperimentalTests/CodexDistTest.cs

@@ -1,5 +1,6 @@
 using BlockchainUtils;
 using CodexClient;
+using CodexClient.Hooks;
 using CodexContractsPlugin;
 using CodexNetDeployer;
 using CodexPlugin;
@@ -16,14 +17,85 @@ using Newtonsoft.Json;
 using NUnit.Framework;
 using NUnit.Framework.Constraints;
 using OverwatchTranscript;
+using Utils;
 
 namespace CodexTests
 {
+    public class CodexLogTrackerProvider  : ICodexHooksProvider
+    {
+        private readonly Action<ICodexNode> addNode;
+
+        public CodexLogTrackerProvider(Action<ICodexNode> addNode)
+        {
+            this.addNode = addNode;
+        }
+
+        // See TestLifecycle.cs DownloadAllLogs()
+        public ICodexNodeHooks CreateHooks(string nodeName)
+        {
+            return new CodexLogTracker(addNode);
+        }
+
+        public class CodexLogTracker : ICodexNodeHooks
+        {
+            private readonly Action<ICodexNode> addNode;
+
+            public CodexLogTracker(Action<ICodexNode> addNode)
+            {
+                this.addNode = addNode;
+            }
+
+            public void OnFileDownloaded(ByteSize size, ContentId cid)
+            {
+            }
+
+            public void OnFileDownloading(ContentId cid)
+            {
+            }
+
+            public void OnFileUploaded(string uid, ByteSize size, ContentId cid)
+            {
+            }
+
+            public void OnFileUploading(string uid, ByteSize size)
+            {
+            }
+
+            public void OnNodeStarted(ICodexNode node, string peerId, string nodeId)
+            {
+                addNode(node);
+            }
+
+            public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount)
+            {
+            }
+
+            public void OnNodeStopping()
+            {
+            }
+
+            public void OnStorageAvailabilityCreated(StorageAvailability response)
+            {
+            }
+
+            public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract)
+            {
+            }
+
+            public void OnStorageContractUpdated(StoragePurchase purchaseStatus)
+            {
+            }
+        }
+    }
+
     public class CodexDistTest : DistTest
     {
         private static readonly Dictionary<TestLifecycle, CodexTranscriptWriter> writers = new Dictionary<TestLifecycle, CodexTranscriptWriter>();
         private static readonly Dictionary<TestLifecycle, BlockCache> blockCaches = new Dictionary<TestLifecycle, BlockCache>();
 
+        // this entire structure is not good and needs to be destroyed at the earliest convenience:
+        private static readonly Dictionary<TestLifecycle, List<ICodexNode>> nodes = new Dictionary<TestLifecycle, List<ICodexNode>>();
+
         public CodexDistTest()
         {
             ProjectPlugin.Load<CodexPlugin.CodexPlugin>();
@@ -43,12 +115,24 @@ namespace CodexTests
         {
             base.LifecycleStart(lifecycle);
             SetupTranscript(lifecycle);
+
+            Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n =>
+            {
+                if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List<ICodexNode>());
+                nodes[lifecycle].Add(n);
+            }));
         }
 
         protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result)
         {
             base.LifecycleStop(lifecycle, result);
             TeardownTranscript(lifecycle, result);
+
+            if (!result.Success)
+            {
+                var codexNodes = nodes[lifecycle];
+                foreach (var node in codexNodes) node.DownloadLog();
+            }
         }
 
         public ICodexNode StartCodex()
@@ -178,7 +262,7 @@ namespace CodexTests
 
             var log = new LogPrefixer(lifecycle.Log, "(Transcript) ");
             var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log));
-            Ci.SetCodexHooksProvider(writer);
+            Ci.AddCodexHooksProvider(writer);
             writers.Add(lifecycle, writer);
         }
 

Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs

@@ -1,4 +1,4 @@
-using CodexPlugin;
+using CodexClient;
 using CodexTests;
 using NUnit.Framework;
 using Utils;

Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs

@@ -1,7 +1,8 @@
-using NUnit.Framework;
+using CodexTests;
+using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.DownloadConnectivityTests
+namespace ExperimentalTests.DownloadConnectivityTests
 {
     [TestFixture]
     public class DetectBlockRetransmitTest : AutoBootstrapDistTest

Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs

@@ -1,9 +1,10 @@
 using CodexClient;
 using CodexContractsPlugin;
+using CodexTests;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.DownloadConnectivityTests
+namespace ExperimentalTests.DownloadConnectivityTests
 {
     [TestFixture]
     public class FullyConnectedDownloadTests : AutoBootstrapDistTest

Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs

@@ -1,10 +1,11 @@
 using CodexClient;
+using CodexTests;
 using FileUtils;
 using Logging;
 using NUnit.Framework;
 using Utils;
 
-namespace CodexTests.DownloadConnectivityTests
+namespace ExperimentalTests.DownloadConnectivityTests
 {
     [TestFixture]
     public class MultiswarmTests : AutoBootstrapDistTest
@@ -206,7 +207,7 @@ namespace CodexTests.DownloadConnectivityTests
             var available = NodePlans.Where(n =>
                 n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
             ).ToArray();
-            if (available.Any()) return RandomUtils.GetOneRandom(available);
+            if (available.Any()) return available.GetOneRandom();
 
             var newNodePlan = new NodePlan(NodePlans.Count);
             NodePlans.Add(newNodePlan);
@@ -218,7 +219,7 @@ namespace CodexTests.DownloadConnectivityTests
             var available = NodePlans.Where(n =>
                 n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
             ).ToArray();
-            if (available.Any()) return RandomUtils.GetOneRandom(available);
+            if (available.Any()) return available.GetOneRandom();
 
             var newNodePlan = new NodePlan(NodePlans.Count);
             NodePlans.Add(newNodePlan);