Merge branch 'experiment/deploy-replication'

Framework/ArgsUniform/ArgsUniform.cs

@@ -234,7 +234,7 @@ namespace ArgsUniform
         private static bool AssignBool(T result, PropertyInfo uniformProperty, object value)
         {
             var s = value.ToString();
-            if (s == "1" || s.ToLowerInvariant() == "true")
+            if (s == "1" || (s != null && s.ToLowerInvariant() == "true"))
             {
                 uniformProperty.SetValue(result, true);
             }

ProjectPlugins/CodexPlugin/CodexAccess.cs

@@ -44,6 +44,11 @@ namespace CodexPlugin
             return result;
         }
 
+        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        {
+            return Http().HttpGetJson<CodexDebugBlockExchangeResponse>("debug/blockexchange");
+        }
+
         public CodexDebugThresholdBreaches GetDebugThresholdBreaches()
         {
             return Http().HttpGetJson<CodexDebugThresholdBreaches>("debug/loop");

ProjectPlugins/CodexPlugin/CodexApiTypes.cs

@@ -122,4 +122,47 @@ namespace CodexPlugin
         public string state { get; set; } = string.Empty;
         public string error { get; set; } = string.Empty;
     }
+
+    public class CodexDebugBlockExchangeResponse
+    {
+        public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
+        public int taskQueue { get; set; }
+        public int pendingBlocks { get; set; }
+
+        public override string ToString()
+        {
+            if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
+
+            return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
+        }
+    }
+
+    public class CodexDebugBlockExchangeResponsePeer
+    {
+        public string peerid { get; set; } = string.Empty;
+        public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
+        public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
+        public int exchanged { get; set; }
+
+        public override string ToString()
+        {
+            return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
+        }
+    }
+
+    public class CodexDebugBlockExchangeResponsePeerHasBlock
+    {
+        public string cid { get; set; } = string.Empty;
+        public bool have { get; set; }
+        public string price { get; set; } = string.Empty;
+    }
+
+    public class CodexDebugBlockExchangeResponsePeerWant
+    {
+        public string block { get; set; } = string.Empty;
+        public int priority { get; set; }
+        public bool cancel { get; set; }
+        public string wantType { get; set; } = string.Empty;
+        public bool sendDontHave { get; set; }
+    }
 }

ProjectPlugins/CodexPlugin/CodexNode.cs

@@ -13,6 +13,7 @@ namespace CodexPlugin
         string GetName();
         CodexDebugResponse GetDebugInfo();
         CodexDebugPeerResponse GetDebugPeer(string peerId);
+        CodexDebugBlockExchangeResponse GetDebugBlockExchange();
         ContentId UploadFile(TrackedFile file);
         TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
         void ConnectToPeer(ICodexNode node);
@@ -81,6 +82,11 @@ namespace CodexPlugin
             return CodexAccess.GetDebugPeer(peerId);
         }
 
+        public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
+        {
+            return CodexAccess.GetDebugBlockExchange();
+        }
+
         public ContentId UploadFile(TrackedFile file)
         {
             using var fileStream = File.OpenRead(file.Filename);

ProjectPlugins/CodexPlugin/CodexSetup.cs

@@ -11,11 +11,7 @@ namespace CodexPlugin
         ICodexSetup At(ILocation location);
         ICodexSetup WithBootstrapNode(ICodexNode node);
         ICodexSetup WithLogLevel(CodexLogLevel level);
-        /// <summary>
+        ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics);
-        /// Sets the log level for codex. The default level is INFO and the
-        /// log level is applied only to the supplied topics.
-        /// </summary>
-        ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics);
         ICodexSetup WithStorageQuota(ByteSize storageQuota);
         ICodexSetup WithBlockTTL(TimeSpan duration);
         ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration);
@@ -28,6 +24,18 @@ namespace CodexPlugin
         ICodexSetup WithSimulateProofFailures(uint failEveryNProofs);
     }
 
+    public class CodexLogCustomTopics
+    {
+        public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p)
+        {
+            DiscV5 = discV5;
+            Libp2p = libp2p;
+        }
+
+        public CodexLogLevel DiscV5 { get; set; }
+        public CodexLogLevel Libp2p { get; set; }
+    }
+
     public class CodexSetup : CodexStartupConfig, ICodexSetup
     {
         public int NumberOfNodes { get; }
@@ -61,10 +69,10 @@ namespace CodexPlugin
             return this;
         }
 
-        public ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics)
+        public ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics)
         {
             LogLevel = level;
-            LogTopics = topics;
+            CustomTopics = customTopics;
             return this;
         }
 

ProjectPlugins/CodexPlugin/CodexStartupConfig.cs

@@ -8,7 +8,7 @@ namespace CodexPlugin
         public string? NameOverride { get; set; }
         public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation;
         public CodexLogLevel LogLevel { get; set; }
-        public string[]? LogTopics { get; set; }
+        public CodexLogCustomTopics? CustomTopics { get; set; }
         public ByteSize? StorageQuota { get; set; }
         public bool MetricsEnabled { get; set; }
         public MarketplaceInitialConfig? MarketplaceConfig { get; set; }
@@ -22,9 +22,41 @@ namespace CodexPlugin
         public string LogLevelWithTopics()
         {
             var level = LogLevel.ToString()!.ToUpperInvariant();
-            if (LogTopics != null && LogTopics.Count() > 0)
+            if (CustomTopics != null)
             {
-                level = $"INFO;{level}: {string.Join(",", LogTopics.Where(s => !string.IsNullOrEmpty(s)))}";
+                var discV5Topics = new[]
+                {
+                    "discv5",
+                    "providers",
+                    "manager",
+                    "cache",
+                };
+                var libp2pTopics = new[]
+                {
+                    "libp2p",
+                    "multistream",
+                    "switch",
+                    "transport",
+                    "tcptransport",
+                    "semaphore",
+                    "asyncstreamwrapper",
+                    "lpstream",
+                    "mplex",
+                    "mplexchannel",
+                    "noise",
+                    "bufferstream",
+                    "mplexcoder",
+                    "secure",
+                    "chronosstream",
+                    "connection",
+                    "connmanager",
+                    "websock",
+                    "ws-session"
+                };
+
+                level = $"{level};" +
+                    $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" +
+                    $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}";
             }
             return level;
         }

Tests/CodexContinuousTests/Configuration.cs

@@ -33,6 +33,10 @@ namespace ContinuousTests
         [Uniform("cleanup", "cl", "CLEANUP", false, "If set to 1 or 'true', the kubernetes namespace will be deleted after the test run has finished.")]
         public bool Cleanup { get; set; } = false;
 
+        [Uniform("full-container-logs", "fcl", "FULLCONTAINERLOGS", false, "If set to 1 or 'true', container logs downloaded on test failure will download from" +
+            " the timestamp of the start of the network deployment. Otherwise, logs will start from the test start timestamp.")]
+        public bool FullContainerLogs { get; set; } = false;
+
         public CodexDeployment CodexDeployment { get; set; } = null!;
     }
 

Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs

@@ -16,11 +16,11 @@ namespace ContinuousTests
             this.log = log;
         }
 
-        public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc)
+        public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine)
         {
             try
             {
-                DownloadLog(targetFile, container, startUtc, endUtc);
+                DownloadLog(targetFile, container, startUtc, endUtc, openingLine);
             }
             catch (Exception ex)
             {
@@ -28,10 +28,11 @@ namespace ContinuousTests
             }
         }
 
-        private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc)
+        private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine)
         {
             log.Log($"Downloading log (from ElasticSearch) for container '{container.Name}' within time range: " +
                 $"{startUtc.ToString("o")} - {endUtc.ToString("o")}");
+            log.Log(openingLine);
 
             var http = CreateElasticSearchHttp();
             var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc);

Tests/CodexContinuousTests/SingleTestRun.cs

@@ -104,12 +104,17 @@ namespace ContinuousTests
             Thread.Sleep(TimeSpan.FromMinutes(1));
 
             var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30));
+            if (config.FullContainerLogs)
+            {
+                effectiveStart = config.CodexDeployment.Metadata.DeployDateTimeUtc.Subtract(TimeSpan.FromSeconds(30));
+            }
             var effectiveEnd = DateTime.UtcNow;
             var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog);
 
             foreach (var node in nodes)
             {
-                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd);
+                var openingLine = $"{node.Container.Pod.PodInfo.Name} = {node.Container.Name} = {node.GetDebugInfo().id}";
+                elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine);
             }
         }
 

Tests/CodexContinuousTests/StartupChecker.cs

@@ -42,6 +42,7 @@ namespace ContinuousTests
             foreach (var container in deployment.CodexContainers)
             {
                 log.Log($"Codex environment variables for '{container.Name}':");
+                log.Log($"Pod name: {container.Pod.PodInfo.Name} - Deployment name: {container.Pod.DeploymentName}");
                 var codexVars = container.Recipe.EnvVars;
                 foreach (var vars in codexVars) log.Log(vars.ToString());
                 log.Log("");

Tests/CodexContinuousTests/deploy-and-run.sh

@@ -0,0 +1,45 @@
+set -e
+
+replication=5
+
+echo "Deploying..."
+cd ../../Tools/CodexNetDeployer
+for i in $( seq 0 $replication)
+do
+    dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --kube-namespace=codex-continuous-tests-$i \
+    --deploy-file=codex-deployment-$i.json \
+    --nodes=5 \
+    --validators=3 \
+    --log-level=Trace \
+    --storage-quota=20480 \
+    --storage-sell=1024 \
+    --min-price=1024 \
+    --max-collateral=1024 \
+    --max-duration=3600000 \
+    --block-ttl=99999999 \
+    --block-mi=99999999 \
+    --block-mn=100 \
+    --metrics=1 \
+    --check-connect=1 \
+    -y
+
+    cp codex-deployment-$i.json ../../Tests/CodexContinuousTests
+done
+echo "Starting tests..."
+cd ../../Tests/CodexContinuousTests
+for i in $( seq 0 $replication)
+do
+    screen -d -m dotnet run \
+    --kube-config=/opt/kubeconfig.yaml \
+    --codex-deployment=codex-deployment-$i.json \
+    --log-path=logs-$i \
+    --data-path=data-$i \
+    --keep=1 \
+    --stop=1 \
+    --filter=TwoClient \
+    --cleanup=1 \
+    --full-container-logs=1 \
+    --target-duration=172800 # 48 hours
+done

Tests/CodexTests/BasicTests/ExampleTests.cs

@@ -1,4 +1,5 @@
 using CodexContractsPlugin;
+using CodexPlugin;
 using DistTestCore;
 using GethPlugin;
 using MetricsPlugin;
@@ -13,7 +14,7 @@ namespace Tests.BasicTests
         [Test]
         public void CodexLogExample()
         {
-            var primary = AddCodex();
+            var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn)));
 
             primary.UploadFile(GenerateTestFile(5.MB()));
 

Tools/CodexNetDeployer/CodexNodeStarter.cs

@@ -36,7 +36,7 @@ namespace CodexNetDeployer
                 codexNode = ci.StartCodexNode(s =>
                 {
                     s.WithName(name);
-                    s.WithLogLevel(config.CodexLogLevel);
+                    s.WithLogLevel(config.CodexLogLevel, new CodexLogCustomTopics(config.Discv5LogLevel, config.Libp2pLogLevel));
                     s.WithStorageQuota(config.StorageQuota!.Value.MB());
                     s.EnableMarketplace(gethNode, contracts, 100.Eth(), config.InitialTestTokens.TestTokens(), validatorsLeft > 0);
                     s.EnableMetrics();

Tools/CodexNetDeployer/Configuration.cs

@@ -36,6 +36,12 @@ namespace CodexNetDeployer
 
         [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")]
         public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug;
+        
+        [Uniform("log-level-libp2p", "lp2p", "LOGLEVELLIBP2P", true, "Log level for all libp2p topics. [Trace, Debug, Info, Warn*, Error]")]
+        public CodexLogLevel Libp2pLogLevel { get; set; } = CodexLogLevel.Warn;
+
+        [Uniform("log-level-discv5", "ldv5", "LOGLEVELDISCV5", true, "Log level for all discv5 topics. [Trace, Debug, Info, Warn*, Error]")]
+        public CodexLogLevel Discv5LogLevel { get; set; } = CodexLogLevel.Warn;
 
         [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")]
         public int InitialTestTokens { get; set; } = int.MaxValue;

Tools/CodexNetDeployer/Program.cs

@@ -36,9 +36,7 @@ public class Program
         var deployment = deployer.Deploy();
 
         Console.WriteLine($"Writing deployment file '{config.DeployFile}'...");
-
         File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented));
-
         Console.WriteLine("Done!");
     }