diff --git a/Framework/ArgsUniform/ArgsUniform.cs b/Framework/ArgsUniform/ArgsUniform.cs index db1a6b2..d987aa7 100644 --- a/Framework/ArgsUniform/ArgsUniform.cs +++ b/Framework/ArgsUniform/ArgsUniform.cs @@ -234,7 +234,7 @@ namespace ArgsUniform private static bool AssignBool(T result, PropertyInfo uniformProperty, object value) { var s = value.ToString(); - if (s == "1" || s.ToLowerInvariant() == "true") + if (s == "1" || (s != null && s.ToLowerInvariant() == "true")) { uniformProperty.SetValue(result, true); } diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 8e0ac40..f9d7772 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -44,6 +44,11 @@ namespace CodexPlugin return result; } + public CodexDebugBlockExchangeResponse GetDebugBlockExchange() + { + return Http().HttpGetJson("debug/blockexchange"); + } + public CodexDebugThresholdBreaches GetDebugThresholdBreaches() { return Http().HttpGetJson("debug/loop"); diff --git a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs index 127f437..b347508 100644 --- a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs @@ -122,4 +122,47 @@ namespace CodexPlugin public string state { get; set; } = string.Empty; public string error { get; set; } = string.Empty; } + + public class CodexDebugBlockExchangeResponse + { + public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty(); + public int taskQueue { get; set; } + public int pendingBlocks { get; set; } + + public override string ToString() + { + if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty"; + + return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}"; + } + } + + public class CodexDebugBlockExchangeResponsePeer + { + public string peerid { get; set; } = string.Empty; + public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty(); + public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty(); + public int exchanged { get; set; } + + public override string ToString() + { + return $"(blocks:{hasBlocks.Length} wants:{wants.Length})"; + } + } + + public class CodexDebugBlockExchangeResponsePeerHasBlock + { + public string cid { get; set; } = string.Empty; + public bool have { get; set; } + public string price { get; set; } = string.Empty; + } + + public class CodexDebugBlockExchangeResponsePeerWant + { + public string block { get; set; } = string.Empty; + public int priority { get; set; } + public bool cancel { get; set; } + public string wantType { get; set; } = string.Empty; + public bool sendDontHave { get; set; } + } } diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index f4ff802..4a5940d 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -13,6 +13,7 @@ namespace CodexPlugin string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); + CodexDebugBlockExchangeResponse GetDebugBlockExchange(); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(ICodexNode node); @@ -81,6 +82,11 @@ namespace CodexPlugin return CodexAccess.GetDebugPeer(peerId); } + public CodexDebugBlockExchangeResponse GetDebugBlockExchange() + { + return CodexAccess.GetDebugBlockExchange(); + } + public ContentId UploadFile(TrackedFile file) { using var fileStream = File.OpenRead(file.Filename); diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index 6ab4d20..0f9f37f 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -11,11 +11,7 @@ namespace CodexPlugin ICodexSetup At(ILocation location); ICodexSetup WithBootstrapNode(ICodexNode node); ICodexSetup WithLogLevel(CodexLogLevel level); - /// - /// Sets the log level for codex. The default level is INFO and the - /// log level is applied only to the supplied topics. - /// - ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics); + ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup WithBlockTTL(TimeSpan duration); ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); @@ -28,6 +24,18 @@ namespace CodexPlugin ICodexSetup WithSimulateProofFailures(uint failEveryNProofs); } + public class CodexLogCustomTopics + { + public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p) + { + DiscV5 = discV5; + Libp2p = libp2p; + } + + public CodexLogLevel DiscV5 { get; set; } + public CodexLogLevel Libp2p { get; set; } + } + public class CodexSetup : CodexStartupConfig, ICodexSetup { public int NumberOfNodes { get; } @@ -61,10 +69,10 @@ namespace CodexPlugin return this; } - public ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics) + public ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics) { LogLevel = level; - LogTopics = topics; + CustomTopics = customTopics; return this; } diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index a3b6314..971dd2c 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -8,7 +8,7 @@ namespace CodexPlugin public string? NameOverride { get; set; } public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation; public CodexLogLevel LogLevel { get; set; } - public string[]? LogTopics { get; set; } + public CodexLogCustomTopics? CustomTopics { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } @@ -22,9 +22,41 @@ namespace CodexPlugin public string LogLevelWithTopics() { var level = LogLevel.ToString()!.ToUpperInvariant(); - if (LogTopics != null && LogTopics.Count() > 0) + if (CustomTopics != null) { - level = $"INFO;{level}: {string.Join(",", LogTopics.Where(s => !string.IsNullOrEmpty(s)))}"; + var discV5Topics = new[] + { + "discv5", + "providers", + "manager", + "cache", + }; + var libp2pTopics = new[] + { + "libp2p", + "multistream", + "switch", + "transport", + "tcptransport", + "semaphore", + "asyncstreamwrapper", + "lpstream", + "mplex", + "mplexchannel", + "noise", + "bufferstream", + "mplexcoder", + "secure", + "chronosstream", + "connection", + "connmanager", + "websock", + "ws-session" + }; + + level = $"{level};" + + $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" + + $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}"; } return level; } diff --git a/Tests/CodexContinuousTests/Configuration.cs b/Tests/CodexContinuousTests/Configuration.cs index ba4d9aa..3e51d92 100644 --- a/Tests/CodexContinuousTests/Configuration.cs +++ b/Tests/CodexContinuousTests/Configuration.cs @@ -33,6 +33,10 @@ namespace ContinuousTests [Uniform("cleanup", "cl", "CLEANUP", false, "If set to 1 or 'true', the kubernetes namespace will be deleted after the test run has finished.")] public bool Cleanup { get; set; } = false; + [Uniform("full-container-logs", "fcl", "FULLCONTAINERLOGS", false, "If set to 1 or 'true', container logs downloaded on test failure will download from" + + " the timestamp of the start of the network deployment. Otherwise, logs will start from the test start timestamp.")] + public bool FullContainerLogs { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; } diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs index c78ed63..6516b76 100644 --- a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs +++ b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs @@ -16,11 +16,11 @@ namespace ContinuousTests this.log = log; } - public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc) + public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine) { try { - DownloadLog(targetFile, container, startUtc, endUtc); + DownloadLog(targetFile, container, startUtc, endUtc, openingLine); } catch (Exception ex) { @@ -28,10 +28,11 @@ namespace ContinuousTests } } - private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc) + private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine) { log.Log($"Downloading log (from ElasticSearch) for container '{container.Name}' within time range: " + $"{startUtc.ToString("o")} - {endUtc.ToString("o")}"); + log.Log(openingLine); var http = CreateElasticSearchHttp(); var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc); diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 85283fa..ad54dd5 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -104,12 +104,17 @@ namespace ContinuousTests Thread.Sleep(TimeSpan.FromMinutes(1)); var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30)); + if (config.FullContainerLogs) + { + effectiveStart = config.CodexDeployment.Metadata.DeployDateTimeUtc.Subtract(TimeSpan.FromSeconds(30)); + } var effectiveEnd = DateTime.UtcNow; var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog); foreach (var node in nodes) { - elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd); + var openingLine = $"{node.Container.Pod.PodInfo.Name} = {node.Container.Name} = {node.GetDebugInfo().id}"; + elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine); } } diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index abc4dba..aae813f 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -42,6 +42,7 @@ namespace ContinuousTests foreach (var container in deployment.CodexContainers) { log.Log($"Codex environment variables for '{container.Name}':"); + log.Log($"Pod name: {container.Pod.PodInfo.Name} - Deployment name: {container.Pod.DeploymentName}"); var codexVars = container.Recipe.EnvVars; foreach (var vars in codexVars) log.Log(vars.ToString()); log.Log(""); diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh new file mode 100644 index 0000000..130c534 --- /dev/null +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -0,0 +1,45 @@ +set -e + +replication=5 + +echo "Deploying..." +cd ../../Tools/CodexNetDeployer +for i in $( seq 0 $replication) +do + dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --kube-namespace=codex-continuous-tests-$i \ + --deploy-file=codex-deployment-$i.json \ + --nodes=5 \ + --validators=3 \ + --log-level=Trace \ + --storage-quota=20480 \ + --storage-sell=1024 \ + --min-price=1024 \ + --max-collateral=1024 \ + --max-duration=3600000 \ + --block-ttl=99999999 \ + --block-mi=99999999 \ + --block-mn=100 \ + --metrics=1 \ + --check-connect=1 \ + -y + + cp codex-deployment-$i.json ../../Tests/CodexContinuousTests +done +echo "Starting tests..." +cd ../../Tests/CodexContinuousTests +for i in $( seq 0 $replication) +do + screen -d -m dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --codex-deployment=codex-deployment-$i.json \ + --log-path=logs-$i \ + --data-path=data-$i \ + --keep=1 \ + --stop=1 \ + --filter=TwoClient \ + --cleanup=1 \ + --full-container-logs=1 \ + --target-duration=172800 # 48 hours +done diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 4049b9a..0db9bbb 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using CodexPlugin; using DistTestCore; using GethPlugin; using MetricsPlugin; @@ -13,7 +14,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = AddCodex(); + var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); primary.UploadFile(GenerateTestFile(5.MB())); diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index f1cfed4..6ed6528 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -36,7 +36,7 @@ namespace CodexNetDeployer codexNode = ci.StartCodexNode(s => { s.WithName(name); - s.WithLogLevel(config.CodexLogLevel); + s.WithLogLevel(config.CodexLogLevel, new CodexLogCustomTopics(config.Discv5LogLevel, config.Libp2pLogLevel)); s.WithStorageQuota(config.StorageQuota!.Value.MB()); s.EnableMarketplace(gethNode, contracts, 100.Eth(), config.InitialTestTokens.TestTokens(), validatorsLeft > 0); s.EnableMetrics(); diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index b65ad72..3a371cd 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -36,6 +36,12 @@ namespace CodexNetDeployer [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")] public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; + + [Uniform("log-level-libp2p", "lp2p", "LOGLEVELLIBP2P", true, "Log level for all libp2p topics. [Trace, Debug, Info, Warn*, Error]")] + public CodexLogLevel Libp2pLogLevel { get; set; } = CodexLogLevel.Warn; + + [Uniform("log-level-discv5", "ldv5", "LOGLEVELDISCV5", true, "Log level for all discv5 topics. [Trace, Debug, Info, Warn*, Error]")] + public CodexLogLevel Discv5LogLevel { get; set; } = CodexLogLevel.Warn; [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")] public int InitialTestTokens { get; set; } = int.MaxValue; diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 0029d56..aad5fab 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -36,9 +36,7 @@ public class Program var deployment = deployer.Deploy(); Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); - File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); - Console.WriteLine("Done!"); }