From 8fb63213bc299682acc31fc9e5fdf91991abc4dc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 5 Oct 2023 08:55:13 +0200 Subject: [PATCH 01/18] Trying deploy replication --- Tools/CodexNetDeployer/Configuration.cs | 3 +++ Tools/CodexNetDeployer/Program.cs | 26 +++++++++++++++++++++---- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index b65ad720..fdb8f271 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -68,6 +68,9 @@ namespace CodexNetDeployer [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; + [Uniform("replication", "rep", "REPLICATION", false, "Number of times to repeat")] + public int Replication { get; set; } = 0; + public List Validate() { var errors = new List(); diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 0029d56b..291c85cb 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -33,13 +33,31 @@ public class Program Console.WriteLine("I think so too."); } - var deployment = deployer.Deploy(); + if (config.Replication == 0) + { + var deployment = deployer.Deploy(); - Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); + Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); + File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); + Console.WriteLine("Done!"); + } + else + { + var originalNamespace = config.KubeNamespace; + var originalDeployFile = config.DeployFile; + for (var i = 0; i < config.Replication; i++) + { + config.KubeNamespace = originalNamespace + "-" + i; + config.DeployFile = originalDeployFile.ToLowerInvariant().Replace(".json", $"-{i}.json"); - File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); + var deployment = deployer.Deploy(); - Console.WriteLine("Done!"); + Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); + File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); + } + + Console.WriteLine("Done!"); + } } private static void PrintHelp() From b365d4c0ba260405d3b6acdad1edb01674079c50 Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 5 Oct 2023 08:58:05 +0200 Subject: [PATCH 02/18] bumps timeout because cluster might need to start more nodes --- Tools/CodexNetDeployer/Deployer.cs | 4 ++-- Tools/CodexNetDeployer/Program.cs | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index d95f6bff..54f9a5c6 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -87,8 +87,8 @@ namespace CodexNetDeployer var configuration = new KubernetesWorkflow.Configuration( kubeConfig, - operationTimeout: TimeSpan.FromSeconds(120), - retryDelay: TimeSpan.FromSeconds(3), + operationTimeout: TimeSpan.FromSeconds(300), + retryDelay: TimeSpan.FromSeconds(10), kubernetesNamespace: config.KubeNamespace); var result = new EntryPoint(log, configuration, string.Empty); diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 291c85cb..8bbf1748 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -23,9 +23,6 @@ public class Program return; } - var deployer = new Deployer(config); - deployer.AnnouncePlugins(); - if (!args.Any(a => a == "-y")) { Console.WriteLine("Does the above config look good? [y/n]"); @@ -35,6 +32,8 @@ public class Program if (config.Replication == 0) { + var deployer = new Deployer(config); + deployer.AnnouncePlugins(); var deployment = deployer.Deploy(); Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); @@ -50,6 +49,8 @@ public class Program config.KubeNamespace = originalNamespace + "-" + i; config.DeployFile = originalDeployFile.ToLowerInvariant().Replace(".json", $"-{i}.json"); + var deployer = new Deployer(config); + deployer.AnnouncePlugins(); var deployment = deployer.Deploy(); Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); From 75e263be319d8ae722333ac1de9b811f5570ff8e Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 07:47:54 +0200 Subject: [PATCH 03/18] Try with script --- Tests/CodexContinuousTests/deploy-and-run.sh | 35 ++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Tests/CodexContinuousTests/deploy-and-run.sh diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh new file mode 100644 index 00000000..e392da43 --- /dev/null +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -0,0 +1,35 @@ +replication=10 + +echo "Deploying..." +cd ../../Tools/CodexNetDeployer +for i in {0..$replication} +do + dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --kube-namespace=codex-continuous-tests-$i \ + --deploy-file=codex-deployment-$i.json \ + --nodes=5 \ + --validators=3 \ + --log-level=Trace \ + --storage-quota=2048 \ + --storage-sell=1024 \ + --min-price=1024 \ + --max-collateral=1024 \ + --max-duration=3600000 \ + --block-ttl=180 \ + --block-mi=120 \ + --block-mn=10000 \ + --metrics=1 \ + --check-connect=1 +done +echo "Starting tests..." +cd ../../Tests/CodexContinousTests +for i in {0..$replication} +do + screen -d -m dotnet run \ + --kube-config=/opt/kubeconfig.yaml \ + --codex-deployment=codex-deployment-$i.json \ + --keep=1 \ + --stop=1 \ + --target-duration=172800 # 48 hours +done From fab855c4bb9ead22d16cac4b72177528ad49c397 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 07:49:55 +0200 Subject: [PATCH 04/18] adds log and data paths --- Tests/CodexContinuousTests/deploy-and-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index e392da43..8c12a4ad 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -29,6 +29,8 @@ do screen -d -m dotnet run \ --kube-config=/opt/kubeconfig.yaml \ --codex-deployment=codex-deployment-$i.json \ + --log-path=logs-$i \ + --data-path=data-$i \ --keep=1 \ --stop=1 \ --target-duration=172800 # 48 hours From c653afa554408d2467971f493ccdc728171237bd Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 07:53:10 +0200 Subject: [PATCH 05/18] trying with script --- Tests/CodexContinuousTests/deploy-and-run.sh | 4 ++- Tools/CodexNetDeployer/Configuration.cs | 4 +-- Tools/CodexNetDeployer/Program.cs | 34 ++++++++++---------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index 8c12a4ad..a4dc6bcf 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -1,3 +1,5 @@ +set -e + replication=10 echo "Deploying..." @@ -23,7 +25,7 @@ do --check-connect=1 done echo "Starting tests..." -cd ../../Tests/CodexContinousTests +cd ../../Tests/CodexContinuousTests for i in {0..$replication} do screen -d -m dotnet run \ diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index fdb8f271..dbf2c89d 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -68,8 +68,8 @@ namespace CodexNetDeployer [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; - [Uniform("replication", "rep", "REPLICATION", false, "Number of times to repeat")] - public int Replication { get; set; } = 0; + //[Uniform("replication", "rep", "REPLICATION", false, "Number of times to repeat")] + //public int Replication { get; set; } = 0; public List Validate() { diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 8bbf1748..2a879b43 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -30,7 +30,7 @@ public class Program Console.WriteLine("I think so too."); } - if (config.Replication == 0) + //if (config.Replication == 0) { var deployer = new Deployer(config); deployer.AnnouncePlugins(); @@ -40,25 +40,25 @@ public class Program File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); Console.WriteLine("Done!"); } - else - { - var originalNamespace = config.KubeNamespace; - var originalDeployFile = config.DeployFile; - for (var i = 0; i < config.Replication; i++) - { - config.KubeNamespace = originalNamespace + "-" + i; - config.DeployFile = originalDeployFile.ToLowerInvariant().Replace(".json", $"-{i}.json"); + //else + //{ + // var originalNamespace = config.KubeNamespace; + // var originalDeployFile = config.DeployFile; + // for (var i = 0; i < config.Replication; i++) + // { + // config.KubeNamespace = originalNamespace + "-" + i; + // config.DeployFile = originalDeployFile.ToLowerInvariant().Replace(".json", $"-{i}.json"); - var deployer = new Deployer(config); - deployer.AnnouncePlugins(); - var deployment = deployer.Deploy(); + // var deployer = new Deployer(config); + // deployer.AnnouncePlugins(); + // var deployment = deployer.Deploy(); - Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); - File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); - } + // Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); + // File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); + // } - Console.WriteLine("Done!"); - } + // Console.WriteLine("Done!"); + //} } private static void PrintHelp() From 092128d77bfbf00f2b7bf2e0a165cef50c8ff86f Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 08:04:11 +0200 Subject: [PATCH 06/18] with working loops this time --- Tests/CodexContinuousTests/deploy-and-run.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index a4dc6bcf..8c482f17 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -4,7 +4,7 @@ replication=10 echo "Deploying..." cd ../../Tools/CodexNetDeployer -for i in {0..$replication} +for i in $( seq 0 $replication) do dotnet run \ --kube-config=/opt/kubeconfig.yaml \ @@ -22,11 +22,12 @@ do --block-mi=120 \ --block-mn=10000 \ --metrics=1 \ - --check-connect=1 + --check-connect=1 \ + -y done echo "Starting tests..." cd ../../Tests/CodexContinuousTests -for i in {0..$replication} +for i in $( seq 0 $replication) do screen -d -m dotnet run \ --kube-config=/opt/kubeconfig.yaml \ From 1a48c40ad7b3def916849f07640e670051391ac0 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 09:48:12 +0200 Subject: [PATCH 07/18] Example of setting codex log topics for libp2p and discv5 --- ProjectPlugins/CodexPlugin/CodexSetup.cs | 22 ++++++++++----- .../CodexPlugin/CodexStartupConfig.cs | 27 ++++++++++++++++--- Tests/CodexTests/BasicTests/ExampleTests.cs | 3 ++- 3 files changed, 41 insertions(+), 11 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index 6ab4d206..0f9f37f8 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -11,11 +11,7 @@ namespace CodexPlugin ICodexSetup At(ILocation location); ICodexSetup WithBootstrapNode(ICodexNode node); ICodexSetup WithLogLevel(CodexLogLevel level); - /// - /// Sets the log level for codex. The default level is INFO and the - /// log level is applied only to the supplied topics. - /// - ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics); + ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics); ICodexSetup WithStorageQuota(ByteSize storageQuota); ICodexSetup WithBlockTTL(TimeSpan duration); ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); @@ -28,6 +24,18 @@ namespace CodexPlugin ICodexSetup WithSimulateProofFailures(uint failEveryNProofs); } + public class CodexLogCustomTopics + { + public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p) + { + DiscV5 = discV5; + Libp2p = libp2p; + } + + public CodexLogLevel DiscV5 { get; set; } + public CodexLogLevel Libp2p { get; set; } + } + public class CodexSetup : CodexStartupConfig, ICodexSetup { public int NumberOfNodes { get; } @@ -61,10 +69,10 @@ namespace CodexPlugin return this; } - public ICodexSetup WithLogLevel(CodexLogLevel level, params string[] topics) + public ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics) { LogLevel = level; - LogTopics = topics; + CustomTopics = customTopics; return this; } diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index a3b63144..7a68bc7a 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -8,7 +8,7 @@ namespace CodexPlugin public string? NameOverride { get; set; } public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation; public CodexLogLevel LogLevel { get; set; } - public string[]? LogTopics { get; set; } + public CodexLogCustomTopics? CustomTopics { get; set; } public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public MarketplaceInitialConfig? MarketplaceConfig { get; set; } @@ -22,9 +22,30 @@ namespace CodexPlugin public string LogLevelWithTopics() { var level = LogLevel.ToString()!.ToUpperInvariant(); - if (LogTopics != null && LogTopics.Count() > 0) + if (CustomTopics != null) { - level = $"INFO;{level}: {string.Join(",", LogTopics.Where(s => !string.IsNullOrEmpty(s)))}"; + var discV5Topics = new[] + { + "discv5", + "providers", + "manager", + "cache", + }; + var libp2pTopics = new[] + { + "libp2p", + "multistream", + "switch", + "transport", + "tcptransport", + "semaphore", + "asyncstreamwrapper", + "lpstream" + }; + + level = $"{level};" + + $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" + + $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)}"; } return level; } diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 4049b9a0..0db9bbb8 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using CodexPlugin; using DistTestCore; using GethPlugin; using MetricsPlugin; @@ -13,7 +14,7 @@ namespace Tests.BasicTests [Test] public void CodexLogExample() { - var primary = AddCodex(); + var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); primary.UploadFile(GenerateTestFile(5.MB())); From ab9257ce564a598afd6213ae93b56cc650ddf97a Mon Sep 17 00:00:00 2001 From: benbierens Date: Sat, 7 Oct 2023 10:29:09 +0200 Subject: [PATCH 08/18] moving the deployment json --- Tests/CodexContinuousTests/deploy-and-run.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index 8c482f17..e98a93a7 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -24,6 +24,8 @@ do --metrics=1 \ --check-connect=1 \ -y + + cp codex-deployment-$i.json ../../Tests/CodexContinuousTests done echo "Starting tests..." cd ../../Tests/CodexContinuousTests From 4f764ac4c781cf317e9ff04e8510d22a8a3eb64b Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 07:29:55 +0200 Subject: [PATCH 09/18] Wires up the codex log topics config options. --- Tools/CodexNetDeployer/CodexNodeStarter.cs | 2 +- Tools/CodexNetDeployer/Configuration.cs | 9 ++++-- Tools/CodexNetDeployer/Program.cs | 35 +++++----------------- 3 files changed, 14 insertions(+), 32 deletions(-) diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index f1cfed48..6ed65280 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -36,7 +36,7 @@ namespace CodexNetDeployer codexNode = ci.StartCodexNode(s => { s.WithName(name); - s.WithLogLevel(config.CodexLogLevel); + s.WithLogLevel(config.CodexLogLevel, new CodexLogCustomTopics(config.Discv5LogLevel, config.Libp2pLogLevel)); s.WithStorageQuota(config.StorageQuota!.Value.MB()); s.EnableMarketplace(gethNode, contracts, 100.Eth(), config.InitialTestTokens.TestTokens(), validatorsLeft > 0); s.EnableMetrics(); diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index dbf2c89d..3a371cd7 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -36,6 +36,12 @@ namespace CodexNetDeployer [Uniform("log-level", "l", "LOGLEVEL", true, "Log level used by each Codex node. [Trace, Debug*, Info, Warn, Error]")] public CodexLogLevel CodexLogLevel { get; set; } = CodexLogLevel.Debug; + + [Uniform("log-level-libp2p", "lp2p", "LOGLEVELLIBP2P", true, "Log level for all libp2p topics. [Trace, Debug, Info, Warn*, Error]")] + public CodexLogLevel Libp2pLogLevel { get; set; } = CodexLogLevel.Warn; + + [Uniform("log-level-discv5", "ldv5", "LOGLEVELDISCV5", true, "Log level for all discv5 topics. [Trace, Debug, Info, Warn*, Error]")] + public CodexLogLevel Discv5LogLevel { get; set; } = CodexLogLevel.Warn; [Uniform("test-tokens", "tt", "TESTTOKENS", true, "Initial amount of test-tokens minted for each Codex node.")] public int InitialTestTokens { get; set; } = int.MaxValue; @@ -68,9 +74,6 @@ namespace CodexNetDeployer [Uniform("check-connect", "cc", "CHECKCONNECT", false, "If true, deployer check ensure peer-connectivity between all deployed nodes after deployment. Default is false.")] public bool CheckPeerConnection { get; set; } = false; - //[Uniform("replication", "rep", "REPLICATION", false, "Number of times to repeat")] - //public int Replication { get; set; } = 0; - public List Validate() { var errors = new List(); diff --git a/Tools/CodexNetDeployer/Program.cs b/Tools/CodexNetDeployer/Program.cs index 2a879b43..aad5fab5 100644 --- a/Tools/CodexNetDeployer/Program.cs +++ b/Tools/CodexNetDeployer/Program.cs @@ -23,6 +23,9 @@ public class Program return; } + var deployer = new Deployer(config); + deployer.AnnouncePlugins(); + if (!args.Any(a => a == "-y")) { Console.WriteLine("Does the above config look good? [y/n]"); @@ -30,35 +33,11 @@ public class Program Console.WriteLine("I think so too."); } - //if (config.Replication == 0) - { - var deployer = new Deployer(config); - deployer.AnnouncePlugins(); - var deployment = deployer.Deploy(); + var deployment = deployer.Deploy(); - Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); - File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); - Console.WriteLine("Done!"); - } - //else - //{ - // var originalNamespace = config.KubeNamespace; - // var originalDeployFile = config.DeployFile; - // for (var i = 0; i < config.Replication; i++) - // { - // config.KubeNamespace = originalNamespace + "-" + i; - // config.DeployFile = originalDeployFile.ToLowerInvariant().Replace(".json", $"-{i}.json"); - - // var deployer = new Deployer(config); - // deployer.AnnouncePlugins(); - // var deployment = deployer.Deploy(); - - // Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); - // File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); - // } - - // Console.WriteLine("Done!"); - //} + Console.WriteLine($"Writing deployment file '{config.DeployFile}'..."); + File.WriteAllText(config.DeployFile, JsonConvert.SerializeObject(deployment, Formatting.Indented)); + Console.WriteLine("Done!"); } private static void PrintHelp() From e2a6a2f7925e6777bd12dc357536770165be88fe Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 07:46:48 +0200 Subject: [PATCH 10/18] fixes possible null-ref --- Framework/ArgsUniform/ArgsUniform.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/ArgsUniform/ArgsUniform.cs b/Framework/ArgsUniform/ArgsUniform.cs index db1a6b2a..d987aa71 100644 --- a/Framework/ArgsUniform/ArgsUniform.cs +++ b/Framework/ArgsUniform/ArgsUniform.cs @@ -234,7 +234,7 @@ namespace ArgsUniform private static bool AssignBool(T result, PropertyInfo uniformProperty, object value) { var s = value.ToString(); - if (s == "1" || s.ToLowerInvariant() == "true") + if (s == "1" || (s != null && s.ToLowerInvariant() == "true")) { uniformProperty.SetValue(result, true); } From 24ba0b4f09a5c1b0ccadaf0295cb5f8626fac886 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 08:04:04 +0200 Subject: [PATCH 11/18] Adds missing libp2p topics --- ProjectPlugins/CodexPlugin/CodexStartupConfig.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index 7a68bc7a..4b651dfa 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -40,7 +40,14 @@ namespace CodexPlugin "tcptransport", "semaphore", "asyncstreamwrapper", - "lpstream" + "lpstream", + "mplex", + "mplexchannel", + "noise", + "bufferstream", + "mplexcoder", + "websock", + "ws-session" }; level = $"{level};" + From f3555f27031c6b214b3f18dfde7624f7f2442925 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 08:31:48 +0200 Subject: [PATCH 12/18] Adds node id to downloaded container logs --- Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs | 7 ++++--- Tests/CodexContinuousTests/SingleTestRun.cs | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs index c78ed637..6516b768 100644 --- a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs +++ b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs @@ -16,11 +16,11 @@ namespace ContinuousTests this.log = log; } - public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc) + public void Download(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine) { try { - DownloadLog(targetFile, container, startUtc, endUtc); + DownloadLog(targetFile, container, startUtc, endUtc, openingLine); } catch (Exception ex) { @@ -28,10 +28,11 @@ namespace ContinuousTests } } - private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc) + private void DownloadLog(LogFile targetFile, RunningContainer container, DateTime startUtc, DateTime endUtc, string openingLine) { log.Log($"Downloading log (from ElasticSearch) for container '{container.Name}' within time range: " + $"{startUtc.ToString("o")} - {endUtc.ToString("o")}"); + log.Log(openingLine); var http = CreateElasticSearchHttp(); var queryTemplate = CreateQueryTemplate(container, startUtc, endUtc); diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 85283fa9..1de32833 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -109,7 +109,8 @@ namespace ContinuousTests foreach (var node in nodes) { - elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd); + var openingLine = $"{node.Container.Name} = {node.GetDebugInfo().id}"; + elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine); } } From 0dcf3b0b16617431e8bc2fd97261730eeebacdd7 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 18:56:00 +0200 Subject: [PATCH 13/18] Adds pod and deployment name --- Tests/CodexContinuousTests/StartupChecker.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index abc4dbaf..aae813f5 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -42,6 +42,7 @@ namespace ContinuousTests foreach (var container in deployment.CodexContainers) { log.Log($"Codex environment variables for '{container.Name}':"); + log.Log($"Pod name: {container.Pod.PodInfo.Name} - Deployment name: {container.Pod.DeploymentName}"); var codexVars = container.Recipe.EnvVars; foreach (var vars in codexVars) log.Log(vars.ToString()); log.Log(""); From e4b7e461e9384d0229b16339cea77f5ff97e3b90 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 18:58:15 +0200 Subject: [PATCH 14/18] Adds pod name to downloaded logs --- Tests/CodexContinuousTests/SingleTestRun.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 1de32833..72586ae3 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -109,7 +109,7 @@ namespace ContinuousTests foreach (var node in nodes) { - var openingLine = $"{node.Container.Name} = {node.GetDebugInfo().id}"; + var openingLine = $"{node.Container.Pod.PodInfo.Name} = {node.Container.Name} = {node.GetDebugInfo().id}"; elasticSearchLogDownloader.Download(fixtureLog.CreateSubfile(), node.Container, effectiveStart, effectiveEnd, openingLine); } } From b0610393afb37ff421a6734fc9250c66257e02b9 Mon Sep 17 00:00:00 2001 From: benbierens Date: Sun, 8 Oct 2023 19:11:31 +0200 Subject: [PATCH 15/18] rigged for twoclient test with full container log download --- Tests/CodexContinuousTests/Configuration.cs | 4 ++++ Tests/CodexContinuousTests/SingleTestRun.cs | 4 ++++ Tests/CodexContinuousTests/deploy-and-run.sh | 13 ++++++++----- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/Tests/CodexContinuousTests/Configuration.cs b/Tests/CodexContinuousTests/Configuration.cs index ba4d9aa9..3e51d927 100644 --- a/Tests/CodexContinuousTests/Configuration.cs +++ b/Tests/CodexContinuousTests/Configuration.cs @@ -33,6 +33,10 @@ namespace ContinuousTests [Uniform("cleanup", "cl", "CLEANUP", false, "If set to 1 or 'true', the kubernetes namespace will be deleted after the test run has finished.")] public bool Cleanup { get; set; } = false; + [Uniform("full-container-logs", "fcl", "FULLCONTAINERLOGS", false, "If set to 1 or 'true', container logs downloaded on test failure will download from" + + " the timestamp of the start of the network deployment. Otherwise, logs will start from the test start timestamp.")] + public bool FullContainerLogs { get; set; } = false; + public CodexDeployment CodexDeployment { get; set; } = null!; } diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 72586ae3..ad54dd57 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -104,6 +104,10 @@ namespace ContinuousTests Thread.Sleep(TimeSpan.FromMinutes(1)); var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30)); + if (config.FullContainerLogs) + { + effectiveStart = config.CodexDeployment.Metadata.DeployDateTimeUtc.Subtract(TimeSpan.FromSeconds(30)); + } var effectiveEnd = DateTime.UtcNow; var elasticSearchLogDownloader = new ElasticSearchLogDownloader(entryPoint.Tools, fixtureLog); diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/CodexContinuousTests/deploy-and-run.sh index e98a93a7..130c534f 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/CodexContinuousTests/deploy-and-run.sh @@ -1,6 +1,6 @@ set -e -replication=10 +replication=5 echo "Deploying..." cd ../../Tools/CodexNetDeployer @@ -13,14 +13,14 @@ do --nodes=5 \ --validators=3 \ --log-level=Trace \ - --storage-quota=2048 \ + --storage-quota=20480 \ --storage-sell=1024 \ --min-price=1024 \ --max-collateral=1024 \ --max-duration=3600000 \ - --block-ttl=180 \ - --block-mi=120 \ - --block-mn=10000 \ + --block-ttl=99999999 \ + --block-mi=99999999 \ + --block-mn=100 \ --metrics=1 \ --check-connect=1 \ -y @@ -38,5 +38,8 @@ do --data-path=data-$i \ --keep=1 \ --stop=1 \ + --filter=TwoClient \ + --cleanup=1 \ + --full-container-logs=1 \ --target-duration=172800 # 48 hours done From f08986f6c0ce3abdf7477668f0213228c754bec5 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 9 Oct 2023 12:33:35 +0200 Subject: [PATCH 16/18] Setup call to debug blockexchange endpoint. --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 5 +++ ProjectPlugins/CodexPlugin/CodexApiTypes.cs | 42 +++++++++++++++++++ .../CodexPlugin/CodexContainerRecipe.cs | 3 +- ProjectPlugins/CodexPlugin/CodexNode.cs | 6 +++ .../CodexPlugin/CodexStartupConfig.cs | 4 ++ 5 files changed, 59 insertions(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 8e0ac40e..f9d77722 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -44,6 +44,11 @@ namespace CodexPlugin return result; } + public CodexDebugBlockExchangeResponse GetDebugBlockExchange() + { + return Http().HttpGetJson("debug/blockexchange"); + } + public CodexDebugThresholdBreaches GetDebugThresholdBreaches() { return Http().HttpGetJson("debug/loop"); diff --git a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs index 127f4374..3fbc1a2d 100644 --- a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs @@ -122,4 +122,46 @@ namespace CodexPlugin public string state { get; set; } = string.Empty; public string error { get; set; } = string.Empty; } + + public class CodexDebugBlockExchangeResponse + { + public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty(); + public int taskQueue { get; set; } + public int pendingBlocks { get; set; } + + public override string ToString() + { + if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty"; + + return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}"; + } + } + + public class CodexDebugBlockExchangeResponsePeer + { + public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty(); + public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty(); + public int exchanged { get; set; } + + public override string ToString() + { + return $"(blocks:{hasBlocks.Length} wants:{wants.Length})"; + } + } + + public class CodexDebugBlockExchangeResponsePeerHasBlock + { + public string cid { get; set; } = string.Empty; + public bool have { get; set; } + public string price { get; set; } = string.Empty; + } + + public class CodexDebugBlockExchangeResponsePeerWant + { + public string block { get; set; } = string.Empty; + public int priority { get; set; } + public bool cancel { get; set; } + public string wantType { get; set; } = string.Empty; + public bool sendDontHave { get; set; } + } } diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index ebef9932..cf72fb6b 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,8 @@ namespace CodexPlugin { private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; + //private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1875e6b-dist-tests"; public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port"; diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index f4ff8027..4a5940d9 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -13,6 +13,7 @@ namespace CodexPlugin string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); + CodexDebugBlockExchangeResponse GetDebugBlockExchange(); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); void ConnectToPeer(ICodexNode node); @@ -81,6 +82,11 @@ namespace CodexPlugin return CodexAccess.GetDebugPeer(peerId); } + public CodexDebugBlockExchangeResponse GetDebugBlockExchange() + { + return CodexAccess.GetDebugBlockExchange(); + } + public ContentId UploadFile(TrackedFile file) { using var fileStream = File.OpenRead(file.Filename); diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index 4b651dfa..971dd2cc 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -46,6 +46,10 @@ namespace CodexPlugin "noise", "bufferstream", "mplexcoder", + "secure", + "chronosstream", + "connection", + "connmanager", "websock", "ws-session" }; From b3ab369dde4c4f04a253ae6a930629582bffb12f Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 9 Oct 2023 12:59:05 +0200 Subject: [PATCH 17/18] adds peer id --- ProjectPlugins/CodexPlugin/CodexApiTypes.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs index 3fbc1a2d..b347508f 100644 --- a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs @@ -139,6 +139,7 @@ namespace CodexPlugin public class CodexDebugBlockExchangeResponsePeer { + public string peerid { get; set; } = string.Empty; public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty(); public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty(); public int exchanged { get; set; } From b4a187f0b620c40e8a640d503a91d7de89b17172 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 9 Oct 2023 16:35:09 +0200 Subject: [PATCH 18/18] undo custom image --- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index cf72fb6b..ebef9932 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,8 +7,7 @@ namespace CodexPlugin { private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - //private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-1875e6b-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; public const string MetricsPortTag = "metrics_port"; public const string DiscoveryPortTag = "discovery-port";