diff --git a/Framework/Core/PluginManager.cs b/Framework/Core/PluginManager.cs index 27b08fe3..c8bac789 100644 --- a/Framework/Core/PluginManager.cs +++ b/Framework/Core/PluginManager.cs @@ -1,6 +1,6 @@ namespace Core { - internal class PluginManager + internal class PluginManager : IPluginAccess { private readonly List pairs = new List(); @@ -14,6 +14,7 @@ ApplyLogPrefix(plugin, tools); } + AwakePlugins(); } internal void AnnouncePlugins() @@ -43,7 +44,7 @@ } } - internal T GetPlugin() where T : IProjectPlugin + public T GetPlugin() where T : IProjectPlugin { return (T)pairs.Single(p => p.Plugin.GetType() == typeof(T)).Plugin; } @@ -55,6 +56,14 @@ return plugin; } + private void AwakePlugins() + { + foreach (var p in pairs) + { + p.Plugin.Awake(this); + } + } + private void ApplyLogPrefix(IProjectPlugin plugin, PluginTools tools) { if (plugin is IHasLogPrefix hasLogPrefix) diff --git a/Framework/Core/ProjectPlugin.cs b/Framework/Core/ProjectPlugin.cs index 72f3c16a..a777ef80 100644 --- a/Framework/Core/ProjectPlugin.cs +++ b/Framework/Core/ProjectPlugin.cs @@ -4,6 +4,7 @@ namespace Core { public interface IProjectPlugin { + void Awake(IPluginAccess access); void Announce(); void Decommission(); } @@ -18,6 +19,11 @@ namespace Core void AddMetadata(IAddMetadata metadata); } + public interface IPluginAccess + { + T GetPlugin() where T : IProjectPlugin; + } + public static class ProjectPlugin { /// diff --git a/Framework/DiscordRewards/CheckConfig.cs b/Framework/DiscordRewards/CheckConfig.cs deleted file mode 100644 index 34425cea..00000000 --- a/Framework/DiscordRewards/CheckConfig.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Utils; - -namespace DiscordRewards -{ - public class CheckConfig - { - public CheckType Type { get; set; } - public ulong MinNumberOfHosts { get; set; } - public ByteSize MinSlotSize { get; set; } = 0.Bytes(); - public TimeSpan MinDuration { get; set; } = TimeSpan.Zero; - } - - public enum CheckType - { - Uninitialized, - HostFilledSlot, - HostFinishedSlot, - ClientPostedContract, - ClientStartedContract, - } -} diff --git a/Framework/DiscordRewards/EventsAndErrors.cs b/Framework/DiscordRewards/EventsAndErrors.cs new file mode 100644 index 00000000..7e18f07f --- /dev/null +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -0,0 +1,34 @@ +namespace DiscordRewards +{ + public class EventsAndErrors + { + public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); + public string[] Errors { get; set; } = Array.Empty(); + public ActiveChainAddresses ActiveChainAddresses { get; set; } = new ActiveChainAddresses(); + + public bool HasAny() + { + return + Errors.Length > 0 || + EventsOverview.Length > 0 || + ActiveChainAddresses.HasAny(); + } + } + + public class ChainEventMessage + { + public ulong BlockNumber { get; set; } + public string Message { get; set; } = string.Empty; + } + + public class ActiveChainAddresses + { + public string[] Hosts { get; set; } = Array.Empty(); + public string[] Clients { get; set; } = Array.Empty(); + + public bool HasAny() + { + return Hosts.Length > 0 || Clients.Length > 0; + } + } +} diff --git a/Framework/DiscordRewards/GiveRewardsCommand.cs b/Framework/DiscordRewards/GiveRewardsCommand.cs deleted file mode 100644 index 3aae088b..00000000 --- a/Framework/DiscordRewards/GiveRewardsCommand.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace DiscordRewards -{ - public class GiveRewardsCommand - { - public RewardUsersCommand[] Rewards { get; set; } = Array.Empty(); - public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); - public string[] Errors { get; set; } = Array.Empty(); - - public bool HasAny() - { - return Rewards.Any() || EventsOverview.Any(); - } - } - - public class RewardUsersCommand - { - public ulong RewardId { get; set; } - public string[] UserAddresses { get; set; } = Array.Empty(); - } - - public class ChainEventMessage - { - public ulong BlockNumber { get; set; } - public string Message { get; set; } = string.Empty; - } -} diff --git a/Framework/DiscordRewards/RewardConfig.cs b/Framework/DiscordRewards/RewardConfig.cs deleted file mode 100644 index dda0dfe1..00000000 --- a/Framework/DiscordRewards/RewardConfig.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace DiscordRewards -{ - public class RewardConfig - { - public const string UsernameTag = ""; - - public RewardConfig(ulong roleId, string message, CheckConfig checkConfig) - { - RoleId = roleId; - Message = message; - CheckConfig = checkConfig; - } - - public ulong RoleId { get; } - public string Message { get; } - public CheckConfig CheckConfig { get; } - } -} diff --git a/Framework/DiscordRewards/RewardRepo.cs b/Framework/DiscordRewards/RewardRepo.cs deleted file mode 100644 index 1c97caf9..00000000 --- a/Framework/DiscordRewards/RewardRepo.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace DiscordRewards -{ - public class RewardRepo - { - private static string Tag => RewardConfig.UsernameTag; - - public RewardConfig[] Rewards { get; } = new RewardConfig[0]; - - // Example configuration, from test server: - //{ - // // Filled any slot - // new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig - // { - // Type = CheckType.HostFilledSlot - // }), - - // // Finished any slot - // new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot - // }), - - // // Finished a sizable slot - // new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }), - - // // Posted any contract - // new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig - // { - // Type = CheckType.ClientPostedContract - // }), - - // // Started any contract - // new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig - // { - // Type = CheckType.ClientStartedContract - // }), - - // // Started a sizable contract - // new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.ClientStartedContract, - // MinNumberOfHosts = 4, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }) - //}; - } -} diff --git a/Framework/Logging/BaseLog.cs b/Framework/Logging/BaseLog.cs index 2c79fd39..56ff66ec 100644 --- a/Framework/Logging/BaseLog.cs +++ b/Framework/Logging/BaseLog.cs @@ -7,8 +7,10 @@ namespace Logging void Log(string message); void Debug(string message = "", int skipFrames = 0); void Error(string message); + void Raw(string message); void AddStringReplace(string from, string to); LogFile CreateSubfile(string addName, string ext = "log"); + string GetFullName(); } public abstract class BaseLog : ILog @@ -28,7 +30,8 @@ namespace Logging } protected bool IsDebug { get; private set; } - protected abstract string GetFullName(); + + public abstract string GetFullName(); public LogFile LogFile { @@ -60,6 +63,11 @@ namespace Logging Log(msg); } + public void Raw(string message) + { + LogFile.WriteRaw(message); + } + public virtual void AddStringReplace(string from, string to) { if (string.IsNullOrWhiteSpace(from)) return; diff --git a/Framework/Logging/ConsoleLog.cs b/Framework/Logging/ConsoleLog.cs index ab67a400..2604e64f 100644 --- a/Framework/Logging/ConsoleLog.cs +++ b/Framework/Logging/ConsoleLog.cs @@ -2,7 +2,7 @@ { public class ConsoleLog : BaseLog { - protected override string GetFullName() + public override string GetFullName() { return "CONSOLE"; } diff --git a/Framework/Logging/FileLog.cs b/Framework/Logging/FileLog.cs index cc5e9952..86bc1985 100644 --- a/Framework/Logging/FileLog.cs +++ b/Framework/Logging/FileLog.cs @@ -9,7 +9,7 @@ public string FullFilename { get; } - protected override string GetFullName() + public override string GetFullName() { return FullFilename; } diff --git a/Framework/Logging/LogPrefixer.cs b/Framework/Logging/LogPrefixer.cs index 6fe6033b..f0f303d6 100644 --- a/Framework/Logging/LogPrefixer.cs +++ b/Framework/Logging/LogPrefixer.cs @@ -17,7 +17,6 @@ public string Prefix { get; set; } = string.Empty; - public LogFile CreateSubfile(string addName, string ext = "log") { return backingLog.CreateSubfile(addName, ext); @@ -42,5 +41,15 @@ { backingLog.AddStringReplace(from, to); } + + public void Raw(string message) + { + backingLog.Raw(message); + } + + public string GetFullName() + { + return backingLog.GetFullName(); + } } } diff --git a/Framework/Logging/LogSplitter.cs b/Framework/Logging/LogSplitter.cs index 563f5812..8de462c9 100644 --- a/Framework/Logging/LogSplitter.cs +++ b/Framework/Logging/LogSplitter.cs @@ -29,11 +29,21 @@ OnAll(l => l.Error(message)); } + public string GetFullName() + { + return targetLogs.First().GetFullName(); + } + public void Log(string message) { OnAll(l => l.Log(message)); } + public void Raw(string message) + { + OnAll(l => l.Raw(message)); + } + private void OnAll(Action action) { foreach (var t in targetLogs) action(t); diff --git a/Framework/Logging/NullLog.cs b/Framework/Logging/NullLog.cs index a9cf8d2d..f8094f32 100644 --- a/Framework/Logging/NullLog.cs +++ b/Framework/Logging/NullLog.cs @@ -2,11 +2,9 @@ { public class NullLog : BaseLog { - public string FullFilename { get; set; } = "NULL"; - - protected override string GetFullName() + public override string GetFullName() { - return FullFilename; + return "NULL"; } public override void Log(string message) diff --git a/Framework/Utils/RandomUtils.cs b/Framework/Utils/RandomUtils.cs index f4f28dd4..9f127167 100644 --- a/Framework/Utils/RandomUtils.cs +++ b/Framework/Utils/RandomUtils.cs @@ -34,10 +34,28 @@ var source = items.ToList(); while (source.Any()) { - result.Add(RandomUtils.PickOneRandom(source)); + result.Add(PickOneRandom(source)); } return result.ToArray(); } } + + public static string GenerateRandomString(long requiredLength) + { + lock (@lock) + { + var result = ""; + while (result.Length < requiredLength) + { + var remaining = requiredLength - result.Length; + var len = Math.Min(1024, remaining); + var bytes = new byte[len]; + random.NextBytes(bytes); + result += string.Join("", bytes.Select(b => b.ToString())); + } + + return result.Substring(0, Convert.ToInt32(requiredLength)); + } + } } } diff --git a/Framework/Utils/Str.cs b/Framework/Utils/Str.cs index aea19ec8..ab8fe46d 100644 --- a/Framework/Utils/Str.cs +++ b/Framework/Utils/Str.cs @@ -4,8 +4,11 @@ { public static string Between(string input, string open, string close) { - var openIndex = input.IndexOf(open) + open.Length; + var openI = input.IndexOf(open); + if (openI == -1) return input; + var openIndex = openI + open.Length; var closeIndex = input.LastIndexOf(close); + if (closeIndex == -1) return input; return input.Substring(openIndex, closeIndex - openIndex); } diff --git a/Framework/WebUtils/Http.cs b/Framework/WebUtils/Http.cs index 7c9a91ad..e4931996 100644 --- a/Framework/WebUtils/Http.cs +++ b/Framework/WebUtils/Http.cs @@ -20,11 +20,6 @@ namespace WebUtils private readonly Action onClientCreated; private readonly string id; - internal Http(string id, ILog log, IWebCallTimeSet timeSet) - : this(id, log, timeSet, DoNothing) - { - } - internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action onClientCreated) { this.id = id; @@ -89,9 +84,5 @@ namespace WebUtils onClientCreated(client); return client; } - - private static void DoNothing(HttpClient client) - { - } } } diff --git a/Framework/WebUtils/HttpFactory.cs b/Framework/WebUtils/HttpFactory.cs index 4d5d7c4c..8120527c 100644 --- a/Framework/WebUtils/HttpFactory.cs +++ b/Framework/WebUtils/HttpFactory.cs @@ -13,16 +13,28 @@ namespace WebUtils { private readonly ILog log; private readonly IWebCallTimeSet defaultTimeSet; + private readonly Action factoryOnClientCreated; public HttpFactory(ILog log) : this (log, new DefaultWebCallTimeSet()) { } + public HttpFactory(ILog log, Action onClientCreated) + : this(log, new DefaultWebCallTimeSet(), onClientCreated) + { + } + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet) + : this(log, defaultTimeSet, DoNothing) + { + } + + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet, Action onClientCreated) { this.log = log; this.defaultTimeSet = defaultTimeSet; + this.factoryOnClientCreated = onClientCreated; } public IHttp CreateHttp(string id, Action onClientCreated) @@ -32,12 +44,20 @@ namespace WebUtils public IHttp CreateHttp(string id, Action onClientCreated, IWebCallTimeSet ts) { - return new Http(id, log, ts, onClientCreated); + return new Http(id, log, ts, (c) => + { + factoryOnClientCreated(c); + onClientCreated(c); + }); } public IHttp CreateHttp(string id) { - return new Http(id, log, defaultTimeSet); + return new Http(id, log, defaultTimeSet, factoryOnClientCreated); + } + + private static void DoNothing(HttpClient client) + { } } } diff --git a/ProjectPlugins/CodexClient/CodexTypes.cs b/ProjectPlugins/CodexClient/CodexTypes.cs index 9085aa2c..f9a0d9d2 100644 --- a/ProjectPlugins/CodexClient/CodexTypes.cs +++ b/ProjectPlugins/CodexClient/CodexTypes.cs @@ -17,6 +17,7 @@ namespace CodexClient { public string Version { get; set; } = string.Empty; public string Revision { get; set; } = string.Empty; + public string Contracts { get; set; } = string.Empty; public bool IsValid() { diff --git a/ProjectPlugins/CodexClient/Mapper.cs b/ProjectPlugins/CodexClient/Mapper.cs index 43bb15d0..16c156bd 100644 --- a/ProjectPlugins/CodexClient/Mapper.cs +++ b/ProjectPlugins/CodexClient/Mapper.cs @@ -168,7 +168,8 @@ namespace CodexClient return new DebugInfoVersion { Version = obj.Version, - Revision = obj.Revision + Revision = obj.Revision, + Contracts = obj.Contracts }; } diff --git a/ProjectPlugins/CodexClient/openapi.yaml b/ProjectPlugins/CodexClient/openapi.yaml index fd66648e..d59cbcfa 100644 --- a/ProjectPlugins/CodexClient/openapi.yaml +++ b/ProjectPlugins/CodexClient/openapi.yaml @@ -124,6 +124,9 @@ components: revision: type: string example: 0c647d8 + contracts: + type: string + example: 0b537c7 PeersTable: type: object diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs index 66adf592..b46f3c96 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsContainerRecipe.cs @@ -1,4 +1,5 @@ -using GethPlugin; +using CodexClient; +using GethPlugin; using KubernetesWorkflow; using KubernetesWorkflow.Recipe; @@ -6,13 +7,17 @@ namespace CodexContractsPlugin { public class CodexContractsContainerRecipe : ContainerRecipeFactory { - public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests"; - public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json"; public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json"; + private readonly DebugInfoVersion versionInfo; public override string AppName => "codex-contracts"; - public override string Image => DockerImage; + public override string Image => GetContractsDockerImage(); + + public CodexContractsContainerRecipe(DebugInfoVersion versionInfo) + { + this.versionInfo = versionInfo; + } protected override void Initialize(StartupConfig startupConfig) { @@ -26,5 +31,10 @@ namespace CodexContractsPlugin AddEnvVar("HARDHAT_NETWORK", "codexdisttestnetwork"); AddEnvVar("KEEP_ALIVE", "1"); } + + private string GetContractsDockerImage() + { + return $"codexstorage/codex-contracts-eth:sha-{versionInfo.Contracts}-dist-tests"; + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs index 416b17aa..1d122d3b 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.cs @@ -16,6 +16,10 @@ namespace CodexContractsPlugin public string LogPrefix => "(CodexContracts) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Loaded Codex-Marketplace SmartContracts"); @@ -23,16 +27,16 @@ namespace CodexContractsPlugin public void AddMetadata(IAddMetadata metadata) { - metadata.Add("codexcontractsid", CodexContractsContainerRecipe.DockerImage); + metadata.Add("codexcontractsid", "dynamic"); } public void Decommission() { } - public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode) + public CodexContractsDeployment DeployContracts(CoreInterface ci, IGethNode gethNode, CodexClient.DebugInfoVersion versionInfo) { - return starter.Deploy(ci, gethNode); + return starter.Deploy(ci, gethNode, versionInfo); } public ICodexContracts WrapDeploy(IGethNode gethNode, CodexContractsDeployment deployment) diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj index 24f87068..34f6cad1 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsPlugin.csproj @@ -13,6 +13,7 @@ + diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs index be74a3b2..c7e900be 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsStarter.cs @@ -1,4 +1,5 @@ -using CodexContractsPlugin.Marketplace; +using CodexClient; +using CodexContractsPlugin.Marketplace; using Core; using GethPlugin; using KubernetesWorkflow; @@ -18,7 +19,7 @@ namespace CodexContractsPlugin this.tools = tools; } - public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode) + public CodexContractsDeployment Deploy(CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo) { Log("Starting Codex SmartContracts container..."); @@ -26,7 +27,10 @@ namespace CodexContractsPlugin var startupConfig = CreateStartupConfig(gethNode); startupConfig.NameOverride = "codex-contracts"; - var containers = workflow.Start(1, new CodexContractsContainerRecipe(), startupConfig).WaitForOnline(); + var recipe = new CodexContractsContainerRecipe(versionInfo); + Log($"Using image: {recipe.Image}"); + + var containers = workflow.Start(1, recipe, startupConfig).WaitForOnline(); if (containers.Containers.Length != 1) throw new InvalidOperationException("Expected 1 Codex contracts container to be created. Test infra failure."); var container = containers.Containers[0]; diff --git a/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs index 7d68a860..d07e25f7 100644 --- a/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexContractsPlugin/CoreInterfaceExtensions.cs @@ -1,13 +1,14 @@ -using Core; +using CodexClient; +using Core; using GethPlugin; namespace CodexContractsPlugin { public static class CoreInterfaceExtensions { - public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode) + public static CodexContractsDeployment DeployCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo) { - return Plugin(ci).DeployContracts(ci, gethNode); + return Plugin(ci).DeployContracts(ci, gethNode, versionInfo); } public static ICodexContracts WrapCodexContractsDeployment(this CoreInterface ci, IGethNode gethNode, CodexContractsDeployment deployment) @@ -15,9 +16,9 @@ namespace CodexContractsPlugin return Plugin(ci).WrapDeploy(gethNode, deployment); } - public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode) + public static ICodexContracts StartCodexContracts(this CoreInterface ci, IGethNode gethNode, DebugInfoVersion versionInfo) { - var deployment = DeployCodexContracts(ci, gethNode); + var deployment = DeployCodexContracts(ci, gethNode, versionInfo); return WrapCodexContractsDeployment(ci, gethNode, deployment); } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index 556ef152..e1354806 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -17,6 +17,10 @@ namespace CodexDiscordBotPlugin public string LogPrefix => "(DiscordBot) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Codex DiscordBot (BiblioTech) loaded."); diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 656c6588..4b190b34 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -10,7 +10,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "1A-F7-DF-C3-E1-C6-98-FF-32-20-21-9B-26-40-B0-51-08-35-C2-E7-DB-41-49-93-60-A9-CE-47-B5-AD-3D-A3"; + private const string OpenApiYamlHash = "06-B9-41-E8-C8-6C-DE-01-86-83-F3-9A-E4-AC-E7-30-D9-E6-64-60-E0-21-81-9E-4E-C5-93-77-2C-71-79-14"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 9dfcf2cf..6a0b76e7 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,6 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; @@ -16,11 +15,15 @@ namespace CodexPlugin // Used by tests for time-constraint assertions. public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); + private readonly CodexDockerImage codexDockerImage; public override string AppName => "codex"; - public override string Image => GetDockerImage(); + public override string Image => codexDockerImage.GetCodexDockerImage(); - public static string DockerImageOverride { get; set; } = string.Empty; + public CodexContainerRecipe(CodexDockerImage codexDockerImage) + { + this.codexDockerImage = codexDockerImage; + } protected override void Initialize(StartupConfig startupConfig) { @@ -163,13 +166,5 @@ namespace CodexPlugin // Default Codex quota: 8 Gb, using +20% to be safe. return 8.GB().Multiply(1.2); } - - private string GetDockerImage() - { - var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); - if (!string.IsNullOrEmpty(image)) return image; - if (!string.IsNullOrEmpty(DockerImageOverride)) return DockerImageOverride; - return DefaultDockerImage; - } } } diff --git a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs new file mode 100644 index 00000000..bbb09c0b --- /dev/null +++ b/ProjectPlugins/CodexPlugin/CodexDockerImage.cs @@ -0,0 +1,17 @@ +namespace CodexPlugin +{ + public class CodexDockerImage + { + private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests"; + + public static string Override { get; set; } = string.Empty; + + public string GetCodexDockerImage() + { + var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); + if (!string.IsNullOrEmpty(image)) return image; + if (!string.IsNullOrEmpty(Override)) return Override; + return DefaultDockerImage; + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 02fbeaa6..0c8f4510 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -13,12 +13,15 @@ namespace CodexPlugin private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory(); private readonly ProcessControlMap processControlMap = new ProcessControlMap(); + private readonly CodexDockerImage codexDockerImage = new CodexDockerImage(); + private readonly CodexContainerRecipe recipe; private readonly CodexWrapper codexWrapper; public CodexPlugin(IPluginTools tools) { this.tools = tools; + recipe = new CodexContainerRecipe(codexDockerImage); codexStarter = CreateCodexStarter(); codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory); } @@ -28,7 +31,7 @@ namespace CodexPlugin if (UseContainers) { Log("Using Containerized Codex instances"); - return new ContainerCodexStarter(tools, processControlMap); + return new ContainerCodexStarter(tools, recipe, processControlMap); } Log("Using Binary Codex instances"); @@ -37,8 +40,14 @@ namespace CodexPlugin public string LogPrefix => "(Codex) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { + // give codex docker image to contracts plugin. + Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}"); } diff --git a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs b/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs index 0627bc1f..4103fb8c 100644 --- a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs @@ -10,12 +10,13 @@ namespace CodexPlugin { private readonly IPluginTools pluginTools; private readonly ProcessControlMap processControlMap; - private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); + private readonly CodexContainerRecipe recipe; private readonly ApiChecker apiChecker; - public ContainerCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap) + public ContainerCodexStarter(IPluginTools pluginTools, CodexContainerRecipe recipe, ProcessControlMap processControlMap) { this.pluginTools = pluginTools; + this.recipe = recipe; this.processControlMap = processControlMap; apiChecker = new ApiChecker(pluginTools); } diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs index 9f1b76c9..90bc1cb1 100644 --- a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs +++ b/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs @@ -40,7 +40,7 @@ namespace CodexNetDeployer Log($"Codex docker image will be built in path '{repoPath}'."); Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,"); Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image."); - CodexContainerRecipe.DockerImageOverride = $"Using docker image locally built in path '{repoPath}'."; + CodexDockerImage.Override = $"Using docker image locally built in path '{repoPath}'."; } public void Build() @@ -62,7 +62,7 @@ namespace CodexNetDeployer Docker("push", customImage); - CodexContainerRecipe.DockerImageOverride = customImage; + CodexDockerImage.Override = customImage; Log("Image pushed. Good to go!"); } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs index afd8d1a2..95b8dde6 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs @@ -23,16 +23,27 @@ namespace CodexPlugin.OverwatchSupport converter = new CodexLogConverter(writer, config, identityMap); } - public void Finalize(string outputFilepath) + public void FinalizeWriter() { log.Log("Finalizing Codex transcript..."); writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); - writer.Write(outputFilepath); + writer.Write(GetOutputFullPath()); log.Log("Done"); } + private string GetOutputFullPath() + { + var outputPath = Path.GetDirectoryName(log.GetFullName()); + if (outputPath == null) throw new Exception("Logfile path is null"); + var filename = Path.GetFileNameWithoutExtension(log.GetFullName()); + if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); + var outputFile = Path.Combine(outputPath, filename + "_" + config.OutputPath); + if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; + return outputFile; + } + public ICodexNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs index 247494c8..112f5b16 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs +++ b/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs @@ -2,11 +2,13 @@ { public class CodexTranscriptWriterConfig { - public CodexTranscriptWriterConfig(bool includeBlockReceivedEvents) + public CodexTranscriptWriterConfig(string outputPath, bool includeBlockReceivedEvents) { + OutputPath = outputPath; IncludeBlockReceivedEvents = includeBlockReceivedEvents; } + public string OutputPath { get; } public bool IncludeBlockReceivedEvents { get; } } } diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs index c290c0d5..e1c60c24 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunPlugin.cs @@ -13,6 +13,10 @@ namespace DeployAndRunPlugin this.tools = tools; } + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log("Deploy-and-Run plugin loaded."); diff --git a/ProjectPlugins/GethPlugin/GethPlugin.cs b/ProjectPlugins/GethPlugin/GethPlugin.cs index 07249eca..76f92314 100644 --- a/ProjectPlugins/GethPlugin/GethPlugin.cs +++ b/ProjectPlugins/GethPlugin/GethPlugin.cs @@ -16,6 +16,10 @@ namespace GethPlugin public string LogPrefix => "(Geth) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Loaded Geth plugin."); diff --git a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs index a796ba06..75e27eca 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsPlugin.cs @@ -18,6 +18,10 @@ namespace MetricsPlugin public string LogPrefix => "(Metrics) "; + public void Awake(IPluginAccess access) + { + } + public void Announce() { tools.GetLog().Log($"Prometheus plugin loaded with '{starter.GetPrometheusId()}'."); diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/CodexContinuousTests/ContinuousTestRunner.cs index fde65aa3..edec4e43 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/CodexContinuousTests/ContinuousTestRunner.cs @@ -27,7 +27,7 @@ namespace ContinuousTests var startTime = DateTime.UtcNow; var overviewLog = new LogSplitter( - new FixtureLog(logConfig, startTime, config.CodexDeployment.Id, "Overview"), + FixtureLog.Create(logConfig, startTime, config.CodexDeployment.Id, "Overview"), new ConsoleLog() ); var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id, diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index f2b3c64f..ae18f7e9 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -33,7 +33,7 @@ namespace ContinuousTests var entryPoint = CreateEntryPoint(); // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network. // Therefore, we use the image of the bootstrap node. - CodexContainerRecipe.DockerImageOverride = bootstrapNode.GetImageName(); + CodexDockerImage.Override = bootstrapNode.GetImageName(); try { diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 0606c8e4..93870406 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -37,7 +37,7 @@ namespace ContinuousTests this.handle = handle; this.cancelToken = cancelToken; testName = handle.Test.GetType().Name; - fixtureLog = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName); + fixtureLog = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName); entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, fixtureLog); ApplyLogReplacements(fixtureLog, startupChecker); @@ -81,17 +81,11 @@ namespace ContinuousTests OverviewLog($" > Test passed. ({Time.FormatDuration(duration)})"); UpdateStatusLogPassed(testStart, duration); - if (!config.KeepPassedTestLogs) - { - fixtureLog.Delete(); - } - resultHandler(true); } catch (Exception ex) { fixtureLog.Error("Test run failed with exception: " + ex); - fixtureLog.MarkAsFailed(); UpdateStatusLogFailed(testStart, duration, ex.ToString()); DownloadContainerLogs(testStart); diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index 0914b3d1..2febf14e 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -22,7 +22,7 @@ namespace ContinuousTests public void Check() { - var log = new FixtureLog(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id, + var log = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id, "StartupChecks"); log.Log("Starting continuous test run..."); IncludeDeploymentConfiguration(log); @@ -90,7 +90,7 @@ namespace ContinuousTests } } - private void CheckCodexNodes(BaseLog log, Configuration config) + private void CheckCodexNodes(ILog log, Configuration config) { throw new NotImplementedException(); diff --git a/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs b/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs index 263a9e8b..4b9139e6 100644 --- a/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs +++ b/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs @@ -50,8 +50,9 @@ namespace CodexReleaseTests.DataTests var blockTtl = TimeSpan.FromMinutes(1.0); var interval = TimeSpan.FromSeconds(10.0); + var bootstrapNode = StartCodex(); var geth = StartGethNode(s => s.IsMiner()); - var contracts = Ci.StartCodexContracts(geth); + var contracts = Ci.StartCodexContracts(geth, bootstrapNode.Version); var node = StartCodex(s => s .EnableMarketplace(geth, contracts, m => m.WithInitial(100.Eth(), 100.Tst())) .WithBlockTTL(blockTtl) diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index e6711e34..b209ccc4 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -3,7 +3,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; -using DistTestCore; using GethPlugin; using Nethereum.Hex.HexConvertors.Extensions; using NUnit.Framework; @@ -13,32 +12,26 @@ namespace CodexReleaseTests.MarketTests { public abstract class MarketplaceAutoBootstrapDistTest : AutoBootstrapDistTest { - private readonly Dictionary handles = new Dictionary(); + private MarketplaceHandle handle = null!; protected const int StartingBalanceTST = 1000; protected const int StartingBalanceEth = 10; - protected override void LifecycleStart(TestLifecycle lifecycle) + [SetUp] + public void SetupMarketplace() { - base.LifecycleStart(lifecycle); var geth = StartGethNode(s => s.IsMiner()); - var contracts = Ci.StartCodexContracts(geth); - handles.Add(lifecycle, new MarketplaceHandle(geth, contracts)); - } - - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) - { - base.LifecycleStop(lifecycle, result); - handles.Remove(lifecycle); + var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); + handle = new MarketplaceHandle(geth, contracts); } protected IGethNode GetGeth() { - return handles[Get()].Geth; + return handle.Geth; } protected ICodexContracts GetContracts() { - return handles[Get()].Contracts; + return handle.Contracts; } protected TimeSpan GetPeriodDuration() @@ -195,6 +188,8 @@ namespace CodexReleaseTests.MarketTests var expectedBalance = StartingBalanceTST.Tst() - GetContractFinalCost(pricePerBytePerSecond, contract, hosts); AssertTstBalance(client, expectedBalance, "Client balance incorrect."); + + Log($"Client has paid for contract. Balance: {expectedBalance}"); } protected void AssertHostsWerePaidForContract(TestToken pricePerBytePerSecond, IStoragePurchaseContract contract, ICodexNodeGroup hosts) @@ -214,7 +209,9 @@ namespace CodexReleaseTests.MarketTests foreach (var pair in expectedBalances) { - AssertTstBalance(pair.Key, pair.Value, "Host was not paid for storage."); + AssertTstBalance(pair.Key, pair.Value, $"Host {pair.Key} was not paid for storage."); + + Log($"Host {pair.Key} was paid for storage. Balance: {pair.Value}"); } } diff --git a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs b/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs index 9b9f4bbf..f6cba228 100644 --- a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs +++ b/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs @@ -1,11 +1,5 @@ -using CodexPlugin; -using CodexTests; +using CodexTests; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Utils; namespace CodexReleaseTests.NodeTests diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/CodexReleaseTests/Parallelism.cs index a1b26c73..d589af32 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/CodexReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -[assembly: LevelOfParallelism(1)] -namespace CodexReleaseTests.DataTests +[assembly: LevelOfParallelism(10)] +namespace CodexReleaseTests { } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 691f19ed..93732405 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -12,70 +12,45 @@ using Assert = NUnit.Framework.Assert; namespace DistTestCore { [Parallelizable(ParallelScope.All)] + [FixtureLifeCycle(LifeCycle.InstancePerTestCase)] public abstract class DistTest { - private const string TestNamespacePrefix = "cdx-"; - private readonly Configuration configuration = new Configuration(); - private readonly Assembly[] testAssemblies; + private static readonly Global global = new Global(); private readonly FixtureLog fixtureLog; private readonly StatusLog statusLog; - private readonly object lifecycleLock = new object(); - private readonly EntryPoint globalEntryPoint; - private readonly Dictionary lifecycles = new Dictionary(); - private readonly string deployId; - + private readonly TestLifecycle lifecycle; + private readonly string deployId = NameUtils.MakeDeployId(); + public DistTest() { - var assemblies = AppDomain.CurrentDomain.GetAssemblies(); - testAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); - - deployId = NameUtils.MakeDeployId(); - - var logConfig = configuration.GetLogConfig(); + var logConfig = global.Configuration.GetLogConfig(); var startTime = DateTime.UtcNow; - fixtureLog = new FixtureLog(logConfig, startTime, deployId); + fixtureLog = FixtureLog.Create(logConfig, startTime, deployId); statusLog = new StatusLog(logConfig, startTime, "dist-tests", deployId); - globalEntryPoint = new EntryPoint(fixtureLog, configuration.GetK8sConfiguration(new DefaultK8sTimeSet(), TestNamespacePrefix), configuration.GetFileManagerFolder()); + fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus()); + + lifecycle = new TestLifecycle(fixtureLog.CreateTestLog(startTime), global.Configuration, + GetWebCallTimeSet(), + GetK8sTimeSet(), + Global.TestNamespacePrefix + Guid.NewGuid().ToString(), + deployId, + ShouldWaitForCleanup() + ); Initialize(fixtureLog); } [OneTimeSetUp] - public void GlobalSetup() + public static void GlobalSetup() { - fixtureLog.Log($"Distributed Tests are starting..."); - globalEntryPoint.Announce(); - - // Previous test run may have been interrupted. - // Begin by cleaning everything up. - try - { - Stopwatch.Measure(fixtureLog, "Global setup", () => - { - globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true); - }); - } - catch (Exception ex) - { - GlobalTestFailure.HasFailed = true; - fixtureLog.Error($"Global setup cleanup failed with: {ex}"); - throw; - } - - fixtureLog.Log("Test framework revision: " + GitInfo.GetStatus()); - fixtureLog.Log("Global setup cleanup successful"); + global.Setup(); } [OneTimeTearDown] - public void GlobalTearDown() + public static void GlobalTearDown() { - globalEntryPoint.Decommission( - // There shouldn't be any of either, but clean everything up regardless. - deleteKubernetesResources: true, - deleteTrackedFiles: true, - waitTillDone: true - ); + global.TearDown(); } [SetUp] @@ -85,10 +60,6 @@ namespace DistTestCore { Assert.Inconclusive("Skip test: Previous test failed during clean up."); } - else - { - CreateNewTestLifecycle(); - } } [TearDown] @@ -109,18 +80,18 @@ namespace DistTestCore { get { - return Get().CoreInterface; + return lifecycle.CoreInterface; } } public TrackedFile GenerateTestFile(ByteSize size, string label = "") { - return Get().GenerateTestFile(size, label); + return lifecycle.GenerateTestFile(size, label); } public TrackedFile GenerateTestFile(Action options, string label = "") { - return Get().GenerateTestFile(options, label); + return lifecycle.GenerateTestFile(options, label); } /// @@ -129,12 +100,22 @@ namespace DistTestCore /// public void ScopedTestFiles(Action action) { - Get().GetFileManager().ScopedFiles(action); + lifecycle.GetFileManager().ScopedFiles(action); } public ILog GetTestLog() { - return Get().Log; + return lifecycle.Log; + } + + public IFileManager GetFileManager() + { + return lifecycle.GetFileManager(); + } + + public string GetTestNamespace() + { + return lifecycle.TestNamespace; } public void Log(string msg) @@ -151,64 +132,24 @@ namespace DistTestCore public void Measure(string name, Action action) { - Stopwatch.Measure(Get().Log, name, action); + Stopwatch.Measure(lifecycle.Log, name, action); } protected TimeRange GetTestRunTimeRange() { - return new TimeRange(Get().TestStart, DateTime.UtcNow); + return new TimeRange(lifecycle.TestStart, DateTime.UtcNow); } protected virtual void Initialize(FixtureLog fixtureLog) { } - protected virtual void LifecycleStart(TestLifecycle lifecycle) - { - } - - protected virtual void LifecycleStop(TestLifecycle lifecycle, DistTestResult testResult) - { - } - protected virtual void CollectStatusLogData(TestLifecycle lifecycle, Dictionary data) { } - protected TestLifecycle Get() - { - lock (lifecycleLock) - { - return lifecycles[GetCurrentTestName()]; - } - } - - private void CreateNewTestLifecycle() - { - var testName = GetCurrentTestName(); - fixtureLog.WriteLogTag(); - Stopwatch.Measure(fixtureLog, $"Setup for {testName}", () => - { - lock (lifecycleLock) - { - var testNamespace = TestNamespacePrefix + Guid.NewGuid().ToString(); - var lifecycle = new TestLifecycle( - fixtureLog.CreateTestLog(), - configuration, - GetWebCallTimeSet(), - GetK8sTimeSet(), - testNamespace, - deployId, - ShouldWaitForCleanup()); - lifecycles.Add(testName, lifecycle); - LifecycleStart(lifecycle); - } - }); - } - private void DisposeTestLifecycle() { - var lifecycle = Get(); var testResult = GetTestResult(); var testDuration = lifecycle.GetTestDuration(); var data = lifecycle.GetPluginMetadata(); @@ -220,9 +161,7 @@ namespace DistTestCore WriteEndTestLog(lifecycle.Log); IncludeLogsOnTestFailure(lifecycle); - LifecycleStop(lifecycle, testResult); lifecycle.DeleteAllResources(); - lifecycles.Remove(GetCurrentTestName()); }); } @@ -236,11 +175,6 @@ namespace DistTestCore Log(result.Message); Log($"{result.StackTrace}"); } - - if (result.Outcome.Status == TestStatus.Failed) - { - log.MarkAsFailed(); - } } private IWebCallTimeSet GetWebCallTimeSet() @@ -284,7 +218,7 @@ namespace DistTestCore var className = currentTest.ClassName; var methodName = currentTest.MethodName; - var testClasses = testAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); + var testClasses = global.TestAssemblies.SelectMany(a => a.GetTypes()).Where(c => c.FullName == className).ToArray(); var testMethods = testClasses.SelectMany(c => c.GetMethods()).Where(m => m.Name == methodName).ToArray(); return testMethods.Select(m => m.GetCustomAttribute()) @@ -293,24 +227,24 @@ namespace DistTestCore .ToArray(); } + protected IDownloadedLog[] DownloadAllLogs() + { + return lifecycle.DownloadAllLogs(); + } + private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) { var testStatus = TestContext.CurrentContext.Result.Outcome.Status; - if (testStatus == TestStatus.Failed) - { - fixtureLog.MarkAsFailed(); - } - if (ShouldDownloadAllLogs(testStatus)) { lifecycle.Log.Log("Downloading all container logs..."); - lifecycle.DownloadAllLogs(); + DownloadAllLogs(); } } private bool ShouldDownloadAllLogs(TestStatus testStatus) { - if (configuration.AlwaysDownloadContainerLogs) return true; + if (global.Configuration.AlwaysDownloadContainerLogs) return true; if (!IsDownloadingLogsEnabled()) return false; if (testStatus == TestStatus.Failed) { @@ -325,7 +259,7 @@ namespace DistTestCore return $"[{TestContext.CurrentContext.Test.Name}]"; } - private DistTestResult GetTestResult() + public DistTestResult GetTestResult() { var success = TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Passed; var status = TestContext.CurrentContext.Result.Outcome.Status.ToString(); diff --git a/Tests/DistTestCore/Global.cs b/Tests/DistTestCore/Global.cs new file mode 100644 index 00000000..4ba1374b --- /dev/null +++ b/Tests/DistTestCore/Global.cs @@ -0,0 +1,60 @@ +using System.Reflection; +using Core; +using Logging; + +namespace DistTestCore +{ + public class Global + { + public const string TestNamespacePrefix = "cdx-"; + public Configuration Configuration { get; } = new Configuration(); + + public Assembly[] TestAssemblies { get; } + private readonly EntryPoint globalEntryPoint; + private readonly ILog log; + + public Global() + { + var assemblies = AppDomain.CurrentDomain.GetAssemblies(); + TestAssemblies = assemblies.Where(a => a.FullName!.ToLowerInvariant().Contains("test")).ToArray(); + + log = new ConsoleLog(); + globalEntryPoint = new EntryPoint( + log, + Configuration.GetK8sConfiguration( + new DefaultK8sTimeSet(), + TestNamespacePrefix + ), + Configuration.GetFileManagerFolder() + ); + } + + public void Setup() + { + try + { + Stopwatch.Measure(log, "Global setup", () => + { + globalEntryPoint.Announce(); + globalEntryPoint.Tools.CreateWorkflow().DeleteNamespacesStartingWith(TestNamespacePrefix, wait: true); + }); + } + catch (Exception ex) + { + GlobalTestFailure.HasFailed = true; + log.Error($"Global setup cleanup failed with: {ex}"); + throw; + } + } + + public void TearDown() + { + globalEntryPoint.Decommission( + // There shouldn't be any of either, but clean everything up regardless. + deleteKubernetesResources: true, + deleteTrackedFiles: true, + waitTillDone: true + ); + } + } +} diff --git a/Tests/DistTestCore/Logs/BaseTestLog.cs b/Tests/DistTestCore/Logs/BaseTestLog.cs index 22485545..e670e0c1 100644 --- a/Tests/DistTestCore/Logs/BaseTestLog.cs +++ b/Tests/DistTestCore/Logs/BaseTestLog.cs @@ -2,27 +2,83 @@ namespace DistTestCore.Logs { - public abstract class BaseTestLog : BaseLog + public abstract class BaseTestLog : ILog { - private bool hasFailed; - private readonly string deployId; + private readonly ILog backingLog; - protected BaseTestLog(string deployId) + protected BaseTestLog(ILog backingLog, string deployId) { - this.deployId = deployId; + this.backingLog = backingLog; + + DeployId = deployId; + } + + public string DeployId { get; } + + public void AddStringReplace(string from, string to) + { + backingLog.AddStringReplace(from, to); + } + + public LogFile CreateSubfile(string addName, string ext = "log") + { + return backingLog.CreateSubfile(addName, ext); + } + + public void Debug(string message = "", int skipFrames = 0) + { + backingLog.Debug(message, skipFrames); + } + + public void Error(string message) + { + backingLog.Error(message); + } + + public string GetFullName() + { + return backingLog.GetFullName(); + } + + public void Log(string message) + { + backingLog.Log(message); + } + + public void Raw(string message) + { + backingLog.Raw(message); } public void WriteLogTag() { var category = NameUtils.GetCategoryName(); var name = NameUtils.GetTestMethodName(); - LogFile.WriteRaw($"{deployId} {category} {name}"); + backingLog.Raw($"{DeployId} {category} {name}"); } - public void MarkAsFailed() + protected static ILog CreateMainLog(string fullName, string name) { - if (hasFailed) return; - hasFailed = true; + ILog log = new FileLog(fullName); + log = ApplyConsoleOutput(log); + return log; + } + + private static ILog ApplyConsoleOutput(ILog log) + { + // If we're running as a release test, we'll split the log output + // to the console as well. + + var testType = Environment.GetEnvironmentVariable("TEST_TYPE"); + if (string.IsNullOrEmpty(testType) || testType.ToLowerInvariant() != "release-tests") + { + return log; + } + + return new LogSplitter( + log, + new ConsoleLog() + ); } } } diff --git a/Tests/DistTestCore/Logs/FixtureLog.cs b/Tests/DistTestCore/Logs/FixtureLog.cs index 6559f81d..5978a602 100644 --- a/Tests/DistTestCore/Logs/FixtureLog.cs +++ b/Tests/DistTestCore/Logs/FixtureLog.cs @@ -4,28 +4,21 @@ namespace DistTestCore.Logs { public class FixtureLog : BaseTestLog { - private readonly string fullName; - private readonly string deployId; - - public FixtureLog(LogConfig config, DateTime start, string deployId, string name = "") : base(deployId) + public FixtureLog(ILog backingLog, string deployId) + : base(backingLog, deployId) { - this.deployId = deployId; - fullName = NameUtils.GetFixtureFullName(config, start, name); } - public TestLog CreateTestLog(string name = "") + public TestLog CreateTestLog(DateTime start, string name = "") { - return new TestLog(fullName, deployId, name); + return TestLog.Create(this, start, name); } - public void DeleteFolder() + public static FixtureLog Create(LogConfig config, DateTime start, string deployId, string name = "") { - Directory.Delete(fullName, true); - } - - protected override string GetFullName() - { - return fullName; + var fullName = NameUtils.GetFixtureFullName(config, start, name); + var log = CreateMainLog(fullName, name); + return new FixtureLog(log, deployId); } } -} \ No newline at end of file +} diff --git a/Tests/DistTestCore/Logs/TestLog.cs b/Tests/DistTestCore/Logs/TestLog.cs index 1f598b65..5c38f951 100644 --- a/Tests/DistTestCore/Logs/TestLog.cs +++ b/Tests/DistTestCore/Logs/TestLog.cs @@ -1,20 +1,21 @@ -namespace DistTestCore.Logs +using Logging; + +namespace DistTestCore.Logs { public class TestLog : BaseTestLog { - private readonly string fullName; - - public TestLog(string folder, string deployId, string name = "") : base(deployId) + public TestLog(ILog backingLog, string methodName, string deployId, string name = "") + : base(backingLog, deployId) { - var methodName = NameUtils.GetTestMethodName(name); - fullName = Path.Combine(folder, methodName); - - Log($"*** Begin: {methodName}"); + backingLog.Log($"*** Begin: {methodName}"); } - protected override string GetFullName() + public static TestLog Create(FixtureLog parentLog, DateTime start, string name = "") { - return fullName; + var methodName = NameUtils.GetTestLogFileName(start, name); + var fullName = Path.Combine(parentLog.GetFullName(), methodName); + var backingLog = CreateMainLog(fullName, name); + return new TestLog(backingLog, methodName, parentLog.DeployId); } } } diff --git a/Tests/DistTestCore/NameUtils.cs b/Tests/DistTestCore/NameUtils.cs index 5e1f8a9f..44919d51 100644 --- a/Tests/DistTestCore/NameUtils.cs +++ b/Tests/DistTestCore/NameUtils.cs @@ -5,6 +5,11 @@ namespace DistTestCore { public static class NameUtils { + public static string GetTestLogFileName(DateTime start, string name = "") + { + return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{GetTestMethodName(name)}"; + } + public static string GetTestMethodName(string name = "") { if (!string.IsNullOrEmpty(name)) return name; @@ -16,7 +21,7 @@ namespace DistTestCore public static string GetFixtureFullName(LogConfig config, DateTime start, string name) { var folder = DetermineFolder(config, start); - var fixtureName = GetFixtureName(name, start); + var fixtureName = GetRawFixtureName(); return Path.Combine(folder, fixtureName); } @@ -25,6 +30,7 @@ namespace DistTestCore var test = TestContext.CurrentContext.Test; if (test.ClassName!.Contains("AdhocContext")) return "none"; var className = test.ClassName!.Substring(test.ClassName.LastIndexOf('.') + 1); + className += FormatArguments(test); return className.Replace('.', '-'); } @@ -54,7 +60,7 @@ namespace DistTestCore private static string FormatArguments(TestContext.TestAdapter test) { - if (test.Arguments == null || !test.Arguments.Any()) return ""; + if (test.Arguments == null || test.Arguments.Length == 0) return ""; return $"[{string.Join(',', test.Arguments.Select(FormatArgument).ToArray())}]"; } @@ -69,6 +75,8 @@ namespace DistTestCore private static string ReplaceInvalidCharacters(string name) { return name + .Replace("codexstorage/nim-codex:", "") + .Replace("-dist-tests", "") .Replace(":", "_") .Replace("/", "_") .Replace("\\", "_"); @@ -82,13 +90,6 @@ namespace DistTestCore Pad(start.Day)); } - private static string GetFixtureName(string name, DateTime start) - { - var className = GetRawFixtureName(); - if (!string.IsNullOrEmpty(name)) className = name; - return $"{Pad(start.Hour)}-{Pad(start.Minute)}-{Pad(start.Second)}Z_{className.Replace('.', '-')}"; - } - private static string Pad(int n) { return n.ToString().PadLeft(2, '0'); diff --git a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs index d39c31ec..93fcf6f7 100644 --- a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs +++ b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs @@ -1,47 +1,35 @@ using CodexClient; using CodexPlugin; -using DistTestCore; using NUnit.Framework; namespace CodexTests { public class AutoBootstrapDistTest : CodexDistTest { - private readonly Dictionary bootstrapNodes = new Dictionary(); + private bool isBooting = false; + + public ICodexNode BootstrapNode { get; private set; } = null!; [SetUp] - public void SetUpBootstrapNode() + public void SetupBootstrapNode() { - var tl = Get(); - if (!bootstrapNodes.ContainsKey(tl)) - { - bootstrapNodes.Add(tl, StartCodex(s => s.WithName("BOOTSTRAP_" + tl.TestNamespace))); - } + isBooting = true; + BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP_" + GetTestNamespace())); + isBooting = false; } [TearDown] public void TearDownBootstrapNode() { - bootstrapNodes.Remove(Get()); + BootstrapNode.Stop(waitTillStopped: false); } protected override void OnCodexSetup(ICodexSetup setup) { + if (isBooting) return; + var node = BootstrapNode; if (node != null) setup.WithBootstrapNode(node); } - - protected ICodexNode? BootstrapNode - { - get - { - var tl = Get(); - if (bootstrapNodes.TryGetValue(tl, out var node)) - { - return node; - } - return null; - } - } } } diff --git a/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs b/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs index f8ef52d5..af213342 100644 --- a/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs +++ b/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs @@ -31,7 +31,7 @@ namespace ExperimentalTests.BasicTests ); var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth")); - var contracts = Ci.StartCodexContracts(geth); + var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); var numberOfHosts = 5; var hosts = StartCodex(numberOfHosts, s => s diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/CodexDistTest.cs index 4cc9c707..422b5bd7 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/CodexDistTest.cs @@ -1,6 +1,5 @@ using BlockchainUtils; using CodexClient; -using CodexClient.Hooks; using CodexContractsPlugin; using CodexNetDeployer; using CodexPlugin; @@ -17,85 +16,14 @@ using Newtonsoft.Json; using NUnit.Framework; using NUnit.Framework.Constraints; using OverwatchTranscript; -using Utils; namespace CodexTests { - public class CodexLogTrackerProvider : ICodexHooksProvider - { - private readonly Action addNode; - - public CodexLogTrackerProvider(Action addNode) - { - this.addNode = addNode; - } - - // See TestLifecycle.cs DownloadAllLogs() - public ICodexNodeHooks CreateHooks(string nodeName) - { - return new CodexLogTracker(addNode); - } - - public class CodexLogTracker : ICodexNodeHooks - { - private readonly Action addNode; - - public CodexLogTracker(Action addNode) - { - this.addNode = addNode; - } - - public void OnFileDownloaded(ByteSize size, ContentId cid) - { - } - - public void OnFileDownloading(ContentId cid) - { - } - - public void OnFileUploaded(string uid, ByteSize size, ContentId cid) - { - } - - public void OnFileUploading(string uid, ByteSize size) - { - } - - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) - { - addNode(node); - } - - public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) - { - } - - public void OnNodeStopping() - { - } - - public void OnStorageAvailabilityCreated(StorageAvailability response) - { - } - - public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) - { - } - - public void OnStorageContractUpdated(StoragePurchase purchaseStatus) - { - } - } - } - public class CodexDistTest : DistTest { - private static readonly object _lock = new object(); - private static readonly Dictionary writers = new Dictionary(); - private static readonly Dictionary blockCaches = new Dictionary(); - - // this entire structure is not good and needs to be destroyed at the earliest convenience: - private static readonly Dictionary> nodes = new Dictionary>(); + private readonly BlockCache blockCache = new BlockCache(); + private readonly List nodes = new List(); + private CodexTranscriptWriter? writer; public CodexDistTest() { @@ -105,38 +33,25 @@ namespace CodexTests ProjectPlugin.Load(); } + [SetUp] + public void SetupCodexDistTest() + { + writer = SetupTranscript(); + } + + [TearDown] + public void TearDownCodexDistTest() + { + TeardownTranscript(); + } + protected override void Initialize(FixtureLog fixtureLog) { var localBuilder = new LocalCodexBuilder(fixtureLog); localBuilder.Intialize(); localBuilder.Build(); - } - protected override void LifecycleStart(TestLifecycle lifecycle) - { - base.LifecycleStart(lifecycle); - SetupTranscript(lifecycle); - - Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(n => - { - lock (_lock) - { - if (!nodes.ContainsKey(lifecycle)) nodes.Add(lifecycle, new List()); - nodes[lifecycle].Add(n); - } - })); - } - - protected override void LifecycleStop(TestLifecycle lifecycle, DistTestResult result) - { - base.LifecycleStop(lifecycle, result); - TeardownTranscript(lifecycle, result); - - if (!result.Success) - { - var codexNodes = nodes[lifecycle]; - foreach (var node in codexNodes) node.DownloadLog(); - } + Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(nodes.Add)); } public ICodexNode StartCodex() @@ -167,7 +82,7 @@ namespace CodexTests public IGethNode StartGethNode(Action setup) { - return Ci.StartGethNode(GetBlockCache(), setup); + return Ci.StartGethNode(blockCache, setup); } public PeerConnectionTestHelpers CreatePeerConnectionTestHelpers() @@ -177,7 +92,7 @@ namespace CodexTests public PeerDownloadTestHelpers CreatePeerDownloadTestHelpers() { - return new PeerDownloadTestHelpers(GetTestLog(), Get().GetFileManager()); + return new PeerDownloadTestHelpers(GetTestLog(), GetFileManager()); } public void AssertBalance(ICodexContracts contracts, ICodexNode codexNode, Constraint constraint, string msg = "") @@ -255,82 +170,47 @@ namespace CodexTests return null; } - private void SetupTranscript(TestLifecycle lifecycle) + private CodexTranscriptWriter? SetupTranscript() { var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; + if (attr == null) return null; var config = new CodexTranscriptWriterConfig( + attr.OutputFilename, attr.IncludeBlockReceivedEvents ); - var log = new LogPrefixer(lifecycle.Log, "(Transcript) "); + var log = new LogPrefixer(GetTestLog(), "(Transcript) "); var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); Ci.AddCodexHooksProvider(writer); - lock (_lock) - { - writers.Add(lifecycle, writer); - } + return writer; } - private void TeardownTranscript(TestLifecycle lifecycle, DistTestResult result) + private void TeardownTranscript() { - var attr = GetTranscriptAttributeOfCurrentTest(); - if (attr == null) return; - - var outputFilepath = GetOutputFullPath(lifecycle, attr); - - CodexTranscriptWriter writer = null!; - lock (_lock) - { - writer = writers[lifecycle]; - writers.Remove(lifecycle); - } + if (writer == null) return; + var result = GetTestResult(); + var log = GetTestLog(); writer.AddResult(result.Success, result.Result); - try { - Stopwatch.Measure(lifecycle.Log, "Transcript.ProcessLogs", () => + Stopwatch.Measure(log, "Transcript.ProcessLogs", () => { - writer.ProcessLogs(lifecycle.DownloadAllLogs()); + writer.ProcessLogs(DownloadAllLogs()); }); - Stopwatch.Measure(lifecycle.Log, $"Transcript.Finalize: {outputFilepath}", () => + Stopwatch.Measure(log, $"Transcript.FinalizeWriter", () => { - writer.IncludeFile(lifecycle.Log.LogFile.Filename); - writer.Finalize(outputFilepath); + writer.IncludeFile(log.GetFullName() + ".log"); + writer.FinalizeWriter(); }); } catch (Exception ex) { - lifecycle.Log.Error("Failure during transcript teardown: " + ex); + log.Error("Failure during transcript teardown: " + ex); } } - - private string GetOutputFullPath(TestLifecycle lifecycle, CreateTranscriptAttribute attr) - { - var outputPath = Path.GetDirectoryName(lifecycle.Log.LogFile.Filename); - if (outputPath == null) throw new Exception("Logfile path is null"); - var filename = Path.GetFileNameWithoutExtension(lifecycle.Log.LogFile.Filename); - if (string.IsNullOrEmpty(filename)) throw new Exception("Logfile name is null or empty"); - var outputFile = Path.Combine(outputPath, filename + "_" + attr.OutputFilename); - if (!outputFile.EndsWith(".owts")) outputFile += ".owts"; - return outputFile; - } - - private BlockCache GetBlockCache() - { - var lifecycle = Get(); - lock (_lock) - { - if (!blockCaches.ContainsKey(lifecycle)) - { - blockCaches[lifecycle] = new BlockCache(); - } - } - return blockCaches[lifecycle]; - } } [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] diff --git a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs new file mode 100644 index 00000000..1b2fdd66 --- /dev/null +++ b/Tests/ExperimentalTests/CodexLogTrackerProvider.cs @@ -0,0 +1,73 @@ +using CodexClient; +using CodexClient.Hooks; +using Utils; + +namespace CodexTests +{ + public class CodexLogTrackerProvider : ICodexHooksProvider + { + private readonly Action addNode; + + public CodexLogTrackerProvider(Action addNode) + { + this.addNode = addNode; + } + + // See TestLifecycle.cs DownloadAllLogs() + public ICodexNodeHooks CreateHooks(string nodeName) + { + return new CodexLogTracker(addNode); + } + + public class CodexLogTracker : ICodexNodeHooks + { + private readonly Action addNode; + + public CodexLogTracker(Action addNode) + { + this.addNode = addNode; + } + + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + { + addNode(node); + } + + public void OnNodeStarting(DateTime startUtc, string image, EthAccount? ethAccount) + { + } + + public void OnNodeStopping() + { + } + + public void OnStorageAvailabilityCreated(StorageAvailability response) + { + } + + public void OnStorageContractSubmitted(StoragePurchaseContract storagePurchaseContract) + { + } + + public void OnStorageContractUpdated(StoragePurchase purchaseStatus) + { + } + } + } +} diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index ec128154..d0acc991 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -21,7 +21,7 @@ namespace ExperimentalTests.DownloadConnectivityTests public void MarketplaceDoesNotInterfereWithPeerDownload() { var geth = StartGethNode(s => s.IsMiner()); - var contracts = Ci.StartCodexContracts(geth); + var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TstWei()))); diff --git a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 901504c7..60a9be9a 100644 --- a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -31,7 +31,7 @@ namespace ExperimentalTests.PeerDiscoveryTests public void MarketplaceDoesNotInterfereWithPeerDiscovery() { var geth = StartGethNode(s => s.IsMiner()); - var contracts = Ci.StartCodexContracts(geth); + var contracts = Ci.StartCodexContracts(geth, BootstrapNode.Version); var nodes = StartCodex(2, s => s.EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), 1000.TstWei()))); diff --git a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs deleted file mode 100644 index abebac0c..00000000 --- a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs +++ /dev/null @@ -1,370 +0,0 @@ -using CodexClient; -using CodexContractsPlugin; -using CodexDiscordBotPlugin; -using CodexPlugin; -using CodexTests; -using Core; -using DiscordRewards; -using DistTestCore; -using GethPlugin; -using KubernetesWorkflow.Types; -using Logging; -using Newtonsoft.Json; -using NUnit.Framework; -using Utils; - -namespace ExperimentalTests.UtilityTests -{ - [TestFixture] - public class DiscordBotTests : AutoBootstrapDistTest - { - private readonly RewardRepo repo = new RewardRepo(); - private readonly TestToken hostInitialBalance = 3000000.TstWei(); - private readonly TestToken clientInitialBalance = 1000000000.TstWei(); - private readonly EthAccount clientAccount = EthAccountGenerator.GenerateNew(); - private readonly List hostAccounts = new List(); - private readonly List rewardsSeen = new List(); - private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); - private readonly List receivedEvents = new List(); - - [Test] - [DontDownloadLogs] - [Ignore("Used to debug testnet bots.")] - public void BotRewardTest() - { - var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth")); - var contracts = Ci.StartCodexContracts(geth); - var gethInfo = CreateGethInfo(geth, contracts); - - var botContainer = StartDiscordBot(gethInfo); - var rewarderContainer = StartRewarderBot(gethInfo, botContainer); - - StartHosts(geth, contracts); - var client = StartClient(geth, contracts); - - var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer); - apiCalls.Start(OnCommand); - - var purchaseContract = ClientPurchasesStorage(client); - purchaseContract.WaitForStorageContractStarted(); - purchaseContract.WaitForStorageContractFinished(); - - // todo: removed from codexclient: - //contracts.WaitUntilNextPeriod(); - //contracts.WaitUntilNextPeriod(); - - //var blocks = 3; - //Log($"Waiting {blocks} blocks for nodes to process payouts..."); - //Thread.Sleep(GethContainerRecipe.BlockInterval * blocks); - - Thread.Sleep(rewarderInterval * 3); - - apiCalls.Stop(); - - AssertEventOccurance("Created as New.", 1); - AssertEventOccurance("SlotFilled", Convert.ToInt32(GetNumberOfRequiredHosts())); - AssertEventOccurance("Transit: New -> Started", 1); - AssertEventOccurance("Transit: Started -> Finished", 1); - - foreach (var r in repo.Rewards) - { - var seen = rewardsSeen.Any(s => r.RoleId == s); - - Log($"{Lookup(r.RoleId)} = {seen}"); - } - - Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId))); - } - - private string Lookup(ulong rewardId) - { - var reward = repo.Rewards.Single(r => r.RoleId == rewardId); - return $"({rewardId})'{reward.Message}'"; - } - - private void AssertEventOccurance(string msg, int expectedCount) - { - Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount), - $"Event '{msg}' did not occure correct number of times."); - } - - private void OnCommand(string timestamp, GiveRewardsCommand call) - { - Log($""); - foreach (var e in call.EventsOverview) - { - Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order."); - } - - receivedEvents.AddRange(call.EventsOverview); - foreach (var e in call.EventsOverview) - { - Log("\tEvent: " + e); - } - foreach (var r in call.Rewards) - { - var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); - if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId); - foreach (var address in r.UserAddresses) - { - var user = IdentifyAccount(address); - Log("\tReward: " + user + ": " + reward.Message); - } - } - Log($""); - } - - private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client) - { - var testFile = GenerateTestFile(GetMinFileSize()); - var contentId = client.UploadFile(testFile); - var purchase = new StoragePurchaseRequest(contentId) - { - PricePerBytePerSecond = 2.TstWei(), - CollateralPerByte = 10.TstWei(), - MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(), - NodeFailureTolerance = 2, - ProofProbability = 5, - Duration = GetMinRequiredRequestDuration(), - Expiry = GetMinRequiredRequestDuration() - TimeSpan.FromMinutes(1) - }; - - return client.Marketplace.RequestStorage(purchase); - } - - private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts) - { - var node = StartCodex(s => s - .WithName("Client") - .EnableMarketplace(geth, contracts, m => m - .WithAccount(clientAccount) - .WithInitial(10.Eth(), clientInitialBalance))); - - Log($"Client {node.EthAccount.EthAddress}"); - return node; - } - - private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) - { - return Ci.DeployRewarderBot(new RewarderBotStartupConfig( - name: "rewarder-bot", - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)), - historyStartUtc: DateTime.UtcNow, - gethInfo: gethInfo, - dataPath: null - )); - } - - private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts) - { - return new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - } - - private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo) - { - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "discord-bot", - token: "aaa", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - return bot.Containers.Single(); - } - - private void StartHosts(IGethNode geth, ICodexContracts contracts) - { - var hosts = StartCodex(GetNumberOfLiveHosts(), s => s - .WithName("Host") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts())) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), hostInitialBalance) - .AsStorageNode() - .AsValidator())); - - var availability = new StorageAvailability( - totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()), - maxDuration: TimeSpan.FromMinutes(30), - minPricePerBytePerSecond: 1.TstWei(), - totalCollateral: hostInitialBalance - ); - - foreach (var host in hosts) - { - hostAccounts.Add(host.EthAccount); - host.Marketplace.MakeStorageAvailable(availability); - } - } - - private int GetNumberOfLiveHosts() - { - return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3; - } - - private ByteSize Mult(ByteSize size, int mult) - { - return new ByteSize(size.SizeInBytes * mult); - } - - private ByteSize GetMinFileSizePlus(int plusMb) - { - return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes); - } - - private ByteSize GetMinFileSize() - { - ulong minSlotSize = 0; - ulong minNumHosts = 0; - foreach (var r in repo.Rewards) - { - var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes); - var h = r.CheckConfig.MinNumberOfHosts; - if (s > minSlotSize) minSlotSize = s; - if (h > minNumHosts) minNumHosts = h; - } - - var minFileSize = (minSlotSize + 1024) * minNumHosts; - return new ByteSize(Convert.ToInt64(minFileSize)); - } - - private uint GetNumberOfRequiredHosts() - { - return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); - } - - private TimeSpan GetMinRequiredRequestDuration() - { - return repo.Rewards.Max(r => r.CheckConfig.MinDuration) + TimeSpan.FromSeconds(10); - } - - private string IdentifyAccount(string address) - { - if (address == clientAccount.EthAddress.Address) return "Client"; - try - { - var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address); - return "Host" + index; - } - catch - { - return "UNKNOWN"; - } - } - - public class RewardApiCalls - { - private readonly ContainerFileMonitor monitor; - - public RewardApiCalls(ILog log, CoreInterface ci, RunningContainer botContainer) - { - monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log"); - } - - public void Start(Action onCommand) - { - monitor.Start(line => ParseLine(line, onCommand)); - } - - public void Stop() - { - monitor.Stop(); - } - - private void ParseLine(string line, Action onCommand) - { - try - { - var timestamp = line.Substring(0, 30); - var json = line.Substring(31); - - var cmd = JsonConvert.DeserializeObject(json); - if (cmd != null) - { - onCommand(timestamp, cmd); - } - } - catch - { - } - } - } - - public class ContainerFileMonitor - { - private readonly ILog log; - private readonly CoreInterface ci; - private readonly RunningContainer botContainer; - private readonly string filePath; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); - private readonly List seenLines = new List(); - private Task worker = Task.CompletedTask; - private Action onNewLine = c => { }; - - public ContainerFileMonitor(ILog log, CoreInterface ci, RunningContainer botContainer, string filePath) - { - this.log = log; - this.ci = ci; - this.botContainer = botContainer; - this.filePath = filePath; - } - - public void Start(Action onNewLine) - { - this.onNewLine = onNewLine; - worker = Task.Run(Worker); - } - - public void Stop() - { - cts.Cancel(); - worker.Wait(); - } - - // did any container crash? that's why it repeats? - - - private void Worker() - { - while (!cts.IsCancellationRequested) - { - Update(); - } - } - - private void Update() - { - Thread.Sleep(TimeSpan.FromSeconds(10)); - if (cts.IsCancellationRequested) return; - - var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath); - var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - foreach (var line in lines) - { - // log.Log("line: " + line); - - if (!seenLines.Contains(line)) - { - seenLines.Add(line); - onNewLine(line); - } - } - } - } - } -} diff --git a/Tests/FrameworkTests/Utils/RandomUtilsTests.cs b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs new file mode 100644 index 00000000..3859f0d3 --- /dev/null +++ b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs @@ -0,0 +1,19 @@ +using NUnit.Framework; +using Utils; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class RandomUtilsTests + { + [Test] + [Combinatorial] + public void TestRandomStringLength( + [Values(1, 5, 10, 1023, 1024, 1025, 2222)] int length) + { + var str = RandomUtils.GenerateRandomString(length); + + Assert.That(str.Length, Is.EqualTo(length)); + } + } +} diff --git a/Tools/AutoClient/CodexWrapper.cs b/Tools/AutoClient/CodexWrapper.cs index e91c3cfa..02fcca3e 100644 --- a/Tools/AutoClient/CodexWrapper.cs +++ b/Tools/AutoClient/CodexWrapper.cs @@ -7,6 +7,7 @@ namespace AutoClient public class CodexWrapper { private readonly App app; + private static readonly Random r = new Random(); public CodexWrapper(App app, ICodexNode node) { @@ -26,11 +27,11 @@ namespace AutoClient var result = Node.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) { CollateralPerByte = app.Config.CollateralPerByte.TstWei(), - Duration = TimeSpan.FromMinutes(app.Config.ContractDurationMinutes), + Duration = GetDuration(), Expiry = TimeSpan.FromMinutes(app.Config.ContractExpiryMinutes), MinRequiredNumberOfNodes = Convert.ToUInt32(app.Config.NumHosts), NodeFailureTolerance = Convert.ToUInt32(app.Config.HostTolerance), - PricePerBytePerSecond = app.Config.PricePerBytePerSecond.TstWei(), + PricePerBytePerSecond = GetPricePerBytePerSecond(), ProofProbability = 15 }); return result; @@ -40,5 +41,25 @@ namespace AutoClient { return Node.GetPurchaseStatus(pid); } + + private TestToken GetPricePerBytePerSecond() + { + var i = app.Config.PricePerBytePerSecond; + i -= 100; + i += r.Next(0, 1000); + + return i.TstWei(); + } + + private TimeSpan GetDuration() + { + var i = app.Config.ContractDurationMinutes; + var day = 60 * 24; + i -= day; + i -= 10; // We don't want to accidentally hit exactly 7 days because that's the limit of the storage node availabilities. + i += r.Next(0, day * 2); + + return TimeSpan.FromMinutes(i); + } } } diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 13aea6fd..65996333 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -59,6 +59,9 @@ namespace AutoClient "/root/codex-testnet-starter/scripts/eth_7.address" + ";" + "/root/codex-testnet-starter/scripts/eth_8.address"; + [Uniform("slowModeDelayMinutes", "smdm", "SLOWMODEDELAYMINUTES", false, "When contract failure threshold is reached, slow down process for each file by this amount of minutes.")] + public int SlowModeDelayMinutes { get; set; } = 60 * 1; + public string LogPath { get diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index 027065e4..448755f0 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -7,6 +7,11 @@ namespace AutoClient.Modes.FolderStore public interface IFileSaverEventHandler { void SaveChanges(); + } + + public interface IFileSaverResultHandler + { + void OnSuccess(); void OnFailure(); } @@ -17,16 +22,18 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; - public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.loadBalancer = loadBalancer; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; } public void Process() @@ -46,9 +53,9 @@ namespace AutoClient.Modes.FolderStore loadBalancer.DispatchOnCodex(instance => { entry.CodexNodeId = instance.Node.GetName(); - handler.SaveChanges(); + saveHandler.SaveChanges(); - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }); } @@ -57,7 +64,7 @@ namespace AutoClient.Modes.FolderStore { loadBalancer.DispatchOnSpecificCodex(instance => { - var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, saveHandler, resultHandler); run.Process(); }, entry.CodexNodeId); } @@ -70,17 +77,19 @@ namespace AutoClient.Modes.FolderStore private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly IFileSaverEventHandler handler; + private readonly IFileSaverEventHandler saveHandler; + private readonly IFileSaverResultHandler resultHandler; private readonly QuotaCheck quotaCheck; - public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler saveHandler, IFileSaverResultHandler resultHandler) { this.log = log; this.instance = instance; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.handler = handler; + this.saveHandler = saveHandler; + this.resultHandler = resultHandler; quotaCheck = new QuotaCheck(log, folderFile, instance); } @@ -127,7 +136,7 @@ namespace AutoClient.Modes.FolderStore Thread.Sleep(TimeSpan.FromMinutes(1.0)); } Log("Could not upload: Insufficient local storage quota."); - handler.OnFailure(); + resultHandler.OnFailure(); return false; } @@ -206,9 +215,9 @@ namespace AutoClient.Modes.FolderStore entry.BasicCid = string.Empty; stats.FailedUploads++; log.Error("Failed to upload: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } - handler.SaveChanges(); + saveHandler.SaveChanges(); } private void CreateNewPurchase() @@ -224,17 +233,18 @@ namespace AutoClient.Modes.FolderStore WaitForStarted(request); stats.StorageRequestStats.SuccessfullyStarted++; - handler.SaveChanges(); + saveHandler.SaveChanges(); Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); + resultHandler.OnSuccess(); } catch (Exception exc) { entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); log.Error("Failed to start new purchase: " + exc); - handler.OnFailure(); + resultHandler.OnFailure(); } } @@ -253,7 +263,7 @@ namespace AutoClient.Modes.FolderStore throw new Exception("CID received from storage request was not protected."); } - handler.SaveChanges(); + saveHandler.SaveChanges(); Log("Saved new purchaseId: " + entry.PurchaseId); return request; } @@ -289,7 +299,7 @@ namespace AutoClient.Modes.FolderStore Log("Request failed to start. State: " + update.State); entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - handler.SaveChanges(); + saveHandler.SaveChanges(); return; } } @@ -297,7 +307,7 @@ namespace AutoClient.Modes.FolderStore } catch (Exception exc) { - handler.OnFailure(); + resultHandler.OnFailure(); Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}"); throw; } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index 2f71b23b..d951aaee 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -1,4 +1,5 @@ using Logging; +using Utils; namespace AutoClient.Modes.FolderStore { @@ -10,14 +11,16 @@ namespace AutoClient.Modes.FolderStore private readonly JsonFile statusFile; private readonly FolderStatus status; private readonly BalanceChecker balanceChecker; + private readonly SlowModeHandler slowModeHandler; private int changeCounter = 0; - private int failureCount = 0; + private int saveFolderJsonCounter = 0; public FolderSaver(App app, LoadBalancer loadBalancer) { this.app = app; this.loadBalancer = loadBalancer; balanceChecker = new BalanceChecker(app); + slowModeHandler = new SlowModeHandler(app); statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); status = statusFile.Load(); @@ -25,10 +28,11 @@ namespace AutoClient.Modes.FolderStore public void Run() { + saveFolderJsonCounter = 0; + var folderFiles = Directory.GetFiles(app.Config.FolderToStore); if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); - var saveFolderJsonCounter = 0; balanceChecker.Check(); foreach (var folderFile in folderFiles) { @@ -40,35 +44,30 @@ namespace AutoClient.Modes.FolderStore SaveFile(folderFile); } - if (failureCount > 3) - { - app.Log.Error("Failure count reached threshold. Stopping..."); - app.Cts.Cancel(); - return; - } - - if (changeCounter > 1) - { - changeCounter = 0; - saveFolderJsonCounter++; - if (saveFolderJsonCounter > 5) - { - saveFolderJsonCounter = 0; - if (failureCount > 0) - { - app.Log.Log($"Failure count is reset. (Was: {failureCount})"); - failureCount = 0; - } - balanceChecker.Check(); - SaveFolderSaverJsonFile(); - } - } + slowModeHandler.Check(); + + CheckAndSaveChanges(); statusFile.Save(status); Thread.Sleep(100); } } + private void CheckAndSaveChanges() + { + if (changeCounter > 1) + { + changeCounter = 0; + saveFolderJsonCounter++; + if (saveFolderJsonCounter > 5) + { + saveFolderJsonCounter = 0; + balanceChecker.Check(); + SaveFolderSaverJsonFile(); + } + } + } + private void SaveFile(string folderFile) { var localFilename = Path.GetFileName(folderFile); @@ -113,7 +112,6 @@ namespace AutoClient.Modes.FolderStore } private const int MinCodexStorageFilesize = 262144; - private readonly Random random = new Random(); private readonly string paddingMessage = $"Codex currently requires a minimum filesize of {MinCodexStorageFilesize} bytes for datasets used in storage contracts. " + $"Anything smaller, and the erasure-coding algorithms used for data durability won't function. Therefore, we apply this padding field to make sure this " + $"file is larger than the minimal size. The following is pseudo-random: "; @@ -125,29 +123,16 @@ namespace AutoClient.Modes.FolderStore if (info.Length < min) { var required = Math.Max(1024, min - info.Length); - status.Padding = paddingMessage + GenerateRandomString(required); + status.Padding = paddingMessage + RandomUtils.GenerateRandomString(required); statusFile.Save(status); } } - private string GenerateRandomString(long required) - { - var result = ""; - while (result.Length < required) - { - var bytes = new byte[1024]; - random.NextBytes(bytes); - result += string.Join("", bytes.Select(b => b.ToString())); - } - - return result; - } - private FileSaver CreateFileSaver(string folderFile, FileStatus entry) { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this); + return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this, slowModeHandler); } public void SaveChanges() @@ -155,10 +140,5 @@ namespace AutoClient.Modes.FolderStore statusFile.Save(status); changeCounter++; } - - public void OnFailure() - { - failureCount++; - } } } diff --git a/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs new file mode 100644 index 00000000..37d3f7ee --- /dev/null +++ b/Tools/AutoClient/Modes/FolderStore/SlowModeHandler.cs @@ -0,0 +1,54 @@ +namespace AutoClient.Modes.FolderStore +{ + public class SlowModeHandler : IFileSaverResultHandler + { + private readonly App app; + private int failureCount = 0; + private bool slowMode = false; + private int recoveryCount = 0; + + public SlowModeHandler(App app) + { + this.app = app; + } + + public void OnSuccess() + { + failureCount = 0; + if (slowMode) + { + recoveryCount++; + if (recoveryCount > 3) + { + Log("Recovery limit reached. Exiting slow mode."); + slowMode = false; + failureCount = 0; + } + } + } + + public void OnFailure() + { + failureCount++; + if (failureCount > 3 && !slowMode) + { + Log("Failure limit reached. Entering slow mode."); + slowMode = true; + recoveryCount = 0; + } + } + + public void Check() + { + if (slowMode) + { + Thread.Sleep(TimeSpan.FromMinutes(app.Config.SlowModeDelayMinutes)); + } + } + + private void Log(string msg) + { + app.Log.Log(msg); + } + } +} diff --git a/Tools/BiblioTech/AdminChecker.cs b/Tools/BiblioTech/AdminChecker.cs index 532220de..96f89691 100644 --- a/Tools/BiblioTech/AdminChecker.cs +++ b/Tools/BiblioTech/AdminChecker.cs @@ -1,7 +1,6 @@ using BiblioTech.Options; using Discord; using Discord.WebSocket; -using Org.BouncyCastle.Utilities; namespace BiblioTech { diff --git a/Tools/BiblioTech/CallDispatcher.cs b/Tools/BiblioTech/CallDispatcher.cs new file mode 100644 index 00000000..b24ea830 --- /dev/null +++ b/Tools/BiblioTech/CallDispatcher.cs @@ -0,0 +1,66 @@ +using Logging; + +namespace BiblioTech +{ + public class CallDispatcher + { + private readonly ILog log; + private readonly object _lock = new object(); + private readonly List queue = new List(); + private readonly AutoResetEvent autoResetEvent = new AutoResetEvent(false); + + public CallDispatcher(ILog log) + { + this.log = log; + } + + public void Add(Action call) + { + lock (_lock) + { + queue.Add(call); + autoResetEvent.Set(); + if (queue.Count > 100) + { + log.Error("Queue overflow!"); + queue.Clear(); + } + } + } + + public void Start() + { + Task.Run(() => + { + while (true) + { + try + { + Worker(); + } + catch (Exception ex) + { + log.Error("Exception in CallDispatcher: " + ex); + } + } + }); + } + + private void Worker() + { + autoResetEvent.WaitOne(); + var tasks = Array.Empty(); + + lock (_lock) + { + tasks = queue.ToArray(); + queue.Clear(); + } + + foreach (var task in tasks) + { + task(); + } + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs new file mode 100644 index 00000000..2d8ec602 --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs @@ -0,0 +1,78 @@ +using BiblioTech.Rewards; +using Discord; +using Logging; +using System.Threading.Tasks; + +namespace BiblioTech.CodexChecking +{ + public class ActiveP2pRoleRemover + { + private readonly Configuration config; + private readonly ILog log; + private readonly CheckRepo repo; + + public ActiveP2pRoleRemover(Configuration config, ILog log, CheckRepo repo) + { + this.config = config; + this.log = log; + this.repo = repo; + } + + public void Start() + { + if (config.ActiveP2pRoleDurationMinutes > 0) + { + Task.Run(Worker); + } + } + + private void Worker() + { + var loopDelay = TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes) / 60; + var min = TimeSpan.FromMinutes(10.0); + if (loopDelay < min) loopDelay = min; + + try + { + while (true) + { + Thread.Sleep(loopDelay); + CheckP2pRoleRemoval(); + } + } + catch (Exception ex) + { + log.Error($"Exception in {nameof(ActiveP2pRoleRemover)}: {ex}"); + Environment.Exit(1); + } + } + + private void CheckP2pRoleRemoval() + { + var expiryMoment = DateTime.UtcNow - TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes); + + Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, expiryMoment), + Program.Config.ActiveP2pParticipantRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + if (report.UploadCheck.CompletedUtc > expiryMoment) return; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return; + + await giver.RemoveActiveP2pParticipant(user.Id); + } + + private bool ShouldRemoveRole(IUser user, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + + if (report.UploadCheck.CompletedUtc > expiryMoment) return false; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return false; + + return true; + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CheckRepo.cs b/Tools/BiblioTech/CodexChecking/CheckRepo.cs new file mode 100644 index 00000000..a8b2860b --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CheckRepo.cs @@ -0,0 +1,78 @@ +using Newtonsoft.Json; + +namespace BiblioTech.CodexChecking +{ + public class CheckRepo + { + private const string modelFilename = "model.json"; + private readonly Configuration config; + private readonly object _lock = new object(); + private CheckRepoModel? model = null; + + public CheckRepo(Configuration config) + { + this.config = config; + } + + public CheckReport GetOrCreate(ulong userId) + { + lock (_lock) + { + if (model == null) LoadModel(); + + var existing = model.Reports.SingleOrDefault(r => r.UserId == userId); + if (existing == null) + { + var newEntry = new CheckReport + { + UserId = userId, + }; + model.Reports.Add(newEntry); + SaveChanges(); + return newEntry; + } + return existing; + } + } + + public void SaveChanges() + { + File.WriteAllText(GetModelFilepath(), JsonConvert.SerializeObject(model, Formatting.Indented)); + } + + private void LoadModel() + { + if (!File.Exists(GetModelFilepath())) + { + model = new CheckRepoModel(); + SaveChanges(); + return; + } + + model = JsonConvert.DeserializeObject(File.ReadAllText(GetModelFilepath())); + } + + private string GetModelFilepath() + { + return Path.Combine(config.ChecksDataPath, modelFilename); + } + } + + public class CheckRepoModel + { + public List Reports { get; set; } = new List(); + } + + public class CheckReport + { + public ulong UserId { get; set; } + public TransferCheck UploadCheck { get; set; } = new TransferCheck(); + public TransferCheck DownloadCheck { get; set; } = new TransferCheck(); + } + + public class TransferCheck + { + public DateTime CompletedUtc { get; set; } = DateTime.MinValue; + public string UniqueData { get; set; } = string.Empty; + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs new file mode 100644 index 00000000..ac83ccd9 --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -0,0 +1,228 @@ +using CodexClient; +using FileUtils; +using Logging; +using Utils; + +namespace BiblioTech.CodexChecking +{ + public interface ICheckResponseHandler + { + Task CheckNotStarted(); + Task NowCompleted(string checkName); + Task GiveRoleReward(); + + Task InvalidData(); + Task CouldNotDownloadCid(); + Task GiveCidToUser(string cid); + Task GiveDataFileToUser(string fileContent); + + Task ToAdminChannel(string msg); + } + + public class CodexTwoWayChecker + { + private readonly ILog log; + private readonly Configuration config; + private readonly CheckRepo repo; + private readonly CodexWrapper codexWrapper; + + public CodexTwoWayChecker(ILog log, Configuration config, CheckRepo repo, CodexWrapper codexWrapper) + { + this.log = log; + this.config = config; + this.repo = repo; + this.codexWrapper = codexWrapper; + } + + public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (IsUniqueDataStale(check)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + var cid = UploadData(check.UniqueData); + await handler.GiveCidToUser(cid); + } + + public async Task VerifyDownloadCheck(ICheckResponseHandler handler, ulong userId, string receivedData) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + await handler.CheckNotStarted(); + return; + } + + Log($"Verifying for downloadCheck: received: '{receivedData}' check: '{check.UniqueData}'"); + if (string.IsNullOrEmpty(receivedData) || receivedData != check.UniqueData) + { + await handler.InvalidData(); + return; + } + + await CheckNowCompleted(handler, check, userId, "DownloadCheck"); + } + + public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (IsUniqueDataStale(check)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + await handler.GiveDataFileToUser(check.UniqueData); + } + + public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId, string receivedCid) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (string.IsNullOrEmpty(receivedCid)) + { + await handler.InvalidData(); + return; + } + + var manifest = GetManifest(receivedCid); + if (manifest == null) + { + await handler.CouldNotDownloadCid(); + return; + } + + if (IsManifestLengthCompatible(handler, check, manifest)) + { + if (IsContentCorrect(handler, check, receivedCid)) + { + await CheckNowCompleted(handler, check, userId, "UploadCheck"); + return; + } + } + + await handler.InvalidData(); + } + + private string GenerateUniqueData() + { + return $"{RandomBusyMessage.Get().Substring(5)}{RandomUtils.GenerateRandomString(12)}"; + } + + private bool IsUniqueDataStale(TransferCheck check) + { + var expiry = DateTime.UtcNow - TimeSpan.FromMinutes(10.0); + + return + string.IsNullOrEmpty(check.UniqueData) || + check.CompletedUtc < expiry; + } + + private string UploadData(string uniqueData) + { + var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString()); + + try + { + File.WriteAllText(filePath, uniqueData); + var file = new TrackedFile(log, filePath, "checkData"); + + return codexWrapper.OnCodex(node => + { + return node.UploadFile(file).Id; + }); + } + catch (Exception ex) + { + log.Error("Exception when uploading data: " + ex); + throw; + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + private Manifest? GetManifest(string receivedCid) + { + try + { + return codexWrapper.OnCodex(node => + { + return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest; + }); + } + catch + { + return null; + } + } + + private bool IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) + { + var dataLength = check.UniqueData.Length; + var manifestLength = manifest.OriginalBytes.SizeInBytes; + + Log($"Checking manifest length: dataLength={dataLength},manifestLength={manifestLength}"); + + return + manifestLength > (dataLength - 1) && + manifestLength < (dataLength + 1); + } + + private bool IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) + { + try + { + var content = codexWrapper.OnCodex(node => + { + var file = node.DownloadContent(new ContentId(receivedCid)); + if (file == null) return string.Empty; + try + { + return File.ReadAllText(file.Filename).Trim(); + } + finally + { + if (File.Exists(file.Filename)) File.Delete(file.Filename); + } + }); + + Log($"Checking content: content={content},check={check.UniqueData}"); + return content == check.UniqueData; + } + catch + { + return false; + } + } + + private async Task CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId, string checkName) + { + await handler.NowCompleted(checkName); + + check.CompletedUtc = DateTime.UtcNow; + repo.SaveChanges(); + + await CheckUserForRoleRewards(handler, userId); + } + + private async Task CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId); + + if (check.UploadCheck.CompletedUtc != DateTime.MinValue && + check.DownloadCheck.CompletedUtc != DateTime.MinValue) + { + await handler.GiveRoleReward(); + } + } + + private void Log(string msg) + { + log.Log(msg); + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs new file mode 100644 index 00000000..3c295d7f --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs @@ -0,0 +1,85 @@ +using CodexClient; +using IdentityModel.Client; +using Logging; +using Utils; +using WebUtils; + +namespace BiblioTech.CodexChecking +{ + public class CodexWrapper + { + private readonly CodexNodeFactory factory; + private readonly ILog log; + private readonly Configuration config; + private readonly object codexLock = new object(); + private ICodexNode? currentCodexNode; + + public CodexWrapper(ILog log, Configuration config) + { + this.log = log; + this.config = config; + + var httpFactory = CreateHttpFactory(); + factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); + } + + public void OnCodex(Action action) + { + lock (codexLock) + { + action(Get()); + } + } + + public T OnCodex(Func func) + { + lock (codexLock) + { + return func(Get()); + } + } + + private ICodexNode Get() + { + if (currentCodexNode == null) + { + currentCodexNode = CreateCodex(); + } + + return currentCodexNode; + } + + private ICodexNode CreateCodex() + { + var endpoint = config.CodexEndpoint; + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); + + var address = new Address( + logName: $"cdx@{host}:{port}", + host: host, + port: port + ); + + var instance = CodexInstance.CreateFromApiEndpoint("ac", address); + return factory.CreateCodexNode(instance); + } + + private HttpFactory CreateHttpFactory() + { + if (string.IsNullOrEmpty(config.CodexEndpointAuth) || !config.CodexEndpointAuth.Contains(":")) + { + return new HttpFactory(log); + } + + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + + return new HttpFactory(log, onClientCreated: client => + { + client.SetBasicAuthentication(tokens[0], tokens[1]); + }); + } + } +} diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs deleted file mode 100644 index 15728d0d..00000000 --- a/Tools/BiblioTech/CodexCidChecker.cs +++ /dev/null @@ -1,204 +0,0 @@ -using CodexClient; -using Logging; -using Utils; - -namespace BiblioTech -{ - public class CodexCidChecker - { - private static readonly string nl = Environment.NewLine; - private readonly Configuration config; - private readonly ILog log; - private readonly Mutex checkMutex = new Mutex(); - private readonly CodexNodeFactory factory; - private ICodexNode? currentCodexNode; - - public CodexCidChecker(Configuration config, ILog log) - { - this.config = config; - this.log = log; - - factory = new CodexNodeFactory(log, dataDir: config.DataPath); - - if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) - { - throw new Exception("Todo: codexnodefactory httpfactory support basicauth!"); - //var tokens = config.CodexEndpointAuth.Split(':'); - //if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); - //client.SetBasicAuthentication(tokens[0], tokens[1]); - } - } - - public CheckResponse PerformCheck(string cid) - { - if (string.IsNullOrEmpty(config.CodexEndpoint)) - { - return new CheckResponse(false, "Codex CID checker is not (yet) available.", ""); - } - - try - { - checkMutex.WaitOne(); - var codex = GetCodex(); - var nodeCheck = CheckCodex(codex); - if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); - - return PerformCheck(codex, cid); - } - catch (Exception ex) - { - return new CheckResponse(false, "Internal server error", ex.ToString()); - } - finally - { - checkMutex.ReleaseMutex(); - } - } - - private CheckResponse PerformCheck(ICodexNode codex, string cid) - { - try - { - var manifest = codex.DownloadManifestOnly(new ContentId(cid)); - return SuccessMessage(manifest); - } - catch (Exception ex) - { - return UnexpectedException(ex); - } - } - - #region Response formatting - - private CheckResponse SuccessMessage(LocalDataset content) - { - return FormatResponse( - success: true, - title: $"Success: '{content.Cid}'", - error: "", - $"size: {content.Manifest.OriginalBytes} bytes", - $"blockSize: {content.Manifest.BlockSize} bytes", - $"protected: {content.Manifest.Protected}" - ); - } - - private CheckResponse UnexpectedException(Exception ex) - { - return FormatResponse( - success: false, - title: "Unexpected error", - error: ex.ToString(), - content: "Details will be sent to the bot-admin channel." - ); - } - - private CheckResponse UnexpectedReturnCode(string response) - { - var msg = "Unexpected return code. Response: " + response; - return FormatResponse( - success: false, - title: "Unexpected return code", - error: msg, - content: msg - ); - } - - private CheckResponse FailedToFetch(string response) - { - var msg = "Failed to download content. Response: " + response; - return FormatResponse( - success: false, - title: "Could not download content", - error: msg, - msg, - $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" - ); - } - - private CheckResponse CidFormatInvalid(string response) - { - return FormatResponse( - success: false, - title: "Invalid format", - error: "", - content: "Provided CID is not formatted correctly." - ); - } - - private CheckResponse FormatResponse(bool success, string title, string error, params string[] content) - { - var msg = string.Join(nl, - new string[] - { - title, - "```" - } - .Concat(content) - .Concat(new string[] - { - "```" - }) - ) + nl + nl; - - return new CheckResponse(success, msg, error); - } - - #endregion - - #region Codex Node API - - private ICodexNode GetCodex() - { - if (currentCodexNode == null) currentCodexNode = CreateCodex(); - return currentCodexNode; - } - - private bool CheckCodex(ICodexNode node) - { - try - { - var info = node.GetDebugInfo(); - if (info == null || string.IsNullOrEmpty(info.Id)) return false; - return true; - } - catch (Exception e) - { - log.Error(e.ToString()); - return false; - } - } - - private ICodexNode CreateCodex() - { - var endpoint = config.CodexEndpoint; - var splitIndex = endpoint.LastIndexOf(':'); - var host = endpoint.Substring(0, splitIndex); - var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); - - var address = new Address( - logName: $"cdx@{host}:{port}", - host: host, - port: port - ); - - var instance = CodexInstance.CreateFromApiEndpoint("ac", address); - return factory.CreateCodexNode(instance); - } - - #endregion - } - - public class CheckResponse - { - public CheckResponse(bool success, string message, string error) - { - Success = success; - Message = message; - Error = error; - } - - public bool Success { get; } - public string Message { get; } - public string Error { get; } - } -} diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index b6f2f7ca..10e1e24b 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -4,6 +4,9 @@ using Discord; using Newtonsoft.Json; using BiblioTech.Rewards; using Logging; +using BiblioTech.CodexChecking; +using Nethereum.Model; +using static Org.BouncyCastle.Math.EC.ECCurve; namespace BiblioTech { @@ -11,13 +14,15 @@ namespace BiblioTech { private readonly DiscordSocketClient client; private readonly CustomReplacement replacement; + private readonly ActiveP2pRoleRemover roleRemover; private readonly BaseCommand[] commands; private readonly ILog log; - public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, params BaseCommand[] commands) + public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, ActiveP2pRoleRemover roleRemover, params BaseCommand[] commands) { this.client = client; this.replacement = replacement; + this.roleRemover = roleRemover; this.commands = commands; this.log = log; client.Ready += Client_Ready; @@ -30,10 +35,15 @@ namespace BiblioTech Program.AdminChecker.SetGuild(guild); log.Log($"Initializing for guild: '{guild.Name}'"); - var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray(); - if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel"); - Program.AdminChecker.SetAdminChannel(adminChannels.First()); - Program.RoleDriver = new RoleDriver(client, log, replacement); + var adminChannel = GetChannel(guild, Program.Config.AdminChannelId); + if (adminChannel == null) throw new Exception("No admin message channel"); + var chainEventsChannel = GetChannel(guild, Program.Config.ChainEventsChannelId); + var rewardsChannel = GetChannel(guild, Program.Config.RewardsChannelId); + + Program.AdminChecker.SetAdminChannel(adminChannel); + Program.RoleDriver = new RoleDriver(client, Program.UserRepo, log, rewardsChannel); + Program.ChainActivityHandler = new ChainActivityHandler(log, Program.UserRepo); + Program.EventsSender = new ChainEventsSender(log, replacement, chainEventsChannel); var builders = commands.Select(c => { @@ -65,6 +75,8 @@ namespace BiblioTech { log.Log($"{cmd.Name} ({cmd.Description}) [{DescribOptions(cmd.Options)}]"); } + + roleRemover.Start(); } catch (HttpException exception) { @@ -72,9 +84,16 @@ namespace BiblioTech log.Error(json); throw; } + Program.Dispatcher.Start(); log.Log("Initialized."); } + private SocketTextChannel? GetChannel(SocketGuild guild, ulong id) + { + if (id == 0) return null; + return guild.TextChannels.SingleOrDefault(c => c.Id == id); + } + private string DescribOptions(IReadOnlyCollection options) { return string.Join(",", options.Select(DescribeOption).ToArray()); diff --git a/Tools/BiblioTech/Commands/AdminCommand.cs b/Tools/BiblioTech/Commands/AdminCommand.cs index 45b3fc29..f9877181 100644 --- a/Tools/BiblioTech/Commands/AdminCommand.cs +++ b/Tools/BiblioTech/Commands/AdminCommand.cs @@ -8,16 +8,10 @@ namespace BiblioTech.Commands private readonly ClearUserAssociationCommand clearCommand = new ClearUserAssociationCommand(); private readonly ReportCommand reportCommand = new ReportCommand(); private readonly WhoIsCommand whoIsCommand = new WhoIsCommand(); - private readonly AddSprCommand addSprCommand; - private readonly ClearSprsCommand clearSprsCommand; - private readonly GetSprCommand getSprCommand; private readonly LogReplaceCommand logReplaceCommand; - public AdminCommand(SprCommand sprCommand, CustomReplacement replacement) + public AdminCommand(CustomReplacement replacement) { - addSprCommand = new AddSprCommand(sprCommand); - clearSprsCommand = new ClearSprsCommand(sprCommand); - getSprCommand = new GetSprCommand(sprCommand); logReplaceCommand = new LogReplaceCommand(replacement); } @@ -30,9 +24,6 @@ namespace BiblioTech.Commands clearCommand, reportCommand, whoIsCommand, - addSprCommand, - clearSprsCommand, - getSprCommand, logReplaceCommand }; @@ -53,9 +44,6 @@ namespace BiblioTech.Commands await clearCommand.CommandHandler(context); await reportCommand.CommandHandler(context); await whoIsCommand.CommandHandler(context); - await addSprCommand.CommandHandler(context); - await clearSprsCommand.CommandHandler(context); - await getSprCommand.CommandHandler(context); await logReplaceCommand.CommandHandler(context); } @@ -144,78 +132,6 @@ namespace BiblioTech.Commands } } - public class AddSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("spr", "Codex SPR", true); - - public AddSprCommand(SprCommand sprCommand) - : base(name: "addspr", - description: "Adds a Codex SPR, to be given to users with '/boot'.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var spr = await stringOption.Parse(context); - - if (!string.IsNullOrEmpty(spr) ) - { - sprCommand.Add(spr); - await context.Followup("A-OK!"); - } - else - { - await context.Followup("SPR is null or empty."); - } - } - } - - public class ClearSprsCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("areyousure", "set to 'true' if you are.", true); - - public ClearSprsCommand(SprCommand sprCommand) - : base(name: "clearsprs", - description: "Clears all Codex SPRs in the bot. Users won't be able to use '/boot' till new ones are added.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var areyousure = await stringOption.Parse(context); - - if (areyousure != "true") return; - - sprCommand.Clear(); - await context.Followup("Cleared all SPRs."); - } - } - - public class GetSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - - public GetSprCommand(SprCommand sprCommand) - : base(name: "getsprs", - description: "Shows all Codex SPRs in the bot.") - { - this.sprCommand = sprCommand; - } - - protected override async Task onSubCommand(CommandContext context) - { - await context.Followup("SPRs: " + string.Join(", ", sprCommand.Get().Select(s => $"'{s}'"))); - } - } - public class LogReplaceCommand : SubCommandOption { private readonly CustomReplacement replacement; diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs deleted file mode 100644 index 1e77ce26..00000000 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ /dev/null @@ -1,111 +0,0 @@ -using BiblioTech.Options; -using Discord; - -namespace BiblioTech.Commands -{ - public class CheckCidCommand : BaseCommand - { - private readonly StringOption cidOption = new StringOption( - name: "cid", - description: "Codex Content-Identifier", - isRequired: true); - private readonly CodexCidChecker checker; - private readonly CidStorage cidStorage; - - public CheckCidCommand(CodexCidChecker checker) - { - this.checker = checker; - this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt")); - } - - public override string Name => "check"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Checks if content is available in the testnet."; - public override CommandOption[] Options => new[] { cidOption }; - - protected override async Task Invoke(CommandContext context) - { - var user = context.Command.User; - var cid = await cidOption.Parse(context); - if (string.IsNullOrEmpty(cid)) - { - await context.Followup("Option 'cid' was not received."); - return; - } - - var response = checker.PerformCheck(cid); - await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); - - if (response.Success) - { - await CheckAltruisticRole(context, user, cid, response.Message); - return; - } - - await context.Followup(response.Message); - } - - private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage) - { - if (cidStorage.TryAddCid(cid, user.Id)) - { - if (await GiveAltruisticRole(context, user, responseMessage)) - { - return; - } - } - else - { - await context.Followup($"{responseMessage}\n\nThis CID has already been used by another user. No role will be granted."); - return; - } - - await context.Followup(responseMessage); - } - - private async Task GiveAltruisticRole(CommandContext context, IUser user, string responseMessage) - { - try - { - await Program.RoleDriver.GiveAltruisticRole(user); - await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!"); - return true; - } - catch (Exception ex) - { - await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}"); - return false; - } - } - } - - public class CidStorage - { - private readonly string filePath; - private static readonly object _lock = new object(); - - public CidStorage(string filePath) - { - this.filePath = filePath; - if (!File.Exists(filePath)) - { - File.WriteAllText(filePath, string.Empty); - } - } - - public bool TryAddCid(string cid, ulong userId) - { - lock (_lock) - { - var existingEntries = File.ReadAllLines(filePath); - if (existingEntries.Any(line => line.Split(',')[0] == cid)) - { - return false; - } - - File.AppendAllLines(filePath, new[] { $"{cid},{userId}" }); - return true; - } - } - } -} diff --git a/Tools/BiblioTech/Commands/CheckDownloadCommand.cs b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs new file mode 100644 index 00000000..0e1aa563 --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs @@ -0,0 +1,58 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckDownloadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption contentOption = new StringOption( + name: "content", + description: "Content of the downloaded file", + isRequired: false); + + public CheckDownloadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkdownload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the download connectivity of your Codex node."; + public override CommandOption[] Options => [contentOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var content = await contentOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(content)) + { + await checker.StartDownloadCheck(handler, user.Id); + } + else + { + if (content.Length > 1024) + { + await context.Followup("Provided content is too long!"); + return; + } + await checker.VerifyDownloadCheck(handler, user.Id, content); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckDownloadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs new file mode 100644 index 00000000..f7b1c4f0 --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -0,0 +1,102 @@ +using System.Linq; +using BiblioTech.CodexChecking; +using BiblioTech.Options; +using Discord; + +namespace BiblioTech.Commands +{ + public class CheckResponseHandler : ICheckResponseHandler + { + private CommandContext context; + private readonly IUser user; + + public CheckResponseHandler(CommandContext context, IUser user) + { + this.context = context; + this.user = user; + } + + public async Task CheckNotStarted() + { + await context.Followup("Run this command without any arguments first, to begin the check process."); + } + + public async Task CouldNotDownloadCid() + { + await context.Followup("Could not download the CID."); + } + + public async Task GiveCidToUser(string cid) + { + await context.Followup( + FormatCatchyMessage("[💾] Please download this CID using your Codex node.", + $"👉 `{cid}`.", + "👉 Then provide the *content of the downloaded file* as argument to this command.")); + } + + public async Task GiveDataFileToUser(string fileContent) + { + await context.SendFile(fileContent, + FormatCatchyMessage("[💿] Please download the attached file.", + "👉 Upload it to your Codex node.", + "👉 Then provide the *CID* as argument to this command.")); + } + + private string FormatCatchyMessage(string title, params string[] content) + { + var entries = new List(); + entries.Add(title); + entries.Add("```"); + entries.AddRange(content); + entries.Add("```"); + return string.Join(Environment.NewLine, entries.ToArray()); + } + + public async Task GiveRoleReward() + { + try + { + await Program.RoleDriver.RunRoleGiver(async r => + { + await r.GiveAltruisticRole(user.Id); + await r.GiveActiveP2pParticipant(user.Id); + }); + await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!"); + } + catch (Exception ex) + { + await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user <@{user.Id}>: {ex.Message}"); + } + } + + public async Task InvalidData() + { + await context.Followup("The received data didn't match. Check has failed."); + } + + public async Task NowCompleted(string checkName) + { + // check if eth address is known for user. + var data = Program.UserRepo.GetUser(user); + if (data.CurrentAddress == null) + { + await context.Followup($"Successfully completed the check!{Environment.NewLine}" + + $"You haven't yet set your ethereum address. Consider using '/set' to set it.{Environment.NewLine}" + + $"(You can find your address in the 'eth.address' file of your Codex node.)"); + + await Program.AdminChecker.SendInAdminChannel($"User <@{user.Id}> has completed check: {checkName}" + + $" - EthAddress not set for user. User was reminded."); + } + else + { + await context.Followup("Successfully completed the check!"); + await Program.AdminChecker.SendInAdminChannel($"User <@{user.Id}> has completed check: {checkName}"); + } + } + + public async Task ToAdminChannel(string msg) + { + await Program.AdminChecker.SendInAdminChannel(msg); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckUploadCommand.cs b/Tools/BiblioTech/Commands/CheckUploadCommand.cs new file mode 100644 index 00000000..b1589b5c --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckUploadCommand.cs @@ -0,0 +1,53 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckUploadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption cidOption = new StringOption( + name: "cid", + description: "Codex Content-Identifier", + isRequired: false); + + public CheckUploadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkupload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the upload connectivity of your Codex node."; + public override CommandOption[] Options => [cidOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var cid = await cidOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(cid)) + { + await checker.StartUploadCheck(handler, user.Id); + } + else + { + await checker.VerifyUploadCheck(handler, user.Id, cid); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckUploadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Commands/SprCommand.cs b/Tools/BiblioTech/Commands/SprCommand.cs deleted file mode 100644 index 4b3235de..00000000 --- a/Tools/BiblioTech/Commands/SprCommand.cs +++ /dev/null @@ -1,48 +0,0 @@ -using BiblioTech.Options; - -namespace BiblioTech.Commands -{ - public class SprCommand : BaseCommand - { - private readonly Random random = new Random(); - private readonly List knownSprs = new List(); - - public override string Name => "boot"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Gets an SPR. (Signed peer record, used for bootstrapping.)"; - - protected override async Task Invoke(CommandContext context) - { - await ReplyWithRandomSpr(context); - } - - public void Add(string spr) - { - if (knownSprs.Contains(spr)) return; - knownSprs.Add(spr); - } - - public void Clear() - { - knownSprs.Clear(); - } - - public string[] Get() - { - return knownSprs.ToArray(); - } - - private async Task ReplyWithRandomSpr(CommandContext context) - { - if (!knownSprs.Any()) - { - await context.Followup("I'm sorry, no SPRs are available... :c"); - return; - } - - var i = random.Next(0, knownSprs.Count); - var spr = knownSprs[i]; - await context.Followup($"Your SPR: `{spr}`"); - } - } -} diff --git a/Tools/BiblioTech/Commands/UserAssociateCommand.cs b/Tools/BiblioTech/Commands/UserAssociateCommand.cs index 61d55fb9..dbe7dced 100644 --- a/Tools/BiblioTech/Commands/UserAssociateCommand.cs +++ b/Tools/BiblioTech/Commands/UserAssociateCommand.cs @@ -69,8 +69,8 @@ namespace BiblioTech.Commands { await context.Followup(new string[] { - "Done! Thank you for joining the test net!", - "By default, the bot will @-mention you with test-net related notifications.", + "Done! Thank you for joining!", + "By default, the bot will @-mention you with discord role notifications.", $"You can enable/disable this behavior with the '/{notifyCommand.Name}' command." }); diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 342fbb08..a8187379 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -26,9 +26,6 @@ namespace BiblioTech [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")] public ulong ChainEventsChannelId { get; set; } - [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] - public ulong AltruisticRoleId { get; set; } - [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")] public int RewardApiPort { get; set; } = 31080; @@ -47,8 +44,40 @@ namespace BiblioTech [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")] public string CodexEndpointAuth { get; set; } = ""; + #region Role Rewards + + /// + /// Awarded when both checkupload and checkdownload have been completed. + /// + [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] + public ulong AltruisticRoleId { get; set; } + + /// + /// Awarded as long as either checkupload or checkdownload were completed within the last ActiveP2pRoleDuration minutes. + /// + [Uniform("active-p2p-role-id", "apri", "ACTIVEP2PROLEID", false, "ID of discord server role for active p2p participants.")] + public ulong ActiveP2pParticipantRoleId { get; set; } + + [Uniform("active-p2p-role-duration", "aprd", "ACTIVEP2PROLEDURATION", false, "Duration in minutes for the active p2p participant role from the last successful check command.")] + public int ActiveP2pRoleDurationMinutes { get; set; } + + /// + /// Awarded as long as the user is hosting at least 1 slot. + /// + [Uniform("active-host-role-id", "ahri", "ACTIVEHOSTROLEID", false, "Id of discord server role for active slot hosters.")] + public ulong ActiveHostRoleId { get; set; } + + /// + /// Awarded as long as the user has at least 1 active storage purchase contract. + /// + [Uniform("active-client-role-id", "acri", "ACTIVECLIENTROLEID", false, "Id of discord server role for users with at least 1 active purchase contract.")] + public ulong ActiveClientRoleId { get; set; } + + #endregion + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); + public string ChecksDataPath => Path.Combine(DataPath, "checks"); public string LogPath => Path.Combine(DataPath, "logs"); public bool DebugNoDiscord => NoDiscord == 1; } diff --git a/Tools/BiblioTech/LoggingRoleDriver.cs b/Tools/BiblioTech/LoggingRoleDriver.cs index f5435523..b1759af9 100644 --- a/Tools/BiblioTech/LoggingRoleDriver.cs +++ b/Tools/BiblioTech/LoggingRoleDriver.cs @@ -15,18 +15,72 @@ namespace BiblioTech this.log = log; } - public async Task GiveAltruisticRole(IUser user) + public async Task RunRoleGiver(Func action) { await Task.CompletedTask; - - log.Log($"Give altruistic role to {user.Id}"); + await action(new LoggingRoleGiver(log)); } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) { await Task.CompletedTask; + } - log.Log(JsonConvert.SerializeObject(rewards, Formatting.None)); + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) + { + await Task.CompletedTask; + } + + private class LoggingRoleGiver : IRoleGiver + { + private readonly ILog log; + + public LoggingRoleGiver(ILog log) + { + this.log = log; + } + + public async Task GiveActiveClient(ulong userId) + { + log.Log($"Giving ActiveClient role to " + userId); + await Task.CompletedTask; + } + + public async Task GiveActiveHost(ulong userId) + { + log.Log($"Giving ActiveHost role to " + userId); + await Task.CompletedTask; + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + log.Log($"Giving ActiveP2p role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + log.Log($"Removing ActiveP2p role from " + userId); + await Task.CompletedTask; + } + + public async Task GiveAltruisticRole(ulong userId) + { + log.Log($"Giving Altruistic role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveClient(ulong userId) + { + log.Log($"Removing ActiveClient role from " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveHost(ulong userId) + { + log.Log($"Removing ActiveHost role from " + userId); + await Task.CompletedTask; + } } } } diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 42a065ae..fbd45b36 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -49,6 +49,23 @@ namespace BiblioTech.Options } } + public async Task SendFile(string fileContent, string message) + { + if (fileContent.Length < 1) throw new Exception("File content is empty."); + + var filename = Guid.NewGuid().ToString() + ".tmp"; + File.WriteAllText(filename, fileContent); + + await Command.FollowupWithFileAsync(filename, "Codex_UploadCheckFile.txt", text: message, ephemeral: true); + + // Detached task for cleaning up the stream resources. + _ = Task.Run(() => + { + Thread.Sleep(TimeSpan.FromMinutes(2)); + File.Delete(filename); + }); + } + private string FormatChunk(string[] chunk) { return string.Join(Environment.NewLine, chunk); diff --git a/Tools/BiblioTech/Options/StringOption.cs b/Tools/BiblioTech/Options/StringOption.cs index 15e04c9a..047bf15a 100644 --- a/Tools/BiblioTech/Options/StringOption.cs +++ b/Tools/BiblioTech/Options/StringOption.cs @@ -12,11 +12,12 @@ namespace BiblioTech.Options public async Task Parse(CommandContext context) { var strData = context.Options.SingleOrDefault(o => o.Name == Name); - if (strData == null) + if (strData == null && IsRequired) { await context.Followup("String option not received."); return null; } + if (strData == null) return null; return strData.Value as string; } } diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 46dee29c..0ce84bde 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -1,9 +1,13 @@ using ArgsUniform; +using BiblioTech.CodexChecking; using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; using Discord.WebSocket; +using DiscordRewards; using Logging; +using Nethereum.Model; +using Newtonsoft.Json; namespace BiblioTech { @@ -12,16 +16,17 @@ namespace BiblioTech private DiscordSocketClient client = null!; private CustomReplacement replacement = null!; + public static CallDispatcher Dispatcher { get; private set; } = null!; public static Configuration Config { get; private set; } = null!; public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; public static IDiscordRoleDriver RoleDriver { get; set; } = null!; + public static ChainActivityHandler ChainActivityHandler { get; set; } = null!; + public static ChainEventsSender EventsSender { get; set; } = null!; public static ILog Log { get; private set; } = null!; public static Task Main(string[] args) { - Log = new ConsoleLog(); - var uniformArgs = new ArgsUniform(PrintHelp, args); Config = uniformArgs.Parse(); @@ -30,9 +35,12 @@ namespace BiblioTech new ConsoleLog() ); + Dispatcher = new CallDispatcher(Log); + EnsurePath(Config.DataPath); EnsurePath(Config.UserDataPath); EnsurePath(Config.EndpointsPath); + EnsurePath(Config.ChecksDataPath); return new Program().MainAsync(args); } @@ -80,18 +88,20 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; - var checker = new CodexCidChecker(Config, Log); + var checkRepo = new CheckRepo(Config); + var codexWrapper = new CodexWrapper(Log, Config); + var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); - var sprCommand = new SprCommand(); - var handler = new CommandHandler(Log, client, replacement, + var roleRemover = new ActiveP2pRoleRemover(Config, Log, checkRepo); + var handler = new CommandHandler(Log, client, replacement, roleRemover, new GetBalanceCommand(associateCommand), new MintCommand(associateCommand), - sprCommand, associateCommand, notifyCommand, - new CheckCidCommand(checker), - new AdminCommand(sprCommand, replacement) + new CheckUploadCommand(checker), + new CheckDownloadCommand(checker), + new AdminCommand(replacement) ); await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); diff --git a/Tools/BiblioTech/RandomBusyMessage.cs b/Tools/BiblioTech/RandomBusyMessage.cs index 290c9273..8f0477a7 100644 --- a/Tools/BiblioTech/RandomBusyMessage.cs +++ b/Tools/BiblioTech/RandomBusyMessage.cs @@ -14,7 +14,9 @@ "Analyzing the wavelengths...", "Charging the flux-capacitor...", "Jumping to hyperspace...", - "Computing the ultimate answer..." + "Computing the ultimate answer...", + "Turning it off and on again...", + "Compiling from sources..." }; public static string Get() diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs new file mode 100644 index 00000000..d35d9fe9 --- /dev/null +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -0,0 +1,134 @@ +using Discord; +using DiscordRewards; +using Logging; + +namespace BiblioTech.Rewards +{ + public class ChainActivityHandler + { + private readonly ILog log; + private readonly UserRepo repo; + private ActiveUserIds? previousIds = null; + + public ChainActivityHandler(ILog log, UserRepo repo) + { + this.log = log; + this.repo = repo; + } + + public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) + { + var activeUserIds = ConvertToUserIds(activeChainAddresses); + if (!HasChanged(activeUserIds)) return; + + await GiveAndRemoveRoles(activeUserIds); + } + + private async Task GiveAndRemoveRoles(ActiveUserIds activeUserIds) + { + await Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, activeUserIds), + whenDone: g => GiveRolesToRemaining(g, activeUserIds), + Program.Config.ActiveClientRoleId, + Program.Config.ActiveHostRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, ActiveUserIds activeIds) + { + if (roleId == Program.Config.ActiveClientRoleId) + { + await CheckUserWithRole(user, activeIds.Clients, giver.RemoveActiveClient); + } + else if (roleId == Program.Config.ActiveHostRoleId) + { + await CheckUserWithRole(user, activeIds.Hosts, giver.RemoveActiveHost); + } + else + { + throw new Exception("Unknown roleId received!"); + } + } + + private async Task CheckUserWithRole(IUser user, List activeUsers, Func removeActiveRole) + { + if (ShouldUserHaveRole(user, activeUsers)) + { + activeUsers.Remove(user.Id); + } + else + { + await removeActiveRole(user.Id); + } + } + + private bool ShouldUserHaveRole(IUser user, List activeUsers) + { + return activeUsers.Any(id => id == user.Id); + } + + private async Task GiveRolesToRemaining(IRoleGiver giver, ActiveUserIds ids) + { + foreach (var client in ids.Clients) await giver.GiveActiveClient(client); + foreach (var host in ids.Hosts) await giver.GiveActiveHost(host); + } + + private bool HasChanged(ActiveUserIds activeUserIds) + { + if (previousIds == null) + { + previousIds = activeUserIds; + return true; + } + + if (!IsEquivalent(previousIds.Hosts, activeUserIds.Hosts)) return true; + if (!IsEquivalent(previousIds.Clients, activeUserIds.Clients)) return true; + return false; + } + + private static bool IsEquivalent(IEnumerable a, IEnumerable b) + { + return a.SequenceEqual(b); + } + + private ActiveUserIds ConvertToUserIds(ActiveChainAddresses activeChainAddresses) + { + return new ActiveUserIds + ( + hosts: Map(activeChainAddresses.Hosts), + clients: Map(activeChainAddresses.Clients) + ); + } + + private ulong[] Map(string[] ethAddresses) + { + var result = new List(); + foreach (var ethAddress in ethAddresses) + { + var userMaybe = repo.GetUserDataForAddressMaybe(new Utils.EthAddress(ethAddress)); + if (userMaybe != null) + { + result.Add(userMaybe.DiscordId); + } + } + + return result.Order().ToArray(); + } + + private void Log(string msg) + { + log.Log(msg); + } + + private class ActiveUserIds + { + public ActiveUserIds(IEnumerable hosts, IEnumerable clients) + { + Hosts = hosts.ToList(); + Clients = clients.ToList(); + } + + public List Hosts { get; } + public List Clients { get; } + } + } +} diff --git a/Tools/BiblioTech/Rewards/CustomReplacement.cs b/Tools/BiblioTech/Rewards/CustomReplacement.cs index dbd8eaf3..912aa789 100644 --- a/Tools/BiblioTech/Rewards/CustomReplacement.cs +++ b/Tools/BiblioTech/Rewards/CustomReplacement.cs @@ -28,14 +28,14 @@ namespace BiblioTech.Rewards public void Add(string from, string to) { - if (replacements.ContainsKey(from)) + AddOrUpdate(from, to); + + var lower = from.ToLowerInvariant(); + if (lower != from) { - replacements[from] = to; - } - else - { - replacements.Add(from, to); + AddOrUpdate(lower, to); } + Save(); } @@ -55,6 +55,18 @@ namespace BiblioTech.Rewards return result; } + private void AddOrUpdate(string from, string to) + { + if (replacements.ContainsKey(from)) + { + replacements[from] = to; + } + else + { + replacements.Add(from, to); + } + } + private void Save() { ReplaceJson[] replaces = replacements.Select(pair => diff --git a/Tools/BiblioTech/Rewards/RewardContext.cs b/Tools/BiblioTech/Rewards/RewardContext.cs deleted file mode 100644 index d05d5f6d..00000000 --- a/Tools/BiblioTech/Rewards/RewardContext.cs +++ /dev/null @@ -1,102 +0,0 @@ -using Discord.WebSocket; -using Discord; -using DiscordRewards; - -namespace BiblioTech.Rewards -{ - public class RewardContext - { - private readonly Dictionary users; - private readonly Dictionary roles; - private readonly SocketTextChannel? rewardsChannel; - - public RewardContext(Dictionary users, Dictionary roles, SocketTextChannel? rewardsChannel) - { - this.users = users; - this.roles = roles; - this.rewardsChannel = rewardsChannel; - } - - public async Task ProcessGiveRewardsCommand(UserReward[] rewards) - { - foreach (var rewardCommand in rewards) - { - if (roles.ContainsKey(rewardCommand.RewardCommand.RewardId)) - { - var role = roles[rewardCommand.RewardCommand.RewardId]; - await ProcessRewardCommand(role, rewardCommand); - } - else - { - Program.Log.Error($"RoleID not found on guild: {rewardCommand.RewardCommand.RewardId}"); - } - } - } - - private async Task ProcessRewardCommand(RoleReward role, UserReward reward) - { - foreach (var user in reward.Users) - { - await GiveReward(role, user); - } - } - - private async Task GiveReward(RoleReward role, UserData user) - { - if (!users.ContainsKey(user.DiscordId)) - { - Program.Log.Log($"User by id '{user.DiscordId}' not found."); - return; - } - - var guildUser = users[user.DiscordId]; - - var alreadyHas = guildUser.RoleIds.ToArray(); - var logMessage = $"Giving reward '{role.SocketRole.Id}' to user '{user.DiscordId}'({user.Name})[" + - $"alreadyHas:{string.Join(",", alreadyHas.Select(a => a.ToString()))}]: "; - - - if (alreadyHas.Any(r => r == role.Reward.RoleId)) - { - logMessage += "Already has role"; - Program.Log.Log(logMessage); - return; - } - - await GiveRole(guildUser, role.SocketRole); - await SendNotification(role, user, guildUser); - await Task.Delay(1000); - logMessage += "Role given. Notification sent."; - Program.Log.Log(logMessage); - } - - private async Task GiveRole(IGuildUser user, SocketRole role) - { - try - { - Program.Log.Log($"Giving role {role.Name}={role.Id} to user {user.DisplayName}"); - await user.AddRoleAsync(role); - } - catch (Exception ex) - { - Program.Log.Error($"Failed to give role '{role.Name}' to user '{user.DisplayName}': {ex}"); - } - } - - private async Task SendNotification(RoleReward reward, UserData userData, IGuildUser user) - { - try - { - if (userData.NotificationsEnabled && rewardsChannel != null) - { - var msg = reward.Reward.Message.Replace(RewardConfig.UsernameTag, $"<@{user.Id}>"); - await rewardsChannel.SendMessageAsync(msg); - } - } - catch (Exception ex) - { - Program.Log.Error($"Failed to notify user '{user.DisplayName}' about role '{reward.SocketRole.Name}': {ex}"); - } - } - } -} diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 3a20a084..43a01ad5 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -4,10 +4,26 @@ using Microsoft.AspNetCore.Mvc; namespace BiblioTech.Rewards { + /// + /// We like callbacks in this interface because we're trying to batch role-modifying operations, + /// So that we're not poking the server lots of times very quickly. + /// public interface IDiscordRoleDriver { - Task GiveRewards(GiveRewardsCommand rewards); - Task GiveAltruisticRole(IUser user); + Task RunRoleGiver(Func action); + Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate); + Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate); + } + + public interface IRoleGiver + { + Task GiveAltruisticRole(ulong userId); + Task GiveActiveP2pParticipant(ulong userId); + Task RemoveActiveP2pParticipant(ulong userId); + Task GiveActiveHost(ulong userId); + Task RemoveActiveHost(ulong userId); + Task GiveActiveClient(ulong userId); + Task RemoveActiveClient(ulong userId); } [Route("api/[controller]")] @@ -21,16 +37,19 @@ namespace BiblioTech.Rewards } [HttpPost] - public async Task Give(GiveRewardsCommand cmd) + public async Task Give(EventsAndErrors cmd) { - try + Program.Dispatcher.Add(() => { - await Program.RoleDriver.GiveRewards(cmd); - } - catch (Exception ex) + Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses).Wait(); + }); + + Program.Dispatcher.Add(() => { - Program.Log.Error("Exception: " + ex); - } + Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors).Wait(); + }); + + await Task.CompletedTask; return "OK"; } } diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index 1964da8e..1cc9f3f2 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -1,6 +1,7 @@ using Discord; using Discord.WebSocket; using DiscordRewards; +using k8s.KubeConfigModels; using Logging; using Newtonsoft.Json; using Utils; @@ -10,145 +11,52 @@ namespace BiblioTech.Rewards public class RoleDriver : IDiscordRoleDriver { private readonly DiscordSocketClient client; + private readonly UserRepo userRepo; private readonly ILog log; private readonly SocketTextChannel? rewardsChannel; - private readonly ChainEventsSender eventsSender; - private readonly RewardRepo repo = new RewardRepo(); - public RoleDriver(DiscordSocketClient client, ILog log, CustomReplacement replacement) + public RoleDriver(DiscordSocketClient client, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) { this.client = client; + this.userRepo = userRepo; this.log = log; - rewardsChannel = GetChannel(Program.Config.RewardsChannelId); - eventsSender = new ChainEventsSender(log, replacement, GetChannel(Program.Config.ChainEventsChannelId)); + this.rewardsChannel = rewardsChannel; } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task RunRoleGiver(Func action) { - log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'"); + var context = OpenRoleModifyContext(); + var mapper = new RoleMapper(context); + await action(mapper); + } - if (rewards.Rewards.Any()) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) + { + await IterateUsersWithRoles(onUserWithRole, g => Task.CompletedTask, rolesToIterate); + } + + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) + { + var context = OpenRoleModifyContext(); + var mapper = new RoleMapper(context); + foreach (var user in context.Users) { - await ProcessRewards(rewards); - } - - await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors); - } - - public async Task GiveAltruisticRole(IUser user) - { - var guild = GetGuild(); - var role = guild.Roles.SingleOrDefault(r => r.Id == Program.Config.AltruisticRoleId); - if (role == null) return; - - var guildUser = guild.Users.SingleOrDefault(u => u.Id == user.Id); - if (guildUser == null) return; - - await guildUser.AddRoleAsync(role); - } - - private async Task ProcessRewards(GiveRewardsCommand rewards) - { - try - { - var guild = GetGuild(); - // We load all role and user information first, - // so we don't ask the server for the same info multiple times. - var context = new RewardContext( - await LoadAllUsers(guild), - LookUpAllRoles(guild, rewards), - rewardsChannel); - - await context.ProcessGiveRewardsCommand(LookUpUsers(rewards)); - } - catch (Exception ex) - { - log.Error("Failed to process rewards: " + ex); - } - } - - private SocketTextChannel? GetChannel(ulong id) - { - if (id == 0) return null; - return GetGuild().TextChannels.SingleOrDefault(c => c.Id == id); - } - - private async Task> LoadAllUsers(SocketGuild guild) - { - log.Log("Loading all users.."); - var result = new Dictionary(); - var users = guild.GetUsersAsync(); - await foreach (var ulist in users) - { - foreach (var u in ulist) + foreach (var role in rolesToIterate) { - result.Add(u.Id, u); - //var roleIds = string.Join(",", u.RoleIds.Select(r => r.ToString()).ToArray()); - //log.Log($" > {u.Id}({u.DisplayName}) has [{roleIds}]"); - } - } - return result; - } - - private Dictionary LookUpAllRoles(SocketGuild guild, GiveRewardsCommand rewards) - { - var result = new Dictionary(); - foreach (var r in rewards.Rewards) - { - if (!result.ContainsKey(r.RewardId)) - { - var rewardConfig = repo.Rewards.SingleOrDefault(rr => rr.RoleId == r.RewardId); - if (rewardConfig == null) + if (user.RoleIds.Contains(role)) { - log.Log($"No Reward is configured for id '{r.RewardId}'."); - } - else - { - var socketRole = guild.GetRole(r.RewardId); - if (socketRole == null) - { - log.Log($"Guild Role by id '{r.RewardId}' not found."); - } - else - { - result.Add(r.RewardId, new RoleReward(socketRole, rewardConfig)); - } + await onUserWithRole(mapper, user, role); } } } - - return result; + await whenDone(mapper); } - private UserReward[] LookUpUsers(GiveRewardsCommand rewards) + private RoleModifyContext OpenRoleModifyContext() { - return rewards.Rewards.Select(LookUpUserData).ToArray(); - } - - private UserReward LookUpUserData(RewardUsersCommand command) - { - return new UserReward(command, - command.UserAddresses - .Select(LookUpUserDataForAddress) - .Where(d => d != null) - .Cast() - .ToArray()); - } - - private UserData? LookUpUserDataForAddress(string address) - { - try - { - var userData = Program.UserRepo.GetUserDataForAddress(new EthAddress(address)); - if (userData != null) log.Log($"User '{userData.Name}' was looked up."); - else log.Log($"Lookup for user was unsuccessful. EthAddress: '{address}'"); - return userData; - } - catch (Exception ex) - { - log.Error("Error during UserData lookup: " + ex); - return null; - } + var context = new RoleModifyContext(GetGuild(), userRepo, log, rewardsChannel); + context.Initialize(); + return context; } private SocketGuild GetGuild() @@ -163,27 +71,48 @@ namespace BiblioTech.Rewards } } - public class RoleReward + public class RoleMapper : IRoleGiver { - public RoleReward(SocketRole socketRole, RewardConfig reward) + private readonly RoleModifyContext context; + + public RoleMapper(RoleModifyContext context) { - SocketRole = socketRole; - Reward = reward; + this.context = context; } - public SocketRole SocketRole { get; } - public RewardConfig Reward { get; } - } - - public class UserReward - { - public UserReward(RewardUsersCommand rewardCommand, UserData[] users) + public async Task GiveActiveClient(ulong userId) { - RewardCommand = rewardCommand; - Users = users; + await context.GiveRole(userId, Program.Config.ActiveClientRoleId); } - public RewardUsersCommand RewardCommand { get; } - public UserData[] Users { get; } + public async Task GiveActiveHost(ulong userId) + { + await context.GiveRole(userId, Program.Config.ActiveHostRoleId); + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + await context.GiveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task GiveAltruisticRole(ulong userId) + { + await context.GiveRole(userId, Program.Config.AltruisticRoleId); + } + + public async Task RemoveActiveClient(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveClientRoleId); + } + + public async Task RemoveActiveHost(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveHostRoleId); + } } } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs new file mode 100644 index 00000000..6fd4fb48 --- /dev/null +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -0,0 +1,135 @@ +using Discord.WebSocket; +using Discord; +using DiscordRewards; +using Nethereum.Model; +using Logging; + +namespace BiblioTech.Rewards +{ + public class RoleModifyContext + { + private Dictionary users = new(); + private Dictionary roles = new(); + private DateTime lastLoad = DateTime.MinValue; + private readonly object _lock = new object(); + + private readonly SocketGuild guild; + private readonly UserRepo userRepo; + private readonly ILog log; + private readonly SocketTextChannel? rewardsChannel; + + public RoleModifyContext(SocketGuild guild, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) + { + this.guild = guild; + this.userRepo = userRepo; + this.log = log; + this.rewardsChannel = rewardsChannel; + } + + public void Initialize() + { + lock (_lock) + { + var span = DateTime.UtcNow - lastLoad; + if (span > TimeSpan.FromMinutes(10)) + { + lastLoad = DateTime.UtcNow; + log.Log("Loading all users and roles..."); + var task = LoadAllUsers(guild); + task.Wait(); + this.users = task.Result; + this.roles = LoadAllRoles(guild); + } + } + } + + public IGuildUser[] Users => users.Values.ToArray(); + + public async Task GiveRole(ulong userId, ulong roleId) + { + Log($"Giving role {roleId} to user {userId}"); + var role = GetRole(roleId); + var guildUser = GetUser(userId); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.AddRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Added role '{role.Name}' for user <@{userId}>."); + + await SendNotification(guildUser, role); + } + + public async Task RemoveRole(ulong userId, ulong roleId) + { + Log($"Removing role {roleId} from user {userId}"); + var role = GetRole(roleId); + var guildUser = GetUser(userId); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.RemoveRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Removed role '{role.Name}' for user <@{userId}>."); + } + + private SocketRole? GetRole(ulong roleId) + { + if (roles.ContainsKey(roleId)) return roles[roleId]; + return null; + } + + private IGuildUser? GetUser(ulong userId) + { + if (users.ContainsKey(userId)) return users[userId]; + return null; + } + + private void Log(string msg) + { + log.Log(msg); + } + + private async Task> LoadAllUsers(SocketGuild guild) + { + var result = new Dictionary(); + var users = guild.GetUsersAsync(); + await foreach (var ulist in users) + { + foreach (var u in ulist) + { + result.Add(u.Id, u); + } + } + return result; + } + + private Dictionary LoadAllRoles(SocketGuild guild) + { + var result = new Dictionary(); + var roles = guild.Roles.ToArray(); + foreach (var role in roles) + { + result.Add(role.Id, role); + } + return result; + } + + private async Task SendNotification(IGuildUser user, SocketRole role) + { + try + { + var userData = userRepo.GetUser(user); + if (userData == null) return; + + if (userData.NotificationsEnabled && rewardsChannel != null) + { + var msg = $"<@{user.Id}> has received '{role.Name}'."; + await rewardsChannel.SendMessageAsync(msg); + } + } + catch (Exception ex) + { + log.Error($"Failed to notify user '{user.DisplayName}' about role '{role.Name}': {ex}"); + } + } + } +} diff --git a/Tools/BiblioTech/UserRepo.cs b/Tools/BiblioTech/UserRepo.cs index d1a766ab..701fdfe9 100644 --- a/Tools/BiblioTech/UserRepo.cs +++ b/Tools/BiblioTech/UserRepo.cs @@ -41,6 +41,12 @@ namespace BiblioTech return cache.Values.ToArray(); } + public UserData GetUser(IUser user) + { + if (cache.Count == 0) LoadAllUserData(); + return GetOrCreate(user); + } + public void AddMintEventForUser(IUser user, EthAddress usedAddress, Transaction? eth, Transaction? tokens) { lock (repoLock) @@ -68,10 +74,10 @@ namespace BiblioTech lock (repoLock) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { - result.Add("User has not joined the test net."); + result.Add("User has not interacted with bot."); } else { @@ -100,36 +106,33 @@ namespace BiblioTech public string[] GetUserReport(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) return new[] { "User has not joined the test net." }; return userData.CreateOverview(); } public string[] GetUserReport(EthAddress ethAddress) { - var userData = GetUserDataForAddress(ethAddress); + var userData = GetUserDataForAddressMaybe(ethAddress); if (userData == null) return new[] { "No user is using this eth address." }; return userData.CreateOverview(); } - public UserData? GetUserDataForAddress(EthAddress? address) + public UserData? GetUserDataForAddressMaybe(EthAddress? address) { if (address == null) return null; - // If this becomes a performance problem, switch to in-memory cached list. - var files = Directory.GetFiles(Program.Config.UserDataPath); - foreach (var file in files) + var lower = address.Address.ToLowerInvariant(); + if (string.IsNullOrEmpty(lower)) return null; + + if (cache.Count == 0) LoadAllUserData(); + foreach (var item in cache.Values) { - try + if (item.CurrentAddress != null && + item.CurrentAddress.Address.ToLowerInvariant() == lower) { - var user = JsonConvert.DeserializeObject(File.ReadAllText(file))!; - if (user.CurrentAddress != null && - user.CurrentAddress.Address == address.Address) - { - return user; - } + return item; } - catch { } } return null; @@ -137,7 +140,7 @@ namespace BiblioTech private SetAddressResponse SetUserAddress(IUser user, EthAddress? address) { - if (GetUserDataForAddress(address) != null) + if (GetUserDataForAddressMaybe(address) != null) { return SetAddressResponse.AddressAlreadyInUse; } @@ -152,13 +155,12 @@ namespace BiblioTech private void SetUserNotification(IUser user, bool notifyEnabled) { - var userData = GetUserData(user); - if (userData == null) return; + var userData = GetOrCreate(user); userData.NotificationsEnabled = notifyEnabled; SaveUserData(userData); } - private UserData? GetUserData(IUser user) + private UserData? GetUserDataMaybe(IUser user) { if (cache.ContainsKey(user.Id)) { @@ -177,7 +179,7 @@ namespace BiblioTech private UserData GetOrCreate(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { return CreateAndSaveNewUserData(user); diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 753a1371..0d78962e 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -64,8 +64,12 @@ namespace CodexNetDeployer var gethDeployment = DeployGeth(ci); var gethNode = ci.WrapGethDeployment(gethDeployment, new BlockCache()); + var bootNode = ci.StartCodexNode(); + var versionInfo = bootNode.GetDebugInfo().Version; + bootNode.Stop(waitTillStopped: true); + Log("Geth started. Deploying Codex contracts..."); - var contractsDeployment = ci.DeployCodexContracts(gethNode); + var contractsDeployment = ci.DeployCodexContracts(gethNode, versionInfo); var contracts = ci.WrapCodexContractsDeployment(gethNode, contractsDeployment); Log("Codex contracts deployed."); diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index 6a5c5759..c5ddd9a0 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -21,7 +21,7 @@ namespace TestNetRewarder return result == "Pong"; } - public async Task SendRewards(GiveRewardsCommand command) + public async Task SendRewards(EventsAndErrors command) { if (command == null) return false; var result = await HttpPostJson(command); diff --git a/Tools/TestNetRewarder/Configuration.cs b/Tools/TestNetRewarder/Configuration.cs index 5dbeac30..b3aa84cb 100644 --- a/Tools/TestNetRewarder/Configuration.cs +++ b/Tools/TestNetRewarder/Configuration.cs @@ -33,6 +33,9 @@ namespace TestNetRewarder [Uniform("proof-submitted-events", "pse", "PROOFSUBMITTEDEVENTS", false, "When greater than zero, chain event summary will include proof-submitted events.")] public int ShowProofSubmittedEvents { get; set; } = 0; // Defaulted to zero, aprox 7 to 10 such events every 2 minutes in testnet (from autoclient alone!) + [Uniform("proof-period-report-hours", "pprh", "PROOFPERIODREPORTHOURS", false, "Frequency in hours with which proof period reports are created.")] + public int ProofReportHours { get; set; } = 24; + public string LogPath { get diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 2b49ef11..649ebf47 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -8,7 +8,6 @@ namespace TestNetRewarder public class Processor : ITimeSegmentHandler { private readonly RequestBuilder builder; - private readonly RewardChecker rewardChecker; private readonly EventsFormatter eventsFormatter; private readonly ChainState chainState; private readonly Configuration config; @@ -23,23 +22,19 @@ namespace TestNetRewarder this.log = log; lastPeriodUpdateUtc = DateTime.UtcNow; + if (config.ProofReportHours < 1) throw new Exception("ProofReportHours must be one or greater"); + builder = new RequestBuilder(); - rewardChecker = new RewardChecker(builder); eventsFormatter = new EventsFormatter(config); - var handler = new ChainStateChangeHandlerMux( - rewardChecker.Handler, - eventsFormatter - ); - - chainState = new ChainState(log, contracts, handler, config.HistoryStartUtc, + chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc, doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); } public async Task Initialize() { var events = eventsFormatter.GetInitializationEvents(config); - var request = builder.Build(events, Array.Empty()); + var request = builder.Build(chainState, events, Array.Empty()); if (request.HasAny()) { await client.SendRewards(request); @@ -54,9 +49,8 @@ namespace TestNetRewarder var numberOfChainEvents = await ProcessEvents(timeRange); var duration = sw.Elapsed; - if (numberOfChainEvents == 0) return TimeSegmentResponse.Underload; - if (numberOfChainEvents > 10) return TimeSegmentResponse.Overload; - if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Overload; + if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Underload; + if (duration > TimeSpan.FromSeconds(3)) return TimeSegmentResponse.Overload; return TimeSegmentResponse.OK; } catch (Exception ex) @@ -76,7 +70,7 @@ namespace TestNetRewarder var events = eventsFormatter.GetEvents(); var errors = eventsFormatter.GetErrors(); - var request = builder.Build(events, errors); + var request = builder.Build(chainState, events, errors); if (request.HasAny()) { await client.SendRewards(request); @@ -87,7 +81,7 @@ namespace TestNetRewarder private void ProcessPeriodUpdate() { if (config.ShowProofPeriodReports < 1) return; - if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(1.0))) return; + if (DateTime.UtcNow < (lastPeriodUpdateUtc + TimeSpan.FromHours(config.ProofReportHours))) return; lastPeriodUpdateUtc = DateTime.UtcNow; eventsFormatter.ProcessPeriodReports(chainState.PeriodMonitor.GetAndClearReports()); diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index ab21f61c..e3ed24e7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -45,6 +45,7 @@ namespace TestNetRewarder Log.Log("Starting TestNet Rewarder..."); var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor); + await EnsureBotOnline(); await processor.Initialize(); while (!CancellationToken.IsCancellationRequested) diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs index b1f641b8..5b859269 100644 --- a/Tools/TestNetRewarder/RequestBuilder.cs +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -1,40 +1,55 @@ -using DiscordRewards; +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; using Utils; namespace TestNetRewarder { - public class RequestBuilder : IRewardGiver + public class RequestBuilder { - private readonly Dictionary> rewards = new Dictionary>(); - - public void Give(RewardConfig reward, EthAddress receiver) + public EventsAndErrors Build(ChainState chainState, ChainEventMessage[] lines, string[] errors) { - if (rewards.ContainsKey(reward.RoleId)) + var activeChainAddresses = CollectActiveAddresses(chainState); + + return new EventsAndErrors { - rewards[reward.RoleId].Add(receiver); + EventsOverview = lines, + Errors = errors, + ActiveChainAddresses = activeChainAddresses + }; + } + + private ActiveChainAddresses CollectActiveAddresses(ChainState chainState) + { + var hosts = new List(); + var clients = new List(); + + foreach (var request in chainState.Requests) + { + CollectAddresses(request, hosts, clients); } - else + + return new ActiveChainAddresses { - rewards.Add(reward.RoleId, new List { receiver }); + Hosts = hosts.ToArray(), + Clients = clients.ToArray() + }; + } + + private void CollectAddresses(IChainStateRequest request, List hosts, List clients) + { + if (request.State != CodexContractsPlugin.RequestState.Started) return; + + AddIfNew(clients, request.Client); + foreach (var host in request.Hosts.GetHosts()) + { + AddIfNew(hosts, host); } } - public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors) + private void AddIfNew(List list, EthAddress address) { - var result = new GiveRewardsCommand - { - Rewards = rewards.Select(p => new RewardUsersCommand - { - RewardId = p.Key, - UserAddresses = p.Value.Select(v => v.Address).ToArray() - }).ToArray(), - EventsOverview = lines, - Errors = errors - }; - - rewards.Clear(); - - return result; + var addr = address.Address; + if (!list.Contains(addr)) list.Add(addr); } } } diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs deleted file mode 100644 index 2f947d8d..00000000 --- a/Tools/TestNetRewarder/RewardCheck.cs +++ /dev/null @@ -1,113 +0,0 @@ -using BlockchainUtils; -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; -using System.Numerics; -using Utils; - -namespace TestNetRewarder -{ - public interface IRewardGiver - { - void Give(RewardConfig reward, EthAddress receiver); - } - - public class RewardCheck : IChainStateChangeHandler - { - private readonly RewardConfig reward; - private readonly IRewardGiver giver; - - public RewardCheck(RewardConfig reward, IRewardGiver giver) - { - this.reward = reward; - this.giver = giver; - } - - public void OnNewRequest(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientPostedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnRequestCancelled(RequestEvent requestEvent) - { - } - - public void OnRequestFailed(RequestEvent requestEvent) - { - } - - public void OnRequestFinished(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.HostFinishedSlot, requestEvent)) - { - foreach (var host in requestEvent.Request.Hosts.GetHosts()) - { - GiveReward(reward, host); - } - } - } - - public void OnRequestFulfilled(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientStartedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex) - { - if (MeetsRequirements(CheckType.HostFilledSlot, requestEvent)) - { - if (host != null) - { - GiveReward(reward, host); - } - } - } - - public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnError(string msg) - { - } - - public void OnProofSubmitted(BlockTimeEntry block, string id) - { - } - - private void GiveReward(RewardConfig reward, EthAddress receiver) - { - giver.Give(reward, receiver); - } - - private bool MeetsRequirements(CheckType type, RequestEvent requestEvent) - { - return - reward.CheckConfig.Type == type && - MeetsDurationRequirement(requestEvent.Request) && - MeetsSizeRequirement(requestEvent.Request); - } - - private bool MeetsSizeRequirement(IChainStateRequest r) - { - var slotSize = r.Request.Ask.SlotSize; - ulong min = Convert.ToUInt64(reward.CheckConfig.MinSlotSize.SizeInBytes); - return slotSize >= min; - } - - private bool MeetsDurationRequirement(IChainStateRequest r) - { - var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); - return duration >= reward.CheckConfig.MinDuration; - } - } -} diff --git a/Tools/TestNetRewarder/RewardChecker.cs b/Tools/TestNetRewarder/RewardChecker.cs deleted file mode 100644 index f8e500cc..00000000 --- a/Tools/TestNetRewarder/RewardChecker.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; - -namespace TestNetRewarder -{ - public class RewardChecker - { - public RewardChecker(IRewardGiver giver) - { - var repo = new RewardRepo(); - var checks = repo.Rewards.Select(r => new RewardCheck(r, giver)).ToArray(); - Handler = new ChainStateChangeHandlerMux(checks); - } - - public IChainStateChangeHandler Handler { get; } - } -}