From 22cf82b99b37f68287d10be445978abc39614516 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 16 May 2024 16:00:19 +0200 Subject: [PATCH 01/28] Adds no-discord debug option to discord bot. --- .../UtilityTests/ClusterSpeedTests.cs | 6 +-- Tools/BiblioTech/Configuration.cs | 28 +++------- Tools/BiblioTech/LoggingRoleDriver.cs | 24 +++++++++ Tools/BiblioTech/Program.cs | 51 ++++++++++++------- 4 files changed, 65 insertions(+), 44 deletions(-) create mode 100644 Tools/BiblioTech/LoggingRoleDriver.cs diff --git a/Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs b/Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs index 9e647ea..7800e13 100644 --- a/Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs +++ b/Tests/CodexTests/UtilityTests/ClusterSpeedTests.cs @@ -3,7 +3,7 @@ using Logging; using NUnit.Framework; using Utils; -namespace CodexTests.ScalabilityTests +namespace CodexTests.UtilityTests { [TestFixture] public class ClusterDiscSpeedTests : DistTest @@ -18,7 +18,7 @@ namespace CodexTests.ScalabilityTests ) { long targetSize = (long)(1024 * 1024 * 1024) * 2; - long bufferSizeBytes = ((long)bufferSizeKb) * 1024; + long bufferSizeBytes = (long)bufferSizeKb * 1024; var filename = nameof(DiscSpeedTest); @@ -28,7 +28,7 @@ namespace CodexTests.ScalabilityTests var writeSpeed = PerformWrite(targetSize, bufferSizeBytes, filename); Thread.Sleep(2000); var readSpeed = PerformRead(targetSize, bufferSizeBytes, filename); - + Log($"Write speed: {writeSpeed} per second."); Log($"Read speed: {readSpeed} per second."); } diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 9164cf7..ffbf4c6 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -34,28 +34,12 @@ namespace BiblioTech [Uniform("mint-tt", "mt", "MINTTT", true, "Amount of TestTokens minted by the mint command.")] public int MintTT { get; set; } = 1073741824; - public string EndpointsPath - { - get - { - return Path.Combine(DataPath, "endpoints"); - } - } + [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")] + public int NoDiscord { get; set; } = 0; - public string UserDataPath - { - get - { - return Path.Combine(DataPath, "users"); - } - } - - public string LogPath - { - get - { - return Path.Combine(DataPath, "logs"); - } - } + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); + public string UserDataPath => Path.Combine(DataPath, "users"); + public string LogPath => Path.Combine(DataPath, "logs"); + public bool DebugNoDiscord => NoDiscord == 1; } } diff --git a/Tools/BiblioTech/LoggingRoleDriver.cs b/Tools/BiblioTech/LoggingRoleDriver.cs new file mode 100644 index 0000000..9275a7a --- /dev/null +++ b/Tools/BiblioTech/LoggingRoleDriver.cs @@ -0,0 +1,24 @@ +using BiblioTech.Rewards; +using DiscordRewards; +using Logging; +using Newtonsoft.Json; + +namespace BiblioTech +{ + public class LoggingRoleDriver : IDiscordRoleDriver + { + private readonly ILog log; + + public LoggingRoleDriver(ILog log) + { + this.log = log; + } + + public async Task GiveRewards(GiveRewardsCommand rewards) + { + await Task.CompletedTask; + + log.Log(JsonConvert.SerializeObject(rewards, Formatting.None)); + } + } +} diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index f289d9b..6ec7dae 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -41,25 +41,15 @@ namespace BiblioTech public async Task MainAsync(string[] args) { Log.Log("Starting Codex Discord Bot..."); - client = new DiscordSocketClient(); - client.Log += ClientLog; - - var notifyCommand = new NotifyCommand(); - var associateCommand = new UserAssociateCommand(notifyCommand); - var sprCommand = new SprCommand(); - var handler = new CommandHandler(client, - new GetBalanceCommand(associateCommand), - new MintCommand(associateCommand), - sprCommand, - associateCommand, - notifyCommand, - new AdminCommand(sprCommand), - new MarketCommand() - ); - - await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); - await client.StartAsync(); - AdminChecker = new AdminChecker(); + if (Config.DebugNoDiscord) + { + Log.Log("Debug option is set. Discord connection disabled!"); + RoleDriver = new LoggingRoleDriver(Log); + } + else + { + await StartDiscordBot(); + } var builder = WebApplication.CreateBuilder(args); builder.WebHost.ConfigureKestrel((context, options) => @@ -75,6 +65,29 @@ namespace BiblioTech await Task.Delay(-1); } + private async Task StartDiscordBot() + { + client = new DiscordSocketClient(); + client.Log += ClientLog; + + var notifyCommand = new NotifyCommand(); + var associateCommand = new UserAssociateCommand(notifyCommand); + var sprCommand = new SprCommand(); + var handler = new CommandHandler(client, + new GetBalanceCommand(associateCommand), + new MintCommand(associateCommand), + sprCommand, + associateCommand, + notifyCommand, + new AdminCommand(sprCommand), + new MarketCommand() + ); + + await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); + await client.StartAsync(); + AdminChecker = new AdminChecker(); + } + private static void PrintHelp() { Log.Log("BiblioTech - Codex Discord Bot"); From d6f7e225beac683813139a720f94ec74230b99a2 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 20 May 2024 16:18:01 +0200 Subject: [PATCH 02/28] successful downloading of bot log --- .../DiscordBotContainerRecipe.cs | 4 +- .../UtilityTests/DiscordBotTests.cs | 172 ++++++++++-------- 2 files changed, 104 insertions(+), 72 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index 9a2e3fe..86ee1e2 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class DiscordBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-bibliotech"; - public override string Image => "codexstorage/codex-discordbot:sha-8c64352"; + public override string Image => "codexstorage/codex-discordbot:sha-22cf82b"; public static string RewardsPort = "bot_rewards_port"; @@ -33,6 +33,8 @@ namespace CodexDiscordBotPlugin AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress); AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi); + AddEnvVar("NODISCORD", "1"); + AddInternalPortAndVar("REWARDAPIPORT", RewardsPort); if (!string.IsNullOrEmpty(config.DataPath)) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index d642525..2223f92 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -2,6 +2,7 @@ using CodexDiscordBotPlugin; using CodexPlugin; using GethPlugin; +using KubernetesWorkflow.Types; using NUnit.Framework; using Utils; @@ -10,86 +11,42 @@ namespace CodexTests.UtilityTests [TestFixture] public class DiscordBotTests : AutoBootstrapDistTest { + private readonly TestToken hostInitialBalance = 3000.TestTokens(); + private readonly TestToken clientInitialBalance = 1000000000.TestTokens(); + private readonly ByteSize fileSize = 11.MB(); + [Test] - [Ignore("Used for debugging bots")] public void BotRewardTest() { - var myAccount = EthAccount.GenerateNew(); - - var sellerInitialBalance = 234.TestTokens(); - var buyerInitialBalance = 100000.TestTokens(); - var fileSize = 11.MB(); + var clientAccount = EthAccount.GenerateNew(); var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); + var gethInfo = CreateGethInfo(geth, contracts); - // start bot and rewarder - var gethInfo = new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "bot", - token: "aaa", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - var botContainer = bot.Containers.Single(); - Ci.DeployRewarderBot(new RewarderBotStartupConfig( - //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, - //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: "1", - historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3), - gethInfo: gethInfo, - dataPath: null - )); + var botContainer = StartDiscordBot(gethInfo); - var numberOfHosts = 3; + var hostAccount = EthAccount.GenerateNew(); + StartHosts(hostAccount, geth, contracts); - for (var i = 0; i < numberOfHosts; i++) - { - var seller = StartCodex(s => s - .WithName("Seller") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(11.GB()) - .EnableMarketplace(geth, contracts, m => m - .WithAccount(myAccount) - .WithInitial(10.Eth(), sellerInitialBalance) - .AsStorageNode() - .AsValidator())); + StartRewarderBot(gethInfo, botContainer); - var availability = new StorageAvailability( - totalSpace: 10.GB(), - maxDuration: TimeSpan.FromMinutes(30), - minPriceForTotalSpace: 1.TestTokens(), - maxCollateral: 20.TestTokens() - ); - seller.Marketplace.MakeStorageAvailable(availability); - } + var client = StartClient(geth, contracts, clientAccount); + var purchaseContract = ClientPurchasesStorage(client); + + //purchaseContract.WaitForStorageContractStarted(); + //purchaseContract.WaitForStorageContractFinished(); + Thread.Sleep(TimeSpan.FromMinutes(5)); + + var botLog = Ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); + var aaaa = 0; + } + + private StoragePurchaseContract ClientPurchasesStorage(ICodexNode client) + { var testFile = GenerateTestFile(fileSize); - - var buyer = StartCodex(s => s - .WithName("Buyer") - .EnableMarketplace(geth, contracts, m => m - .WithAccount(myAccount) - .WithInitial(10.Eth(), buyerInitialBalance))); - - var contentId = buyer.UploadFile(testFile); - + var contentId = client.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) { PricePerSlotPerSecond = 2.TestTokens(), @@ -101,11 +58,84 @@ namespace CodexTests.UtilityTests Expiry = TimeSpan.FromMinutes(5) }; - var purchaseContract = buyer.Marketplace.RequestStorage(purchase); + return client.Marketplace.RequestStorage(purchase); + } - purchaseContract.WaitForStorageContractStarted(); + private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts, EthAccount clientAccount) + { + return StartCodex(s => s + .WithName("Client") + .EnableMarketplace(geth, contracts, m => m + .WithAccount(clientAccount) + .WithInitial(10.Eth(), clientInitialBalance))); + } - purchaseContract.WaitForStorageContractFinished(); + private void StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) + { + Ci.DeployRewarderBot(new RewarderBotStartupConfig( + discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, + discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, + intervalMinutes: "10", + historyStartUtc: DateTime.UtcNow, + gethInfo: gethInfo, + dataPath: null + )); + } + + private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts) + { + return new DiscordBotGethInfo( + host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, + port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, + privKey: geth.StartResult.Account.PrivateKey, + marketplaceAddress: contracts.Deployment.MarketplaceAddress, + tokenAddress: contracts.Deployment.TokenAddress, + abi: contracts.Deployment.Abi + ); + } + + private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo) + { + var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( + name: "bot", + token: "aaa", + serverName: "ThatBen's server", + adminRoleName: "bottest-admins", + adminChannelName: "admin-channel", + rewardChannelName: "rewards-channel", + kubeNamespace: "notneeded", + gethInfo: gethInfo + )); + return bot.Containers.Single(); + } + + private void StartHosts(EthAccount hostAccount, IGethNode geth, ICodexContracts contracts) + { + var numberOfHosts = 5; + var hosts = StartCodex(numberOfHosts, s => s + .WithName("Host") + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) + .WithStorageQuota(11.GB()) + .EnableMarketplace(geth, contracts, m => m + .WithAccount(hostAccount) + .WithInitial(10.Eth(), hostInitialBalance) + .AsStorageNode() + .AsValidator())); + + var availability = new StorageAvailability( + totalSpace: 10.GB(), + maxDuration: TimeSpan.FromMinutes(30), + minPriceForTotalSpace: 1.TestTokens(), + maxCollateral: 20.TestTokens() + ); + + foreach (var host in hosts) + { + host.Marketplace.MakeStorageAvailable(availability); + } } } } From fa1b560a917a1e457a3f822ab4b232d8e1c2f0dc Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 21 May 2024 12:58:17 +0200 Subject: [PATCH 03/28] Adds waiting for debug-mode start message --- .../CodexDiscordBotPlugin.cs | 45 ++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index f347805..b2bb597 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -1,11 +1,13 @@ using Core; using KubernetesWorkflow; using KubernetesWorkflow.Types; +using Utils; namespace CodexDiscordBotPlugin { public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata { + private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!"; private readonly IPluginTools tools; public CodexDiscordBotPlugin(IPluginTools tools) @@ -46,7 +48,9 @@ namespace CodexDiscordBotPlugin var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; startupConfig.Add(config); - return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); + var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); + WaitForStartupMessage(workflow, pod); + return pod; } private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) @@ -55,5 +59,44 @@ namespace CodexDiscordBotPlugin startupConfig.Add(config); return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); } + + private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod) + { + var finder = new LogLineFinder(ExpectedStartupMessage, workflow); + Time.WaitUntil(() => + { + finder.FindLine(pod); + return finder.Found; + }, nameof(WaitForStartupMessage)); + } + + public class LogLineFinder : LogHandler + { + private readonly string message; + private readonly IStartupWorkflow workflow; + + public LogLineFinder(string message, IStartupWorkflow workflow) + { + this.message = message; + this.workflow = workflow; + } + + public void FindLine(RunningPod pod) + { + Found = false; + foreach (var c in pod.Containers) + { + workflow.DownloadContainerLog(c, this); + if (Found) return; + } + } + + public bool Found { get; private set; } + + protected override void ProcessLine(string line) + { + if (!Found && line.Contains(message)) Found = true; + } + } } } From a236544ee94e3babc6b2141c1dcd1910ed16b96f Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 21 May 2024 16:10:14 +0200 Subject: [PATCH 04/28] parameterizing the bot test --- Tests/CodexTests/CodexTests.csproj | 1 + .../UtilityTests/DiscordBotTests.cs | 120 ++++++++++++++++-- 2 files changed, 107 insertions(+), 14 deletions(-) diff --git a/Tests/CodexTests/CodexTests.csproj b/Tests/CodexTests/CodexTests.csproj index 495caf6..10d1a22 100644 --- a/Tests/CodexTests/CodexTests.csproj +++ b/Tests/CodexTests/CodexTests.csproj @@ -13,6 +13,7 @@ + diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 2223f92..f781795 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -1,8 +1,11 @@ using CodexContractsPlugin; using CodexDiscordBotPlugin; using CodexPlugin; +using Core; +using DiscordRewards; using GethPlugin; using KubernetesWorkflow.Types; +using Newtonsoft.Json; using NUnit.Framework; using Utils; @@ -11,9 +14,9 @@ namespace CodexTests.UtilityTests [TestFixture] public class DiscordBotTests : AutoBootstrapDistTest { - private readonly TestToken hostInitialBalance = 3000.TestTokens(); + private readonly RewardRepo repo = new RewardRepo(); + private readonly TestToken hostInitialBalance = 3000000.TestTokens(); private readonly TestToken clientInitialBalance = 1000000000.TestTokens(); - private readonly ByteSize fileSize = 11.MB(); [Test] public void BotRewardTest() @@ -35,23 +38,39 @@ namespace CodexTests.UtilityTests var purchaseContract = ClientPurchasesStorage(client); - //purchaseContract.WaitForStorageContractStarted(); - //purchaseContract.WaitForStorageContractFinished(); - Thread.Sleep(TimeSpan.FromMinutes(5)); + var apiCalls = new RewardApiCalls(Ci, botContainer); - var botLog = Ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); - var aaaa = 0; + Time.WaitUntil(() => apiCalls.Get().Length > 4, TimeSpan.FromMinutes(10), TimeSpan.FromSeconds(10), "Waiting for API calls"); + + var calls = apiCalls.Get(); + foreach (var call in calls) + { + var line = ""; + if (call.Averages.Any()) line += $"{call.Averages.Length} average. "; + if (call.EventsOverview.Any()) line += $"{call.EventsOverview.Length} events. "; + foreach (var r in call.Rewards) + { + var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); + var isClient = r.UserAddresses.Any(a => a == clientAccount.EthAddress.Address); + var isHost = r.UserAddresses.Any(a => a == hostAccount.EthAddress.Address); + if (isHost && isClient) throw new Exception("what?"); + var name = isClient ? "Client" : "Host"; + + line += name + " = " + reward.Message; + } + Log(line); + } } private StoragePurchaseContract ClientPurchasesStorage(ICodexNode client) { - var testFile = GenerateTestFile(fileSize); + var testFile = GenerateTestFile(GetMinFileSize()); var contentId = client.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) { PricePerSlotPerSecond = 2.TestTokens(), RequiredCollateral = 10.TestTokens(), - MinRequiredNumberOfNodes = 5, + MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(), NodeFailureTolerance = 2, ProofProbability = 5, Duration = TimeSpan.FromMinutes(6), @@ -111,14 +130,13 @@ namespace CodexTests.UtilityTests private void StartHosts(EthAccount hostAccount, IGethNode geth, ICodexContracts contracts) { - var numberOfHosts = 5; - var hosts = StartCodex(numberOfHosts, s => s + var hosts = StartCodex(GetNumberOfLiveHosts(), s => s .WithName("Host") .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) { ContractClock = CodexLogLevel.Trace, }) - .WithStorageQuota(11.GB()) + .WithStorageQuota(GetFileSizePlus(50)) .EnableMarketplace(geth, contracts, m => m .WithAccount(hostAccount) .WithInitial(10.Eth(), hostInitialBalance) @@ -126,10 +144,10 @@ namespace CodexTests.UtilityTests .AsValidator())); var availability = new StorageAvailability( - totalSpace: 10.GB(), + totalSpace: GetFileSizePlus(5), maxDuration: TimeSpan.FromMinutes(30), minPriceForTotalSpace: 1.TestTokens(), - maxCollateral: 20.TestTokens() + maxCollateral: hostInitialBalance ); foreach (var host in hosts) @@ -137,5 +155,79 @@ namespace CodexTests.UtilityTests host.Marketplace.MakeStorageAvailable(availability); } } + + private int GetNumberOfLiveHosts() + { + return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3; + } + + private ByteSize GetFileSizePlus(int plusMb) + { + return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes); + } + + private ByteSize GetMinFileSize() + { + ulong minSlotSize = 0; + ulong minNumHosts = 0; + foreach (var r in repo.Rewards) + { + var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes); + var h = r.CheckConfig.MinNumberOfHosts; + if (s > minSlotSize) minSlotSize = s; + if (h > minNumHosts) minNumHosts = h; + } + + var minFileSize = (minSlotSize * minNumHosts) + 1024; + return new ByteSize(Convert.ToInt64(minFileSize)); + } + + private uint GetNumberOfRequiredHosts() + { + return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); + } + + public class RewardApiCalls + { + private readonly CoreInterface ci; + private readonly RunningContainer botContainer; + private readonly Dictionary commands = new Dictionary(); + + public RewardApiCalls(CoreInterface ci, RunningContainer botContainer) + { + this.ci = ci; + this.botContainer = botContainer; + } + + public void Update() + { + var botLog = ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); + + var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + foreach (var line in lines) AddToCache(line); + } + + public GiveRewardsCommand[] Get() + { + Update(); + return commands.Select(c => c.Value).ToArray(); + } + + private void AddToCache(string line) + { + try + { + var timestamp = line.Substring(0, 30); + if (commands.ContainsKey(timestamp)) return; + var json = line.Substring(31); + + var cmd = JsonConvert.DeserializeObject(json); + if (cmd != null) commands.Add(timestamp, cmd); + } + catch + { + } + } + } } } From 5143361dcd4e3f457862fa8616676efe72d7c65b Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 23 May 2024 11:37:57 +0200 Subject: [PATCH 05/28] wip --- .../UtilityTests/DiscordBotTests.cs | 160 ++++++++++++++---- 1 file changed, 128 insertions(+), 32 deletions(-) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 8099ec7..cfb1dfa 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -5,6 +5,7 @@ using Core; using DiscordRewards; using GethPlugin; using KubernetesWorkflow.Types; +using Logging; using Newtonsoft.Json; using NUnit.Framework; using Utils; @@ -17,49 +18,54 @@ namespace CodexTests.UtilityTests private readonly RewardRepo repo = new RewardRepo(); private readonly TestToken hostInitialBalance = 3000000.TstWei(); private readonly TestToken clientInitialBalance = 1000000000.TstWei(); + private readonly EthAccount clientAccount = EthAccount.GenerateNew(); + private readonly EthAccount hostAccount = EthAccount.GenerateNew(); [Test] public void BotRewardTest() { - var clientAccount = EthAccount.GenerateNew(); - var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); var gethInfo = CreateGethInfo(geth, contracts); + var monitor = new ChainMonitor(contracts, geth, GetTestLog()); + monitor.Start(); + var botContainer = StartDiscordBot(gethInfo); - var hostAccount = EthAccount.GenerateNew(); - StartHosts(hostAccount, geth, contracts); + StartHosts(geth, contracts); StartRewarderBot(gethInfo, botContainer); - var client = StartClient(geth, contracts, clientAccount); + var client = StartClient(geth, contracts); var purchaseContract = ClientPurchasesStorage(client); var apiCalls = new RewardApiCalls(Ci, botContainer); + apiCalls.Start(OnCommand); - Time.WaitUntil(() => apiCalls.Get().Length > 4, TimeSpan.FromMinutes(10), TimeSpan.FromSeconds(10), "Waiting for API calls"); + Thread.Sleep(TimeSpan.FromMinutes(10)); - var calls = apiCalls.Get(); - foreach (var call in calls) + apiCalls.Stop(); + monitor.Stop(); + } + + private void OnCommand(GiveRewardsCommand call) + { + var line = ""; + if (call.Averages.Any()) line += $"{call.Averages.Length} average. "; + if (call.EventsOverview.Any()) line += $"{call.EventsOverview.Length} events. "; + foreach (var r in call.Rewards) { - var line = ""; - if (call.Averages.Any()) line += $"{call.Averages.Length} average. "; - if (call.EventsOverview.Any()) line += $"{call.EventsOverview.Length} events. "; - foreach (var r in call.Rewards) - { - var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); - var isClient = r.UserAddresses.Any(a => a == clientAccount.EthAddress.Address); - var isHost = r.UserAddresses.Any(a => a == hostAccount.EthAddress.Address); - if (isHost && isClient) throw new Exception("what?"); - var name = isClient ? "Client" : "Host"; + var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); + var isClient = r.UserAddresses.Any(a => a == clientAccount.EthAddress.Address); + var isHost = r.UserAddresses.Any(a => a == hostAccount.EthAddress.Address); + if (isHost && isClient) throw new Exception("what?"); + var name = isClient ? "Client" : "Host"; - line += name + " = " + reward.Message; - } - Log(line); + line += name + " = " + reward.Message; } + Log(line); } private StoragePurchaseContract ClientPurchasesStorage(ICodexNode client) @@ -80,7 +86,7 @@ namespace CodexTests.UtilityTests return client.Marketplace.RequestStorage(purchase); } - private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts, EthAccount clientAccount) + private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts) { return StartCodex(s => s .WithName("Client") @@ -128,7 +134,7 @@ namespace CodexTests.UtilityTests return bot.Containers.Single(); } - private void StartHosts(EthAccount hostAccount, IGethNode geth, ICodexContracts contracts) + private void StartHosts(IGethNode geth, ICodexContracts contracts) { var hosts = StartCodex(GetNumberOfLiveHosts(), s => s .WithName("Host") @@ -192,6 +198,9 @@ namespace CodexTests.UtilityTests private readonly CoreInterface ci; private readonly RunningContainer botContainer; private readonly Dictionary commands = new Dictionary(); + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private Task worker = Task.CompletedTask; + private Action onCommand = c => { }; public RewardApiCalls(CoreInterface ci, RunningContainer botContainer) { @@ -199,18 +208,37 @@ namespace CodexTests.UtilityTests this.botContainer = botContainer; } - public void Update() + public void Start(Action onCommand) { - var botLog = ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); - - var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - foreach (var line in lines) AddToCache(line); + this.onCommand = onCommand; + worker = Task.Run(Worker); } - public GiveRewardsCommand[] Get() + public void Stop() { - Update(); - return commands.Select(c => c.Value).ToArray(); + cts.Cancel(); + worker.Wait(); + } + + private void Worker() + { + while (!cts.IsCancellationRequested) + { + Update(); + } + } + + private void Update() + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (cts.IsCancellationRequested) return; + + var botLog = ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); + var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); + foreach (var line in lines) + { + AddToCache(line); + } } private void AddToCache(string line) @@ -222,12 +250,80 @@ namespace CodexTests.UtilityTests var json = line.Substring(31); var cmd = JsonConvert.DeserializeObject(json); - if (cmd != null) commands.Add(timestamp, cmd); + if (cmd != null) + { + commands.Add(timestamp, cmd); + onCommand(cmd); + } } catch { } } } + + public class ChainMonitor + { + private readonly ICodexContracts contracts; + private readonly IGethNode geth; + private readonly ILog log; + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private Task worker = Task.CompletedTask; + private DateTime last = DateTime.UtcNow; + + public ChainMonitor(ICodexContracts contracts, IGethNode geth, ILog log) + { + this.contracts = contracts; + this.geth = geth; + this.log = log; + } + + public void Start() + { + last = DateTime.UtcNow; + worker = Task.Run(Worker); + } + + public void Stop() + { + cts.Cancel(); + worker.Wait(); + } + + private void Worker() + { + while (!cts.IsCancellationRequested) + { + Thread.Sleep(TimeSpan.FromSeconds(10)); + if (cts.IsCancellationRequested) return; + + Update(); + + } + } + + private void Update() + { + var start = last; + var stop = DateTime.UtcNow; + last = stop; + + var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop)); + + + LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range); + LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range); + LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range); + LogEvents(nameof(contracts.GetSlotFilledEvents), contracts.GetSlotFilledEvents, range); + LogEvents(nameof(contracts.GetSlotFreedEvents), contracts.GetSlotFreedEvents, range); + } + + private void LogEvents(string n, Func f, BlockInterval r) + { + var a = (object[])f(r); + + a.ToList().ForEach(request => log.Log(n + " - " + JsonConvert.SerializeObject(request))); + } + } } } From 69aa3a998f518f53408fc86bb48bfa61c14c5a00 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 24 May 2024 15:34:42 +0200 Subject: [PATCH 06/28] Fixes bot test contract --- .../DiscordBotStartupConfig.cs | 4 +- .../RewarderBotContainerRecipe.cs | 2 +- ProjectPlugins/CodexPlugin/CodexNode.cs | 25 ++++++-- .../CodexPlugin/CodexNodeFactory.cs | 14 ++--- .../CodexTests/BasicTests/MarketplaceTests.cs | 4 +- .../UtilityTests/DiscordBotTests.cs | 63 +++++++++++++------ 6 files changed, 76 insertions(+), 36 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs index bbef0b1..b990e60 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs @@ -27,7 +27,7 @@ public class RewarderBotStartupConfig { - public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) + public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) { DiscordBotHost = discordBotHost; DiscordBotPort = discordBotPort; @@ -39,7 +39,7 @@ public string DiscordBotHost { get; } public int DiscordBotPort { get; } - public string IntervalMinutes { get; } + public int IntervalMinutes { get; } public DateTime HistoryStartUtc { get; } public DiscordBotGethInfo GethInfo { get; } public string? DataPath { get; set; } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index 816c910..ae31e69 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -17,7 +17,7 @@ namespace CodexDiscordBotPlugin AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost); AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString()); - AddEnvVar("INTERVALMINUTES", config.IntervalMinutes); + AddEnvVar("INTERVALMINUTES", config.IntervalMinutes.ToString()); var offset = new DateTimeOffset(config.HistoryStartUtc); AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString()); diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 36fb1dc..33c249a 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -23,6 +23,7 @@ namespace CodexPlugin CrashWatcher CrashWatcher { get; } PodInfo GetPodInfo(); ITransferSpeeds TransferSpeeds { get; } + EthAccount EthAccount { get; } void Stop(bool waitTillStopped); } @@ -30,13 +31,13 @@ namespace CodexPlugin { private const string UploadFailedMessage = "Unable to store block"; private readonly IPluginTools tools; - private readonly EthAddress? ethAddress; + private readonly EthAccount? ethAccount; private readonly TransferSpeeds transferSpeeds; - public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAddress? ethAddress) + public CodexNode(IPluginTools tools, CodexAccess codexAccess, CodexNodeGroup group, IMarketplaceAccess marketplaceAccess, EthAccount? ethAccount) { this.tools = tools; - this.ethAddress = ethAddress; + this.ethAccount = ethAccount; CodexAccess = codexAccess; Group = group; Marketplace = marketplaceAccess; @@ -66,8 +67,17 @@ namespace CodexPlugin { get { - if (ethAddress == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it."); - return ethAddress; + EnsureMarketplace(); + return ethAccount!.EthAddress; + } + } + + public EthAccount EthAccount + { + get + { + EnsureMarketplace(); + return ethAccount!; } } @@ -197,6 +207,11 @@ namespace CodexPlugin } } + private void EnsureMarketplace() + { + if (ethAccount == null) throw new Exception("Marketplace is not enabled for this Codex node. Please start it with the option '.EnableMarketplace(...)' to enable it."); + } + private void Log(string msg) { tools.GetLog().Log($"{GetName()}: {msg}"); diff --git a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs index 18483de..425d489 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs @@ -22,22 +22,22 @@ namespace CodexPlugin public CodexNode CreateOnlineCodexNode(CodexAccess access, CodexNodeGroup group) { - var ethAddress = GetEthAddress(access); - var marketplaceAccess = GetMarketplaceAccess(access, ethAddress); - return new CodexNode(tools, access, group, marketplaceAccess, ethAddress); + var ethAccount = GetEthAccount(access); + var marketplaceAccess = GetMarketplaceAccess(access, ethAccount); + return new CodexNode(tools, access, group, marketplaceAccess, ethAccount); } - private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAddress? ethAddress) + private IMarketplaceAccess GetMarketplaceAccess(CodexAccess codexAccess, EthAccount? ethAccount) { - if (ethAddress == null) return new MarketplaceUnavailable(); + if (ethAccount == null) return new MarketplaceUnavailable(); return new MarketplaceAccess(tools.GetLog(), codexAccess); } - private EthAddress? GetEthAddress(CodexAccess access) + private EthAccount? GetEthAccount(CodexAccess access) { var ethAccount = access.Container.Containers.Single().Recipe.Additionals.Get(); if (ethAccount == null) return null; - return ethAccount.EthAddress; + return ethAccount; } public CrashWatcher CreateCrashWatcher(RunningContainer c) diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 7cff4aa..2ecc325 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -65,8 +65,8 @@ namespace CodexTests.BasicTests MinRequiredNumberOfNodes = 5, NodeFailureTolerance = 2, ProofProbability = 5, - Duration = TimeSpan.FromMinutes(5), - Expiry = TimeSpan.FromMinutes(4) + Duration = TimeSpan.FromMinutes(6), + Expiry = TimeSpan.FromMinutes(5) }; var purchaseContract = client.Marketplace.RequestStorage(purchase); diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index cfb1dfa..ab63112 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -19,7 +19,7 @@ namespace CodexTests.UtilityTests private readonly TestToken hostInitialBalance = 3000000.TstWei(); private readonly TestToken clientInitialBalance = 1000000000.TstWei(); private readonly EthAccount clientAccount = EthAccount.GenerateNew(); - private readonly EthAccount hostAccount = EthAccount.GenerateNew(); + private readonly List hostAccounts = new List(); [Test] public void BotRewardTest() @@ -44,28 +44,27 @@ namespace CodexTests.UtilityTests var apiCalls = new RewardApiCalls(Ci, botContainer); apiCalls.Start(OnCommand); - Thread.Sleep(TimeSpan.FromMinutes(10)); + purchaseContract.WaitForStorageContractFinished(); apiCalls.Stop(); monitor.Stop(); + + Log("Done!"); } private void OnCommand(GiveRewardsCommand call) { - var line = ""; - if (call.Averages.Any()) line += $"{call.Averages.Length} average. "; - if (call.EventsOverview.Any()) line += $"{call.EventsOverview.Length} events. "; + if (call.Averages.Any()) Log($"{call.Averages.Length} average."); + if (call.EventsOverview.Any()) Log($"{call.EventsOverview.Length} events."); foreach (var r in call.Rewards) { var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); - var isClient = r.UserAddresses.Any(a => a == clientAccount.EthAddress.Address); - var isHost = r.UserAddresses.Any(a => a == hostAccount.EthAddress.Address); - if (isHost && isClient) throw new Exception("what?"); - var name = isClient ? "Client" : "Host"; - - line += name + " = " + reward.Message; + foreach (var address in r.UserAddresses) + { + var user = IdentifyAccount(address); + Log(user + ": " + reward.Message); + } } - Log(line); } private StoragePurchaseContract ClientPurchasesStorage(ICodexNode client) @@ -88,11 +87,14 @@ namespace CodexTests.UtilityTests private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts) { - return StartCodex(s => s + var node = StartCodex(s => s .WithName("Client") .EnableMarketplace(geth, contracts, m => m .WithAccount(clientAccount) .WithInitial(10.Eth(), clientInitialBalance))); + + Log($"Client {node.EthAccount.EthAddress}"); + return node; } private void StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) @@ -100,7 +102,7 @@ namespace CodexTests.UtilityTests Ci.DeployRewarderBot(new RewarderBotStartupConfig( discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: "10", + intervalMinutes: 1, historyStartUtc: DateTime.UtcNow, gethInfo: gethInfo, dataPath: null @@ -142,23 +144,27 @@ namespace CodexTests.UtilityTests { ContractClock = CodexLogLevel.Trace, }) - .WithStorageQuota(GetFileSizePlus(50)) + .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts())) .EnableMarketplace(geth, contracts, m => m - .WithAccount(hostAccount) .WithInitial(10.Eth(), hostInitialBalance) .AsStorageNode() .AsValidator())); var availability = new StorageAvailability( - totalSpace: GetFileSizePlus(5), + totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()), maxDuration: TimeSpan.FromMinutes(30), minPriceForTotalSpace: 1.TstWei(), maxCollateral: hostInitialBalance ); + var i = 0; foreach (var host in hosts) { + hostAccounts.Add(host.EthAccount); host.Marketplace.MakeStorageAvailable(availability); + + Log($"Host{i} {host.EthAccount.EthAddress}"); + i++; } } @@ -167,7 +173,12 @@ namespace CodexTests.UtilityTests return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3; } - private ByteSize GetFileSizePlus(int plusMb) + private ByteSize Mult(ByteSize size, int mult) + { + return new ByteSize(size.SizeInBytes * mult); + } + + private ByteSize GetMinFileSizePlus(int plusMb) { return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes); } @@ -184,7 +195,7 @@ namespace CodexTests.UtilityTests if (h > minNumHosts) minNumHosts = h; } - var minFileSize = (minSlotSize * minNumHosts) + 1024; + var minFileSize = ((minSlotSize + 1024) * minNumHosts); return new ByteSize(Convert.ToInt64(minFileSize)); } @@ -193,6 +204,20 @@ namespace CodexTests.UtilityTests return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); } + private string IdentifyAccount(string address) + { + if (address == clientAccount.EthAddress.Address) return "Client"; + try + { + var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address); + return "Host" + index; + } + catch + { + return "UNKNOWN"; + } + } + public class RewardApiCalls { private readonly CoreInterface ci; From ba9e4b098f06a034ca2b4309ea1676a89a438875 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 24 May 2024 16:11:51 +0200 Subject: [PATCH 07/28] checking that all rewards are sent. --- .../CodexPlugin/CodexContainerRecipe.cs | 5 ++- ProjectPlugins/CodexPlugin/CodexSetup.cs | 37 +++++++++++++++++-- ProjectPlugins/GethPlugin/EthAccount.cs | 5 +++ .../UtilityTests/DiscordBotTests.cs | 27 +++++++++++--- 4 files changed, 64 insertions(+), 10 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 7bc308a..07b94ac 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -109,8 +109,9 @@ namespace CodexPlugin // Custom scripting in the Codex test image will write this variable to a private-key file, // and pass the correct filename to Codex. - AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey); - Additional(marketplaceSetup.EthAccount); + var account = marketplaceSetup.EthAccountSetup.GetNew(); + AddEnvVar("PRIV_KEY", account.PrivateKey); + Additional(account); SetCommandOverride(marketplaceSetup); if (marketplaceSetup.IsValidator) diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index 04aaf42..44454c7 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -169,7 +169,7 @@ namespace CodexPlugin public bool IsValidator { get; private set; } public Ether InitialEth { get; private set; } = 0.Eth(); public TestToken InitialTestTokens { get; private set; } = 0.Tst(); - public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew(); + public EthAccountSetup EthAccountSetup { get; private set; } = new EthAccountSetup(); public IMarketplaceSetup AsStorageNode() { @@ -185,7 +185,7 @@ namespace CodexPlugin public IMarketplaceSetup WithAccount(EthAccount account) { - EthAccount = account; + EthAccountSetup.Pin(account); return this; } @@ -201,10 +201,41 @@ namespace CodexPlugin var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit. result += IsStorageNode ? "(storageNode)" : "()"; result += IsValidator ? "(validator)" : "() "; - result += $"Address: '{EthAccount.EthAddress}' "; + result += $"Address: '{EthAccountSetup}' "; result += $"{InitialEth.Eth} / {InitialTestTokens}"; result += "] "; return result; } } + + public class EthAccountSetup + { + private readonly List accounts = new List(); + private bool pinned = false; + + public void Pin(EthAccount account) + { + accounts.Add(account); + pinned = true; + } + + public EthAccount GetNew() + { + if (pinned) return accounts.Last(); + + var a = EthAccount.GenerateNew(); + accounts.Add(a); + return a; + } + + public EthAccount[] GetAll() + { + return accounts.ToArray(); + } + + public override string ToString() + { + return string.Join(",", accounts.Select(a => a.ToString()).ToArray()); + } + } } diff --git a/ProjectPlugins/GethPlugin/EthAccount.cs b/ProjectPlugins/GethPlugin/EthAccount.cs index 9f1318b..60bf40d 100644 --- a/ProjectPlugins/GethPlugin/EthAccount.cs +++ b/ProjectPlugins/GethPlugin/EthAccount.cs @@ -24,5 +24,10 @@ namespace GethPlugin return new EthAccount(ethAddress, account.PrivateKey); } + + public override string ToString() + { + return EthAddress.ToString(); + } } } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index ab63112..d7ebd8d 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -20,6 +20,8 @@ namespace CodexTests.UtilityTests private readonly TestToken clientInitialBalance = 1000000000.TstWei(); private readonly EthAccount clientAccount = EthAccount.GenerateNew(); private readonly List hostAccounts = new List(); + private readonly List rewardsSeen = new List(); + private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); [Test] public void BotRewardTest() @@ -50,6 +52,24 @@ namespace CodexTests.UtilityTests monitor.Stop(); Log("Done!"); + + Thread.Sleep(rewarderInterval * 2); + + Log("Seen:"); + foreach (var seen in rewardsSeen) + { + Log(seen.ToString()); + } + Log(""); + + foreach (var r in repo.Rewards) + { + var seen = rewardsSeen.Any(s => r.RoleId == s); + + Log($"{r.RoleId} = {seen}"); + } + + Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId))); } private void OnCommand(GiveRewardsCommand call) @@ -59,6 +79,7 @@ namespace CodexTests.UtilityTests foreach (var r in call.Rewards) { var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); + if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId); foreach (var address in r.UserAddresses) { var user = IdentifyAccount(address); @@ -102,7 +123,7 @@ namespace CodexTests.UtilityTests Ci.DeployRewarderBot(new RewarderBotStartupConfig( discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: 1, + intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)), historyStartUtc: DateTime.UtcNow, gethInfo: gethInfo, dataPath: null @@ -157,14 +178,10 @@ namespace CodexTests.UtilityTests maxCollateral: hostInitialBalance ); - var i = 0; foreach (var host in hosts) { hostAccounts.Add(host.EthAccount); host.Marketplace.MakeStorageAvailable(availability); - - Log($"Host{i} {host.EthAccount.EthAddress}"); - i++; } } From ccc6c815e4ef30e8ef9096e768c4e9313990bd53 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 29 May 2024 14:05:16 +0200 Subject: [PATCH 08/28] Logging of entire chain state as seen by rewarder bot --- .../CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs | 1 + .../CodexDiscordBotPlugin/DiscordBotStartupConfig.cs | 4 +++- Tests/CodexTests/UtilityTests/DiscordBotTests.cs | 3 ++- Tools/TestNetRewarder/ChainState.cs | 11 +++++++++++ Tools/TestNetRewarder/HistoricState.cs | 5 +++++ Tools/TestNetRewarder/Processor.cs | 2 ++ 6 files changed, 24 insertions(+), 2 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index b2bb597..b2e184b 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -56,6 +56,7 @@ namespace CodexDiscordBotPlugin private RunningPod StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) { var startupConfig = new StartupConfig(); + startupConfig.NameOverride = config.Name; startupConfig.Add(config); return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs index b990e60..e77552e 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs @@ -27,8 +27,9 @@ public class RewarderBotStartupConfig { - public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) + public RewarderBotStartupConfig(string name, string discordBotHost, int discordBotPort, int intervalMinutes, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) { + Name = name; DiscordBotHost = discordBotHost; DiscordBotPort = discordBotPort; IntervalMinutes = intervalMinutes; @@ -37,6 +38,7 @@ DataPath = dataPath; } + public string Name { get; } public string DiscordBotHost { get; } public int DiscordBotPort { get; } public int IntervalMinutes { get; } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index d7ebd8d..671d6c0 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -121,6 +121,7 @@ namespace CodexTests.UtilityTests private void StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) { Ci.DeployRewarderBot(new RewarderBotStartupConfig( + name: "rewarder-bot", discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)), @@ -145,7 +146,7 @@ namespace CodexTests.UtilityTests private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo) { var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "bot", + name: "discord-bot", token: "aaa", serverName: "ThatBen's server", adminRoleName: "bottest-admins", diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index c3ce8a2..50b2501 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -59,6 +59,17 @@ namespace TestNetRewarder public SlotFilledEventDTO[] SlotFilledEvents { get; } public SlotFreedEventDTO[] SlotFreedEvents { get; } + public string EntireString() + { + return + $"NewRequests: {JsonConvert.SerializeObject(NewRequests)}" + + $"FulfilledE: {JsonConvert.SerializeObject(RequestFulfilledEvents)}" + + $"CancelledE: {JsonConvert.SerializeObject(RequestCancelledEvents)}" + + $"FilledE: {JsonConvert.SerializeObject(SlotFilledEvents)}" + + $"FreedE: {JsonConvert.SerializeObject(SlotFreedEvents)}" + + $"Historic: {historicState.EntireString()}"; + } + public string[] GenerateOverview() { var entries = new List(); diff --git a/Tools/TestNetRewarder/HistoricState.cs b/Tools/TestNetRewarder/HistoricState.cs index 36d335d..3ae5882 100644 --- a/Tools/TestNetRewarder/HistoricState.cs +++ b/Tools/TestNetRewarder/HistoricState.cs @@ -29,6 +29,11 @@ namespace TestNetRewarder r.State == RequestState.Failed ); } + + public string EntireString() + { + return JsonConvert.SerializeObject(StorageRequests); + } } public class StorageRequest diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index c18247b..a756b93 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -58,6 +58,8 @@ namespace TestNetRewarder private async Task ProcessChainState(ChainState chainState) { + log.Log($"Processing chain state: '{chainState.EntireString()}'"); + var outgoingRewards = new List(); foreach (var reward in rewardRepo.Rewards) { From 117a30bb829051da76bcacd2ed1b6364689926bb Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 30 May 2024 10:55:33 +0200 Subject: [PATCH 09/28] wip --- .../RewarderBotContainerRecipe.cs | 2 +- Tests/CodexTests/CodexTests.csproj | 1 + .../UtilityTests/DiscordBotTests.cs | 143 ++++++++++++++---- 3 files changed, 112 insertions(+), 34 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index ae31e69..8bf4344 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2"; + public override string Image => "codexstorage/codex-rewarderbot:sha-ccc6c81"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/Tests/CodexTests/CodexTests.csproj b/Tests/CodexTests/CodexTests.csproj index 10d1a22..4a9a3ed 100644 --- a/Tests/CodexTests/CodexTests.csproj +++ b/Tests/CodexTests/CodexTests.csproj @@ -19,6 +19,7 @@ + diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 671d6c0..29403eb 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -37,17 +37,19 @@ namespace CodexTests.UtilityTests StartHosts(geth, contracts); - StartRewarderBot(gethInfo, botContainer); + var rewarderContainer = StartRewarderBot(gethInfo, botContainer); var client = StartClient(geth, contracts); - var purchaseContract = ClientPurchasesStorage(client); - var apiCalls = new RewardApiCalls(Ci, botContainer); apiCalls.Start(OnCommand); + var rewarderLog = new RewarderLogMonitor(Ci, rewarderContainer.Containers.Single()); + rewarderLog.Start(l => Log("Rewarder ChainState: " + l)); + var purchaseContract = ClientPurchasesStorage(client); purchaseContract.WaitForStorageContractFinished(); + rewarderLog.Stop(); apiCalls.Stop(); monitor.Stop(); @@ -118,9 +120,9 @@ namespace CodexTests.UtilityTests return node; } - private void StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) + private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) { - Ci.DeployRewarderBot(new RewarderBotStartupConfig( + return Ci.DeployRewarderBot(new RewarderBotStartupConfig( name: "rewarder-bot", discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, @@ -238,22 +240,113 @@ namespace CodexTests.UtilityTests public class RewardApiCalls { - private readonly CoreInterface ci; - private readonly RunningContainer botContainer; + private readonly ContainerFileMonitor monitor; private readonly Dictionary commands = new Dictionary(); - private readonly CancellationTokenSource cts = new CancellationTokenSource(); - private Task worker = Task.CompletedTask; - private Action onCommand = c => { }; public RewardApiCalls(CoreInterface ci, RunningContainer botContainer) { - this.ci = ci; - this.botContainer = botContainer; + monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/discordbot.log"); } public void Start(Action onCommand) { - this.onCommand = onCommand; + monitor.Start(line => ParseLine(line, onCommand)); + } + + public void Stop() + { + monitor.Stop(); + } + + private void ParseLine(string line, Action onCommand) + { + try + { + var timestamp = line.Substring(0, 30); + if (commands.ContainsKey(timestamp)) return; + var json = line.Substring(31); + + var cmd = JsonConvert.DeserializeObject(json); + if (cmd != null) + { + commands.Add(timestamp, cmd); + onCommand(cmd); + } + } + catch + { + } + } + } + + public class RewarderLogMonitor + { + private readonly ContainerFileMonitor monitor; + private readonly Dictionary commands = new Dictionary(); + + public RewarderLogMonitor(CoreInterface ci, RunningContainer botContainer) + { + monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/testnetrewarder.log"); + } + + public void Start(Action onCommand) + { + monitor.Start(l => ProcessLine(l, onCommand)); + } + + public void Stop() + { + monitor.Stop(); + } + + private void ProcessLine(string line, Action log) + { + // Processing chain state: ' + // NewRequests: [] + // FulfilledE: [] + // CancelledE: [] + // FilledE: [] + // FreedE: [] + // Historic: [{"Request":{"RequestId":"hvWlli4gHP5tUJoG5b8zh3tMBIr0wpcr/lHgIU/KRmU=","ClientAddress":{"Address":"0x760e722469cdeb086b78edd2b4b670621f43a923"},"Client":"0x760e722469Cdeb086b78EdD2b4b670621F43a923","Ask":{"Slots":4,"SlotSize":33554432,"Duration":360,"ProofProbability":5,"Reward":2,"Collateral":10,"MaxSlotLoss":2},"Content":{"Cid":"zDvZRwzm8DzbqdUcy7gS1wFKAsBjUHTQEUt1am1L6XGEWtsJ5U2X","MerkleRoot":"/KjFr+FezQ8m3huhdaPDy6zVgZs3ODtfE78K5eyieQo="},"Expiry":300,"Nonce":"2tANtfgVcMBuLLxe+4MbxMFWWO36Yv+l2yC4tpqdelI="},"Hosts":[{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"}],"State":"New"}] + // ' + + if (!line.Contains("Processing chain state: ")) return; + + log(Between(line, "'")); + //var tokens = state.Split("Historic: ", StringSplitOptions.RemoveEmptyEntries); + //var historics = JsonConvert.DeserializeObject(tokens[1]); + + //log(tokens[0] + " historic: " + string.Join(",", historics!.Select(h => h.Request.RequestId + " = " + h.State))); + } + + private string Between(string s, string lim) + { + var start = s.IndexOf(lim) + lim.Length; + var end = s.LastIndexOf(lim); + return s.Substring(start, end - start); + } + } + + public class ContainerFileMonitor + { + private readonly CoreInterface ci; + private readonly RunningContainer botContainer; + private readonly string filePath; + private readonly CancellationTokenSource cts = new CancellationTokenSource(); + private readonly List seenLines = new List(); + private Task worker = Task.CompletedTask; + private Action onNewLine = c => { }; + + public ContainerFileMonitor(CoreInterface ci, RunningContainer botContainer, string filePath) + { + this.ci = ci; + this.botContainer = botContainer; + this.filePath = filePath; + } + + public void Start(Action onNewLine) + { + this.onNewLine = onNewLine; worker = Task.Run(Worker); } @@ -276,32 +369,16 @@ namespace CodexTests.UtilityTests Thread.Sleep(TimeSpan.FromSeconds(10)); if (cts.IsCancellationRequested) return; - var botLog = ci.ExecuteContainerCommand(botContainer, "cat", "/app/datapath/logs/discordbot.log"); + var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath); var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); foreach (var line in lines) { - AddToCache(line); - } - } - - private void AddToCache(string line) - { - try - { - var timestamp = line.Substring(0, 30); - if (commands.ContainsKey(timestamp)) return; - var json = line.Substring(31); - - var cmd = JsonConvert.DeserializeObject(json); - if (cmd != null) + if (!seenLines.Contains(line)) { - commands.Add(timestamp, cmd); - onCommand(cmd); + seenLines.Add(line); + onNewLine(line); } } - catch - { - } } } From 12dc7efd5b8fd14756728e77432e14069ac4f810 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 30 May 2024 11:17:13 +0200 Subject: [PATCH 10/28] wip --- .../UtilityTests/DiscordBotTests.cs | 35 +++++++++--------- Tools/TestNetRewarder/ChainState.cs | 36 +++++++++++++++---- Tools/TestNetRewarder/HistoricState.cs | 9 +++++ Tools/TestNetRewarder/Processor.cs | 2 +- 4 files changed, 58 insertions(+), 24 deletions(-) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 29403eb..e9dba30 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using CodexContractsPlugin.Marketplace; using CodexDiscordBotPlugin; using CodexPlugin; using Core; @@ -8,6 +9,7 @@ using KubernetesWorkflow.Types; using Logging; using Newtonsoft.Json; using NUnit.Framework; +using TestNetRewarder; using Utils; namespace CodexTests.UtilityTests @@ -301,28 +303,29 @@ namespace CodexTests.UtilityTests private void ProcessLine(string line, Action log) { - // Processing chain state: ' - // NewRequests: [] - // FulfilledE: [] - // CancelledE: [] - // FilledE: [] - // FreedE: [] - // Historic: [{"Request":{"RequestId":"hvWlli4gHP5tUJoG5b8zh3tMBIr0wpcr/lHgIU/KRmU=","ClientAddress":{"Address":"0x760e722469cdeb086b78edd2b4b670621f43a923"},"Client":"0x760e722469Cdeb086b78EdD2b4b670621F43a923","Ask":{"Slots":4,"SlotSize":33554432,"Duration":360,"ProofProbability":5,"Reward":2,"Collateral":10,"MaxSlotLoss":2},"Content":{"Cid":"zDvZRwzm8DzbqdUcy7gS1wFKAsBjUHTQEUt1am1L6XGEWtsJ5U2X","MerkleRoot":"/KjFr+FezQ8m3huhdaPDy6zVgZs3ODtfE78K5eyieQo="},"Expiry":300,"Nonce":"2tANtfgVcMBuLLxe+4MbxMFWWO36Yv+l2yC4tpqdelI="},"Hosts":[{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"},{"Address":"0x0000000000000000000000000000000000000000"}],"State":"New"}] - // ' + //$"ChainState=[{JsonConvert.SerializeObject(this)}]" + + //$"HistoricState=[{historicState.EntireString()}]"; - if (!line.Contains("Processing chain state: ")) return; + var stateOpenTag = "ChainState=["; + var historicOpenTag = "]HistoricState=["; - log(Between(line, "'")); - //var tokens = state.Split("Historic: ", StringSplitOptions.RemoveEmptyEntries); - //var historics = JsonConvert.DeserializeObject(tokens[1]); + if (!line.Contains(stateOpenTag)) return; + if (!line.Contains(historicOpenTag)) return; - //log(tokens[0] + " historic: " + string.Join(",", historics!.Select(h => h.Request.RequestId + " = " + h.State))); + var stateStr = Between(line, stateOpenTag, historicOpenTag); + var historicStr = Between(line, historicOpenTag, "]"); + + var chainState = JsonConvert.DeserializeObject(stateStr); + var historicState = JsonConvert.DeserializeObject(historicStr)!; + chainState!.Set(new HistoricState(historicState)); + + log(string.Join(",", chainState!.GenerateOverview())); } - private string Between(string s, string lim) + private string Between(string s, string open, string close) { - var start = s.IndexOf(lim) + lim.Length; - var end = s.LastIndexOf(lim); + var start = s.IndexOf(open) + open.Length; + var end = s.LastIndexOf(close); return s.Substring(start, end - start); } } diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index 50b2501..50fa87d 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -8,7 +8,7 @@ namespace TestNetRewarder { public class ChainState { - private readonly HistoricState historicState; + private HistoricState historicState; private readonly string[] colorIcons = new[] { "🔴", @@ -50,9 +50,30 @@ namespace TestNetRewarder SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange); } + public ChainState( + Request[] newRequests, + RequestFulfilledEventDTO[] requestFulfilledEvents, + RequestCancelledEventDTO[] requestCancelledEvents, + SlotFilledEventDTO[] slotFilledEvents, + SlotFreedEventDTO[] slotFreedEvents) + { + NewRequests = newRequests; + RequestFulfilledEvents = requestFulfilledEvents; + RequestCancelledEvents = requestCancelledEvents; + SlotFilledEvents = slotFilledEvents; + SlotFreedEvents = slotFreedEvents; + + historicState = new HistoricState(); + StartedRequests = Array.Empty(); + FinishedRequests = Array.Empty(); + } + public Request[] NewRequests { get; } + [JsonIgnore] public StorageRequest[] AllRequests => historicState.StorageRequests; + [JsonIgnore] public StorageRequest[] StartedRequests { get; private set; } + [JsonIgnore] public StorageRequest[] FinishedRequests { get; private set; } public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; } public RequestCancelledEventDTO[] RequestCancelledEvents { get; } @@ -62,12 +83,13 @@ namespace TestNetRewarder public string EntireString() { return - $"NewRequests: {JsonConvert.SerializeObject(NewRequests)}" + - $"FulfilledE: {JsonConvert.SerializeObject(RequestFulfilledEvents)}" + - $"CancelledE: {JsonConvert.SerializeObject(RequestCancelledEvents)}" + - $"FilledE: {JsonConvert.SerializeObject(SlotFilledEvents)}" + - $"FreedE: {JsonConvert.SerializeObject(SlotFreedEvents)}" + - $"Historic: {historicState.EntireString()}"; + $"ChainState=[{JsonConvert.SerializeObject(this)}]" + + $"HistoricState=[{historicState.EntireString()}]"; + } + + public void Set(HistoricState h) + { + historicState = h; } public string[] GenerateOverview() diff --git a/Tools/TestNetRewarder/HistoricState.cs b/Tools/TestNetRewarder/HistoricState.cs index 3ae5882..8487aa0 100644 --- a/Tools/TestNetRewarder/HistoricState.cs +++ b/Tools/TestNetRewarder/HistoricState.cs @@ -34,6 +34,15 @@ namespace TestNetRewarder { return JsonConvert.SerializeObject(StorageRequests); } + + public HistoricState() + { + } + + public HistoricState(StorageRequest[] requests) + { + storageRequests.AddRange(requests); + } } public class StorageRequest diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index a756b93..678d4f8 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -58,7 +58,7 @@ namespace TestNetRewarder private async Task ProcessChainState(ChainState chainState) { - log.Log($"Processing chain state: '{chainState.EntireString()}'"); + log.Log(chainState.EntireString()); var outgoingRewards = new List(); foreach (var reward in rewardRepo.Rewards) From f1d453251cb532c30c2bb0ed70f6e7c718776952 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 30 May 2024 11:33:16 +0200 Subject: [PATCH 11/28] cleanup --- .../RewarderBotContainerRecipe.cs | 2 +- .../CodexPlugin/MarketplaceAccess.cs | 134 +---------------- .../CodexPlugin/StoragePurchaseContract.cs | 140 ++++++++++++++++++ .../UtilityTests/DiscordBotTests.cs | 2 +- 4 files changed, 145 insertions(+), 133 deletions(-) create mode 100644 ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index 8bf4344..55b60a2 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-ccc6c81"; + public override string Image => "codexstorage/codex-rewarderbot:sha-12dc7ef"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs index 8944d95..11f3b60 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs @@ -1,5 +1,4 @@ using Logging; -using Newtonsoft.Json; using Utils; namespace CodexPlugin @@ -7,7 +6,7 @@ namespace CodexPlugin public interface IMarketplaceAccess { string MakeStorageAvailable(StorageAvailability availability); - StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase); + IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase); } public class MarketplaceAccess : IMarketplaceAccess @@ -21,7 +20,7 @@ namespace CodexPlugin this.codexAccess = codexAccess; } - public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) + public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) { purchase.Log(log); @@ -68,7 +67,7 @@ namespace CodexPlugin throw new NotImplementedException(); } - public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) + public IStoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) { Unavailable(); throw new NotImplementedException(); @@ -80,131 +79,4 @@ namespace CodexPlugin throw new InvalidOperationException(); } } - - public class StoragePurchaseContract - { - private readonly ILog log; - private readonly CodexAccess codexAccess; - private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30); - private readonly DateTime contractPendingUtc = DateTime.UtcNow; - private DateTime? contractSubmittedUtc = DateTime.UtcNow; - private DateTime? contractStartedUtc; - private DateTime? contractFinishedUtc; - - public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase) - { - this.log = log; - this.codexAccess = codexAccess; - PurchaseId = purchaseId; - Purchase = purchase; - } - - public string PurchaseId { get; } - public StoragePurchaseRequest Purchase { get; } - - public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc; - public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc; - public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc; - - public void WaitForStorageContractSubmitted() - { - WaitForStorageContractState(gracePeriod, "submitted", sleep: 200); - contractSubmittedUtc = DateTime.UtcNow; - LogSubmittedDuration(); - AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted)); - } - - public void WaitForStorageContractStarted() - { - var timeout = Purchase.Expiry + gracePeriod; - - WaitForStorageContractState(timeout, "started"); - contractStartedUtc = DateTime.UtcNow; - LogStartedDuration(); - AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted)); - } - - public void WaitForStorageContractFinished() - { - if (!contractStartedUtc.HasValue) - { - WaitForStorageContractStarted(); - } - var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value; - var timeout = (Purchase.Duration - currentContractTime) + gracePeriod; - WaitForStorageContractState(timeout, "finished"); - contractFinishedUtc = DateTime.UtcNow; - LogFinishedDuration(); - AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished)); - } - - public StoragePurchase GetPurchaseStatus(string purchaseId) - { - return codexAccess.GetPurchaseStatus(purchaseId); - } - - private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000) - { - var lastState = ""; - var waitStart = DateTime.UtcNow; - - Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'."); - while (lastState != desiredState) - { - var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId); - var statusJson = JsonConvert.SerializeObject(purchaseStatus); - if (purchaseStatus != null && purchaseStatus.State != lastState) - { - lastState = purchaseStatus.State; - log.Debug("Purchase status: " + statusJson); - } - - Thread.Sleep(sleep); - - if (lastState == "errored") - { - FrameworkAssert.Fail("Contract errored: " + statusJson); - } - - if (DateTime.UtcNow - waitStart > timeout) - { - FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}"); - } - } - } - - private void LogSubmittedDuration() - { - Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " + - $"( < {Time.FormatDuration(gracePeriod)})"); - } - - private void LogStartedDuration() - { - Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " + - $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})"); - } - - private void LogFinishedDuration() - { - Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " + - $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})"); - } - - private void AssertDuration(TimeSpan? span, TimeSpan max, string message) - { - if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)"); - if (span.Value.TotalDays >= max.TotalSeconds) - { - throw new Exception(nameof(MarketplaceAccess) + - $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " + - message); - } - } - - private void Log(string msg) - { - log.Log($"[{PurchaseId}] {msg}"); - } - } } diff --git a/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs new file mode 100644 index 0000000..3a9e0b9 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs @@ -0,0 +1,140 @@ +using Logging; +using Newtonsoft.Json; +using Utils; + +namespace CodexPlugin +{ + public interface IStoragePurchaseContract + { + void WaitForStorageContractSubmitted(); + void WaitForStorageContractStarted(); + void WaitForStorageContractFinished(); + } + + public class StoragePurchaseContract : IStoragePurchaseContract + { + private readonly ILog log; + private readonly CodexAccess codexAccess; + private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(30); + private readonly DateTime contractPendingUtc = DateTime.UtcNow; + private DateTime? contractSubmittedUtc = DateTime.UtcNow; + private DateTime? contractStartedUtc; + private DateTime? contractFinishedUtc; + + public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase) + { + this.log = log; + this.codexAccess = codexAccess; + PurchaseId = purchaseId; + Purchase = purchase; + } + + public string PurchaseId { get; } + public StoragePurchaseRequest Purchase { get; } + + public TimeSpan? PendingToSubmitted => contractSubmittedUtc - contractPendingUtc; + public TimeSpan? SubmittedToStarted => contractStartedUtc - contractSubmittedUtc; + public TimeSpan? SubmittedToFinished => contractFinishedUtc - contractSubmittedUtc; + + public void WaitForStorageContractSubmitted() + { + WaitForStorageContractState(gracePeriod, "submitted", sleep: 200); + contractSubmittedUtc = DateTime.UtcNow; + LogSubmittedDuration(); + AssertDuration(PendingToSubmitted, gracePeriod, nameof(PendingToSubmitted)); + } + + public void WaitForStorageContractStarted() + { + var timeout = Purchase.Expiry + gracePeriod; + + WaitForStorageContractState(timeout, "started"); + contractStartedUtc = DateTime.UtcNow; + LogStartedDuration(); + AssertDuration(SubmittedToStarted, timeout, nameof(SubmittedToStarted)); + } + + public void WaitForStorageContractFinished() + { + if (!contractStartedUtc.HasValue) + { + WaitForStorageContractStarted(); + } + var currentContractTime = DateTime.UtcNow - contractSubmittedUtc!.Value; + var timeout = (Purchase.Duration - currentContractTime) + gracePeriod; + WaitForStorageContractState(timeout, "finished"); + contractFinishedUtc = DateTime.UtcNow; + LogFinishedDuration(); + AssertDuration(SubmittedToFinished, timeout, nameof(SubmittedToFinished)); + } + + public StoragePurchase GetPurchaseStatus(string purchaseId) + { + return codexAccess.GetPurchaseStatus(purchaseId); + } + + private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000) + { + var lastState = ""; + var waitStart = DateTime.UtcNow; + + Log($"Waiting for {Time.FormatDuration(timeout)} to reach state '{desiredState}'."); + while (lastState != desiredState) + { + var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId); + var statusJson = JsonConvert.SerializeObject(purchaseStatus); + if (purchaseStatus != null && purchaseStatus.State != lastState) + { + lastState = purchaseStatus.State; + log.Debug("Purchase status: " + statusJson); + } + + Thread.Sleep(sleep); + + if (lastState == "errored") + { + FrameworkAssert.Fail("Contract errored: " + statusJson); + } + + if (DateTime.UtcNow - waitStart > timeout) + { + FrameworkAssert.Fail($"Contract did not reach '{desiredState}' within {Time.FormatDuration(timeout)} timeout. {statusJson}"); + } + } + } + + private void LogSubmittedDuration() + { + Log($"Pending to Submitted in {Time.FormatDuration(PendingToSubmitted)} " + + $"( < {Time.FormatDuration(gracePeriod)})"); + } + + private void LogStartedDuration() + { + Log($"Submitted to Started in {Time.FormatDuration(SubmittedToStarted)} " + + $"( < {Time.FormatDuration(Purchase.Expiry + gracePeriod)})"); + } + + private void LogFinishedDuration() + { + Log($"Submitted to Finished in {Time.FormatDuration(SubmittedToFinished)} " + + $"( < {Time.FormatDuration(Purchase.Duration + gracePeriod)})"); + } + + private void AssertDuration(TimeSpan? span, TimeSpan max, string message) + { + if (span == null) throw new ArgumentNullException(nameof(MarketplaceAccess) + ": " + message + " (IsNull)"); + if (span.Value.TotalDays >= max.TotalSeconds) + { + throw new Exception(nameof(MarketplaceAccess) + + $": Duration out of range. Max: {Time.FormatDuration(max)} but was: {Time.FormatDuration(span.Value)} " + + message); + } + } + + private void Log(string msg) + { + log.Log($"[{PurchaseId}] {msg}"); + } + } +} diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index e9dba30..4a56778 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -92,7 +92,7 @@ namespace CodexTests.UtilityTests } } - private StoragePurchaseContract ClientPurchasesStorage(ICodexNode client) + private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client) { var testFile = GenerateTestFile(GetMinFileSize()); var contentId = client.UploadFile(testFile); From f94a67adb4463d96f75059d9a39ad76104e03a02 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 May 2024 10:06:34 +0200 Subject: [PATCH 12/28] Extracts event methods from CodexContractsAccess --- .../ChainMonitor/ChainEvents.cs | 7 ++ .../ChainMonitor/ChainState.cs | 7 ++ .../CodexContractsAccess.cs | 79 +------------ .../CodexContractsEvents.cs | 106 ++++++++++++++++++ .../CodexTests/BasicTests/MarketplaceTests.cs | 7 +- .../UtilityTests/DiscordBotTests.cs | 11 +- Tools/TestNetRewarder/ChainState.cs | 22 ++-- 7 files changed, 148 insertions(+), 91 deletions(-) create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs create mode 100644 ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs new file mode 100644 index 0000000..b7a6e24 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs @@ -0,0 +1,7 @@ +namespace CodexContractsPlugin.ChainMonitor +{ + public class ChainEvents + { + + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs new file mode 100644 index 0000000..6dc95f1 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -0,0 +1,7 @@ +namespace CodexContractsPlugin.ChainMonitor +{ + public class ChainState + { + + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 197d4f9..58fbb29 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -2,10 +2,8 @@ using GethPlugin; using Logging; using Nethereum.ABI; -using Nethereum.Hex.HexTypes; using Nethereum.Util; using NethereumWorkflow; -using NethereumWorkflow.BlockUtils; using Newtonsoft.Json; using Newtonsoft.Json.Converters; using Utils; @@ -22,13 +20,10 @@ namespace CodexContractsPlugin TestToken GetTestTokenBalance(IHasEthAddress owner); TestToken GetTestTokenBalance(EthAddress ethAddress); - Request[] GetStorageRequests(BlockInterval blockRange); + ICodexContractsEvents GetEvents(TimeRange timeRange); + ICodexContractsEvents GetEvents(BlockInterval blockInterval); EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex); RequestState GetRequestState(Request request); - RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange); - RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange); - SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange); - SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange); } [JsonConverter(typeof(StringEnumConverter))] @@ -81,65 +76,14 @@ namespace CodexContractsPlugin return balance.TstWei(); } - public Request[] GetStorageRequests(BlockInterval blockRange) + public ICodexContractsEvents GetEvents(TimeRange timeRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); - var i = StartInteraction(); - return events - .Select(e => - { - var requestEvent = i.GetRequest(Deployment.MarketplaceAddress, e.Event.RequestId); - var result = requestEvent.ReturnValue1; - result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); - result.RequestId = e.Event.RequestId; - return result; - }) - .ToArray(); + return GetEvents(gethNode.ConvertTimeRangeToBlockRange(timeRange)); } - public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange) + public ICodexContractsEvents GetEvents(BlockInterval blockInterval) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); - return events.Select(e => - { - var result = e.Event; - result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); - return result; - }).ToArray(); - } - - public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange) - { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); - return events.Select(e => - { - var result = e.Event; - result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); - return result; - }).ToArray(); - } - - public SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange) - { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); - return events.Select(e => - { - var result = e.Event; - result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); - result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash); - return result; - }).ToArray(); - } - - public SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange) - { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); - return events.Select(e => - { - var result = e.Event; - result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); - return result; - }).ToArray(); + return new CodexContractsEvents(log, gethNode, Deployment, blockInterval); } public EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex) @@ -170,17 +114,6 @@ namespace CodexContractsPlugin return gethNode.Call(Deployment.MarketplaceAddress, func); } - private BlockTimeEntry GetBlock(ulong number) - { - return gethNode.GetBlockForNumber(number); - } - - private EthAddress GetEthAddressFromTransaction(string transactionHash) - { - var transaction = gethNode.GetTransaction(transactionHash); - return new EthAddress(transaction.From); - } - private ContractInteractions StartInteraction() { return new ContractInteractions(log, gethNode); diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs new file mode 100644 index 0000000..4b2c2f6 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs @@ -0,0 +1,106 @@ +using CodexContractsPlugin.Marketplace; +using GethPlugin; +using Logging; +using Nethereum.Hex.HexTypes; +using NethereumWorkflow.BlockUtils; +using Utils; + +namespace CodexContractsPlugin +{ + public interface ICodexContractsEvents + { + Request[] GetStorageRequests(); + RequestFulfilledEventDTO[] GetRequestFulfilledEvents(); + RequestCancelledEventDTO[] GetRequestCancelledEvents(); + SlotFilledEventDTO[] GetSlotFilledEvents(); + SlotFreedEventDTO[] GetSlotFreedEvents(); + } + + public class CodexContractsEvents : ICodexContractsEvents + { + private readonly ILog log; + private readonly IGethNode gethNode; + private readonly CodexContractsDeployment deployment; + private readonly BlockInterval blockInterval; + + public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval) + { + this.log = log; + this.gethNode = gethNode; + this.deployment = deployment; + this.blockInterval = blockInterval; + } + + public Request[] GetStorageRequests() + { + var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var i = new ContractInteractions(log, gethNode); + return events + .Select(e => + { + var requestEvent = i.GetRequest(deployment.MarketplaceAddress, e.Event.RequestId); + var result = requestEvent.ReturnValue1; + result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); + result.RequestId = e.Event.RequestId; + return result; + }) + .ToArray(); + } + + public RequestFulfilledEventDTO[] GetRequestFulfilledEvents() + { + var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + return events.Select(e => + { + var result = e.Event; + result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); + return result; + }).ToArray(); + } + + public RequestCancelledEventDTO[] GetRequestCancelledEvents() + { + var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + return events.Select(e => + { + var result = e.Event; + result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); + return result; + }).ToArray(); + } + + public SlotFilledEventDTO[] GetSlotFilledEvents() + { + var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + return events.Select(e => + { + var result = e.Event; + result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); + result.Host = GetEthAddressFromTransaction(e.Log.TransactionHash); + return result; + }).ToArray(); + } + + public SlotFreedEventDTO[] GetSlotFreedEvents() + { + var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + return events.Select(e => + { + var result = e.Event; + result.Block = GetBlock(e.Log.BlockNumber.ToUlong()); + return result; + }).ToArray(); + } + + private BlockTimeEntry GetBlock(ulong number) + { + return gethNode.GetBlockForNumber(number); + } + + private EthAddress GetEthAddressFromTransaction(string transactionHash) + { + var transaction = gethNode.GetTransaction(transactionHash); + return new EthAddress(transaction.From); + } + } +} diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 2ecc325..bd3ce41 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -89,8 +89,8 @@ namespace CodexTests.BasicTests { Time.Retry(() => { - var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); - var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange); + var events = contracts.GetEvents(GetTestRunTimeRange()); + var slotFilledEvents = events.GetSlotFilledEvents(); Debug($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}"); @@ -107,7 +107,8 @@ namespace CodexTests.BasicTests private Request GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth) { - var requests = contracts.GetStorageRequests(geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange())); + var events = contracts.GetEvents(GetTestRunTimeRange()); + var requests = events.GetStorageRequests(); Assert.That(requests.Length, Is.EqualTo(1)); return requests.Single(); } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 4a56778..998699f 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -434,11 +434,12 @@ namespace CodexTests.UtilityTests var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop)); - LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range); - LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range); - LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range); - LogEvents(nameof(contracts.GetSlotFilledEvents), contracts.GetSlotFilledEvents, range); - LogEvents(nameof(contracts.GetSlotFreedEvents), contracts.GetSlotFreedEvents, range); + throw new Exception(); + //LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range); + //LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range); + //LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range); + //LogEvents(nameof(contracts.GetSlotFilledEvents), contracts.GetSlotFilledEvents, range); + //LogEvents(nameof(contracts.GetSlotFreedEvents), contracts.GetSlotFreedEvents, range); } private void LogEvents(string n, Func f, BlockInterval r) diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index 50fa87d..cef0808 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -37,17 +37,19 @@ namespace TestNetRewarder { this.historicState = historicState; - NewRequests = contracts.GetStorageRequests(blockRange); - historicState.CleanUpOldRequests(); - historicState.ProcessNewRequests(NewRequests); - historicState.UpdateStorageRequests(contracts); + throw new Exception("This is getting a rewrite"); - StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray(); - FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray(); - RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); - RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange); - SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange); - SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange); + //NewRequests = contracts.GetStorageRequests(blockRange); + //historicState.CleanUpOldRequests(); + //historicState.ProcessNewRequests(NewRequests); + //historicState.UpdateStorageRequests(contracts); + + //StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray(); + //FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray(); + //RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); + //RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange); + //SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange); + //SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange); } public ChainState( From f5da80dc9c4113fc3ddd6d3f5cfe0b01c978cf38 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 May 2024 10:16:57 +0200 Subject: [PATCH 13/28] Sets up ChainEvents object --- .../NethereumWorkflow/NethereumInteraction.cs | 1 + Framework/Utils/BlockInterval.cs | 4 +- .../ChainMonitor/ChainEvents.cs | 49 ++++++++++++++++++- .../CodexContractsEvents.cs | 16 +++--- 4 files changed, 61 insertions(+), 9 deletions(-) diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index f965c35..01f26bf 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -117,6 +117,7 @@ namespace NethereumWorkflow } return new BlockInterval( + timeRange: timeRange, from: fromBlock.Value, to: toBlock.Value ); diff --git a/Framework/Utils/BlockInterval.cs b/Framework/Utils/BlockInterval.cs index 79229bf..2daa8f2 100644 --- a/Framework/Utils/BlockInterval.cs +++ b/Framework/Utils/BlockInterval.cs @@ -2,7 +2,7 @@ { public class BlockInterval { - public BlockInterval(ulong from, ulong to) + public BlockInterval(TimeRange timeRange, ulong from, ulong to) { if (from < to) { @@ -14,10 +14,12 @@ From = to; To = from; } + TimeRange = timeRange; } public ulong From { get; } public ulong To { get; } + public TimeRange TimeRange { get; } public override string ToString() { diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs index b7a6e24..51a84d0 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs @@ -1,7 +1,54 @@ -namespace CodexContractsPlugin.ChainMonitor +using CodexContractsPlugin.Marketplace; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor { public class ChainEvents { + private ChainEvents( + BlockInterval blockInterval, + Request[] requests, + RequestFulfilledEventDTO[] fulfilled, + RequestCancelledEventDTO[] cancelled, + SlotFilledEventDTO[] slotFilled, + SlotFreedEventDTO[] slotFreed + ) + { + BlockInterval = blockInterval; + Requests = requests; + Fulfilled = fulfilled; + Cancelled = cancelled; + SlotFilled = slotFilled; + SlotFreed = slotFreed; + } + public BlockInterval BlockInterval { get; } + public Request[] Requests { get; } + public RequestFulfilledEventDTO[] Fulfilled { get; } + public RequestCancelledEventDTO[] Cancelled { get; } + public SlotFilledEventDTO[] SlotFilled { get; } + public SlotFreedEventDTO[] SlotFreed { get; } + + public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval) + { + return FromContractEvents(contracts.GetEvents(blockInterval)); + } + + public static ChainEvents FromTimeRange(ICodexContracts contracts, TimeRange timeRange) + { + return FromContractEvents(contracts.GetEvents(timeRange)); + } + + public static ChainEvents FromContractEvents(ICodexContractsEvents events) + { + return new ChainEvents( + events.BlockInterval, + events.GetStorageRequests(), + events.GetRequestFulfilledEvents(), + events.GetRequestCancelledEvents(), + events.GetSlotFilledEvents(), + events.GetSlotFreedEvents() + ); + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs index 4b2c2f6..cf7b2d7 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsEvents.cs @@ -9,6 +9,7 @@ namespace CodexContractsPlugin { public interface ICodexContractsEvents { + BlockInterval BlockInterval { get; } Request[] GetStorageRequests(); RequestFulfilledEventDTO[] GetRequestFulfilledEvents(); RequestCancelledEventDTO[] GetRequestCancelledEvents(); @@ -21,19 +22,20 @@ namespace CodexContractsPlugin private readonly ILog log; private readonly IGethNode gethNode; private readonly CodexContractsDeployment deployment; - private readonly BlockInterval blockInterval; public CodexContractsEvents(ILog log, IGethNode gethNode, CodexContractsDeployment deployment, BlockInterval blockInterval) { this.log = log; this.gethNode = gethNode; this.deployment = deployment; - this.blockInterval = blockInterval; + BlockInterval = blockInterval; } + + public BlockInterval BlockInterval { get; } public Request[] GetStorageRequests() { - var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var events = gethNode.GetEvents(deployment.MarketplaceAddress, BlockInterval); var i = new ContractInteractions(log, gethNode); return events .Select(e => @@ -49,7 +51,7 @@ namespace CodexContractsPlugin public RequestFulfilledEventDTO[] GetRequestFulfilledEvents() { - var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var events = gethNode.GetEvents(deployment.MarketplaceAddress, BlockInterval); return events.Select(e => { var result = e.Event; @@ -60,7 +62,7 @@ namespace CodexContractsPlugin public RequestCancelledEventDTO[] GetRequestCancelledEvents() { - var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var events = gethNode.GetEvents(deployment.MarketplaceAddress, BlockInterval); return events.Select(e => { var result = e.Event; @@ -71,7 +73,7 @@ namespace CodexContractsPlugin public SlotFilledEventDTO[] GetSlotFilledEvents() { - var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var events = gethNode.GetEvents(deployment.MarketplaceAddress, BlockInterval); return events.Select(e => { var result = e.Event; @@ -83,7 +85,7 @@ namespace CodexContractsPlugin public SlotFreedEventDTO[] GetSlotFreedEvents() { - var events = gethNode.GetEvents(deployment.MarketplaceAddress, blockInterval); + var events = gethNode.GetEvents(deployment.MarketplaceAddress, BlockInterval); return events.Select(e => { var result = e.Event; From a846d51c0c75a906ffb8d01cc1bac727271fdf7d Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 31 May 2024 11:19:50 +0200 Subject: [PATCH 14/28] Implements ChainState in CodexContracts plugin --- Framework/Utils/BlockInterval.cs | 1 + .../ChainMonitor/ChainEvents.cs | 14 ++ .../ChainMonitor/ChainState.cs | 130 +++++++++++++++++- .../ChainMonitor/ChainStateRequest.cs | 41 ++++++ .../Marketplace/Customizations.cs | 15 +- 5 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs diff --git a/Framework/Utils/BlockInterval.cs b/Framework/Utils/BlockInterval.cs index 2daa8f2..6443048 100644 --- a/Framework/Utils/BlockInterval.cs +++ b/Framework/Utils/BlockInterval.cs @@ -20,6 +20,7 @@ public ulong From { get; } public ulong To { get; } public TimeRange TimeRange { get; } + public ulong NumberOfBlocks => To - From; public override string ToString() { diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs index 51a84d0..25d0851 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainEvents.cs @@ -29,6 +29,20 @@ namespace CodexContractsPlugin.ChainMonitor public SlotFilledEventDTO[] SlotFilled { get; } public SlotFreedEventDTO[] SlotFreed { get; } + public IHasBlock[] All + { + get + { + var all = new List(); + all.AddRange(Requests); + all.AddRange(Fulfilled); + all.AddRange(Cancelled); + all.AddRange(SlotFilled); + all.AddRange(SlotFreed); + return all.ToArray(); + } + } + public static ChainEvents FromBlockInterval(ICodexContracts contracts, BlockInterval blockInterval) { return FromContractEvents(contracts.GetEvents(blockInterval)); diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 6dc95f1..b3870dc 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -1,7 +1,135 @@ -namespace CodexContractsPlugin.ChainMonitor +using CodexContractsPlugin.Marketplace; +using Logging; +using System.Numerics; +using Utils; + +namespace CodexContractsPlugin.ChainMonitor { + public interface IChainStateChangeHandler + { + void OnNewRequest(IChainStateRequest request); + void OnRequestStarted(IChainStateRequest request); + void OnRequestFinished(IChainStateRequest request); + void OnRequestFulfilled(IChainStateRequest request); + void OnRequestCancelled(IChainStateRequest request); + void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex); + void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex); + } + public class ChainState { + private readonly List requests = new List(); + private readonly ILog log; + private readonly IChainStateChangeHandler handler; + private ChainState(ILog log, IChainStateChangeHandler changeHandler, TimeRange timeRange) + { + this.log = log; + handler = changeHandler; + TotalSpan = timeRange; + } + + public static ChainState FromEvents(ILog log, ChainEvents events, IChainStateChangeHandler changeHandler) + { + var state = new ChainState(log, changeHandler, events.BlockInterval.TimeRange); + state.Apply(events); + return state; + } + + public TimeRange TotalSpan { get; private set; } + public IChainStateRequest[] Requests => requests.ToArray(); + + public void Apply(ChainEvents events) + { + if (events.BlockInterval.TimeRange.From < TotalSpan.From) + throw new Exception("Attempt to update ChainState with set of events from before its current record."); + + log.Log($"ChainState updating: {events.BlockInterval}"); + + // Run through each block and apply the events to the state in order. + var span = events.BlockInterval.TimeRange.Duration; + var numBlocks = events.BlockInterval.NumberOfBlocks; + var spanPerBlock = span / numBlocks; + + var eventUtc = events.BlockInterval.TimeRange.From; + for (var b = events.BlockInterval.From; b < events.BlockInterval.To; b++) + { + var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray(); + ApplyEvents(blockEvents, eventUtc); + + eventUtc += spanPerBlock; + } + } + + private void ApplyEvents(IHasBlock[] blockEvents, DateTime eventsUtc) + { + foreach (var e in blockEvents) + { + dynamic d = e; + ApplyEvent(d); + } + + ApplyTimeImplicitEvents(eventsUtc); + } + + private void ApplyEvent(Request request) + { + if (requests.Any(r => Equal(r.Request.RequestId, request.RequestId))) + throw new Exception("Received NewRequest event for id that already exists."); + + var newRequest = new ChainStateRequest(log, request, RequestState.New); + requests.Add(newRequest); + + handler.OnNewRequest(newRequest); + } + + private void ApplyEvent(RequestFulfilledEventDTO request) + { + var r = FindRequest(request.RequestId); + r.UpdateState(RequestState.Started); + handler.OnRequestFulfilled(r); + } + + private void ApplyEvent(RequestCancelledEventDTO request) + { + var r = FindRequest(request.RequestId); + r.UpdateState(RequestState.Cancelled); + handler.OnRequestCancelled(r); + } + + private void ApplyEvent(SlotFilledEventDTO request) + { + var r = FindRequest(request.RequestId); + handler.OnSlotFilled(r, request.SlotIndex); + } + + private void ApplyEvent(SlotFreedEventDTO request) + { + var r = FindRequest(request.RequestId); + handler.OnSlotFreed(r, request.SlotIndex); + } + + private void ApplyTimeImplicitEvents(DateTime eventsUtc) + { + foreach (var r in requests) + { + if (r.State == RequestState.Started + && r.FinishedUtc < eventsUtc) + { + r.UpdateState(RequestState.Finished); + handler.OnRequestFinished(r); + } + } + } + + private ChainStateRequest FindRequest(byte[] requestId) + { + return requests.Single(r => Equal(r.Request.RequestId, requestId)); + } + + private bool Equal(byte[] a, byte[] b) + { + return a.SequenceEqual(b); + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs new file mode 100644 index 0000000..ae735b4 --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs @@ -0,0 +1,41 @@ +using CodexContractsPlugin.Marketplace; +using Logging; + +namespace CodexContractsPlugin.ChainMonitor +{ + public interface IChainStateRequest + { + Request Request { get; } + RequestState State { get; } + DateTime ExpiryUtc { get; } + DateTime FinishedUtc { get; } + } + + public class ChainStateRequest : IChainStateRequest + { + private readonly ILog log; + + public ChainStateRequest(ILog log, Request request, RequestState state) + { + this.log = log; + Request = request; + State = state; + + ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry); + FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration); + + log.Log($"Request created as {State}."); + } + + public Request Request { get; } + public RequestState State { get; private set; } + public DateTime ExpiryUtc { get; } + public DateTime FinishedUtc { get; } + + public void UpdateState(RequestState newState) + { + log.Log($"Request transit: {State} -> {newState}"); + State = newState; + } + } +} diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index 68427b6..5094e12 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -5,7 +5,12 @@ using Newtonsoft.Json; namespace CodexContractsPlugin.Marketplace { - public partial class Request : RequestBase + public interface IHasBlock + { + BlockTimeEntry Block { get; set; } + } + + public partial class Request : RequestBase, IHasBlock { [JsonIgnore] public BlockTimeEntry Block { get; set; } @@ -14,26 +19,26 @@ namespace CodexContractsPlugin.Marketplace public EthAddress ClientAddress { get { return new EthAddress(Client); } } } - public partial class RequestFulfilledEventDTO + public partial class RequestFulfilledEventDTO : IHasBlock { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class RequestCancelledEventDTO + public partial class RequestCancelledEventDTO : IHasBlock { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class SlotFilledEventDTO + public partial class SlotFilledEventDTO : IHasBlock { [JsonIgnore] public BlockTimeEntry Block { get; set; } public EthAddress Host { get; set; } } - public partial class SlotFreedEventDTO + public partial class SlotFreedEventDTO : IHasBlock { [JsonIgnore] public BlockTimeEntry Block { get; set; } From ac7d32320145a9195520b43a67a5e2e2514a018d Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 3 Jun 2024 10:38:48 +0200 Subject: [PATCH 15/28] wip --- .../DoNothingChainEventHandler.cs | 35 ++++++++++++++ .../UtilityTests/DiscordBotTests.cs | 47 ++++++++++++------- 2 files changed, 64 insertions(+), 18 deletions(-) create mode 100644 ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs new file mode 100644 index 0000000..65e038b --- /dev/null +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs @@ -0,0 +1,35 @@ +using System.Numerics; + +namespace CodexContractsPlugin.ChainMonitor +{ + public class DoNothingChainEventHandler : IChainStateChangeHandler + { + public void OnNewRequest(IChainStateRequest request) + { + } + + public void OnRequestCancelled(IChainStateRequest request) + { + } + + public void OnRequestFinished(IChainStateRequest request) + { + } + + public void OnRequestFulfilled(IChainStateRequest request) + { + } + + public void OnRequestStarted(IChainStateRequest request) + { + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + } + } +} diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 998699f..c73cd23 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; using CodexContractsPlugin.Marketplace; using CodexDiscordBotPlugin; using CodexPlugin; @@ -43,6 +44,14 @@ namespace CodexTests.UtilityTests var client = StartClient(geth, contracts); + + var events = ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange()); + var chainState = CodexContractsPlugin.ChainMonitor.ChainState.FromEvents( + GetTestLog(), + events, + new DoNothingChainEventHandler()); + + var apiCalls = new RewardApiCalls(Ci, botContainer); apiCalls.Start(OnCommand); var rewarderLog = new RewarderLogMonitor(Ci, rewarderContainer.Containers.Single()); @@ -59,6 +68,8 @@ namespace CodexTests.UtilityTests Thread.Sleep(rewarderInterval * 2); + chainState.Apply(ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange())); + Log("Seen:"); foreach (var seen in rewardsSeen) { @@ -78,8 +89,8 @@ namespace CodexTests.UtilityTests private void OnCommand(GiveRewardsCommand call) { - if (call.Averages.Any()) Log($"{call.Averages.Length} average."); - if (call.EventsOverview.Any()) Log($"{call.EventsOverview.Length} events."); + if (call.Averages.Any()) Log($"API call: {call.Averages.Length} average."); + if (call.EventsOverview.Any()) Log($"API call: {call.EventsOverview.Length} events."); foreach (var r in call.Rewards) { var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); @@ -87,7 +98,7 @@ namespace CodexTests.UtilityTests foreach (var address in r.UserAddresses) { var user = IdentifyAccount(address); - Log(user + ": " + reward.Message); + Log("API call: " + user + ": " + reward.Message); } } } @@ -306,20 +317,20 @@ namespace CodexTests.UtilityTests //$"ChainState=[{JsonConvert.SerializeObject(this)}]" + //$"HistoricState=[{historicState.EntireString()}]"; - var stateOpenTag = "ChainState=["; - var historicOpenTag = "]HistoricState=["; + //var stateOpenTag = "ChainState=["; + //var historicOpenTag = "]HistoricState=["; - if (!line.Contains(stateOpenTag)) return; - if (!line.Contains(historicOpenTag)) return; + //if (!line.Contains(stateOpenTag)) return; + //if (!line.Contains(historicOpenTag)) return; - var stateStr = Between(line, stateOpenTag, historicOpenTag); - var historicStr = Between(line, historicOpenTag, "]"); + //var stateStr = Between(line, stateOpenTag, historicOpenTag); + //var historicStr = Between(line, historicOpenTag, "]"); - var chainState = JsonConvert.DeserializeObject(stateStr); - var historicState = JsonConvert.DeserializeObject(historicStr)!; - chainState!.Set(new HistoricState(historicState)); + //var chainState = JsonConvert.DeserializeObject(stateStr); + //var historicState = JsonConvert.DeserializeObject(historicStr)!; + //chainState!.Set(new HistoricState(historicState)); - log(string.Join(",", chainState!.GenerateOverview())); + //log(string.Join(",", chainState!.GenerateOverview())); } private string Between(string s, string open, string close) @@ -427,14 +438,14 @@ namespace CodexTests.UtilityTests private void Update() { - var start = last; - var stop = DateTime.UtcNow; - last = stop; + //var start = last; + //var stop = DateTime.UtcNow; + //last = stop; - var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop)); + //var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop)); - throw new Exception(); + //throw new Exception(); //LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range); //LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range); //LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range); From cc2513bd2f0b0fbfdeeddf9042986bcc3b43950a Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Jun 2024 14:04:25 +0200 Subject: [PATCH 16/28] Better chain state representation --- .../ChainMonitor/ChainState.cs | 32 +++- .../ChainMonitor/ChainStateRequest.cs | 9 +- .../Marketplace/Customizations.cs | 11 ++ .../CodexPlugin/StoragePurchaseContract.cs | 1 + .../UtilityTests/DiscordBotTests.cs | 158 ++---------------- 5 files changed, 63 insertions(+), 148 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index b3870dc..e3993f2 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin.ChainMonitor private ChainState(ILog log, IChainStateChangeHandler changeHandler, TimeRange timeRange) { - this.log = log; + this.log = new LogPrefixer(log, "(ChainState) "); handler = changeHandler; TotalSpan = timeRange; } @@ -39,7 +39,21 @@ namespace CodexContractsPlugin.ChainMonitor public TimeRange TotalSpan { get; private set; } public IChainStateRequest[] Requests => requests.ToArray(); - public void Apply(ChainEvents events) + public void Update(ICodexContracts contracts) + { + Update(contracts, DateTime.UtcNow); + } + + public void Update(ICodexContracts contracts, DateTime toUtc) + { + var span = new TimeRange(TotalSpan.To, toUtc); + var events = ChainEvents.FromTimeRange(contracts, span); + Apply(events); + + TotalSpan = new TimeRange(TotalSpan.From, span.To); + } + + private void Apply(ChainEvents events) { if (events.BlockInterval.TimeRange.From < TotalSpan.From) throw new Exception("Attempt to update ChainState with set of events from before its current record."); @@ -52,7 +66,7 @@ namespace CodexContractsPlugin.ChainMonitor var spanPerBlock = span / numBlocks; var eventUtc = events.BlockInterval.TimeRange.From; - for (var b = events.BlockInterval.From; b < events.BlockInterval.To; b++) + for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++) { var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray(); ApplyEvents(blockEvents, eventUtc); @@ -86,6 +100,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(RequestFulfilledEventDTO request) { var r = FindRequest(request.RequestId); + if (r == null) return; r.UpdateState(RequestState.Started); handler.OnRequestFulfilled(r); } @@ -93,6 +108,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(RequestCancelledEventDTO request) { var r = FindRequest(request.RequestId); + if (r == null) return; r.UpdateState(RequestState.Cancelled); handler.OnRequestCancelled(r); } @@ -100,12 +116,16 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(SlotFilledEventDTO request) { var r = FindRequest(request.RequestId); + if (r == null) return; + r.Log("SlotFilled"); handler.OnSlotFilled(r, request.SlotIndex); } private void ApplyEvent(SlotFreedEventDTO request) { var r = FindRequest(request.RequestId); + if (r == null) return; + r.Log("SlotFreed"); handler.OnSlotFreed(r, request.SlotIndex); } @@ -122,9 +142,11 @@ namespace CodexContractsPlugin.ChainMonitor } } - private ChainStateRequest FindRequest(byte[] requestId) + private ChainStateRequest? FindRequest(byte[] requestId) { - return requests.Single(r => Equal(r.Request.RequestId, requestId)); + var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId)); + if (r == null) log.Log("Unable to find request by ID!"); + return r; } private bool Equal(byte[] a, byte[] b) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs index ae735b4..4783e3a 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin.ChainMonitor ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry); FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration); - log.Log($"Request created as {State}."); + log.Log($"Created as {State}."); } public Request Request { get; } @@ -34,8 +34,13 @@ namespace CodexContractsPlugin.ChainMonitor public void UpdateState(RequestState newState) { - log.Log($"Request transit: {State} -> {newState}"); + Log($"Transit: {State} -> {newState}"); State = newState; } + + public void Log(string msg) + { + log.Log($"Request '{Request.Id}': {msg}"); + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index 5094e12..557e021 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -17,6 +17,17 @@ namespace CodexContractsPlugin.Marketplace public byte[] RequestId { get; set; } public EthAddress ClientAddress { get { return new EthAddress(Client); } } + + [JsonIgnore] + public string Id + { + get + { + var id = ""; + foreach (var b in RequestId) id += b.ToString(); + return id; + } + } } public partial class RequestFulfilledEventDTO : IHasBlock diff --git a/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs index 3a9e0b9..d812f51 100644 --- a/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs +++ b/ProjectPlugins/CodexPlugin/StoragePurchaseContract.cs @@ -6,6 +6,7 @@ namespace CodexPlugin { public interface IStoragePurchaseContract { + string PurchaseId { get; } void WaitForStorageContractSubmitted(); void WaitForStorageContractStarted(); void WaitForStorageContractFinished(); diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index c73cd23..3ae9474 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -1,16 +1,13 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; -using CodexContractsPlugin.Marketplace; using CodexDiscordBotPlugin; using CodexPlugin; using Core; using DiscordRewards; using GethPlugin; using KubernetesWorkflow.Types; -using Logging; using Newtonsoft.Json; using NUnit.Framework; -using TestNetRewarder; using Utils; namespace CodexTests.UtilityTests @@ -33,9 +30,6 @@ namespace CodexTests.UtilityTests var contracts = Ci.StartCodexContracts(geth); var gethInfo = CreateGethInfo(geth, contracts); - var monitor = new ChainMonitor(contracts, geth, GetTestLog()); - monitor.Start(); - var botContainer = StartDiscordBot(gethInfo); StartHosts(geth, contracts); @@ -44,49 +38,45 @@ namespace CodexTests.UtilityTests var client = StartClient(geth, contracts); - var events = ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange()); - var chainState = CodexContractsPlugin.ChainMonitor.ChainState.FromEvents( + var chainState = ChainState.FromEvents( GetTestLog(), events, new DoNothingChainEventHandler()); - var apiCalls = new RewardApiCalls(Ci, botContainer); apiCalls.Start(OnCommand); - var rewarderLog = new RewarderLogMonitor(Ci, rewarderContainer.Containers.Single()); - rewarderLog.Start(l => Log("Rewarder ChainState: " + l)); var purchaseContract = ClientPurchasesStorage(client); + chainState.Update(contracts); + Assert.That(chainState.Requests.Length, Is.EqualTo(1)); + + purchaseContract.WaitForStorageContractStarted(); + chainState.Update(contracts); + purchaseContract.WaitForStorageContractFinished(); - rewarderLog.Stop(); - apiCalls.Stop(); - monitor.Stop(); - - Log("Done!"); - Thread.Sleep(rewarderInterval * 2); - - chainState.Apply(ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange())); - - Log("Seen:"); - foreach (var seen in rewardsSeen) - { - Log(seen.ToString()); - } - Log(""); + + apiCalls.Stop(); + chainState.Update(contracts); foreach (var r in repo.Rewards) { var seen = rewardsSeen.Any(s => r.RoleId == s); - Log($"{r.RoleId} = {seen}"); + Log($"{Lookup(r.RoleId)} = {seen}"); } Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId))); } + private string Lookup(ulong rewardId) + { + var reward = repo.Rewards.Single(r => r.RoleId == rewardId); + return $"({rewardId})'{reward.Message}'"; + } + private void OnCommand(GiveRewardsCommand call) { if (call.Averages.Any()) Log($"API call: {call.Averages.Length} average."); @@ -292,55 +282,6 @@ namespace CodexTests.UtilityTests } } - public class RewarderLogMonitor - { - private readonly ContainerFileMonitor monitor; - private readonly Dictionary commands = new Dictionary(); - - public RewarderLogMonitor(CoreInterface ci, RunningContainer botContainer) - { - monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/testnetrewarder.log"); - } - - public void Start(Action onCommand) - { - monitor.Start(l => ProcessLine(l, onCommand)); - } - - public void Stop() - { - monitor.Stop(); - } - - private void ProcessLine(string line, Action log) - { - //$"ChainState=[{JsonConvert.SerializeObject(this)}]" + - //$"HistoricState=[{historicState.EntireString()}]"; - - //var stateOpenTag = "ChainState=["; - //var historicOpenTag = "]HistoricState=["; - - //if (!line.Contains(stateOpenTag)) return; - //if (!line.Contains(historicOpenTag)) return; - - //var stateStr = Between(line, stateOpenTag, historicOpenTag); - //var historicStr = Between(line, historicOpenTag, "]"); - - //var chainState = JsonConvert.DeserializeObject(stateStr); - //var historicState = JsonConvert.DeserializeObject(historicStr)!; - //chainState!.Set(new HistoricState(historicState)); - - //log(string.Join(",", chainState!.GenerateOverview())); - } - - private string Between(string s, string open, string close) - { - var start = s.IndexOf(open) + open.Length; - var end = s.LastIndexOf(close); - return s.Substring(start, end - start); - } - } - public class ContainerFileMonitor { private readonly CoreInterface ci; @@ -395,70 +336,5 @@ namespace CodexTests.UtilityTests } } } - - public class ChainMonitor - { - private readonly ICodexContracts contracts; - private readonly IGethNode geth; - private readonly ILog log; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); - private Task worker = Task.CompletedTask; - private DateTime last = DateTime.UtcNow; - - public ChainMonitor(ICodexContracts contracts, IGethNode geth, ILog log) - { - this.contracts = contracts; - this.geth = geth; - this.log = log; - } - - public void Start() - { - last = DateTime.UtcNow; - worker = Task.Run(Worker); - } - - public void Stop() - { - cts.Cancel(); - worker.Wait(); - } - - private void Worker() - { - while (!cts.IsCancellationRequested) - { - Thread.Sleep(TimeSpan.FromSeconds(10)); - if (cts.IsCancellationRequested) return; - - Update(); - - } - } - - private void Update() - { - //var start = last; - //var stop = DateTime.UtcNow; - //last = stop; - - //var range = geth.ConvertTimeRangeToBlockRange(new TimeRange(start, stop)); - - - //throw new Exception(); - //LogEvents(nameof(contracts.GetStorageRequests), contracts.GetStorageRequests, range); - //LogEvents(nameof(contracts.GetRequestFulfilledEvents), contracts.GetRequestFulfilledEvents, range); - //LogEvents(nameof(contracts.GetRequestCancelledEvents), contracts.GetRequestCancelledEvents, range); - //LogEvents(nameof(contracts.GetSlotFilledEvents), contracts.GetSlotFilledEvents, range); - //LogEvents(nameof(contracts.GetSlotFreedEvents), contracts.GetSlotFreedEvents, range); - } - - private void LogEvents(string n, Func f, BlockInterval r) - { - var a = (object[])f(r); - - a.ToList().ForEach(request => log.Log(n + " - " + JsonConvert.SerializeObject(request))); - } - } } } From 1c856f7615ad3e4c69783406ee491f40dfdbe079 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 10 Jun 2024 14:17:19 +0200 Subject: [PATCH 17/28] Fixes log --- .../CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs index 4783e3a..50088e7 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin.ChainMonitor ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry); FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration); - log.Log($"Created as {State}."); + Log($"Created as {State}."); } public Request Request { get; } From d3488dc907c9aab8440e70dcac6d39d38e41f0fb Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jun 2024 10:22:30 +0200 Subject: [PATCH 18/28] Adds block numbers to chainstate event logging --- .../ChainMonitor/ChainState.cs | 24 ++++++++++++------- .../ChainMonitor/ChainStateRequest.cs | 6 ++--- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index e3993f2..952d4ec 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -29,6 +29,12 @@ namespace CodexContractsPlugin.ChainMonitor TotalSpan = timeRange; } + public static ChainState FromTimeRange(ILog log, ICodexContracts contracts, TimeRange timeRange, IChainStateChangeHandler changeHandler) + { + var events = ChainEvents.FromTimeRange(contracts, timeRange); + return FromEvents(log, events, changeHandler); + } + public static ChainState FromEvents(ILog log, ChainEvents events, IChainStateChangeHandler changeHandler) { var state = new ChainState(log, changeHandler, events.BlockInterval.TimeRange); @@ -69,13 +75,13 @@ namespace CodexContractsPlugin.ChainMonitor for (var b = events.BlockInterval.From; b <= events.BlockInterval.To; b++) { var blockEvents = events.All.Where(e => e.Block.BlockNumber == b).ToArray(); - ApplyEvents(blockEvents, eventUtc); + ApplyEvents(b, blockEvents, eventUtc); eventUtc += spanPerBlock; } } - private void ApplyEvents(IHasBlock[] blockEvents, DateTime eventsUtc) + private void ApplyEvents(ulong blockNumber, IHasBlock[] blockEvents, DateTime eventsUtc) { foreach (var e in blockEvents) { @@ -83,7 +89,7 @@ namespace CodexContractsPlugin.ChainMonitor ApplyEvent(d); } - ApplyTimeImplicitEvents(eventsUtc); + ApplyTimeImplicitEvents(blockNumber, eventsUtc); } private void ApplyEvent(Request request) @@ -101,7 +107,7 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; - r.UpdateState(RequestState.Started); + r.UpdateState(request.Block.BlockNumber, RequestState.Started); handler.OnRequestFulfilled(r); } @@ -109,7 +115,7 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; - r.UpdateState(RequestState.Cancelled); + r.UpdateState(request.Block.BlockNumber, RequestState.Cancelled); handler.OnRequestCancelled(r); } @@ -117,7 +123,7 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; - r.Log("SlotFilled"); + r.Log($"[{request.Block.BlockNumber}] SlotFilled"); handler.OnSlotFilled(r, request.SlotIndex); } @@ -125,18 +131,18 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; - r.Log("SlotFreed"); + r.Log($"[{request.Block.BlockNumber}] SlotFreed"); handler.OnSlotFreed(r, request.SlotIndex); } - private void ApplyTimeImplicitEvents(DateTime eventsUtc) + private void ApplyTimeImplicitEvents(ulong blockNumber, DateTime eventsUtc) { foreach (var r in requests) { if (r.State == RequestState.Started && r.FinishedUtc < eventsUtc) { - r.UpdateState(RequestState.Finished); + r.UpdateState(blockNumber, RequestState.Finished); handler.OnRequestFinished(r); } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs index 50088e7..419e78c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs @@ -24,7 +24,7 @@ namespace CodexContractsPlugin.ChainMonitor ExpiryUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Expiry); FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration); - Log($"Created as {State}."); + Log($"[{request.Block.BlockNumber}] Created as {State}."); } public Request Request { get; } @@ -32,9 +32,9 @@ namespace CodexContractsPlugin.ChainMonitor public DateTime ExpiryUtc { get; } public DateTime FinishedUtc { get; } - public void UpdateState(RequestState newState) + public void UpdateState(ulong blockNumber, RequestState newState) { - Log($"Transit: {State} -> {newState}"); + Log($"[{blockNumber}] Transit: {State} -> {newState}"); State = newState; } From c38a2242ba813e74d9f2168da622df4e7e42da6e Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 14 Jun 2024 11:05:29 +0200 Subject: [PATCH 19/28] wip --- .../ChainMonitor/ChainState.cs | 28 ++-- Tools/TestNetRewarder/ChainState.cs | 153 +----------------- Tools/TestNetRewarder/Configuration.cs | 9 ++ Tools/TestNetRewarder/MarketTracker.cs | 40 ++++- Tools/TestNetRewarder/Processor.cs | 109 +++++-------- Tools/TestNetRewarder/Program.cs | 10 +- Tools/TestNetRewarder/RewardChecker.cs | 102 ++++++++++++ Tools/TestNetRewarder/TimeSegmenter.cs | 55 ++++--- 8 files changed, 237 insertions(+), 269 deletions(-) create mode 100644 Tools/TestNetRewarder/RewardChecker.cs diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 952d4ec..875ea8e 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -20,37 +20,29 @@ namespace CodexContractsPlugin.ChainMonitor { private readonly List requests = new List(); private readonly ILog log; + private readonly ICodexContracts contracts; private readonly IChainStateChangeHandler handler; - private ChainState(ILog log, IChainStateChangeHandler changeHandler, TimeRange timeRange) + public ChainState(ILog log, ICodexContracts contracts, IChainStateChangeHandler changeHandler, DateTime startUtc) { this.log = new LogPrefixer(log, "(ChainState) "); + this.contracts = contracts; handler = changeHandler; - TotalSpan = timeRange; - } - - public static ChainState FromTimeRange(ILog log, ICodexContracts contracts, TimeRange timeRange, IChainStateChangeHandler changeHandler) - { - var events = ChainEvents.FromTimeRange(contracts, timeRange); - return FromEvents(log, events, changeHandler); - } - - public static ChainState FromEvents(ILog log, ChainEvents events, IChainStateChangeHandler changeHandler) - { - var state = new ChainState(log, changeHandler, events.BlockInterval.TimeRange); - state.Apply(events); - return state; + StartUtc = startUtc; + TotalSpan = new TimeRange(startUtc, startUtc); } public TimeRange TotalSpan { get; private set; } public IChainStateRequest[] Requests => requests.ToArray(); - public void Update(ICodexContracts contracts) + public DateTime StartUtc { get; } + + public void Update() { - Update(contracts, DateTime.UtcNow); + Update(DateTime.UtcNow); } - public void Update(ICodexContracts contracts, DateTime toUtc) + public void Update(DateTime toUtc) { var span = new TimeRange(TotalSpan.To, toUtc); var events = ChainEvents.FromTimeRange(contracts, span); diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index cef0808..a22537f 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -6,9 +6,8 @@ using Utils; namespace TestNetRewarder { - public class ChainState + public class Keepers { - private HistoricState historicState; private readonly string[] colorIcons = new[] { "🔴", @@ -32,155 +31,5 @@ namespace TestNetRewarder "🔶", "🔷" }; - - public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange) - { - this.historicState = historicState; - - throw new Exception("This is getting a rewrite"); - - //NewRequests = contracts.GetStorageRequests(blockRange); - //historicState.CleanUpOldRequests(); - //historicState.ProcessNewRequests(NewRequests); - //historicState.UpdateStorageRequests(contracts); - - //StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray(); - //FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFinished).ToArray(); - //RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); - //RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange); - //SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange); - //SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange); - } - - public ChainState( - Request[] newRequests, - RequestFulfilledEventDTO[] requestFulfilledEvents, - RequestCancelledEventDTO[] requestCancelledEvents, - SlotFilledEventDTO[] slotFilledEvents, - SlotFreedEventDTO[] slotFreedEvents) - { - NewRequests = newRequests; - RequestFulfilledEvents = requestFulfilledEvents; - RequestCancelledEvents = requestCancelledEvents; - SlotFilledEvents = slotFilledEvents; - SlotFreedEvents = slotFreedEvents; - - historicState = new HistoricState(); - StartedRequests = Array.Empty(); - FinishedRequests = Array.Empty(); - } - - public Request[] NewRequests { get; } - [JsonIgnore] - public StorageRequest[] AllRequests => historicState.StorageRequests; - [JsonIgnore] - public StorageRequest[] StartedRequests { get; private set; } - [JsonIgnore] - public StorageRequest[] FinishedRequests { get; private set; } - public RequestFulfilledEventDTO[] RequestFulfilledEvents { get; } - public RequestCancelledEventDTO[] RequestCancelledEvents { get; } - public SlotFilledEventDTO[] SlotFilledEvents { get; } - public SlotFreedEventDTO[] SlotFreedEvents { get; } - - public string EntireString() - { - return - $"ChainState=[{JsonConvert.SerializeObject(this)}]" + - $"HistoricState=[{historicState.EntireString()}]"; - } - - public void Set(HistoricState h) - { - historicState = h; - } - - public string[] GenerateOverview() - { - var entries = new List(); - - entries.AddRange(NewRequests.Select(ToPair)); - entries.AddRange(RequestFulfilledEvents.Select(ToPair)); - entries.AddRange(RequestCancelledEvents.Select(ToPair)); - entries.AddRange(SlotFilledEvents.Select(ToPair)); - entries.AddRange(SlotFreedEvents.Select(ToPair)); - entries.AddRange(FinishedRequests.Select(ToPair)); - - entries.Sort(new StringUtcComparer()); - - return entries.Select(ToLine).ToArray(); - } - - private StringBlockNumberPair ToPair(Request r) - { - return new StringBlockNumberPair("NewRequest", JsonConvert.SerializeObject(r), r.Block, r.RequestId); - } - - public StringBlockNumberPair ToPair(StorageRequest r) - { - return new StringBlockNumberPair("FinishedRequest", JsonConvert.SerializeObject(r), r.Request.Block, r.Request.RequestId); - } - - private StringBlockNumberPair ToPair(RequestFulfilledEventDTO r) - { - return new StringBlockNumberPair("Fulfilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId); - } - - private StringBlockNumberPair ToPair(RequestCancelledEventDTO r) - { - return new StringBlockNumberPair("Cancelled", JsonConvert.SerializeObject(r), r.Block, r.RequestId); - } - - private StringBlockNumberPair ToPair(SlotFilledEventDTO r) - { - return new StringBlockNumberPair("SlotFilled", JsonConvert.SerializeObject(r), r.Block, r.RequestId); - } - - private StringBlockNumberPair ToPair(SlotFreedEventDTO r) - { - return new StringBlockNumberPair("SlotFreed", JsonConvert.SerializeObject(r), r.Block, r.RequestId); - } - - private string ToLine(StringBlockNumberPair pair) - { - var nl = Environment.NewLine; - var colorIcon = GetColorIcon(pair.RequestId); - return $"{colorIcon} {pair.Block} ({pair.Name}){nl}" + - $"```json{nl}" + - $"{pair.Str}{nl}" + - $"```"; - } - - private string GetColorIcon(byte[] requestId) - { - var index = requestId[0] % colorIcons.Length; - return colorIcons[index]; - } - - public class StringBlockNumberPair - { - public StringBlockNumberPair(string name, string str, BlockTimeEntry block, byte[] requestId) - { - Name = name; - Str = str; - Block = block; - RequestId = requestId; - } - - public string Name { get; } - public string Str { get; } - public BlockTimeEntry Block { get; } - public byte[] RequestId { get; } - } - - public class StringUtcComparer : IComparer - { - public int Compare(StringBlockNumberPair? x, StringBlockNumberPair? y) - { - if (x == null && y == null) return 0; - if (x == null) return 1; - if (y == null) return -1; - return x.Block.BlockNumber.CompareTo(y.Block.BlockNumber); - } - } } } diff --git a/Tools/TestNetRewarder/Configuration.cs b/Tools/TestNetRewarder/Configuration.cs index c88a0aa..5d6ae51 100644 --- a/Tools/TestNetRewarder/Configuration.cs +++ b/Tools/TestNetRewarder/Configuration.cs @@ -40,5 +40,14 @@ namespace TestNetRewarder return TimeSpan.FromMinutes(IntervalMinutes); } } + + public DateTime HistoryStartUtc + { + get + { + if (CheckHistoryTimestamp == 0) throw new Exception("'check-history' unix timestamp is required. Set it to the start/launch moment of the testnet."); + return DateTimeOffset.FromUnixTimeSeconds(CheckHistoryTimestamp).UtcDateTime; + } + } } } diff --git a/Tools/TestNetRewarder/MarketTracker.cs b/Tools/TestNetRewarder/MarketTracker.cs index 17e5e82..c0abad3 100644 --- a/Tools/TestNetRewarder/MarketTracker.cs +++ b/Tools/TestNetRewarder/MarketTracker.cs @@ -1,10 +1,11 @@ -using CodexContractsPlugin.Marketplace; +using CodexContractsPlugin.ChainMonitor; +using CodexContractsPlugin.Marketplace; using DiscordRewards; using System.Numerics; namespace TestNetRewarder { - public class MarketTracker + public class MarketTracker : IChainStateChangeHandler { private readonly List buffer = new List(); @@ -120,5 +121,40 @@ namespace TestNetRewarder } return Array.Empty(); } + + public void OnNewRequest(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestStarted(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestFinished(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestFulfilled(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestCancelled(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); + } } } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 678d4f8..b358665 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -1,39 +1,37 @@ -using DiscordRewards; +using CodexContractsPlugin; +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; using GethPlugin; using Logging; using Newtonsoft.Json; +using System.Numerics; using Utils; namespace TestNetRewarder { - public class Processor + public class Processor : ITimeSegmentHandler, IChainStateChangeHandler { - private static readonly HistoricState historicState = new HistoricState(); - private static readonly RewardRepo rewardRepo = new RewardRepo(); - private static readonly MarketTracker marketTracker = new MarketTracker(); + private readonly RewardChecker rewardChecker = new RewardChecker(); + private readonly MarketTracker marketTracker = new MarketTracker(); + private readonly ChainState chainState; + private readonly Configuration config; private readonly ILog log; private BlockInterval? lastBlockRange; - public Processor(ILog log) + public Processor(Configuration config, ICodexContracts contracts, ILog log) { + this.config = config; this.log = log; + + chainState = new ChainState(log, contracts, this, config.HistoryStartUtc); } - public async Task ProcessTimeSegment(TimeRange timeRange) + public async Task OnNewSegment(TimeRange timeRange) { - var connector = GethConnector.GethConnector.Initialize(log); - if (connector == null) throw new Exception("Invalid Geth information"); - try { - var blockRange = connector.GethNode.ConvertTimeRangeToBlockRange(timeRange); - if (!IsNewBlockRange(blockRange)) - { - log.Log($"Block range {blockRange} was previously processed. Skipping..."); - return; - } - - var chainState = new ChainState(historicState, connector.CodexContracts, blockRange); + chainState.Update(timeRange.To); + await ProcessChainState(chainState); } catch (Exception ex) @@ -43,19 +41,6 @@ namespace TestNetRewarder } } - private bool IsNewBlockRange(BlockInterval blockRange) - { - if (lastBlockRange == null || - lastBlockRange.From != blockRange.From || - lastBlockRange.To != blockRange.To) - { - lastBlockRange = blockRange; - return true; - } - - return false; - } - private async Task ProcessChainState(ChainState chainState) { log.Log(chainState.EntireString()); @@ -92,57 +77,39 @@ namespace TestNetRewarder return marketTracker.ProcessChainState(chainState); } - private async Task SendRewardsCommand(List outgoingRewards, MarketAverage[] marketAverages, string[] eventsOverview) + public void OnNewRequest(IChainStateRequest request) { - var cmd = new GiveRewardsCommand - { - Rewards = outgoingRewards.ToArray(), - Averages = marketAverages.ToArray(), - EventsOverview = eventsOverview - }; - - log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); - return await Program.BotClient.SendRewards(cmd); + throw new NotImplementedException(); } - private void ProcessReward(List outgoingRewards, RewardConfig reward, ChainState chainState) + public void OnRequestStarted(IChainStateRequest request) { - var winningAddresses = PerformCheck(reward, chainState); - foreach (var win in winningAddresses) - { - log.Log($"Address '{win.Address}' wins '{reward.Message}'"); - } - if (winningAddresses.Any()) - { - outgoingRewards.Add(new RewardUsersCommand - { - RewardId = reward.RoleId, - UserAddresses = winningAddresses.Select(a => a.Address).ToArray() - }); - } + throw new NotImplementedException(); } - private EthAddress[] PerformCheck(RewardConfig reward, ChainState chainState) + public void OnRequestFinished(IChainStateRequest request) { - var check = GetCheck(reward.CheckConfig); - return check.Check(chainState).Distinct().ToArray(); + throw new NotImplementedException(); } - private ICheck GetCheck(CheckConfig config) + public void OnRequestFulfilled(IChainStateRequest request) { - switch (config.Type) - { - case CheckType.FilledSlot: - return new FilledAnySlotCheck(); - case CheckType.FinishedSlot: - return new FinishedSlotCheck(config.MinSlotSize, config.MinDuration); - case CheckType.PostedContract: - return new PostedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); - case CheckType.StartedContract: - return new StartedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); - } + throw new NotImplementedException(); + } - throw new Exception("Unknown check type: " + config.Type); + public void OnRequestCancelled(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); } } } diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index 0aaa41c..1106ff7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -1,5 +1,6 @@ using ArgsUniform; using Logging; +using Nethereum.Model; using Utils; namespace TestNetRewarder @@ -27,8 +28,11 @@ namespace TestNetRewarder new ConsoleLog() ); + var connector = GethConnector.GethConnector.Initialize(Log); + if (connector == null) throw new Exception("Invalid Geth information"); + BotClient = new BotClient(Config, Log); - processor = new Processor(Log); + processor = new Processor(Config, connector.CodexContracts, Log); EnsurePath(Config.DataPath); EnsurePath(Config.LogPath); @@ -41,12 +45,12 @@ namespace TestNetRewarder EnsureGethOnline(); Log.Log("Starting TestNet Rewarder..."); - var segmenter = new TimeSegmenter(Log, Config); + var segmenter = new TimeSegmenter(Log, Config, processor); while (!CancellationToken.IsCancellationRequested) { await EnsureBotOnline(); - await segmenter.WaitForNextSegment(processor.ProcessTimeSegment); + await segmenter.ProcessNextSegment(); await Task.Delay(100, CancellationToken); } } diff --git a/Tools/TestNetRewarder/RewardChecker.cs b/Tools/TestNetRewarder/RewardChecker.cs new file mode 100644 index 0000000..6315196 --- /dev/null +++ b/Tools/TestNetRewarder/RewardChecker.cs @@ -0,0 +1,102 @@ +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; +using GethPlugin; +using Nethereum.Model; +using Newtonsoft.Json; +using System.Numerics; + +namespace TestNetRewarder +{ + public class RewardChecker : IChainStateChangeHandler + { + private static readonly RewardRepo rewardRepo = new RewardRepo(); + + private async Task SendRewardsCommand(List outgoingRewards, MarketAverage[] marketAverages, string[] eventsOverview) + { + var cmd = new GiveRewardsCommand + { + Rewards = outgoingRewards.ToArray(), + Averages = marketAverages.ToArray(), + EventsOverview = eventsOverview + }; + + log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); + return await Program.BotClient.SendRewards(cmd); + } + + private void ProcessReward(List outgoingRewards, RewardConfig reward, ChainState chainState) + { + var winningAddresses = PerformCheck(reward, chainState); + foreach (var win in winningAddresses) + { + log.Log($"Address '{win.Address}' wins '{reward.Message}'"); + } + if (winningAddresses.Any()) + { + outgoingRewards.Add(new RewardUsersCommand + { + RewardId = reward.RoleId, + UserAddresses = winningAddresses.Select(a => a.Address).ToArray() + }); + } + } + + private EthAddress[] PerformCheck(RewardConfig reward, ChainState chainState) + { + var check = GetCheck(reward.CheckConfig); + return check.Check(chainState).Distinct().ToArray(); + } + + private ICheck GetCheck(CheckConfig config) + { + switch (config.Type) + { + case CheckType.FilledSlot: + return new FilledAnySlotCheck(); + case CheckType.FinishedSlot: + return new FinishedSlotCheck(config.MinSlotSize, config.MinDuration); + case CheckType.PostedContract: + return new PostedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); + case CheckType.StartedContract: + return new StartedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); + } + + throw new Exception("Unknown check type: " + config.Type); + } + + public void OnNewRequest(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestStarted(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestFinished(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestFulfilled(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnRequestCancelled(IChainStateRequest request) + { + throw new NotImplementedException(); + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + throw new NotImplementedException(); + } + } +} diff --git a/Tools/TestNetRewarder/TimeSegmenter.cs b/Tools/TestNetRewarder/TimeSegmenter.cs index 44062f8..24899ed 100644 --- a/Tools/TestNetRewarder/TimeSegmenter.cs +++ b/Tools/TestNetRewarder/TimeSegmenter.cs @@ -3,51 +3,60 @@ using Utils; namespace TestNetRewarder { + public interface ITimeSegmentHandler + { + Task OnNewSegment(TimeRange timeRange); + } + public class TimeSegmenter { private readonly ILog log; + private readonly ITimeSegmentHandler handler; private readonly TimeSpan segmentSize; - private DateTime start; + private DateTime latest; - public TimeSegmenter(ILog log, Configuration configuration) + public TimeSegmenter(ILog log, Configuration configuration, ITimeSegmentHandler handler) { this.log = log; - + this.handler = handler; if (configuration.IntervalMinutes < 0) configuration.IntervalMinutes = 1; - if (configuration.CheckHistoryTimestamp == 0) throw new Exception("'check-history' unix timestamp is required. Set it to the start/launch moment of the testnet."); segmentSize = configuration.Interval; - start = DateTimeOffset.FromUnixTimeSeconds(configuration.CheckHistoryTimestamp).UtcDateTime; + latest = configuration.HistoryStartUtc; - log.Log("Starting time segments at " + start); + log.Log("Starting time segments at " + latest); log.Log("Segment size: " + Time.FormatDuration(segmentSize)); } - public async Task WaitForNextSegment(Func onSegment) + public async Task ProcessNextSegment() { - var now = DateTime.UtcNow; - var end = start + segmentSize; - var waited = false; - if (end > now) - { - // Wait for the entire time segment to be in the past. - var delay = end - now; - waited = true; - log.Log($"Waiting till time segment is in the past... {Time.FormatDuration(delay)}"); - await Task.Delay(delay, Program.CancellationToken); - } - await Task.Delay(TimeSpan.FromSeconds(3), Program.CancellationToken); + var end = latest + segmentSize; + var waited = await WaitUntilTimeSegmentInPast(end); if (Program.CancellationToken.IsCancellationRequested) return; var postfix = "(Catching up...)"; if (waited) postfix = "(Real-time)"; + log.Log($"Time segment [{latest} to {end}] {postfix}"); + + var range = new TimeRange(latest, end); + latest = end; - log.Log($"Time segment [{start} to {end}] {postfix}"); - var range = new TimeRange(start, end); - start = end; + await handler.OnNewSegment(range); + } - await onSegment(range); + private async Task WaitUntilTimeSegmentInPast(DateTime end) + { + await Task.Delay(TimeSpan.FromSeconds(3), Program.CancellationToken); + + var now = DateTime.UtcNow; + while (end > now) + { + var delay = (end - now) + TimeSpan.FromSeconds(3); + await Task.Delay(delay, Program.CancellationToken); + return true; + } + return false; } } } From bed57dd35b6ad854301f22103c6bc9664d8e5a23 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 17 Jun 2024 15:34:08 +0200 Subject: [PATCH 20/28] Finish replace of old chainstate object in rewarder bot --- Framework/DiscordRewards/CheckConfig.cs | 8 +- .../DiscordRewards/GiveRewardsCommand.cs | 5 + Framework/DiscordRewards/RewardRepo.cs | 12 +- .../ChainMonitor/ChainState.cs | 3 +- .../ChainMonitor/ChainStateRequest.cs | 34 +++++ .../DoNothingChainEventHandler.cs | 4 - .../UtilityTests/DiscordBotTests.cs | 11 +- Tools/TestNetRewarder/BufferLogger.cs | 52 +++++++ Tools/TestNetRewarder/ChainChangeMux.cs | 45 ++++++ Tools/TestNetRewarder/ChainState.cs | 35 ----- Tools/TestNetRewarder/Checks.cs | 141 ----------------- Tools/TestNetRewarder/MarketBuffer.cs | 70 +++++++++ Tools/TestNetRewarder/MarketTracker.cs | 143 ++++-------------- Tools/TestNetRewarder/Processor.cs | 113 ++++---------- Tools/TestNetRewarder/Program.cs | 11 +- Tools/TestNetRewarder/RequestBuilder.cs | 40 +++++ Tools/TestNetRewarder/RewardCheck.cs | 98 ++++++++++++ Tools/TestNetRewarder/RewardChecker.cs | 97 +----------- 18 files changed, 427 insertions(+), 495 deletions(-) create mode 100644 Tools/TestNetRewarder/BufferLogger.cs create mode 100644 Tools/TestNetRewarder/ChainChangeMux.cs delete mode 100644 Tools/TestNetRewarder/ChainState.cs delete mode 100644 Tools/TestNetRewarder/Checks.cs create mode 100644 Tools/TestNetRewarder/MarketBuffer.cs create mode 100644 Tools/TestNetRewarder/RequestBuilder.cs create mode 100644 Tools/TestNetRewarder/RewardCheck.cs diff --git a/Framework/DiscordRewards/CheckConfig.cs b/Framework/DiscordRewards/CheckConfig.cs index 9c4fccb..34425ce 100644 --- a/Framework/DiscordRewards/CheckConfig.cs +++ b/Framework/DiscordRewards/CheckConfig.cs @@ -13,9 +13,9 @@ namespace DiscordRewards public enum CheckType { Uninitialized, - FilledSlot, - FinishedSlot, - PostedContract, - StartedContract, + HostFilledSlot, + HostFinishedSlot, + ClientPostedContract, + ClientStartedContract, } } diff --git a/Framework/DiscordRewards/GiveRewardsCommand.cs b/Framework/DiscordRewards/GiveRewardsCommand.cs index 48dabcc..dffb4ed 100644 --- a/Framework/DiscordRewards/GiveRewardsCommand.cs +++ b/Framework/DiscordRewards/GiveRewardsCommand.cs @@ -5,6 +5,11 @@ public RewardUsersCommand[] Rewards { get; set; } = Array.Empty(); public MarketAverage[] Averages { get; set; } = Array.Empty(); public string[] EventsOverview { get; set; } = Array.Empty(); + + public bool HasAny() + { + return Rewards.Any() || Averages.Any() || EventsOverview.Any(); + } } public class RewardUsersCommand diff --git a/Framework/DiscordRewards/RewardRepo.cs b/Framework/DiscordRewards/RewardRepo.cs index 51ac3fc..28f6173 100644 --- a/Framework/DiscordRewards/RewardRepo.cs +++ b/Framework/DiscordRewards/RewardRepo.cs @@ -11,19 +11,19 @@ namespace DiscordRewards // Filled any slot new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig { - Type = CheckType.FilledSlot + Type = CheckType.HostFilledSlot }), // Finished any slot new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig { - Type = CheckType.FinishedSlot + Type = CheckType.HostFinishedSlot }), // Finished a sizable slot new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig { - Type = CheckType.FinishedSlot, + Type = CheckType.HostFinishedSlot, MinSlotSize = 10.MB(), MinDuration = TimeSpan.FromMinutes(5.0), }), @@ -31,19 +31,19 @@ namespace DiscordRewards // Posted any contract new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig { - Type = CheckType.PostedContract + Type = CheckType.ClientPostedContract }), // Started any contract new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig { - Type = CheckType.StartedContract + Type = CheckType.ClientStartedContract }), // Started a sizable contract new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig { - Type = CheckType.StartedContract, + Type = CheckType.ClientStartedContract, MinNumberOfHosts = 4, MinSlotSize = 10.MB(), MinDuration = TimeSpan.FromMinutes(5.0), diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 875ea8e..0d6af67 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -8,7 +8,6 @@ namespace CodexContractsPlugin.ChainMonitor public interface IChainStateChangeHandler { void OnNewRequest(IChainStateRequest request); - void OnRequestStarted(IChainStateRequest request); void OnRequestFinished(IChainStateRequest request); void OnRequestFulfilled(IChainStateRequest request); void OnRequestCancelled(IChainStateRequest request); @@ -115,6 +114,7 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; + r.Hosts.Add(request.Host, (int)request.SlotIndex); r.Log($"[{request.Block.BlockNumber}] SlotFilled"); handler.OnSlotFilled(r, request.SlotIndex); } @@ -123,6 +123,7 @@ namespace CodexContractsPlugin.ChainMonitor { var r = FindRequest(request.RequestId); if (r == null) return; + r.Hosts.RemoveHost((int)request.SlotIndex); r.Log($"[{request.Block.BlockNumber}] SlotFreed"); handler.OnSlotFreed(r, request.SlotIndex); } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs index 419e78c..d908d19 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateRequest.cs @@ -1,4 +1,5 @@ using CodexContractsPlugin.Marketplace; +using GethPlugin; using Logging; namespace CodexContractsPlugin.ChainMonitor @@ -9,6 +10,8 @@ namespace CodexContractsPlugin.ChainMonitor RequestState State { get; } DateTime ExpiryUtc { get; } DateTime FinishedUtc { get; } + EthAddress Client { get; } + RequestHosts Hosts { get; } } public class ChainStateRequest : IChainStateRequest @@ -25,12 +28,17 @@ namespace CodexContractsPlugin.ChainMonitor FinishedUtc = request.Block.Utc + TimeSpan.FromSeconds((double)request.Ask.Duration); Log($"[{request.Block.BlockNumber}] Created as {State}."); + + Client = new EthAddress(request.Client); + Hosts = new RequestHosts(); } public Request Request { get; } public RequestState State { get; private set; } public DateTime ExpiryUtc { get; } public DateTime FinishedUtc { get; } + public EthAddress Client { get; } + public RequestHosts Hosts { get; } public void UpdateState(ulong blockNumber, RequestState newState) { @@ -43,4 +51,30 @@ namespace CodexContractsPlugin.ChainMonitor log.Log($"Request '{Request.Id}': {msg}"); } } + + public class RequestHosts + { + private readonly Dictionary hosts = new Dictionary(); + + public void Add(EthAddress host, int index) + { + hosts.Add(index, host); + } + + public void RemoveHost(int index) + { + hosts.Remove(index); + } + + public EthAddress? GetHost(int index) + { + if (!hosts.ContainsKey(index)) return null; + return hosts[index]; + } + + public EthAddress[] GetHosts() + { + return hosts.Values.ToArray(); + } + } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs index 65e038b..8a3709c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs @@ -20,10 +20,6 @@ namespace CodexContractsPlugin.ChainMonitor { } - public void OnRequestStarted(IChainStateRequest request) - { - } - public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) { } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 3ae9474..7aafab6 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -39,27 +39,24 @@ namespace CodexTests.UtilityTests var client = StartClient(geth, contracts); var events = ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange()); - var chainState = ChainState.FromEvents( - GetTestLog(), - events, - new DoNothingChainEventHandler()); + var chainState = new ChainState(GetTestLog(), contracts, new DoNothingChainEventHandler(), GetTestRunTimeRange().From); var apiCalls = new RewardApiCalls(Ci, botContainer); apiCalls.Start(OnCommand); var purchaseContract = ClientPurchasesStorage(client); - chainState.Update(contracts); + chainState.Update(); Assert.That(chainState.Requests.Length, Is.EqualTo(1)); purchaseContract.WaitForStorageContractStarted(); - chainState.Update(contracts); + chainState.Update(); purchaseContract.WaitForStorageContractFinished(); Thread.Sleep(rewarderInterval * 2); apiCalls.Stop(); - chainState.Update(contracts); + chainState.Update(); foreach (var r in repo.Rewards) { diff --git a/Tools/TestNetRewarder/BufferLogger.cs b/Tools/TestNetRewarder/BufferLogger.cs new file mode 100644 index 0000000..f7f9925 --- /dev/null +++ b/Tools/TestNetRewarder/BufferLogger.cs @@ -0,0 +1,52 @@ +using Logging; + +namespace TestNetRewarder +{ + public class BufferLogger : ILog + { + private readonly List lines = new List(); + + public void AddStringReplace(string from, string to) + { + throw new NotImplementedException(); + } + + public LogFile CreateSubfile(string ext = "log") + { + throw new NotImplementedException(); + } + + public void Debug(string message = "", int skipFrames = 0) + { + lines.Add(message); + Trim(); + } + + public void Error(string message) + { + lines.Add($"Error: {message}"); + Trim(); + } + + public void Log(string message) + { + lines.Add(message); + Trim(); + } + + public string[] Get() + { + var result = lines.ToArray(); + lines.Clear(); + return result; + } + + private void Trim() + { + if (lines.Count > 100) + { + lines.RemoveRange(0, 50); + } + } + } +} diff --git a/Tools/TestNetRewarder/ChainChangeMux.cs b/Tools/TestNetRewarder/ChainChangeMux.cs new file mode 100644 index 0000000..f70490b --- /dev/null +++ b/Tools/TestNetRewarder/ChainChangeMux.cs @@ -0,0 +1,45 @@ +using CodexContractsPlugin.ChainMonitor; +using System.Numerics; + +namespace TestNetRewarder +{ + public class ChainChangeMux : IChainStateChangeHandler + { + private readonly IChainStateChangeHandler[] handlers; + + public ChainChangeMux(params IChainStateChangeHandler[] handlers) + { + this.handlers = handlers; + } + + public void OnNewRequest(IChainStateRequest request) + { + foreach (var handler in handlers) handler.OnNewRequest(request); + } + + public void OnRequestCancelled(IChainStateRequest request) + { + foreach (var handler in handlers) handler.OnRequestCancelled(request); + } + + public void OnRequestFinished(IChainStateRequest request) + { + foreach (var handler in handlers) handler.OnRequestFinished(request); + } + + public void OnRequestFulfilled(IChainStateRequest request) + { + foreach (var handler in handlers) handler.OnRequestFulfilled(request); + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + foreach (var handler in handlers) handler.OnSlotFilled(request, slotIndex); + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + foreach (var handler in handlers) handler.OnSlotFreed(request, slotIndex); + } + } +} diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs deleted file mode 100644 index a22537f..0000000 --- a/Tools/TestNetRewarder/ChainState.cs +++ /dev/null @@ -1,35 +0,0 @@ -using CodexContractsPlugin; -using CodexContractsPlugin.Marketplace; -using NethereumWorkflow.BlockUtils; -using Newtonsoft.Json; -using Utils; - -namespace TestNetRewarder -{ - public class Keepers - { - private readonly string[] colorIcons = new[] - { - "🔴", - "🟠", - "🟡", - "🟢", - "🔵", - "🟣", - "🟤", - "⚫", - "⚪", - "🟥", - "🟧", - "🟨", - "🟩", - "🟦", - "🟪", - "🟫", - "⬛", - "⬜", - "🔶", - "🔷" - }; - } -} diff --git a/Tools/TestNetRewarder/Checks.cs b/Tools/TestNetRewarder/Checks.cs deleted file mode 100644 index 43102e0..0000000 --- a/Tools/TestNetRewarder/Checks.cs +++ /dev/null @@ -1,141 +0,0 @@ -using CodexContractsPlugin.Marketplace; -using GethPlugin; -using NethereumWorkflow; -using Utils; - -namespace TestNetRewarder -{ - public interface ICheck - { - EthAddress[] Check(ChainState state); - } - - public class FilledAnySlotCheck : ICheck - { - public EthAddress[] Check(ChainState state) - { - return state.SlotFilledEvents.Select(e => e.Host).ToArray(); - } - } - - public class FinishedSlotCheck : ICheck - { - private readonly ByteSize minSize; - private readonly TimeSpan minDuration; - - public FinishedSlotCheck(ByteSize minSize, TimeSpan minDuration) - { - this.minSize = minSize; - this.minDuration = minDuration; - } - - public EthAddress[] Check(ChainState state) - { - return state.FinishedRequests - .Where(r => - MeetsSizeRequirement(r) && - MeetsDurationRequirement(r)) - .SelectMany(r => r.Hosts) - .ToArray(); - } - - private bool MeetsSizeRequirement(StorageRequest r) - { - var slotSize = r.Request.Ask.SlotSize.ToDecimal(); - decimal min = minSize.SizeInBytes; - return slotSize >= min; - } - - private bool MeetsDurationRequirement(StorageRequest r) - { - var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); - return duration >= minDuration; - } - } - - public class PostedContractCheck : ICheck - { - private readonly ulong minNumberOfHosts; - private readonly ByteSize minSlotSize; - private readonly TimeSpan minDuration; - - public PostedContractCheck(ulong minNumberOfHosts, ByteSize minSlotSize, TimeSpan minDuration) - { - this.minNumberOfHosts = minNumberOfHosts; - this.minSlotSize = minSlotSize; - this.minDuration = minDuration; - } - - public EthAddress[] Check(ChainState state) - { - return state.NewRequests - .Where(r => - MeetsNumSlotsRequirement(r) && - MeetsSizeRequirement(r) && - MeetsDurationRequirement(r)) - .Select(r => r.ClientAddress) - .ToArray(); - } - - private bool MeetsNumSlotsRequirement(Request r) - { - return r.Ask.Slots >= minNumberOfHosts; - } - - private bool MeetsSizeRequirement(Request r) - { - var slotSize = r.Ask.SlotSize.ToDecimal(); - decimal min = minSlotSize.SizeInBytes; - return slotSize >= min; - } - - private bool MeetsDurationRequirement(Request r) - { - var duration = TimeSpan.FromSeconds((double)r.Ask.Duration); - return duration >= minDuration; - } - } - - public class StartedContractCheck : ICheck - { - private readonly ulong minNumberOfHosts; - private readonly ByteSize minSlotSize; - private readonly TimeSpan minDuration; - - public StartedContractCheck(ulong minNumberOfHosts, ByteSize minSlotSize, TimeSpan minDuration) - { - this.minNumberOfHosts = minNumberOfHosts; - this.minSlotSize = minSlotSize; - this.minDuration = minDuration; - } - - public EthAddress[] Check(ChainState state) - { - return state.StartedRequests - .Where(r => - MeetsNumSlotsRequirement(r) && - MeetsSizeRequirement(r) && - MeetsDurationRequirement(r)) - .Select(r => r.Request.ClientAddress) - .ToArray(); - } - - private bool MeetsNumSlotsRequirement(StorageRequest r) - { - return r.Request.Ask.Slots >= minNumberOfHosts; - } - - private bool MeetsSizeRequirement(StorageRequest r) - { - var slotSize = r.Request.Ask.SlotSize.ToDecimal(); - decimal min = minSlotSize.SizeInBytes; - return slotSize >= min; - } - - private bool MeetsDurationRequirement(StorageRequest r) - { - var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); - return duration >= minDuration; - } - } -} diff --git a/Tools/TestNetRewarder/MarketBuffer.cs b/Tools/TestNetRewarder/MarketBuffer.cs new file mode 100644 index 0000000..4557378 --- /dev/null +++ b/Tools/TestNetRewarder/MarketBuffer.cs @@ -0,0 +1,70 @@ +using CodexContractsPlugin.ChainMonitor; +using CodexContractsPlugin.Marketplace; +using DiscordRewards; +using System.Numerics; + +namespace TestNetRewarder +{ + public class MarketBuffer + { + private readonly List requests = new List(); + private readonly TimeSpan bufferSpan; + + public MarketBuffer(TimeSpan bufferSpan) + { + this.bufferSpan = bufferSpan; + } + + public void Add(IChainStateRequest request) + { + requests.Add(request); + } + + public void Update() + { + var now = DateTime.UtcNow; + requests.RemoveAll(r => (now - r.FinishedUtc) > bufferSpan); + } + + public MarketAverage? GetAverage() + { + if (requests.Count == 0) return null; + + return new MarketAverage + { + NumberOfFinished = requests.Count, + TimeRangeSeconds = (int)bufferSpan.TotalSeconds, + Price = Average(s => s.Request.Ask.Reward), + Duration = Average(s => s.Request.Ask.Duration), + Size = Average(s => GetTotalSize(s.Request.Ask)), + Collateral = Average(s => s.Request.Ask.Collateral), + ProofProbability = Average(s => s.Request.Ask.ProofProbability) + }; + } + + private float Average(Func getValue) + { + return Average(s => Convert.ToInt32(getValue(s))); + } + + private float Average(Func getValue) + { + var sum = 0.0f; + float count = requests.Count; + foreach (var r in requests) + { + sum += getValue(r); + } + + if (count < 1.0f) return 0.0f; + return sum / count; + } + + private int GetTotalSize(Ask ask) + { + var nSlots = Convert.ToInt32(ask.Slots); + var slotSize = Convert.ToInt32(ask.SlotSize); + return nSlots * slotSize; + } + } +} diff --git a/Tools/TestNetRewarder/MarketTracker.cs b/Tools/TestNetRewarder/MarketTracker.cs index c0abad3..669dd3a 100644 --- a/Tools/TestNetRewarder/MarketTracker.cs +++ b/Tools/TestNetRewarder/MarketTracker.cs @@ -1,160 +1,73 @@ using CodexContractsPlugin.ChainMonitor; -using CodexContractsPlugin.Marketplace; using DiscordRewards; +using Logging; using System.Numerics; namespace TestNetRewarder { public class MarketTracker : IChainStateChangeHandler { - private readonly List buffer = new List(); + private readonly List buffers = new List(); + private readonly ILog log; - public MarketAverage[] ProcessChainState(ChainState chainState) + public MarketTracker(Configuration config, ILog log) { - var intervalCounts = GetInsightCounts(); - if (!intervalCounts.Any()) return Array.Empty(); + var intervals = GetInsightCounts(config); - UpdateBuffer(chainState, intervalCounts.Max()); - var result = intervalCounts - .Select(GenerateMarketAverage) - .Where(a => a != null) - .Cast() - .ToArray(); - - if (!result.Any()) result = Array.Empty(); - return result; - } - - private void UpdateBuffer(ChainState chainState, int maxNumberOfIntervals) - { - buffer.Add(chainState); - while (buffer.Count > maxNumberOfIntervals) + foreach (var i in intervals) { - buffer.RemoveAt(0); - } - } - - private MarketAverage? GenerateMarketAverage(int numberOfIntervals) - { - var states = SelectStates(numberOfIntervals); - return CreateAverage(states); - } - - private ChainState[] SelectStates(int numberOfIntervals) - { - if (numberOfIntervals < 1) return Array.Empty(); - if (numberOfIntervals > buffer.Count) return Array.Empty(); - return buffer.TakeLast(numberOfIntervals).ToArray(); - } - - private MarketAverage? CreateAverage(ChainState[] states) - { - try - { - return new MarketAverage - { - NumberOfFinished = CountNumberOfFinishedRequests(states), - TimeRangeSeconds = GetTotalTimeRange(states), - Price = Average(states, s => s.Request.Ask.Reward), - Duration = Average(states, s => s.Request.Ask.Duration), - Size = Average(states, s => GetTotalSize(s.Request.Ask)), - Collateral = Average(states, s => s.Request.Ask.Collateral), - ProofProbability = Average(states, s => s.Request.Ask.ProofProbability) - }; - } - catch (Exception ex) - { - Program.Log.Error($"Exception in CreateAverage: {ex}"); - return null; - } - } - - private int GetTotalSize(Ask ask) - { - var nSlots = Convert.ToInt32(ask.Slots); - var slotSize = Convert.ToInt32(ask.SlotSize); - return nSlots * slotSize; - } - - private float Average(ChainState[] states, Func getValue) - { - return Average(states, s => Convert.ToInt32(getValue(s))); - } - - private float Average(ChainState[] states, Func getValue) - { - var sum = 0.0f; - var count = 0.0f; - foreach (var state in states) - { - foreach (var finishedRequest in state.FinishedRequests) - { - sum += getValue(finishedRequest); - count++; - } + buffers.Add(new MarketBuffer( + config.Interval * i + )); } - if (count < 1.0f) return 0.0f; - return sum / count; + this.log = log; } - private int GetTotalTimeRange(ChainState[] states) + public MarketAverage[] GetAverages() { - return Convert.ToInt32((Program.Config.Interval * states.Length).TotalSeconds); - } + foreach (var b in buffers) b.Update(); - private int CountNumberOfFinishedRequests(ChainState[] states) - { - return states.Sum(s => s.FinishedRequests.Length); - } - - private int[] GetInsightCounts() - { - try - { - var tokens = Program.Config.MarketInsights.Split(';').ToArray(); - return tokens.Select(t => Convert.ToInt32(t)).ToArray(); - } - catch (Exception ex) - { - Program.Log.Error($"Exception when parsing MarketInsights config parameters: {ex}"); - } - return Array.Empty(); + return buffers.Select(b => b.GetAverage()).Where(a => a != null).Cast().ToArray(); } public void OnNewRequest(IChainStateRequest request) { - throw new NotImplementedException(); - } - - public void OnRequestStarted(IChainStateRequest request) - { - throw new NotImplementedException(); } public void OnRequestFinished(IChainStateRequest request) { - throw new NotImplementedException(); + foreach (var b in buffers) b.Add(request); } public void OnRequestFulfilled(IChainStateRequest request) { - throw new NotImplementedException(); } public void OnRequestCancelled(IChainStateRequest request) { - throw new NotImplementedException(); } public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) { - throw new NotImplementedException(); } public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) { - throw new NotImplementedException(); + } + + private int[] GetInsightCounts(Configuration config) + { + try + { + var tokens = config.MarketInsights.Split(';').ToArray(); + return tokens.Select(t => Convert.ToInt32(t)).ToArray(); + } + catch (Exception ex) + { + log.Error($"Exception when parsing MarketInsights config parameters: {ex}"); + } + return Array.Empty(); } } } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index b358665..8cbc4db 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -1,29 +1,36 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; -using GethPlugin; using Logging; -using Newtonsoft.Json; -using System.Numerics; using Utils; namespace TestNetRewarder { - public class Processor : ITimeSegmentHandler, IChainStateChangeHandler + public class Processor : ITimeSegmentHandler { - private readonly RewardChecker rewardChecker = new RewardChecker(); - private readonly MarketTracker marketTracker = new MarketTracker(); + private readonly RequestBuilder builder; + private readonly RewardChecker rewardChecker; + private readonly MarketTracker marketTracker; + private readonly BufferLogger bufferLogger; private readonly ChainState chainState; - private readonly Configuration config; + private readonly BotClient client; private readonly ILog log; - private BlockInterval? lastBlockRange; - public Processor(Configuration config, ICodexContracts contracts, ILog log) + public Processor(Configuration config, BotClient client, ICodexContracts contracts, ILog log) { - this.config = config; + this.client = client; this.log = log; - chainState = new ChainState(log, contracts, this, config.HistoryStartUtc); + builder = new RequestBuilder(); + rewardChecker = new RewardChecker(builder); + marketTracker = new MarketTracker(config, log); + bufferLogger = new BufferLogger(); + + var handler = new ChainChangeMux( + rewardChecker.Handler, + marketTracker + ); + + chainState = new ChainState(new LogSplitter(log, bufferLogger), contracts, handler, config.HistoryStartUtc); } public async Task OnNewSegment(TimeRange timeRange) @@ -31,8 +38,15 @@ namespace TestNetRewarder try { chainState.Update(timeRange.To); - - await ProcessChainState(chainState); + + var averages = marketTracker.GetAverages(); + var lines = bufferLogger.Get(); + + var request = builder.Build(averages, lines); + if (request.HasAny()) + { + await client.SendRewards(request); + } } catch (Exception ex) { @@ -40,76 +54,5 @@ namespace TestNetRewarder throw; } } - - private async Task ProcessChainState(ChainState chainState) - { - log.Log(chainState.EntireString()); - - var outgoingRewards = new List(); - foreach (var reward in rewardRepo.Rewards) - { - ProcessReward(outgoingRewards, reward, chainState); - } - - var marketAverages = GetMarketAverages(chainState); - var eventsOverview = GenerateEventsOverview(chainState); - - log.Log($"Found {outgoingRewards.Count} rewards. " + - $"Found {marketAverages.Length} market averages. " + - $"Found {eventsOverview.Length} events."); - - if (outgoingRewards.Any() || marketAverages.Any() || eventsOverview.Any()) - { - if (!await SendRewardsCommand(outgoingRewards, marketAverages, eventsOverview)) - { - log.Error("Failed to send reward command."); - } - } - } - - private string[] GenerateEventsOverview(ChainState chainState) - { - return chainState.GenerateOverview(); - } - - private MarketAverage[] GetMarketAverages(ChainState chainState) - { - return marketTracker.ProcessChainState(chainState); - } - - public void OnNewRequest(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestStarted(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestFinished(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestFulfilled(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestCancelled(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) - { - throw new NotImplementedException(); - } - - public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) - { - throw new NotImplementedException(); - } } } diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index 1106ff7..0056ffe 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -1,16 +1,15 @@ using ArgsUniform; using Logging; -using Nethereum.Model; using Utils; namespace TestNetRewarder { public class Program { - public static Configuration Config { get; private set; } = null!; - public static ILog Log { get; private set; } = null!; - public static CancellationToken CancellationToken { get; private set; } - public static BotClient BotClient { get; private set; } = null!; + public static CancellationToken CancellationToken; + private static Configuration Config = null!; + private static ILog Log = null!; + private static BotClient BotClient = null!; private static Processor processor = null!; private static DateTime lastCheck = DateTime.MinValue; @@ -32,7 +31,7 @@ namespace TestNetRewarder if (connector == null) throw new Exception("Invalid Geth information"); BotClient = new BotClient(Config, Log); - processor = new Processor(Config, connector.CodexContracts, Log); + processor = new Processor(Config, BotClient, connector.CodexContracts, Log); EnsurePath(Config.DataPath); EnsurePath(Config.LogPath); diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs new file mode 100644 index 0000000..ea06eaa --- /dev/null +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -0,0 +1,40 @@ +using DiscordRewards; +using GethPlugin; + +namespace TestNetRewarder +{ + public class RequestBuilder : IRewardGiver + { + private readonly Dictionary> rewards = new Dictionary>(); + + public void Give(RewardConfig reward, EthAddress receiver) + { + if (rewards.ContainsKey(reward.RoleId)) + { + rewards[reward.RoleId].Add(receiver); + } + else + { + rewards.Add(reward.RoleId, new List { receiver }); + } + } + + public GiveRewardsCommand Build(MarketAverage[] marketAverages, string[] lines) + { + var result = new GiveRewardsCommand + { + Rewards = rewards.Select(p => new RewardUsersCommand + { + RewardId = p.Key, + UserAddresses = p.Value.Select(v => v.Address).ToArray() + }).ToArray(), + Averages = marketAverages, + EventsOverview = lines + }; + + rewards.Clear(); + + return result; + } + } +} diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs new file mode 100644 index 0000000..9be87d2 --- /dev/null +++ b/Tools/TestNetRewarder/RewardCheck.cs @@ -0,0 +1,98 @@ +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; +using GethPlugin; +using NethereumWorkflow; +using System.Numerics; + +namespace TestNetRewarder +{ + public interface IRewardGiver + { + void Give(RewardConfig reward, EthAddress receiver); + } + + public class RewardCheck : IChainStateChangeHandler + { + private readonly RewardConfig reward; + private readonly IRewardGiver giver; + + public RewardCheck(RewardConfig reward, IRewardGiver giver) + { + this.reward = reward; + this.giver = giver; + } + + public void OnNewRequest(IChainStateRequest request) + { + if (MeetsRequirements(CheckType.ClientPostedContract, request)) + { + GiveReward(reward, request.Client); + } + } + + public void OnRequestCancelled(IChainStateRequest request) + { + } + + public void OnRequestFinished(IChainStateRequest request) + { + if (MeetsRequirements(CheckType.HostFinishedSlot, request)) + { + foreach (var host in request.Hosts.GetHosts()) + { + GiveReward(reward, host); + } + } + } + + public void OnRequestFulfilled(IChainStateRequest request) + { + if (MeetsRequirements(CheckType.ClientStartedContract, request)) + { + GiveReward(reward, request.Client); + } + } + + public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) + { + if (MeetsRequirements(CheckType.HostFilledSlot, request)) + { + var host = request.Hosts.GetHost((int)slotIndex); + if (host != null) + { + GiveReward(reward, host); + } + } + } + + public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) + { + } + + private void GiveReward(RewardConfig reward, EthAddress receiver) + { + giver.Give(reward, receiver); + } + + private bool MeetsRequirements(CheckType type, IChainStateRequest request) + { + return + reward.CheckConfig.Type == type && + MeetsDurationRequirement(request) && + MeetsSizeRequirement(request); + } + + private bool MeetsSizeRequirement(IChainStateRequest r) + { + var slotSize = r.Request.Ask.SlotSize.ToDecimal(); + decimal min = reward.CheckConfig.MinSlotSize.SizeInBytes; + return slotSize >= min; + } + + private bool MeetsDurationRequirement(IChainStateRequest r) + { + var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); + return duration >= reward.CheckConfig.MinDuration; + } + } +} diff --git a/Tools/TestNetRewarder/RewardChecker.cs b/Tools/TestNetRewarder/RewardChecker.cs index 6315196..0d22baf 100644 --- a/Tools/TestNetRewarder/RewardChecker.cs +++ b/Tools/TestNetRewarder/RewardChecker.cs @@ -1,102 +1,17 @@ using CodexContractsPlugin.ChainMonitor; using DiscordRewards; -using GethPlugin; -using Nethereum.Model; -using Newtonsoft.Json; -using System.Numerics; namespace TestNetRewarder { - public class RewardChecker : IChainStateChangeHandler + public class RewardChecker { - private static readonly RewardRepo rewardRepo = new RewardRepo(); - - private async Task SendRewardsCommand(List outgoingRewards, MarketAverage[] marketAverages, string[] eventsOverview) + public RewardChecker(IRewardGiver giver) { - var cmd = new GiveRewardsCommand - { - Rewards = outgoingRewards.ToArray(), - Averages = marketAverages.ToArray(), - EventsOverview = eventsOverview - }; - - log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); - return await Program.BotClient.SendRewards(cmd); + var repo = new RewardRepo(); + var checks = repo.Rewards.Select(r => new RewardCheck(r, giver)).ToArray(); + Handler = new ChainChangeMux(checks); } - private void ProcessReward(List outgoingRewards, RewardConfig reward, ChainState chainState) - { - var winningAddresses = PerformCheck(reward, chainState); - foreach (var win in winningAddresses) - { - log.Log($"Address '{win.Address}' wins '{reward.Message}'"); - } - if (winningAddresses.Any()) - { - outgoingRewards.Add(new RewardUsersCommand - { - RewardId = reward.RoleId, - UserAddresses = winningAddresses.Select(a => a.Address).ToArray() - }); - } - } - - private EthAddress[] PerformCheck(RewardConfig reward, ChainState chainState) - { - var check = GetCheck(reward.CheckConfig); - return check.Check(chainState).Distinct().ToArray(); - } - - private ICheck GetCheck(CheckConfig config) - { - switch (config.Type) - { - case CheckType.FilledSlot: - return new FilledAnySlotCheck(); - case CheckType.FinishedSlot: - return new FinishedSlotCheck(config.MinSlotSize, config.MinDuration); - case CheckType.PostedContract: - return new PostedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); - case CheckType.StartedContract: - return new StartedContractCheck(config.MinNumberOfHosts, config.MinSlotSize, config.MinDuration); - } - - throw new Exception("Unknown check type: " + config.Type); - } - - public void OnNewRequest(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestStarted(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestFinished(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestFulfilled(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnRequestCancelled(IChainStateRequest request) - { - throw new NotImplementedException(); - } - - public void OnSlotFilled(IChainStateRequest request, BigInteger slotIndex) - { - throw new NotImplementedException(); - } - - public void OnSlotFreed(IChainStateRequest request, BigInteger slotIndex) - { - throw new NotImplementedException(); - } + public IChainStateChangeHandler Handler { get; } } } From cb4cdfe69a621e954babe580e5a853743e8b0630 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 17 Jun 2024 15:59:54 +0200 Subject: [PATCH 21/28] better logging --- .../ChainMonitor/ChainState.cs | 4 ++-- .../UtilityTests/DiscordBotTests.cs | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 0d6af67..bc284eb 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -115,7 +115,7 @@ namespace CodexContractsPlugin.ChainMonitor var r = FindRequest(request.RequestId); if (r == null) return; r.Hosts.Add(request.Host, (int)request.SlotIndex); - r.Log($"[{request.Block.BlockNumber}] SlotFilled"); + r.Log($"[{request.Block.BlockNumber}] SlotFilled (host:'{request.Host}', slotIndex:{request.SlotIndex})"); handler.OnSlotFilled(r, request.SlotIndex); } @@ -124,7 +124,7 @@ namespace CodexContractsPlugin.ChainMonitor var r = FindRequest(request.RequestId); if (r == null) return; r.Hosts.RemoveHost((int)request.SlotIndex); - r.Log($"[{request.Block.BlockNumber}] SlotFreed"); + r.Log($"[{request.Block.BlockNumber}] SlotFreed (slotIndex:{request.SlotIndex})"); handler.OnSlotFreed(r, request.SlotIndex); } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 7aafab6..18ba663 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -4,6 +4,7 @@ using CodexDiscordBotPlugin; using CodexPlugin; using Core; using DiscordRewards; +using DistTestCore; using GethPlugin; using KubernetesWorkflow.Types; using Newtonsoft.Json; @@ -24,6 +25,7 @@ namespace CodexTests.UtilityTests private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); [Test] + [DontDownloadLogs] public void BotRewardTest() { var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); @@ -31,11 +33,9 @@ namespace CodexTests.UtilityTests var gethInfo = CreateGethInfo(geth, contracts); var botContainer = StartDiscordBot(gethInfo); - - StartHosts(geth, contracts); - var rewarderContainer = StartRewarderBot(gethInfo, botContainer); + StartHosts(geth, contracts); var client = StartClient(geth, contracts); var events = ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange()); @@ -76,8 +76,15 @@ namespace CodexTests.UtilityTests private void OnCommand(GiveRewardsCommand call) { - if (call.Averages.Any()) Log($"API call: {call.Averages.Length} average."); - if (call.EventsOverview.Any()) Log($"API call: {call.EventsOverview.Length} events."); + Log($"API call:"); + foreach (var a in call.Averages) + { + Log("Average: " + JsonConvert.SerializeObject(a)); + } + foreach (var e in call.EventsOverview) + { + Log("Event: " + e); + } foreach (var r in call.Rewards) { var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); @@ -85,7 +92,7 @@ namespace CodexTests.UtilityTests foreach (var address in r.UserAddresses) { var user = IdentifyAccount(address); - Log("API call: " + user + ": " + reward.Message); + Log("Reward: " + user + ": " + reward.Message); } } } From f7c45d17d740adf43482ac9e8e1f2fbf25be69da Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 18 Jun 2024 09:57:38 +0200 Subject: [PATCH 22/28] wip --- .../CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index 55b60a2..2d11cbd 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,8 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-12dc7ef"; + public override string Image => "thatbenbierens/codex-rewardbot:newstate"; + //"codexstorage/codex-rewarderbot:sha-12dc7ef"; protected override void Initialize(StartupConfig startupConfig) { From cc3eddf02d671a3bc236f4b01f3af6f28cc9eaf8 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 20 Jun 2024 12:15:59 +0200 Subject: [PATCH 23/28] wip --- .../Marketplace/Customizations.cs | 4 +--- Tests/CodexTests/UtilityTests/DiscordBotTests.cs | 11 ++++++----- Tools/TestNetRewarder/Processor.cs | 8 +++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index 557e021..4331143 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -23,9 +23,7 @@ namespace CodexContractsPlugin.Marketplace { get { - var id = ""; - foreach (var b in RequestId) id += b.ToString(); - return id; + return BitConverter.ToString(RequestId).Replace("-", "").ToLowerInvariant(); } } } diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 18ba663..66601c8 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -53,7 +53,7 @@ namespace CodexTests.UtilityTests purchaseContract.WaitForStorageContractFinished(); - Thread.Sleep(rewarderInterval * 2); + Thread.Sleep(rewarderInterval * 3); apiCalls.Stop(); chainState.Update(); @@ -76,14 +76,14 @@ namespace CodexTests.UtilityTests private void OnCommand(GiveRewardsCommand call) { - Log($"API call:"); + Log($""); foreach (var a in call.Averages) { - Log("Average: " + JsonConvert.SerializeObject(a)); + Log("\tAverage: " + JsonConvert.SerializeObject(a)); } foreach (var e in call.EventsOverview) { - Log("Event: " + e); + Log("\tEvent: " + e); } foreach (var r in call.Rewards) { @@ -92,9 +92,10 @@ namespace CodexTests.UtilityTests foreach (var address in r.UserAddresses) { var user = IdentifyAccount(address); - Log("Reward: " + user + ": " + reward.Message); + Log("\tReward: " + user + ": " + reward.Message); } } + Log($""); } private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client) diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 8cbc4db..b30ca79 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -40,7 +40,7 @@ namespace TestNetRewarder chainState.Update(timeRange.To); var averages = marketTracker.GetAverages(); - var lines = bufferLogger.Get(); + var lines = RemoveFirstLine(bufferLogger.Get()); var request = builder.Build(averages, lines); if (request.HasAny()) @@ -54,5 +54,11 @@ namespace TestNetRewarder throw; } } + + private string[] RemoveFirstLine(string[] lines) + { + if (!lines.Any()) return Array.Empty(); + return lines.Skip(1).ToArray(); + } } } From 32e702802956eccee3a086ab816eaa9eb99b25ea Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jun 2024 08:56:20 +0200 Subject: [PATCH 24/28] wip --- .../UtilityTests/DiscordBotTests.cs | 55 ++++++++++++------- Tools/TestNetRewarder/BufferLogger.cs | 11 ---- Tools/TestNetRewarder/Processor.cs | 9 ++- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 66601c8..01c8fc5 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -7,6 +7,7 @@ using DiscordRewards; using DistTestCore; using GethPlugin; using KubernetesWorkflow.Types; +using Logging; using Newtonsoft.Json; using NUnit.Framework; using Utils; @@ -38,25 +39,37 @@ namespace CodexTests.UtilityTests StartHosts(geth, contracts); var client = StartClient(geth, contracts); - var events = ChainEvents.FromTimeRange(contracts, GetTestRunTimeRange()); - var chainState = new ChainState(GetTestLog(), contracts, new DoNothingChainEventHandler(), GetTestRunTimeRange().From); + //var chainState = new ChainState(GetTestLog(), contracts, new DoNothingChainEventHandler(), GetTestRunTimeRange().From); - var apiCalls = new RewardApiCalls(Ci, botContainer); + //var running = true; + //var task = Task.Run(() => + //{ + // while (running) + // { + // Thread.Sleep(TimeSpan.FromMinutes(1)); + // chainState.Update(); + // } + //}); + + var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer); apiCalls.Start(OnCommand); var purchaseContract = ClientPurchasesStorage(client); - chainState.Update(); - Assert.That(chainState.Requests.Length, Is.EqualTo(1)); + //chainState.Update(); + //Assert.That(chainState.Requests.Length, Is.EqualTo(1)); purchaseContract.WaitForStorageContractStarted(); - chainState.Update(); + //chainState.Update(); purchaseContract.WaitForStorageContractFinished(); Thread.Sleep(rewarderInterval * 3); - + + //running = false; + //task.Wait(); + apiCalls.Stop(); - chainState.Update(); + //chainState.Update(); foreach (var r in repo.Rewards) { @@ -74,9 +87,9 @@ namespace CodexTests.UtilityTests return $"({rewardId})'{reward.Message}'"; } - private void OnCommand(GiveRewardsCommand call) + private void OnCommand(string timestamp, GiveRewardsCommand call) { - Log($""); + Log($""); foreach (var a in call.Averages) { Log("\tAverage: " + JsonConvert.SerializeObject(a)); @@ -249,14 +262,13 @@ namespace CodexTests.UtilityTests public class RewardApiCalls { private readonly ContainerFileMonitor monitor; - private readonly Dictionary commands = new Dictionary(); - public RewardApiCalls(CoreInterface ci, RunningContainer botContainer) + public RewardApiCalls(ILog log, CoreInterface ci, RunningContainer botContainer) { - monitor = new ContainerFileMonitor(ci, botContainer, "/app/datapath/logs/discordbot.log"); + monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log"); } - public void Start(Action onCommand) + public void Start(Action onCommand) { monitor.Start(line => ParseLine(line, onCommand)); } @@ -266,19 +278,17 @@ namespace CodexTests.UtilityTests monitor.Stop(); } - private void ParseLine(string line, Action onCommand) + private void ParseLine(string line, Action onCommand) { try { var timestamp = line.Substring(0, 30); - if (commands.ContainsKey(timestamp)) return; var json = line.Substring(31); var cmd = JsonConvert.DeserializeObject(json); if (cmd != null) { - commands.Add(timestamp, cmd); - onCommand(cmd); + onCommand(timestamp, cmd); } } catch @@ -289,6 +299,7 @@ namespace CodexTests.UtilityTests public class ContainerFileMonitor { + private readonly ILog log; private readonly CoreInterface ci; private readonly RunningContainer botContainer; private readonly string filePath; @@ -297,8 +308,9 @@ namespace CodexTests.UtilityTests private Task worker = Task.CompletedTask; private Action onNewLine = c => { }; - public ContainerFileMonitor(CoreInterface ci, RunningContainer botContainer, string filePath) + public ContainerFileMonitor(ILog log, CoreInterface ci, RunningContainer botContainer, string filePath) { + this.log = log; this.ci = ci; this.botContainer = botContainer; this.filePath = filePath; @@ -316,6 +328,9 @@ namespace CodexTests.UtilityTests worker.Wait(); } + // did any container crash? that's why it repeats? + + private void Worker() { while (!cts.IsCancellationRequested) @@ -333,6 +348,8 @@ namespace CodexTests.UtilityTests var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); foreach (var line in lines) { + // log.Log("line: " + line); + if (!seenLines.Contains(line)) { seenLines.Add(line); diff --git a/Tools/TestNetRewarder/BufferLogger.cs b/Tools/TestNetRewarder/BufferLogger.cs index f7f9925..1f32332 100644 --- a/Tools/TestNetRewarder/BufferLogger.cs +++ b/Tools/TestNetRewarder/BufferLogger.cs @@ -19,19 +19,16 @@ namespace TestNetRewarder public void Debug(string message = "", int skipFrames = 0) { lines.Add(message); - Trim(); } public void Error(string message) { lines.Add($"Error: {message}"); - Trim(); } public void Log(string message) { lines.Add(message); - Trim(); } public string[] Get() @@ -40,13 +37,5 @@ namespace TestNetRewarder lines.Clear(); return result; } - - private void Trim() - { - if (lines.Count > 100) - { - lines.RemoveRange(0, 50); - } - } } } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index b30ca79..f43a924 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -50,15 +50,18 @@ namespace TestNetRewarder } catch (Exception ex) { - log.Error("Exception processing time segment: " + ex); + var msg = "Exception processing time segment: " + ex; + log.Error(msg); + bufferLogger.Error(msg); throw; } } private string[] RemoveFirstLine(string[] lines) { - if (!lines.Any()) return Array.Empty(); - return lines.Skip(1).ToArray(); + //if (!lines.Any()) return Array.Empty(); + //return lines.Skip(1).ToArray(); + return lines; } } } From 2b7ba61543044fedf477a4af0f71da2045bf3682 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jun 2024 09:18:00 +0200 Subject: [PATCH 25/28] Cleanup --- .../CodexDiscordBotPlugin.cs | 5 ++- .../UtilityTests/DiscordBotTests.cs | 43 ++++++++----------- 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index b2e184b..556ef15 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -50,6 +50,7 @@ namespace CodexDiscordBotPlugin startupConfig.Add(config); var pod = workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig).WaitForOnline(); WaitForStartupMessage(workflow, pod); + workflow.CreateCrashWatcher(pod.Containers.Single()).Start(); return pod; } @@ -58,7 +59,9 @@ namespace CodexDiscordBotPlugin var startupConfig = new StartupConfig(); startupConfig.NameOverride = config.Name; startupConfig.Add(config); - return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); + var pod = workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig).WaitForOnline(); + workflow.CreateCrashWatcher(pod.Containers.Single()).Start(); + return pod; } private void WaitForStartupMessage(IStartupWorkflow workflow, RunningPod pod) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index 01c8fc5..f78a46c 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -1,5 +1,4 @@ using CodexContractsPlugin; -using CodexContractsPlugin.ChainMonitor; using CodexDiscordBotPlugin; using CodexPlugin; using Core; @@ -24,7 +23,9 @@ namespace CodexTests.UtilityTests private readonly List hostAccounts = new List(); private readonly List rewardsSeen = new List(); private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); - + private readonly List receivedEvents = new List(); + private readonly List receivedAverages = new List(); + [Test] [DontDownloadLogs] public void BotRewardTest() @@ -39,37 +40,20 @@ namespace CodexTests.UtilityTests StartHosts(geth, contracts); var client = StartClient(geth, contracts); - //var chainState = new ChainState(GetTestLog(), contracts, new DoNothingChainEventHandler(), GetTestRunTimeRange().From); - - //var running = true; - //var task = Task.Run(() => - //{ - // while (running) - // { - // Thread.Sleep(TimeSpan.FromMinutes(1)); - // chainState.Update(); - // } - //}); - var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer); apiCalls.Start(OnCommand); var purchaseContract = ClientPurchasesStorage(client); - //chainState.Update(); - //Assert.That(chainState.Requests.Length, Is.EqualTo(1)); - purchaseContract.WaitForStorageContractStarted(); - //chainState.Update(); - purchaseContract.WaitForStorageContractFinished(); - Thread.Sleep(rewarderInterval * 3); - //running = false; - //task.Wait(); - apiCalls.Stop(); - //chainState.Update(); + + Assert.That(receivedEvents.Count(e => e.Contains("Created as New.")), Is.EqualTo(1)); + Assert.That(receivedEvents.Count(e => e.Contains("SlotFilled")), Is.EqualTo(GetNumberOfRequiredHosts())); + Assert.That(receivedEvents.Count(e => e.Contains("Transit: New -> Started")), Is.EqualTo(1)); + Assert.That(receivedEvents.Count(e => e.Contains("Transit: Started -> Finished")), Is.EqualTo(1)); foreach (var r in repo.Rewards) { @@ -90,10 +74,12 @@ namespace CodexTests.UtilityTests private void OnCommand(string timestamp, GiveRewardsCommand call) { Log($""); + receivedAverages.AddRange(call.Averages); foreach (var a in call.Averages) { Log("\tAverage: " + JsonConvert.SerializeObject(a)); } + receivedEvents.AddRange(call.EventsOverview); foreach (var e in call.EventsOverview) { Log("\tEvent: " + e); @@ -122,8 +108,8 @@ namespace CodexTests.UtilityTests MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(), NodeFailureTolerance = 2, ProofProbability = 5, - Duration = TimeSpan.FromMinutes(6), - Expiry = TimeSpan.FromMinutes(5) + Duration = GetMinRequiredRequestDuration(), + Expiry = GetMinRequiredRequestDuration() - TimeSpan.FromMinutes(1) }; return client.Marketplace.RequestStorage(purchase); @@ -245,6 +231,11 @@ namespace CodexTests.UtilityTests return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); } + private TimeSpan GetMinRequiredRequestDuration() + { + return repo.Rewards.Max(r => r.CheckConfig.MinDuration) + TimeSpan.FromSeconds(10); + } + private string IdentifyAccount(string address) { if (address == clientAccount.EthAddress.Address) return "Client"; From 0c4d3be91250c3e317554cd6523252dc7cc10d52 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jun 2024 09:33:52 +0200 Subject: [PATCH 26/28] Fixes crash in marketBuffer --- Tests/CodexTests/UtilityTests/DiscordBotTests.cs | 14 ++++++++++---- Tools/TestNetRewarder/MarketBuffer.cs | 6 +++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index f78a46c..bcaacbe 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -50,10 +50,10 @@ namespace CodexTests.UtilityTests apiCalls.Stop(); - Assert.That(receivedEvents.Count(e => e.Contains("Created as New.")), Is.EqualTo(1)); - Assert.That(receivedEvents.Count(e => e.Contains("SlotFilled")), Is.EqualTo(GetNumberOfRequiredHosts())); - Assert.That(receivedEvents.Count(e => e.Contains("Transit: New -> Started")), Is.EqualTo(1)); - Assert.That(receivedEvents.Count(e => e.Contains("Transit: Started -> Finished")), Is.EqualTo(1)); + AssertEventOccurance("Created as New.", 1); + AssertEventOccurance("SlotFilled", Convert.ToInt32(GetNumberOfRequiredHosts())); + AssertEventOccurance("Transit: New -> Started", 1); + AssertEventOccurance("Transit: Started -> Finished", 1); foreach (var r in repo.Rewards) { @@ -71,6 +71,12 @@ namespace CodexTests.UtilityTests return $"({rewardId})'{reward.Message}'"; } + private void AssertEventOccurance(string msg, int expectedCount) + { + Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount), + $"Event '{msg}' did not occure correct number of times."); + } + private void OnCommand(string timestamp, GiveRewardsCommand call) { Log($""); diff --git a/Tools/TestNetRewarder/MarketBuffer.cs b/Tools/TestNetRewarder/MarketBuffer.cs index 4557378..b03ad7a 100644 --- a/Tools/TestNetRewarder/MarketBuffer.cs +++ b/Tools/TestNetRewarder/MarketBuffer.cs @@ -44,7 +44,11 @@ namespace TestNetRewarder private float Average(Func getValue) { - return Average(s => Convert.ToInt32(getValue(s))); + return Average(s => + { + var value = getValue(s); + return (int)value; + }); } private float Average(Func getValue) From a905f0ce5335ad9f42019ca430a9d5ff2eb3bc16 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jun 2024 09:59:24 +0200 Subject: [PATCH 27/28] fixes crash in market buffer again --- Tools/TestNetRewarder/MarketBuffer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/TestNetRewarder/MarketBuffer.cs b/Tools/TestNetRewarder/MarketBuffer.cs index b03ad7a..d8ec91d 100644 --- a/Tools/TestNetRewarder/MarketBuffer.cs +++ b/Tools/TestNetRewarder/MarketBuffer.cs @@ -67,7 +67,7 @@ namespace TestNetRewarder private int GetTotalSize(Ask ask) { var nSlots = Convert.ToInt32(ask.Slots); - var slotSize = Convert.ToInt32(ask.SlotSize); + var slotSize = (int)ask.SlotSize; return nSlots * slotSize; } } From 4be9b9df9a450ef3ac6ac9b4caa7659123182695 Mon Sep 17 00:00:00 2001 From: benbierens Date: Fri, 21 Jun 2024 10:19:09 +0200 Subject: [PATCH 28/28] Finished! --- Tests/CodexTests/UtilityTests/DiscordBotTests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index bcaacbe..c6b6b41 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -55,6 +55,8 @@ namespace CodexTests.UtilityTests AssertEventOccurance("Transit: New -> Started", 1); AssertEventOccurance("Transit: Started -> Finished", 1); + AssertMarketAverage(); + foreach (var r in repo.Rewards) { var seen = rewardsSeen.Any(s => r.RoleId == s); @@ -77,6 +79,20 @@ namespace CodexTests.UtilityTests $"Event '{msg}' did not occure correct number of times."); } + private void AssertMarketAverage() + { + Assert.That(receivedAverages.Count, Is.EqualTo(1)); + var a = receivedAverages.Single(); + + Assert.That(a.NumberOfFinished, Is.EqualTo(1)); + Assert.That(a.TimeRangeSeconds, Is.EqualTo(5760)); + Assert.That(a.Price, Is.EqualTo(2.0f).Within(0.1f)); + Assert.That(a.Size, Is.EqualTo(GetMinFileSize().SizeInBytes).Within(1.0f)); + Assert.That(a.Duration, Is.EqualTo(GetMinRequiredRequestDuration().TotalSeconds).Within(1.0f)); + Assert.That(a.Collateral, Is.EqualTo(10.0f).Within(0.1f)); + Assert.That(a.ProofProbability, Is.EqualTo(5.0f).Within(0.1f)); + } + private void OnCommand(string timestamp, GiveRewardsCommand call) { Log($"");