From 7d33c1c113e1d6606507706caf4bb585ee89f8aa Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 31 Jan 2024 11:52:02 -0500 Subject: [PATCH 01/19] Sets up rewarder bot container recipe --- .../CodexDiscordBotPlugin.cs | 13 +++++++ .../CoreInterfaceExtensions.cs | 5 +++ .../DiscordBotStartupConfig.cs | 20 ++++++++++ .../RewarderBotContainerRecipe.cs | 39 +++++++++++++++++++ 4 files changed, 77 insertions(+) create mode 100644 ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs index 91c38e4..cad50c1 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs @@ -35,6 +35,12 @@ namespace CodexDiscordBotPlugin return StartContainer(workflow, config); } + public RunningContainers DeployRewarder(RewarderBotStartupConfig config) + { + var workflow = tools.CreateWorkflow(); + return StartRewarderContainer(workflow, config); + } + private RunningContainers StartContainer(IStartupWorkflow workflow, DiscordBotStartupConfig config) { var startupConfig = new StartupConfig(); @@ -42,5 +48,12 @@ namespace CodexDiscordBotPlugin startupConfig.Add(config); return workflow.Start(1, new DiscordBotContainerRecipe(), startupConfig); } + + private RunningContainers StartRewarderContainer(IStartupWorkflow workflow, RewarderBotStartupConfig config) + { + var startupConfig = new StartupConfig(); + startupConfig.Add(config); + return workflow.Start(1, new RewarderBotContainerRecipe(), startupConfig); + } } } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs index 1c3d673..c17cec1 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs @@ -10,6 +10,11 @@ namespace CodexDiscordBotPlugin return Plugin(ci).Deploy(config); } + public static RunningContainers DeployRewarderBot(this CoreInterface ci, RewarderBotStartupConfig config) + { + return Plugin(ci).DeployRewarder(config); + } + private static CodexDiscordBotPlugin Plugin(CoreInterface ci) { return ci.GetPlugin(); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs index 85c103f..1e8165d 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs @@ -23,6 +23,26 @@ public string? DataPath { get; set; } } + public class RewarderBotStartupConfig + { + public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, TimeSpan interval, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) + { + DiscordBotHost = discordBotHost; + DiscordBotPort = discordBotPort; + Interval = interval; + HistoryStartUtc = historyStartUtc; + GethInfo = gethInfo; + DataPath = dataPath; + } + + public string DiscordBotHost { get; } + public int DiscordBotPort { get; } + public TimeSpan Interval { get; } + public DateTime HistoryStartUtc { get; } + public DiscordBotGethInfo GethInfo { get; } + public string? DataPath { get; set; } + } + public class DiscordBotGethInfo { public DiscordBotGethInfo(string host, int port, string privKey, string marketplaceAddress, string tokenAddress, string abi) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs new file mode 100644 index 0000000..fd5159e --- /dev/null +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -0,0 +1,39 @@ +using KubernetesWorkflow.Recipe; +using KubernetesWorkflow; +using Utils; + +namespace CodexDiscordBotPlugin +{ + public class RewarderBotContainerRecipe : ContainerRecipeFactory + { + public override string AppName => "discordbot-rewarder"; + public override string Image => "codexstorage/codex-rewarderbot"; + + protected override void Initialize(StartupConfig startupConfig) + { + var config = startupConfig.Get(); + + SetSchedulingAffinity(notIn: "false"); + + AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost); + AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString()); + AddEnvVar("INTERVALMINUTES", config.Interval.TotalMinutes.ToString()); + var offset = new DateTimeOffset(config.HistoryStartUtc); + AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString()); + + var gethInfo = config.GethInfo; + AddEnvVar("GETH_HOST", gethInfo.Host); + AddEnvVar("GETH_HTTP_PORT", gethInfo.Port.ToString()); + AddEnvVar("GETH_PRIVATE_KEY", gethInfo.PrivKey); + AddEnvVar("CODEXCONTRACTS_MARKETPLACEADDRESS", gethInfo.MarketplaceAddress); + AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress); + AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi); + + if (!string.IsNullOrEmpty(config.DataPath)) + { + AddEnvVar("DATAPATH", config.DataPath); + AddVolume(config.DataPath, 1.GB()); + } + } + } +} From dd5baeda46c92863661ee9ca1d377562e4b0eafc Mon Sep 17 00:00:00 2001 From: benbierens Date: Thu, 1 Feb 2024 17:02:10 -0500 Subject: [PATCH 02/19] Adds reward api port to bot config --- Tools/BiblioTech/Configuration.cs | 4 +++- Tools/BiblioTech/Rewards/RewardsApi.cs | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index f324d23..d09d646 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -19,9 +19,11 @@ namespace BiblioTech [Uniform("admin-channel-name", "ac", "ADMINCHANNELNAME", true, "Name of the Discord server channel where admin commands are allowed.")] public string AdminChannelName { get; set; } = "admin-channel"; - [Uniform("rewards-channel-name", "ac", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")] + [Uniform("rewards-channel-name", "rc", "REWARDSCHANNELNAME", false, "Name of the Discord server channel where participation rewards will be announced.")] public string RewardsChannelName { get; set; } = ""; + [Uniform("reward-api-port", "rp", "REWARDAPIPORT", false, "TCP listen port for the reward API.")] + public int RewardApiPort { get; set; } = 31080; public string EndpointsPath { diff --git a/Tools/BiblioTech/Rewards/RewardsApi.cs b/Tools/BiblioTech/Rewards/RewardsApi.cs index b534490..1b4a87c 100644 --- a/Tools/BiblioTech/Rewards/RewardsApi.cs +++ b/Tools/BiblioTech/Rewards/RewardsApi.cs @@ -25,9 +25,11 @@ namespace BiblioTech.Rewards public void Start() { cts = new CancellationTokenSource(); - listener.Prefixes.Add($"http://*:31080/"); + var uri = $"http://*:{Program.Config.RewardApiPort}/"; + listener.Prefixes.Add(uri); listener.Start(); taskFactory.Run(ConnectionDispatcher, nameof(ConnectionDispatcher)); + Program.Log.Log($"Reward API listening on '{uri}'"); } public void Stop() From 3c210f96fca591b49607437b2e30a1f397c2b380 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 09:11:36 +0100 Subject: [PATCH 03/19] debugging reward api --- .../DiscordBotContainerRecipe.cs | 5 + .../DiscordBotStartupConfig.cs | 8 +- .../RewarderBotContainerRecipe.cs | 2 +- Tests/CodexTests/BasicTests/ExampleTests.cs | 93 ++++++++----------- Tests/CodexTests/CodexTests.csproj | 1 + Tools/BiblioTech/Rewards/RewardsApi.cs | 3 - Tools/CodexNetDeployer/Configuration.cs | 4 + Tools/CodexNetDeployer/Deployer.cs | 3 +- 8 files changed, 56 insertions(+), 63 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index ea69f3b..b981216 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -9,6 +9,8 @@ namespace CodexDiscordBotPlugin public override string AppName => "discordbot-bibliotech"; public override string Image => "thatbenbierens/codex-discordbot:initial"; + public static string RewardsPort = "bot_rewards_port"; + protected override void Initialize(StartupConfig startupConfig) { var config = startupConfig.Get(); @@ -19,6 +21,7 @@ namespace CodexDiscordBotPlugin AddEnvVar("SERVERNAME", config.ServerName); AddEnvVar("ADMINROLE", config.AdminRoleName); AddEnvVar("ADMINCHANNELNAME", config.AdminChannelName); + AddEnvVar("REWARDSCHANNELNAME", config.RewardChannelName); AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml"); AddEnvVar("KUBENAMESPACE", config.KubeNamespace); @@ -30,6 +33,8 @@ namespace CodexDiscordBotPlugin AddEnvVar("CODEXCONTRACTS_TOKENADDRESS", gethInfo.TokenAddress); AddEnvVar("CODEXCONTRACTS_ABI", gethInfo.Abi); + AddInternalPortAndVar("REWARDAPIPORT", RewardsPort); + if (!string.IsNullOrEmpty(config.DataPath)) { AddEnvVar("DATAPATH", config.DataPath); diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs index 1e8165d..dfc6ad1 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs @@ -2,7 +2,7 @@ { public class DiscordBotStartupConfig { - public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo) + public DiscordBotStartupConfig(string name, string token, string serverName, string adminRoleName, string adminChannelName, string kubeNamespace, DiscordBotGethInfo gethInfo, string rewardChannelName) { Name = name; Token = token; @@ -11,6 +11,7 @@ AdminChannelName = adminChannelName; KubeNamespace = kubeNamespace; GethInfo = gethInfo; + RewardChannelName = rewardChannelName; } public string Name { get; } @@ -18,6 +19,7 @@ public string ServerName { get; } public string AdminRoleName { get; } public string AdminChannelName { get; } + public string RewardChannelName { get; } public string KubeNamespace { get; } public DiscordBotGethInfo GethInfo { get; } public string? DataPath { get; set; } @@ -25,7 +27,7 @@ public class RewarderBotStartupConfig { - public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, TimeSpan interval, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) + public RewarderBotStartupConfig(string discordBotHost, int discordBotPort, string interval, DateTime historyStartUtc, DiscordBotGethInfo gethInfo, string? dataPath) { DiscordBotHost = discordBotHost; DiscordBotPort = discordBotPort; @@ -37,7 +39,7 @@ public string DiscordBotHost { get; } public int DiscordBotPort { get; } - public TimeSpan Interval { get; } + public string Interval { get; } public DateTime HistoryStartUtc { get; } public DiscordBotGethInfo GethInfo { get; } public string? DataPath { get; set; } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index fd5159e..3c8ee3b 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -17,7 +17,7 @@ namespace CodexDiscordBotPlugin AddEnvVar("DISCORDBOTHOST", config.DiscordBotHost); AddEnvVar("DISCORDBOTPORT", config.DiscordBotPort.ToString()); - AddEnvVar("INTERVALMINUTES", config.Interval.TotalMinutes.ToString()); + AddEnvVar("INTERVALMINUTES", config.Interval); var offset = new DateTimeOffset(config.HistoryStartUtc); AddEnvVar("CHECKHISTORY", offset.ToUnixTimeSeconds().ToString()); diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index c73a913..17c911d 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -1,8 +1,8 @@ using CodexContractsPlugin; +using CodexDiscordBotPlugin; using CodexPlugin; using DistTestCore; using GethPlugin; -using MetricsPlugin; using Nethereum.Hex.HexConvertors.Extensions; using NUnit.Framework; using Utils; @@ -13,44 +13,7 @@ namespace CodexTests.BasicTests public class ExampleTests : CodexDistTest { [Test] - public void CodexLogExample() - { - var primary = AddCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); - - var cid = primary.UploadFile(GenerateTestFile(5.MB())); - - var content = primary.LocalFiles(); - CollectionAssert.Contains(content.Select(c => c.Cid), cid); - - var log = Ci.DownloadLog(primary); - - log.AssertLogContains("Uploaded file"); - } - - [Test] - public void TwoMetricsExample() - { - var group = AddCodex(2, s => s.EnableMetrics()); - var group2 = AddCodex(2, s => s.EnableMetrics()); - - var primary = group[0]; - var secondary = group[1]; - var primary2 = group2[0]; - var secondary2 = group2[1]; - - var metrics = Ci.GetMetricsFor(primary, primary2); - - primary.ConnectToPeer(secondary); - primary2.ConnectToPeer(secondary2); - - Thread.Sleep(TimeSpan.FromMinutes(2)); - - metrics[0].AssertThat("libp2p_peers", Is.EqualTo(1)); - metrics[1].AssertThat("libp2p_peers", Is.EqualTo(1)); - } - - [Test] - public void MarketplaceExample() + public void BotRewardTest() { var sellerInitialBalance = 234.TestTokens(); var buyerInitialBalance = 1000.TestTokens(); @@ -80,6 +43,42 @@ namespace CodexTests.BasicTests AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance)); + // start bot and rewarder + var gethInfo = new DiscordBotGethInfo( + host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, + port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, + privKey: geth.StartResult.Account.PrivateKey, + marketplaceAddress: contracts.Deployment.MarketplaceAddress, + tokenAddress: contracts.Deployment.TokenAddress, + abi: contracts.Deployment.Abi + ); + var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( + name: "bot", + token: "MTE2NDEyNzk3MDU4NDE3NDU5Mw.GTpoV6.aDR7zxMNf7vDgMjKASJBQs-RtNP_lYJEY-OglI", + serverName: "ThatBen's server", + adminRoleName: "bottest-admins", + adminChannelName: "admin-channel", + rewardChannelName: "rewards-channel", + kubeNamespace: "notneeded", + gethInfo: gethInfo + )); + var botContainer = bot.Containers.Single(); + Ci.DeployRewarderBot(new RewarderBotStartupConfig( + //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, + //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, + discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, + discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, + interval: "60", + historyStartUtc: DateTime.UtcNow.AddHours(-1), + gethInfo: gethInfo, + dataPath: null + )); + + var sellerAddress = seller.EthAddress; + var buyerAddress = buyer.EthAddress; + + var i = 0; + var contentId = buyer.UploadFile(testFile); var purchaseContract = buyer.Marketplace.RequestStorage(contentId, pricePerSlotPerSecond: 2.TestTokens(), @@ -126,21 +125,5 @@ namespace CodexTests.BasicTests //CheckLogForErrors(seller, buyer); } - [Test] - public void GethBootstrapTest() - { - var boot = Ci.StartGethNode(s => s.WithName("boot").IsMiner()); - var disconnected = Ci.StartGethNode(s => s.WithName("disconnected")); - var follow = Ci.StartGethNode(s => s.WithBootstrapNode(boot).WithName("follow")); - - Thread.Sleep(12000); - - var bootN = boot.GetSyncedBlockNumber(); - var discN = disconnected.GetSyncedBlockNumber(); - var followN = follow.GetSyncedBlockNumber(); - - Assert.That(bootN, Is.EqualTo(followN)); - Assert.That(discN, Is.LessThan(bootN)); - } } } diff --git a/Tests/CodexTests/CodexTests.csproj b/Tests/CodexTests/CodexTests.csproj index 2c04d03..495caf6 100644 --- a/Tests/CodexTests/CodexTests.csproj +++ b/Tests/CodexTests/CodexTests.csproj @@ -14,6 +14,7 @@ + diff --git a/Tools/BiblioTech/Rewards/RewardsApi.cs b/Tools/BiblioTech/Rewards/RewardsApi.cs index 1b4a87c..71055f9 100644 --- a/Tools/BiblioTech/Rewards/RewardsApi.cs +++ b/Tools/BiblioTech/Rewards/RewardsApi.cs @@ -58,9 +58,6 @@ namespace BiblioTech.Rewards { Program.Log.Error("Exception during HTTP handler: " + ex); } - // Whatever happens, everything's always OK. - context.Response.StatusCode = 200; - context.Response.OutputStream.Close(); }, nameof(HandleConnection)); } } diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index 1d93402..b4d23e2 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -112,6 +112,9 @@ namespace CodexNetDeployer [Uniform("dbot-adminchannelname", "dbotacn", "DBOTADMINCHANNELNAME", false, "Required if discord-bot is true. Name of the Discord channel in which admin commands are allowed.")] public string DiscordBotAdminChannelName { get; set; } = string.Empty; + + [Uniform("dbot-rewardchannelname", "dbotrcn", "DBOTREWARDCHANNELNAME", false, "Required if discord-bot is true. Name of the Discord channel in which reward updates are posted.")] + public string DiscordBotRewardChannelName { get; set; } = string.Empty; [Uniform("dbot-datapath", "dbotdp", "DBOTDATAPATH", false, "Optional. Path in container where bot will save all data.")] public string DiscordBotDataPath { get; set; } = string.Empty; @@ -159,6 +162,7 @@ namespace CodexNetDeployer StringIsSet(nameof(DiscordBotServerName), DiscordBotServerName, errors); StringIsSet(nameof(DiscordBotAdminRoleName), DiscordBotAdminRoleName, errors); StringIsSet(nameof(DiscordBotAdminChannelName), DiscordBotAdminChannelName, errors); + StringIsSet(nameof(DiscordBotRewardChannelName), DiscordBotRewardChannelName, errors); } return errors; diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 1461784..066fc11 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -142,7 +142,8 @@ namespace CodexNetDeployer adminRoleName: config.DiscordBotAdminRoleName, adminChannelName: config.DiscordBotAdminChannelName, kubeNamespace: config.KubeNamespace, - gethInfo: info) + gethInfo: info, + rewardChannelName: config.DiscordBotRewardChannelName) { DataPath = config.DiscordBotDataPath }); From da4101a04251b84e24b0deeabac4812799bce330 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 14:56:49 +0100 Subject: [PATCH 04/19] Debugging time segmenter --- .../NethereumWorkflow/BlockTimeFinder.cs | 71 ++++++++++----- .../NethereumWorkflow/NethereumInteraction.cs | 9 +- Tools/BiblioTech/BiblioTech.csproj | 2 +- Tools/BiblioTech/CommandHandler.cs | 6 +- Tools/BiblioTech/Program.cs | 17 +++- .../BiblioTech/Properties/launchSettings.json | 12 +++ Tools/BiblioTech/Rewards/RewardController.cs | 35 +++++++ Tools/BiblioTech/Rewards/RewardsApi.cs | 91 ------------------- .../{RoleController.cs => RoleDriver.cs} | 12 ++- Tools/TestNetRewarder/BotClient.cs | 30 ++++-- Tools/TestNetRewarder/Processor.cs | 9 +- Tools/TestNetRewarder/Program.cs | 1 + Tools/TestNetRewarder/TimeSegmenter.cs | 1 + Tools/TestNetRewarder/build-docker.bat | 2 + 14 files changed, 161 insertions(+), 137 deletions(-) create mode 100644 Tools/BiblioTech/Properties/launchSettings.json create mode 100644 Tools/BiblioTech/Rewards/RewardController.cs delete mode 100644 Tools/BiblioTech/Rewards/RewardsApi.cs rename Tools/BiblioTech/Rewards/{RoleController.cs => RoleDriver.cs} (93%) create mode 100644 Tools/TestNetRewarder/build-docker.bat diff --git a/Framework/NethereumWorkflow/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockTimeFinder.cs index e7b8c4a..f7dd543 100644 --- a/Framework/NethereumWorkflow/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockTimeFinder.cs @@ -52,7 +52,13 @@ namespace NethereumWorkflow return closestBefore.BlockNumber; } - FetchBlocksAround(moment); + var newBlocks = FetchBlocksAround(moment); + if (newBlocks == 0) + { + log.Log("Didn't find any new blocks."); + if (closestBefore != null) return closestBefore.BlockNumber; + throw new Exception("Failed to find highest before."); + } return GetHighestBlockBefore(moment); } @@ -71,11 +77,17 @@ namespace NethereumWorkflow return closestAfter.BlockNumber; } - FetchBlocksAround(moment); + var newBlocks = FetchBlocksAround(moment); + if (newBlocks == 0) + { + log.Log("Didn't find any new blocks."); + if (closestAfter != null) return closestAfter.BlockNumber; + throw new Exception("Failed to find lowest before."); + } return GetLowestBlockAfter(moment); } - private void FetchBlocksAround(DateTime moment) + private int FetchBlocksAround(DateTime moment) { var timePerBlock = EstimateTimePerBlock(); log.Debug("Fetching blocks around " + moment.ToString("o") + " timePerBlock: " + timePerBlock.TotalSeconds); @@ -85,42 +97,55 @@ namespace NethereumWorkflow var max = entries.Keys.Max(); var blockDifference = CalculateBlockDifference(moment, timePerBlock, max); - FetchUp(max, blockDifference); - FetchDown(max, blockDifference); + return + FetchUp(max, blockDifference) + + FetchDown(max, blockDifference); } - private void FetchDown(ulong max, ulong blockDifference) + private int FetchDown(ulong max, ulong blockDifference) { - var target = max - blockDifference - 1; + var target = GetTarget(max, blockDifference); var fetchDown = FetchRange; + var newBlocks = 0; while (fetchDown > 0) { if (!entries.ContainsKey(target)) { - var newBlock = AddBlockNumber(target); - if (newBlock == null) return; + var newBlock = AddBlockNumber("FD" + fetchDown, target); + if (newBlock == null) return newBlocks; + newBlocks++; fetchDown--; } target--; - if (target <= 0) return; + if (target <= 0) return newBlocks; } + return newBlocks; } - private void FetchUp(ulong max, ulong blockDifference) + private int FetchUp(ulong max, ulong blockDifference) { - var target = max - blockDifference; + var target = GetTarget(max, blockDifference); var fetchUp = FetchRange; + var newBlocks = 0; while (fetchUp > 0) { if (!entries.ContainsKey(target)) { - var newBlock = AddBlockNumber(target); - if (newBlock == null) return; + var newBlock = AddBlockNumber("FU" + fetchUp, target); + if (newBlock == null) return newBlocks; + newBlocks++; fetchUp--; } target++; - if (target >= max) return; + if (target >= max) return newBlocks; } + return newBlocks; + } + + private ulong GetTarget(ulong max, ulong blockDifference) + { + if (max <= blockDifference) return 1; + return max - blockDifference; } private ulong CalculateBlockDifference(DateTime moment, TimeSpan timePerBlock, ulong max) @@ -155,13 +180,14 @@ namespace NethereumWorkflow } } - private BlockTimeEntry? AddBlockNumber(decimal blockNumber) + private BlockTimeEntry? AddBlockNumber(string a, decimal blockNumber) { - return AddBlockNumber(Convert.ToUInt64(blockNumber)); + return AddBlockNumber(a, Convert.ToUInt64(blockNumber)); } - private BlockTimeEntry? AddBlockNumber(ulong blockNumber) + private BlockTimeEntry? AddBlockNumber(string a, ulong blockNumber) { + log.Log(a + " - Adding blockNumber: " + blockNumber); if (entries.ContainsKey(blockNumber)) { return entries[blockNumber]; @@ -190,9 +216,10 @@ namespace NethereumWorkflow { var min = entries.Keys.Min(); var max = entries.Keys.Max(); + log.Log("min/max: " + min + " / " + max); var clippedMin = Math.Max(max - 100, min); var minTime = entries[min].Utc; - var clippedMinBlock = AddBlockNumber(clippedMin); + var clippedMinBlock = AddBlockNumber("EST", clippedMin); if (clippedMinBlock != null) minTime = clippedMinBlock.Utc; var maxTime = entries[max].Utc; @@ -212,7 +239,7 @@ namespace NethereumWorkflow if (!entries.Any()) { AddCurrentBlock(); - AddBlockNumber(entries.Single().Key - 1); + AddBlockNumber("INIT", entries.Single().Key - 1); } } @@ -225,7 +252,7 @@ namespace NethereumWorkflow { var number = Time.Wait(web3.Eth.Blocks.GetBlockNumber.SendRequestAsync()); var blockNumber = number.ToDecimal(); - return AddBlockNumber(blockNumber); + return AddBlockNumber("CUR", blockNumber); } private DateTime? GetTimestampFromBlock(ulong blockNumber) @@ -238,7 +265,7 @@ namespace NethereumWorkflow } catch (Exception ex) { - int i = 0; + log.Error(nameof(GetTimestampFromBlock) + " Exception: " + ex); throw; } } diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index 0a8c2e4..0f56145 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -3,7 +3,6 @@ using Nethereum.ABI.FunctionEncoding.Attributes; using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using Nethereum.Web3; -using System.Runtime.CompilerServices; using Utils; namespace NethereumWorkflow @@ -90,14 +89,18 @@ namespace NethereumWorkflow { var blockTimeFinder = new BlockTimeFinder(web3, log); - var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From); - var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To); + var lowest = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From); + var highest = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To); + + var fromBlock = Math.Min(lowest, highest); + var toBlock = Math.Max(lowest, highest); return GetEvents(address, fromBlock, toBlock); } public List> GetEvents(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new() { + log.Debug($"Getting events of type [{typeof(TEvent).Name}] in block range [{fromBlockNumber} - {toBlockNumber}]"); var eventHandler = web3.Eth.GetEvent(address); var from = new BlockParameter(fromBlockNumber); var to = new BlockParameter(toBlockNumber); diff --git a/Tools/BiblioTech/BiblioTech.csproj b/Tools/BiblioTech/BiblioTech.csproj index 19422e4..2f796a8 100644 --- a/Tools/BiblioTech/BiblioTech.csproj +++ b/Tools/BiblioTech/BiblioTech.csproj @@ -1,4 +1,4 @@ - + Exe diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index f21e486..7907177 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -26,12 +26,10 @@ namespace BiblioTech Program.AdminChecker.SetGuild(guild); Program.Log.Log($"Initializing for guild: '{guild.Name}'"); - var roleController = new RoleController(client); - var rewardsApi = new RewardsApi(roleController); - var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray(); if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel"); Program.AdminChecker.SetAdminChannel(adminChannels.First()); + Program.RoleDriver = new RoleDriver(client); var builders = commands.Select(c => { @@ -62,8 +60,6 @@ namespace BiblioTech var json = JsonConvert.SerializeObject(exception.Errors, Formatting.Indented); Program.Log.Error(json); } - - rewardsApi.Start(); } private async Task SlashCommandHandler(SocketSlashCommand command) diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 7d6d690..8f7c98c 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -1,5 +1,6 @@ using ArgsUniform; using BiblioTech.Commands; +using BiblioTech.Rewards; using Discord; using Discord.WebSocket; using Logging; @@ -13,6 +14,7 @@ namespace BiblioTech public static Configuration Config { get; private set; } = null!; public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; + public static IDiscordRoleDriver RoleDriver { get; set; } = null!; public static ILog Log { get; private set; } = null!; public static Task Main(string[] args) @@ -29,10 +31,10 @@ namespace BiblioTech EnsurePath(Config.UserDataPath); EnsurePath(Config.EndpointsPath); - return new Program().MainAsync(); + return new Program().MainAsync(args); } - public async Task MainAsync() + public async Task MainAsync(string[] args) { Log.Log("Starting Codex Discord Bot..."); client = new DiscordSocketClient(); @@ -52,10 +54,19 @@ namespace BiblioTech await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); await client.StartAsync(); - AdminChecker = new AdminChecker(); + var builder = WebApplication.CreateBuilder(args); + builder.WebHost.ConfigureKestrel((context, options) => + { + options.ListenAnyIP(Config.RewardApiPort); + }); + builder.Services.AddControllers(); + var app = builder.Build(); + app.MapControllers(); + Log.Log("Running..."); + await app.RunAsync(); await Task.Delay(-1); } diff --git a/Tools/BiblioTech/Properties/launchSettings.json b/Tools/BiblioTech/Properties/launchSettings.json new file mode 100644 index 0000000..74fbfe0 --- /dev/null +++ b/Tools/BiblioTech/Properties/launchSettings.json @@ -0,0 +1,12 @@ +{ + "profiles": { + "BiblioTech": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "https://localhost:52960;http://localhost:52961" + } + } +} \ No newline at end of file diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs new file mode 100644 index 0000000..c9a19de --- /dev/null +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -0,0 +1,35 @@ +using DiscordRewards; +using Microsoft.AspNetCore.Mvc; + +namespace BiblioTech.Rewards +{ + public interface IDiscordRoleDriver + { + Task GiveRewards(GiveRewardsCommand rewards); + } + + [Route("api/[controller]")] + [ApiController] + public class RewardController : ControllerBase + { + [HttpGet] + public string Ping() + { + return "Pong"; + } + + [HttpPost] + public async Task Give(GiveRewardsCommand cmd) + { + try + { + await Program.RoleDriver.GiveRewards(cmd); + } + catch (Exception ex) + { + Program.Log.Error("Exception: " + ex); + } + return "OK"; + } + } +} diff --git a/Tools/BiblioTech/Rewards/RewardsApi.cs b/Tools/BiblioTech/Rewards/RewardsApi.cs deleted file mode 100644 index 71055f9..0000000 --- a/Tools/BiblioTech/Rewards/RewardsApi.cs +++ /dev/null @@ -1,91 +0,0 @@ -using DiscordRewards; -using Newtonsoft.Json; -using System.Net; -using TaskFactory = Utils.TaskFactory; - -namespace BiblioTech.Rewards -{ - public interface IDiscordRoleController - { - Task GiveRewards(GiveRewardsCommand rewards); - } - - public class RewardsApi - { - private readonly HttpListener listener = new HttpListener(); - private readonly TaskFactory taskFactory = new TaskFactory(); - private readonly IDiscordRoleController roleController; - private CancellationTokenSource cts = new CancellationTokenSource(); - - public RewardsApi(IDiscordRoleController roleController) - { - this.roleController = roleController; - } - - public void Start() - { - cts = new CancellationTokenSource(); - var uri = $"http://*:{Program.Config.RewardApiPort}/"; - listener.Prefixes.Add(uri); - listener.Start(); - taskFactory.Run(ConnectionDispatcher, nameof(ConnectionDispatcher)); - Program.Log.Log($"Reward API listening on '{uri}'"); - } - - public void Stop() - { - listener.Stop(); - cts.Cancel(); - taskFactory.WaitAll(); - } - - private void ConnectionDispatcher() - { - while (!cts.Token.IsCancellationRequested) - { - var wait = listener.GetContextAsync(); - wait.Wait(cts.Token); - if (wait.IsCompletedSuccessfully) - { - taskFactory.Run(() => - { - var context = wait.Result; - try - { - HandleConnection(context).Wait(); - } - catch (Exception ex) - { - Program.Log.Error("Exception during HTTP handler: " + ex); - } - }, nameof(HandleConnection)); - } - } - } - - private async Task HandleConnection(HttpListenerContext context) - { - using var reader = new StreamReader(context.Request.InputStream); - var content = reader.ReadToEnd(); - - if (content == "Ping") - { - using var writer = new StreamWriter(context.Response.OutputStream); - writer.Write("Pong"); - return; - } - - if (!content.StartsWith("{")) return; - var rewards = JsonConvert.DeserializeObject(content); - if (rewards != null) - { - await ProcessRewards(rewards); - } - } - - private async Task ProcessRewards(GiveRewardsCommand rewards) - { - await roleController.GiveRewards(rewards); - } - } -} diff --git a/Tools/BiblioTech/Rewards/RoleController.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs similarity index 93% rename from Tools/BiblioTech/Rewards/RoleController.cs rename to Tools/BiblioTech/Rewards/RoleDriver.cs index e59e804..2478f52 100644 --- a/Tools/BiblioTech/Rewards/RoleController.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -4,13 +4,13 @@ using DiscordRewards; namespace BiblioTech.Rewards { - public class RoleController : IDiscordRoleController + public class RoleDriver : IDiscordRoleDriver { private readonly DiscordSocketClient client; private readonly SocketTextChannel? rewardsChannel; private readonly RewardRepo repo = new RewardRepo(); - public RoleController(DiscordSocketClient client) + public RoleDriver(DiscordSocketClient client) { this.client = client; @@ -107,7 +107,13 @@ namespace BiblioTech.Rewards private SocketGuild GetGuild() { - return client.Guilds.Single(g => g.Name == Program.Config.ServerName); + var guild = client.Guilds.SingleOrDefault(g => g.Name == Program.Config.ServerName); + if (guild == null) + { + throw new Exception($"Unable to find guild by name: '{Program.Config.ServerName}'. " + + $"Known guilds: [{string.Join(",", client.Guilds.Select(g => g.Name))}]"); + } + return guild; } } diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index 91e6c83..e908f89 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -1,4 +1,5 @@ -using DiscordRewards; +using CodexContractsPlugin.Marketplace; +using DiscordRewards; using Logging; using Newtonsoft.Json; @@ -17,13 +18,30 @@ namespace TestNetRewarder public async Task IsOnline() { - return await HttpPost("Ping") == "Ping"; + var result = await HttpGet(); + log.Log("Is DiscordBot online: " + result); + return result == "Pong"; } - public async Task SendRewards(GiveRewardsCommand command) + public async Task SendRewards(GiveRewardsCommand command) { - if (command == null || command.Rewards == null || !command.Rewards.Any()) return; - await HttpPost(JsonConvert.SerializeObject(command)); + if (command == null || command.Rewards == null || !command.Rewards.Any()) return false; + return await HttpPost(JsonConvert.SerializeObject(command)) == "OK"; + } + + private async Task HttpGet() + { + try + { + var client = new HttpClient(); + var response = await client.GetAsync(GetUrl()); + return await response.Content.ReadAsStringAsync(); + } + catch (Exception ex) + { + log.Error(ex.ToString()); + return string.Empty; + } } private async Task HttpPost(string content) @@ -43,7 +61,7 @@ namespace TestNetRewarder private string GetUrl() { - return $"{configuration.DiscordHost}:{configuration.DiscordPort}"; + return $"{configuration.DiscordHost}:{configuration.DiscordPort}/api/reward"; } } } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index e56486b..bd1f20e 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -44,11 +44,14 @@ namespace TestNetRewarder if (outgoingRewards.Any()) { - await SendRewardsCommand(outgoingRewards); + if (!await SendRewardsCommand(outgoingRewards)) + { + log.Error("Failed to send reward command."); + } } } - private async Task SendRewardsCommand(List outgoingRewards) + private async Task SendRewardsCommand(List outgoingRewards) { var cmd = new GiveRewardsCommand { @@ -56,7 +59,7 @@ namespace TestNetRewarder }; log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); - await Program.BotClient.SendRewards(cmd); + return await Program.BotClient.SendRewards(cmd); } private void ProcessReward(List outgoingRewards, RewardConfig reward, ChainState chainState) diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index 3d10cff..c37c8a3 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -59,6 +59,7 @@ namespace TestNetRewarder var blockNumber = gc.GethNode.GetSyncedBlockNumber(); if (blockNumber == null || blockNumber < 1) throw new Exception("Geth connection failed."); + Log.Log("Geth OK. Block number: " + blockNumber); } private static async Task EnsureBotOnline() diff --git a/Tools/TestNetRewarder/TimeSegmenter.cs b/Tools/TestNetRewarder/TimeSegmenter.cs index a9ad71a..5a555f7 100644 --- a/Tools/TestNetRewarder/TimeSegmenter.cs +++ b/Tools/TestNetRewarder/TimeSegmenter.cs @@ -33,6 +33,7 @@ namespace TestNetRewarder // Wait for the entire time segment to be in the past. var delay = (end - now).Add(TimeSpan.FromSeconds(3)); waited = true; + log.Log($"Waiting till time segment is in the past... {Time.FormatDuration(delay)}"); await Task.Delay(delay, Program.CancellationToken); } diff --git a/Tools/TestNetRewarder/build-docker.bat b/Tools/TestNetRewarder/build-docker.bat new file mode 100644 index 0000000..32f207b --- /dev/null +++ b/Tools/TestNetRewarder/build-docker.bat @@ -0,0 +1,2 @@ +docker build -f docker/Dockerfile -t thatbenbierens/codex-rewardbot:initial ../.. +docker push thatbenbierens/codex-rewardbot:initial From 1bf693938c655f7f3943d39904fc68c1dc308cbe Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 15:20:12 +0100 Subject: [PATCH 05/19] Block range type --- .../NethereumWorkflow/BlockTimeFinder.cs | 32 ++++++++++++------- .../NethereumWorkflow/NethereumInteraction.cs | 25 ++++++++------- Framework/Utils/BlockRange.cs | 22 +++++++++++++ ProjectPlugins/GethPlugin/GethNode.cs | 13 ++++++-- 4 files changed, 66 insertions(+), 26 deletions(-) create mode 100644 Framework/Utils/BlockRange.cs diff --git a/Framework/NethereumWorkflow/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockTimeFinder.cs index f7dd543..de26305 100644 --- a/Framework/NethereumWorkflow/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockTimeFinder.cs @@ -19,22 +19,35 @@ namespace NethereumWorkflow this.log = log; } + public BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange) + { + var lowest = GetLowestBlockNumberAfter(timeRange.From); + var highest = GetHighestBlockNumberBefore(timeRange.To); + + var fromBlock = Math.Min(lowest, highest); + var toBlock = Math.Max(lowest, highest); + + return new BlockRange(fromBlock, toBlock); + } + public ulong GetHighestBlockNumberBefore(DateTime moment) { - log.Log("Looking for highest block before " + moment.ToString("o")); AssertMomentIsInPast(moment); Initialize(); - return GetHighestBlockBefore(moment); + var result = GetHighestBlockBefore(moment); + log.Log($"Highest block before [{moment.ToString("o")}] = {result}"); + return result; } public ulong GetLowestBlockNumberAfter(DateTime moment) { - log.Log("Looking for lowest block after " + moment.ToString("o")); AssertMomentIsInPast(moment); Initialize(); - return GetLowestBlockAfter(moment); + var result = GetLowestBlockAfter(moment); + log.Log($"Lowest block after [{moment.ToString("o")}] = {result}"); + return result; } private ulong GetHighestBlockBefore(DateTime moment) @@ -48,14 +61,13 @@ namespace NethereumWorkflow closestAfter.Utc > moment && closestBefore.BlockNumber + 1 == closestAfter.BlockNumber) { - log.Log("Found highest-Before: " + closestBefore); return closestBefore.BlockNumber; } var newBlocks = FetchBlocksAround(moment); if (newBlocks == 0) { - log.Log("Didn't find any new blocks."); + log.Debug("Didn't find any new blocks."); if (closestBefore != null) return closestBefore.BlockNumber; throw new Exception("Failed to find highest before."); } @@ -73,14 +85,13 @@ namespace NethereumWorkflow closestAfter.Utc > moment && closestBefore.BlockNumber + 1 == closestAfter.BlockNumber) { - log.Log("Found lowest-after: " + closestAfter); return closestAfter.BlockNumber; } var newBlocks = FetchBlocksAround(moment); if (newBlocks == 0) { - log.Log("Didn't find any new blocks."); + log.Debug("Didn't find any new blocks."); if (closestAfter != null) return closestAfter.BlockNumber; throw new Exception("Failed to find lowest before."); } @@ -187,7 +198,7 @@ namespace NethereumWorkflow private BlockTimeEntry? AddBlockNumber(string a, ulong blockNumber) { - log.Log(a + " - Adding blockNumber: " + blockNumber); + log.Debug(a + " - Adding blockNumber: " + blockNumber); if (entries.ContainsKey(blockNumber)) { return entries[blockNumber]; @@ -203,7 +214,7 @@ namespace NethereumWorkflow var time = GetTimestampFromBlock(blockNumber); if (time == null) { - log.Log("Failed to get block for number: " + blockNumber); + log.Debug("Failed to get block for number: " + blockNumber); return null; } var entry = new BlockTimeEntry(blockNumber, time.Value); @@ -216,7 +227,6 @@ namespace NethereumWorkflow { var min = entries.Keys.Min(); var max = entries.Keys.Max(); - log.Log("min/max: " + min + " / " + max); var clippedMin = Math.Max(max - 100, min); var minTime = entries[min].Utc; var clippedMinBlock = AddBlockNumber("EST", clippedMin); diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index 0f56145..a13263b 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -4,6 +4,7 @@ using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using Nethereum.Web3; using Utils; +using BlockRange = Utils.BlockRange; namespace NethereumWorkflow { @@ -88,24 +89,24 @@ namespace NethereumWorkflow public List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new() { var blockTimeFinder = new BlockTimeFinder(web3, log); - - var lowest = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From); - var highest = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To); - - var fromBlock = Math.Min(lowest, highest); - var toBlock = Math.Max(lowest, highest); - - return GetEvents(address, fromBlock, toBlock); + var blockRange = blockTimeFinder.ConvertTimeRangeToBlockRange(timeRange); + return GetEvents(address, blockRange); } - public List> GetEvents(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new() + public List> GetEvents(string address, BlockRange blockRange) where TEvent : IEventDTO, new() { - log.Debug($"Getting events of type [{typeof(TEvent).Name}] in block range [{fromBlockNumber} - {toBlockNumber}]"); + log.Debug($"Getting events of type [{typeof(TEvent).Name}] in block range [{blockRange.From} - {blockRange.To}]"); var eventHandler = web3.Eth.GetEvent(address); - var from = new BlockParameter(fromBlockNumber); - var to = new BlockParameter(toBlockNumber); + var from = new BlockParameter(blockRange.From); + var to = new BlockParameter(blockRange.To); var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to)); return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter)); } + + public BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange) + { + var blockTimeFinder = new BlockTimeFinder(web3, log); + return blockTimeFinder.ConvertTimeRangeToBlockRange(timeRange); + } } } diff --git a/Framework/Utils/BlockRange.cs b/Framework/Utils/BlockRange.cs new file mode 100644 index 0000000..23424a5 --- /dev/null +++ b/Framework/Utils/BlockRange.cs @@ -0,0 +1,22 @@ +namespace Utils +{ + public class BlockRange + { + public BlockRange(ulong from, ulong to) + { + if (from < to) + { + From = from; + To = to; + } + else + { + From = to; + To = from; + } + } + + public ulong From { get; } + public ulong To { get; } + } +} diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 2ae834d..8783665 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -6,6 +6,7 @@ using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using NethereumWorkflow; using Utils; +using BlockRange = Utils.BlockRange; namespace GethPlugin { @@ -24,8 +25,9 @@ namespace GethPlugin decimal? GetSyncedBlockNumber(); bool IsContractAvailable(string abi, string contractAddress); GethBootstrapNode GetBootstrapRecord(); - List> GetEvents(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new(); + List> GetEvents(string address, BlockRange blockRange) where TEvent : IEventDTO, new(); List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new(); + BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange); } public class DeploymentGethNode : BaseGethNode, IGethNode @@ -144,9 +146,9 @@ namespace GethPlugin return StartInteraction().IsContractAvailable(abi, contractAddress); } - public List> GetEvents(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new() + public List> GetEvents(string address, BlockRange blockRange) where TEvent : IEventDTO, new() { - return StartInteraction().GetEvents(address, fromBlockNumber, toBlockNumber); + return StartInteraction().GetEvents(address, blockRange); } public List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new() @@ -154,6 +156,11 @@ namespace GethPlugin return StartInteraction().GetEvents(address, timeRange); } + public BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange) + { + return StartInteraction().ConvertTimeRangeToBlockRange(timeRange); + } + protected abstract NethereumInteraction StartInteraction(); } } From 42d3c5cd2c3a52a39aa9cede13014f4ec5b430d6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 15:41:48 +0100 Subject: [PATCH 06/19] debugging rewards --- Framework/Utils/BlockRange.cs | 5 +++ .../CodexContractsAccess.cs | 30 +++++++++--------- Tests/CodexTests/BasicTests/ExampleTests.cs | 9 +++--- Tools/TestNetRewarder/ChainState.cs | 14 ++++----- Tools/TestNetRewarder/Processor.cs | 31 +++++++++++++++++-- Tools/TestNetRewarder/Program.cs | 2 +- 6 files changed, 61 insertions(+), 30 deletions(-) diff --git a/Framework/Utils/BlockRange.cs b/Framework/Utils/BlockRange.cs index 23424a5..fa0574e 100644 --- a/Framework/Utils/BlockRange.cs +++ b/Framework/Utils/BlockRange.cs @@ -18,5 +18,10 @@ public ulong From { get; } public ulong To { get; } + + public override string ToString() + { + return $"[{From} - {To}]"; + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 97083fe..0113020 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -19,13 +19,13 @@ namespace CodexContractsPlugin TestToken GetTestTokenBalance(IHasEthAddress owner); TestToken GetTestTokenBalance(EthAddress ethAddress); - Request[] GetStorageRequests(TimeRange timeRange); + Request[] GetStorageRequests(BlockRange blockRange); EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex); RequestState GetRequestState(Request request); - RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange); - RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange); - SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange); - SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange); + RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockRange blockRange); + RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockRange blockRange); + SlotFilledEventDTO[] GetSlotFilledEvents(BlockRange blockRange); + SlotFreedEventDTO[] GetSlotFreedEvents(BlockRange blockRange); } public enum RequestState @@ -77,9 +77,9 @@ namespace CodexContractsPlugin return balance.TestTokens(); } - public Request[] GetStorageRequests(TimeRange timeRange) + public Request[] GetStorageRequests(BlockRange blockRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, timeRange); + var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); var i = StartInteraction(); return events .Select(e => @@ -93,9 +93,9 @@ namespace CodexContractsPlugin .ToArray(); } - public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(TimeRange timeRange) + public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockRange blockRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, timeRange); + var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => { var result = e.Event; @@ -104,9 +104,9 @@ namespace CodexContractsPlugin }).ToArray(); } - public RequestCancelledEventDTO[] GetRequestCancelledEvents(TimeRange timeRange) + public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockRange blockRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, timeRange); + var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => { var result = e.Event; @@ -115,9 +115,9 @@ namespace CodexContractsPlugin }).ToArray(); } - public SlotFilledEventDTO[] GetSlotFilledEvents(TimeRange timeRange) + public SlotFilledEventDTO[] GetSlotFilledEvents(BlockRange blockRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, timeRange); + var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => { var result = e.Event; @@ -127,9 +127,9 @@ namespace CodexContractsPlugin }).ToArray(); } - public SlotFreedEventDTO[] GetSlotFreedEvents(TimeRange timeRange) + public SlotFreedEventDTO[] GetSlotFreedEvents(BlockRange blockRange) { - var events = gethNode.GetEvents(Deployment.MarketplaceAddress, timeRange); + var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => { var result = e.Event; diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 17c911d..5599104 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -89,7 +89,9 @@ namespace CodexTests.BasicTests purchaseContract.WaitForStorageContractStarted(fileSize); - var requests = contracts.GetStorageRequests(GetTestRunTimeRange()); + var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); + + var requests = contracts.GetStorageRequests(blockRange); Assert.That(requests.Length, Is.EqualTo(1)); var request = requests.Single(); Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started)); @@ -98,10 +100,10 @@ namespace CodexTests.BasicTests AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); + var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1)); CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId); - var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); + var filledSlotEvents = contracts.GetSlotFilledEvents(blockRange); Assert.That(filledSlotEvents.Length, Is.EqualTo(1)); var filledSlotEvent = filledSlotEvents.Single(); Assert.That(filledSlotEvent.SlotIndex.IsZero); @@ -124,6 +126,5 @@ namespace CodexTests.BasicTests //CheckLogForErrors(seller, buyer); } - } } diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index 13356fc..0bb1fdd 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -1,6 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; -using Utils; +using BlockRange = Utils.BlockRange; namespace TestNetRewarder { @@ -8,18 +8,18 @@ namespace TestNetRewarder { private readonly HistoricState historicState; - public ChainState(HistoricState historicState, ICodexContracts contracts, TimeRange timeRange) + public ChainState(HistoricState historicState, ICodexContracts contracts, BlockRange blockRange) { - NewRequests = contracts.GetStorageRequests(timeRange); + NewRequests = contracts.GetStorageRequests(blockRange); historicState.ProcessNewRequests(NewRequests); historicState.UpdateStorageRequests(contracts); StartedRequests = historicState.StorageRequests.Where(r => r.RecentlyStarted).ToArray(); FinishedRequests = historicState.StorageRequests.Where(r => r.RecentlyFininshed).ToArray(); - RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(timeRange); - RequestCancelledEvents = contracts.GetRequestCancelledEvents(timeRange); - SlotFilledEvents = contracts.GetSlotFilledEvents(timeRange); - SlotFreedEvents = contracts.GetSlotFreedEvents(timeRange); + RequestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); + RequestCancelledEvents = contracts.GetRequestCancelledEvents(blockRange); + SlotFilledEvents = contracts.GetSlotFilledEvents(blockRange); + SlotFreedEvents = contracts.GetSlotFreedEvents(blockRange); this.historicState = historicState; } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index bd1f20e..9c0ddc2 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -11,20 +11,28 @@ namespace TestNetRewarder private static readonly HistoricState historicState = new HistoricState(); private static readonly RewardRepo rewardRepo = new RewardRepo(); private readonly ILog log; + private BlockRange? lastBlockRange; public Processor(ILog log) { this.log = log; } - public async Task ProcessTimeSegment(TimeRange range) + public async Task ProcessTimeSegment(TimeRange timeRange) { try { var connector = GethConnector.GethConnector.Initialize(log); if (connector == null) return; - var chainState = new ChainState(historicState, connector.CodexContracts, range); + var blockRange = connector.GethNode.ConvertTimeRangeToBlockRange(timeRange); + if (!IsNewBlockRange(blockRange)) + { + log.Log($"Block range {blockRange} was previously processed. Skipping..."); + return; + } + + var chainState = new ChainState(historicState, connector.CodexContracts, blockRange); await ProcessTimeSegment(chainState); } @@ -34,6 +42,19 @@ namespace TestNetRewarder } } + private bool IsNewBlockRange(BlockRange blockRange) + { + if (lastBlockRange == null || + lastBlockRange.From != blockRange.From || + lastBlockRange.To != blockRange.To) + { + lastBlockRange = blockRange; + return true; + } + + return false; + } + private async Task ProcessTimeSegment(ChainState chainState) { var outgoingRewards = new List(); @@ -58,13 +79,17 @@ namespace TestNetRewarder Rewards = outgoingRewards.ToArray() }; - log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); + log.Log("Sending rewards: " + JsonConvert.SerializeObject(cmd)); return await Program.BotClient.SendRewards(cmd); } private void ProcessReward(List outgoingRewards, RewardConfig reward, ChainState chainState) { var winningAddresses = PerformCheck(reward, chainState); + foreach (var win in winningAddresses) + { + log.Log($"Address '{win.Address}' wins '{reward.Message}'"); + } if (winningAddresses.Any()) { outgoingRewards.Add(new RewardUsersCommand diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index c37c8a3..fbf322c 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -47,7 +47,7 @@ namespace TestNetRewarder { await EnsureBotOnline(); await segmenter.WaitForNextSegment(processor.ProcessTimeSegment); - await Task.Delay(1000, CancellationToken); + await Task.Delay(100, CancellationToken); } } From 23b7bbd548f000e7a0eb40a3214f95b8c88f0b75 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 15:57:28 +0100 Subject: [PATCH 07/19] Fixes sending of rewards to bot --- Tools/TestNetRewarder/BotClient.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index e908f89..c112af4 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -2,6 +2,7 @@ using DiscordRewards; using Logging; using Newtonsoft.Json; +using System.Net.Http.Json; namespace TestNetRewarder { @@ -26,7 +27,9 @@ namespace TestNetRewarder public async Task SendRewards(GiveRewardsCommand command) { if (command == null || command.Rewards == null || !command.Rewards.Any()) return false; - return await HttpPost(JsonConvert.SerializeObject(command)) == "OK"; + var result = await HttpPostJson(command); + log.Log("Reward response: " + result); + return result == "OK"; } private async Task HttpGet() @@ -44,12 +47,13 @@ namespace TestNetRewarder } } - private async Task HttpPost(string content) + private async Task HttpPostJson(T body) { try { - var client = new HttpClient(); - var response = await client.PostAsync(GetUrl(), new StringContent(content)); + using var client = new HttpClient(); + using var content = JsonContent.Create(body); + using var response = await client.PostAsync(GetUrl(), content); return await response.Content.ReadAsStringAsync(); } catch (Exception ex) From a6fce1084dbaa664cb0e2a4dc99713a0e5faefdf Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 19 Feb 2024 15:59:49 +0100 Subject: [PATCH 08/19] Check connection no more than once every 30 seconds --- Tools/TestNetRewarder/Program.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index fbf322c..0aaa41c 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -1,5 +1,4 @@ using ArgsUniform; -using GethConnector; using Logging; using Utils; @@ -12,6 +11,7 @@ namespace TestNetRewarder public static CancellationToken CancellationToken { get; private set; } public static BotClient BotClient { get; private set; } = null!; private static Processor processor = null!; + private static DateTime lastCheck = DateTime.MinValue; public static Task Main(string[] args) { @@ -65,6 +65,9 @@ namespace TestNetRewarder private static async Task EnsureBotOnline() { var start = DateTime.UtcNow; + var timeSince = start - lastCheck; + if (timeSince.TotalSeconds < 30.0) return; + while (! await BotClient.IsOnline() && !CancellationToken.IsCancellationRequested) { await Task.Delay(5000); @@ -77,6 +80,8 @@ namespace TestNetRewarder throw new Exception(msg); } } + + lastCheck = start; } private static void PrintHelp() From 01d6b8f227e9285aea4bb78abbbca650f4bcbab1 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 15:39:42 +0100 Subject: [PATCH 09/19] Applies blockInterval --- .../NethereumWorkflow/NethereumInteraction.cs | 40 ++++++++++--------- .../Utils/{BlockRange.cs => BlockInterval.cs} | 4 +- .../CodexContractsAccess.cs | 20 +++++----- ProjectPlugins/GethPlugin/GethNode.cs | 11 +++-- Tests/CodexTests/BasicTests/ExampleTests.cs | 17 ++++---- Tools/TestNetRewarder/ChainState.cs | 4 +- Tools/TestNetRewarder/Processor.cs | 4 +- 7 files changed, 51 insertions(+), 49 deletions(-) rename Framework/Utils/{BlockRange.cs => BlockInterval.cs} (84%) diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index 3240d9b..197cf2d 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -89,26 +89,9 @@ namespace NethereumWorkflow } } - public List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new() + public List> GetEvents(string address, BlockInterval blockRange) where TEvent : IEventDTO, new() { - var wrapper = new Web3Wrapper(web3, log); - var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log); - - var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From); - var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To); - - if (!fromBlock.HasValue) - { - log.Error("Failed to find lowest block for time range: " + timeRange); - throw new Exception("Failed"); - } - if (!toBlock.HasValue) - { - log.Error("Failed to find highest block for time range: " + timeRange); - throw new Exception("Failed"); - } - - return GetEvents(address, fromBlock.Value, toBlock.Value); + return GetEvents(address, blockRange.From, blockRange.To); } public List> GetEvents(string address, ulong fromBlockNumber, ulong toBlockNumber) where TEvent : IEventDTO, new() @@ -119,5 +102,24 @@ namespace NethereumWorkflow var blockFilter = Time.Wait(eventHandler.CreateFilterBlockRangeAsync(from, to)); return Time.Wait(eventHandler.GetAllChangesAsync(blockFilter)); } + + public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange) + { + var wrapper = new Web3Wrapper(web3, log); + var blockTimeFinder = new BlockTimeFinder(blockCache, wrapper, log); + + var fromBlock = blockTimeFinder.GetLowestBlockNumberAfter(timeRange.From); + var toBlock = blockTimeFinder.GetHighestBlockNumberBefore(timeRange.To); + + if (fromBlock == null || toBlock == null) + { + throw new Exception("Failed to convert time range to block range."); + } + + return new BlockInterval( + from: fromBlock.Value, + to: toBlock.Value + ); + } } } diff --git a/Framework/Utils/BlockRange.cs b/Framework/Utils/BlockInterval.cs similarity index 84% rename from Framework/Utils/BlockRange.cs rename to Framework/Utils/BlockInterval.cs index fa0574e..79229bf 100644 --- a/Framework/Utils/BlockRange.cs +++ b/Framework/Utils/BlockInterval.cs @@ -1,8 +1,8 @@ namespace Utils { - public class BlockRange + public class BlockInterval { - public BlockRange(ulong from, ulong to) + public BlockInterval(ulong from, ulong to) { if (from < to) { diff --git a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs index 0113020..4279347 100644 --- a/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs +++ b/ProjectPlugins/CodexContractsPlugin/CodexContractsAccess.cs @@ -19,13 +19,13 @@ namespace CodexContractsPlugin TestToken GetTestTokenBalance(IHasEthAddress owner); TestToken GetTestTokenBalance(EthAddress ethAddress); - Request[] GetStorageRequests(BlockRange blockRange); + Request[] GetStorageRequests(BlockInterval blockRange); EthAddress? GetSlotHost(Request storageRequest, decimal slotIndex); RequestState GetRequestState(Request request); - RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockRange blockRange); - RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockRange blockRange); - SlotFilledEventDTO[] GetSlotFilledEvents(BlockRange blockRange); - SlotFreedEventDTO[] GetSlotFreedEvents(BlockRange blockRange); + RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange); + RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange); + SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange); + SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange); } public enum RequestState @@ -77,7 +77,7 @@ namespace CodexContractsPlugin return balance.TestTokens(); } - public Request[] GetStorageRequests(BlockRange blockRange) + public Request[] GetStorageRequests(BlockInterval blockRange) { var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); var i = StartInteraction(); @@ -93,7 +93,7 @@ namespace CodexContractsPlugin .ToArray(); } - public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockRange blockRange) + public RequestFulfilledEventDTO[] GetRequestFulfilledEvents(BlockInterval blockRange) { var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => @@ -104,7 +104,7 @@ namespace CodexContractsPlugin }).ToArray(); } - public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockRange blockRange) + public RequestCancelledEventDTO[] GetRequestCancelledEvents(BlockInterval blockRange) { var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => @@ -115,7 +115,7 @@ namespace CodexContractsPlugin }).ToArray(); } - public SlotFilledEventDTO[] GetSlotFilledEvents(BlockRange blockRange) + public SlotFilledEventDTO[] GetSlotFilledEvents(BlockInterval blockRange) { var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => @@ -127,7 +127,7 @@ namespace CodexContractsPlugin }).ToArray(); } - public SlotFreedEventDTO[] GetSlotFreedEvents(BlockRange blockRange) + public SlotFreedEventDTO[] GetSlotFreedEvents(BlockInterval blockRange) { var events = gethNode.GetEvents(Deployment.MarketplaceAddress, blockRange); return events.Select(e => diff --git a/ProjectPlugins/GethPlugin/GethNode.cs b/ProjectPlugins/GethPlugin/GethNode.cs index 8783665..bfa3d0f 100644 --- a/ProjectPlugins/GethPlugin/GethNode.cs +++ b/ProjectPlugins/GethPlugin/GethNode.cs @@ -6,7 +6,6 @@ using Nethereum.Contracts; using Nethereum.RPC.Eth.DTOs; using NethereumWorkflow; using Utils; -using BlockRange = Utils.BlockRange; namespace GethPlugin { @@ -25,9 +24,9 @@ namespace GethPlugin decimal? GetSyncedBlockNumber(); bool IsContractAvailable(string abi, string contractAddress); GethBootstrapNode GetBootstrapRecord(); - List> GetEvents(string address, BlockRange blockRange) where TEvent : IEventDTO, new(); + List> GetEvents(string address, BlockInterval blockRange) where TEvent : IEventDTO, new(); List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new(); - BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange); + BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange); } public class DeploymentGethNode : BaseGethNode, IGethNode @@ -146,17 +145,17 @@ namespace GethPlugin return StartInteraction().IsContractAvailable(abi, contractAddress); } - public List> GetEvents(string address, BlockRange blockRange) where TEvent : IEventDTO, new() + public List> GetEvents(string address, BlockInterval blockRange) where TEvent : IEventDTO, new() { return StartInteraction().GetEvents(address, blockRange); } public List> GetEvents(string address, TimeRange timeRange) where TEvent : IEventDTO, new() { - return StartInteraction().GetEvents(address, timeRange); + return StartInteraction().GetEvents(address, ConvertTimeRangeToBlockRange(timeRange)); } - public BlockRange ConvertTimeRangeToBlockRange(TimeRange timeRange) + public BlockInterval ConvertTimeRangeToBlockRange(TimeRange timeRange) { return StartInteraction().ConvertTimeRangeToBlockRange(timeRange); } diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 0470643..6fdb974 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -1,7 +1,6 @@ using CodexContractsPlugin; using CodexDiscordBotPlugin; using CodexPlugin; -using DistTestCore; using GethPlugin; using Nethereum.Hex.HexConvertors.Extensions; using NUnit.Framework; @@ -110,9 +109,11 @@ namespace CodexTests.BasicTests AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - var request = GetOnChainStorageRequest(contracts); + var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); + + var request = GetOnChainStorageRequest(contracts, blockRange); AssertStorageRequest(request, purchase, contracts, buyer); - AssertSlotFilledEvents(contracts, purchase, request, seller); + AssertSlotFilledEvents(contracts, purchase, request, seller, blockRange); AssertContractSlot(contracts, request, 0, seller); purchaseContract.WaitForStorageContractFinished(); @@ -139,15 +140,15 @@ namespace CodexTests.BasicTests Assert.That(discN, Is.LessThan(bootN)); } - private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller) + private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller, BlockInterval blockRange) { // Expect 1 fulfilled event for the purchase. - var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); + var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(blockRange); Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1)); CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId); // Expect 1 filled-slot event for each slot in the purchase. - var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); + var filledSlotEvents = contracts.GetSlotFilledEvents(blockRange); Assert.That(filledSlotEvents.Length, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); for (var i = 0; i < purchase.MinRequiredNumberOfNodes; i++) { @@ -164,9 +165,9 @@ namespace CodexTests.BasicTests Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); } - private Request GetOnChainStorageRequest(ICodexContracts contracts) + private Request GetOnChainStorageRequest(ICodexContracts contracts, BlockInterval blockRange) { - var requests = contracts.GetStorageRequests(GetTestRunTimeRange()); + var requests = contracts.GetStorageRequests(blockRange); Assert.That(requests.Length, Is.EqualTo(1)); return requests.Single(); } diff --git a/Tools/TestNetRewarder/ChainState.cs b/Tools/TestNetRewarder/ChainState.cs index 0bb1fdd..40e4bf6 100644 --- a/Tools/TestNetRewarder/ChainState.cs +++ b/Tools/TestNetRewarder/ChainState.cs @@ -1,6 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; -using BlockRange = Utils.BlockRange; +using Utils; namespace TestNetRewarder { @@ -8,7 +8,7 @@ namespace TestNetRewarder { private readonly HistoricState historicState; - public ChainState(HistoricState historicState, ICodexContracts contracts, BlockRange blockRange) + public ChainState(HistoricState historicState, ICodexContracts contracts, BlockInterval blockRange) { NewRequests = contracts.GetStorageRequests(blockRange); historicState.ProcessNewRequests(NewRequests); diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 9c0ddc2..425c0e3 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -11,7 +11,7 @@ namespace TestNetRewarder private static readonly HistoricState historicState = new HistoricState(); private static readonly RewardRepo rewardRepo = new RewardRepo(); private readonly ILog log; - private BlockRange? lastBlockRange; + private BlockInterval? lastBlockRange; public Processor(ILog log) { @@ -42,7 +42,7 @@ namespace TestNetRewarder } } - private bool IsNewBlockRange(BlockRange blockRange) + private bool IsNewBlockRange(BlockInterval blockRange) { if (lastBlockRange == null || lastBlockRange.From != blockRange.From || From b25c74752207503b419f9f06ec0d1bcc1dafb2bb Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 15:49:08 +0100 Subject: [PATCH 10/19] Ready for test. Need new bot images --- Framework/DiscordRewards/RewardRepo.cs | 14 ++-- Tests/CodexTests/BasicTests/ExampleTests.cs | 73 +++++++++++---------- 2 files changed, 45 insertions(+), 42 deletions(-) diff --git a/Framework/DiscordRewards/RewardRepo.cs b/Framework/DiscordRewards/RewardRepo.cs index 97b80a1..51ac3fc 100644 --- a/Framework/DiscordRewards/RewardRepo.cs +++ b/Framework/DiscordRewards/RewardRepo.cs @@ -21,11 +21,11 @@ namespace DiscordRewards }), // Finished a sizable slot - new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot!", new CheckConfig + new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig { Type = CheckType.FinishedSlot, - MinSlotSize = 1.GB(), - MinDuration = TimeSpan.FromHours(24.0), + MinSlotSize = 10.MB(), + MinDuration = TimeSpan.FromMinutes(5.0), }), // Posted any contract @@ -41,12 +41,12 @@ namespace DiscordRewards }), // Started a sizable contract - new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time!", new CheckConfig + new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig { - Type = CheckType.FinishedSlot, + Type = CheckType.StartedContract, MinNumberOfHosts = 4, - MinSlotSize = 1.GB(), - MinDuration = TimeSpan.FromHours(24.0), + MinSlotSize = 10.MB(), + MinDuration = TimeSpan.FromMinutes(5.0), }) }; } diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 6fdb974..c0102d7 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -10,7 +10,7 @@ using Request = CodexContractsPlugin.Marketplace.Request; namespace CodexTests.BasicTests { [TestFixture] - public class ExampleTests : CodexDistTest + public class ExampleTests : AutoBootstrapDistTest { [Test] public void BotRewardTest() @@ -22,33 +22,39 @@ namespace CodexTests.BasicTests var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); - var seller = AddCodex(s => s - .WithName("Seller") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(11.GB()) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), sellerInitialBalance) - .AsStorageNode() - .AsValidator())); + var myAccount = EthAccount.GenerateNew(); + var numberOfHosts = 3; - AssertBalance(contracts, seller, Is.EqualTo(sellerInitialBalance)); + for (var i = 0; i < numberOfHosts; i++) + { + var seller = AddCodex(s => s + .WithName("Seller") + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) + .WithStorageQuota(11.GB()) + .EnableMarketplace(geth, contracts, m => m + .WithAccount(myAccount) + .WithInitial(10.Eth(), sellerInitialBalance) + .AsStorageNode() + .AsValidator())); - var availability = new StorageAvailability( - totalSpace: 10.GB(), - maxDuration: TimeSpan.FromMinutes(30), - minPriceForTotalSpace: 1.TestTokens(), - maxCollateral: 20.TestTokens() - ); - seller.Marketplace.MakeStorageAvailable(availability); + AssertBalance(contracts, seller, Is.EqualTo(sellerInitialBalance)); + + var availability = new StorageAvailability( + totalSpace: 10.GB(), + maxDuration: TimeSpan.FromMinutes(30), + minPriceForTotalSpace: 1.TestTokens(), + maxCollateral: 20.TestTokens() + ); + seller.Marketplace.MakeStorageAvailable(availability); + } var testFile = GenerateTestFile(fileSize); var buyer = AddCodex(s => s .WithName("Buyer") - .WithBootstrapNode(seller) .EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), buyerInitialBalance))); @@ -85,11 +91,6 @@ namespace CodexTests.BasicTests dataPath: null )); - var sellerAddress = seller.EthAddress; - var buyerAddress = buyer.EthAddress; - - var i = 0; - var contentId = buyer.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) @@ -107,20 +108,22 @@ namespace CodexTests.BasicTests purchaseContract.WaitForStorageContractStarted(); - AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed."); + //AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); + //var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); - var request = GetOnChainStorageRequest(contracts, blockRange); - AssertStorageRequest(request, purchase, contracts, buyer); - AssertSlotFilledEvents(contracts, purchase, request, seller, blockRange); - AssertContractSlot(contracts, request, 0, seller); + //var request = GetOnChainStorageRequest(contracts, blockRange); + //AssertStorageRequest(request, purchase, contracts, buyer); + //AssertSlotFilledEvents(contracts, purchase, request, seller, blockRange); + //AssertContractSlot(contracts, request, 0, seller); purchaseContract.WaitForStorageContractFinished(); - AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); - AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); - Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished)); + var hold = 0; + + //AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); + //AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); + //Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished)); } [Test] From 8bc63c1fdbcf010d9f36f368739ba0a12efb8e8c Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 29 Mar 2024 11:24:11 +0100 Subject: [PATCH 11/19] Configurable testtoken and eth amounts for mint command --- .../DiscordBotContainerRecipe.cs | 4 +--- .../RewarderBotContainerRecipe.cs | 2 +- Tools/BiblioTech/Commands/MintCommand.cs | 20 +++++++++---------- Tools/BiblioTech/Configuration.cs | 6 ++++++ 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index b981216..3e3951a 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class DiscordBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-bibliotech"; - public override string Image => "thatbenbierens/codex-discordbot:initial"; + public override string Image => "codexstorage/codex-discordbot:sha-b25c747"; public static string RewardsPort = "bot_rewards_port"; @@ -40,8 +40,6 @@ namespace CodexDiscordBotPlugin AddEnvVar("DATAPATH", config.DataPath); AddVolume(config.DataPath, 1.GB()); } - - AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "discordbot-sa-kubeconfig"); } } } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index 3c8ee3b..dd95945 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot"; + public override string Image => "codexstorage/codex-rewarderbot:sha-b25c747"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/Tools/BiblioTech/Commands/MintCommand.cs b/Tools/BiblioTech/Commands/MintCommand.cs index 5b328b7..faf62c8 100644 --- a/Tools/BiblioTech/Commands/MintCommand.cs +++ b/Tools/BiblioTech/Commands/MintCommand.cs @@ -6,8 +6,6 @@ namespace BiblioTech.Commands { public class MintCommand : BaseGethCommand { - private readonly Ether defaultEthToSend = 10.Eth(); - private readonly TestToken defaultTestTokensToMint = 1024.TestTokens(); private readonly UserOption optionalUser = new UserOption( description: "If set, mint tokens for this user. (Optional, admin-only)", isRequired: false); @@ -47,9 +45,10 @@ namespace BiblioTech.Commands { if (ShouldMintTestTokens(contracts, addr)) { - var transaction = contracts.MintTestTokens(addr, defaultTestTokensToMint); - report.Add($"Minted {defaultTestTokensToMint} {FormatTransactionLink(transaction)}"); - return new Transaction(defaultTestTokensToMint, transaction); + var tokens = Program.Config.MintTT.TestTokens(); + var transaction = contracts.MintTestTokens(addr, tokens); + report.Add($"Minted {tokens} {FormatTransactionLink(transaction)}"); + return new Transaction(tokens, transaction); } report.Add("TestToken balance over threshold. (No TestTokens minted.)"); @@ -60,9 +59,10 @@ namespace BiblioTech.Commands { if (ShouldSendEth(gethNode, addr)) { - var transaction = gethNode.SendEth(addr, defaultEthToSend); - report.Add($"Sent {defaultEthToSend} {FormatTransactionLink(transaction)}"); - return new Transaction(defaultEthToSend, transaction); + var eth = Program.Config.SendEth.Eth(); + var transaction = gethNode.SendEth(addr, eth); + report.Add($"Sent {eth} {FormatTransactionLink(transaction)}"); + return new Transaction(eth, transaction); } report.Add("Eth balance is over threshold. (No Eth sent.)"); return null; @@ -71,13 +71,13 @@ namespace BiblioTech.Commands private bool ShouldMintTestTokens(ICodexContracts contracts, EthAddress addr) { var testTokens = contracts.GetTestTokenBalance(addr); - return testTokens.Amount < 64m; + return testTokens.Amount < Program.Config.MintTT; } private bool ShouldSendEth(IGethNode gethNode, EthAddress addr) { var eth = gethNode.GetEthBalance(addr); - return eth.Eth < 1.0m; + return eth.Eth < Program.Config.SendEth; } private string FormatTransactionLink(string transaction) diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index d09d646..2183a54 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -25,6 +25,12 @@ namespace BiblioTech [Uniform("reward-api-port", "rp", "REWARDAPIPORT", false, "TCP listen port for the reward API.")] public int RewardApiPort { get; set; } = 31080; + [Uniform("send-eth", "se", "SENDETH", false, "Amount of Eth send by the mint command. Default: 10.")] + public int SendEth { get; set; } = 10; + + [Uniform("mint-tt", "mt", "MINTTT", false, "Amount of TestTokens minted by the mint command. Default: 1073741824")] + public int MintTT { get; set; } = 1073741824; + public string EndpointsPath { get From 376fd6097439cbcf8da7e5f3c16a42fe945acef9 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 29 Mar 2024 12:07:48 +0100 Subject: [PATCH 12/19] debugging image --- .../DiscordBotContainerRecipe.cs | 2 +- Tests/CodexTests/BasicTests/ExampleTests.cs | 67 ++++++++++--------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index 3e3951a..5633cd1 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class DiscordBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-bibliotech"; - public override string Image => "codexstorage/codex-discordbot:sha-b25c747"; + public override string Image => "codexstorage/codex-discordbot:sha-8bc63c1"; public static string RewardsPort = "bot_rewards_port"; diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index c0102d7..67548ee 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -10,11 +10,13 @@ using Request = CodexContractsPlugin.Marketplace.Request; namespace CodexTests.BasicTests { [TestFixture] - public class ExampleTests : AutoBootstrapDistTest + public class ExampleTests : CodexDistTest { [Test] public void BotRewardTest() { + var myAccount = EthAccount.GenerateNew(); + var sellerInitialBalance = 234.TestTokens(); var buyerInitialBalance = 100000.TestTokens(); var fileSize = 10.MB(); @@ -22,7 +24,37 @@ namespace CodexTests.BasicTests var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); - var myAccount = EthAccount.GenerateNew(); + // start bot and rewarder + var gethInfo = new DiscordBotGethInfo( + host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, + port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, + privKey: geth.StartResult.Account.PrivateKey, + marketplaceAddress: contracts.Deployment.MarketplaceAddress, + tokenAddress: contracts.Deployment.TokenAddress, + abi: contracts.Deployment.Abi + ); + var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( + name: "bot", + token: "aaa", + serverName: "ThatBen's server", + adminRoleName: "bottest-admins", + adminChannelName: "admin-channel", + rewardChannelName: "rewards-channel", + kubeNamespace: "notneeded", + gethInfo: gethInfo + )); + var botContainer = bot.Containers.Single(); + Ci.DeployRewarderBot(new RewarderBotStartupConfig( + //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, + //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, + discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, + discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, + interval: "60", + historyStartUtc: DateTime.UtcNow.AddHours(-1), + gethInfo: gethInfo, + dataPath: null + )); + var numberOfHosts = 3; for (var i = 0; i < numberOfHosts; i++) @@ -60,37 +92,6 @@ namespace CodexTests.BasicTests AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance)); - // start bot and rewarder - var gethInfo = new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "bot", - token: "MTE2NDEyNzk3MDU4NDE3NDU5Mw.GTpoV6.aDR7zxMNf7vDgMjKASJBQs-RtNP_lYJEY-OglI", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - var botContainer = bot.Containers.Single(); - Ci.DeployRewarderBot(new RewarderBotStartupConfig( - //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, - //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - interval: "60", - historyStartUtc: DateTime.UtcNow.AddHours(-1), - gethInfo: gethInfo, - dataPath: null - )); - var contentId = buyer.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) From a6b0b16909e380212328c0e8fcf36f2e59d9ec29 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 11:22:28 +0200 Subject: [PATCH 13/19] ready for bot test --- .../DiscordBotContainerRecipe.cs | 2 +- .../RewarderBotContainerRecipe.cs | 2 +- .../CodexTests/BasicTests/MarketplaceTests.cs | 57 ++++++++++--------- 3 files changed, 31 insertions(+), 30 deletions(-) diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs index 5633cd1..9a2e3fe 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class DiscordBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-bibliotech"; - public override string Image => "codexstorage/codex-discordbot:sha-8bc63c1"; + public override string Image => "codexstorage/codex-discordbot:sha-8c64352"; public static string RewardsPort = "bot_rewards_port"; diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index dd95945..bfe15c5 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-b25c747"; + public override string Image => "codexstorage/codex-rewarderbot:sha-8c64352"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index fbd40d8..79afa43 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -3,7 +3,6 @@ using CodexContractsPlugin.Marketplace; using CodexDiscordBotPlugin; using CodexPlugin; using GethPlugin; -using Nethereum.Hex.HexConvertors.Extensions; using NUnit.Framework; using Utils; @@ -19,7 +18,7 @@ namespace CodexTests.BasicTests var sellerInitialBalance = 234.TestTokens(); var buyerInitialBalance = 100000.TestTokens(); - var fileSize = 10.MB(); + var fileSize = 11.MB(); var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); @@ -88,6 +87,7 @@ namespace CodexTests.BasicTests var buyer = AddCodex(s => s .WithName("Buyer") .EnableMarketplace(geth, contracts, m => m + .WithAccount(myAccount) .WithInitial(10.Eth(), buyerInitialBalance))); AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance)); @@ -101,8 +101,8 @@ namespace CodexTests.BasicTests MinRequiredNumberOfNodes = 5, NodeFailureTolerance = 2, ProofProbability = 5, - Duration = TimeSpan.FromMinutes(5), - Expiry = TimeSpan.FromMinutes(4) + Duration = TimeSpan.FromMinutes(6), + Expiry = TimeSpan.FromMinutes(5) }; var purchaseContract = buyer.Marketplace.RequestStorage(purchase); @@ -127,11 +127,12 @@ namespace CodexTests.BasicTests //Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished)); } - private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase) + private void WaitForAllSlotFilledEvents(IGethNode gethNode, ICodexContracts contracts, StoragePurchaseRequest purchase) { Time.Retry(() => { - var slotFilledEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); + var blockRange = gethNode.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); + var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange); Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}"); @@ -139,23 +140,23 @@ namespace CodexTests.BasicTests }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events"); } - private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller) - { - // Expect 1 fulfilled event for the purchase. - var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); - Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1)); - CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId); + //private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller) + //{ + // // Expect 1 fulfilled event for the purchase. + // var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); + // Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1)); + // CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId); - // Expect 1 filled-slot event for each slot in the purchase. - var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); - Assert.That(filledSlotEvents.Length, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); - for (var i = 0; i < purchase.MinRequiredNumberOfNodes; i++) - { - var filledSlotEvent = filledSlotEvents.Single(e => e.SlotIndex == i); - Assert.That(filledSlotEvent.RequestId.ToHex(), Is.EqualTo(request.RequestId.ToHex())); - Assert.That(filledSlotEvent.Host, Is.EqualTo(seller.EthAddress)); - } - } + // // Expect 1 filled-slot event for each slot in the purchase. + // var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); + // Assert.That(filledSlotEvents.Length, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); + // for (var i = 0; i < purchase.MinRequiredNumberOfNodes; i++) + // { + // var filledSlotEvent = filledSlotEvents.Single(e => e.SlotIndex == i); + // Assert.That(filledSlotEvent.RequestId.ToHex(), Is.EqualTo(request.RequestId.ToHex())); + // Assert.That(filledSlotEvent.Host, Is.EqualTo(seller.EthAddress)); + // } + //} private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer) { @@ -164,12 +165,12 @@ namespace CodexTests.BasicTests Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); } - private Request GetOnChainStorageRequest(ICodexContracts contracts) - { - var requests = contracts.GetStorageRequests(GetTestRunTimeRange()); - Assert.That(requests.Length, Is.EqualTo(1)); - return requests.Single(); - } + //private Request GetOnChainStorageRequest(ICodexContracts contracts) + //{ + // var requests = contracts.GetStorageRequests(GetTestRunTimeRange()); + // Assert.That(requests.Length, Is.EqualTo(1)); + // return requests.Single(); + //} private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex, ICodexNode expectedSeller) { From be268ace427f763b191d47141634ee41badd1dbc Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 13:21:21 +0200 Subject: [PATCH 14/19] Find bug in blocktime finder --- .../CodexTests/BasicTests/MarketplaceTests.cs | 4 --- .../NethereumWorkflow/BlockTimeFinderTests.cs | 35 +++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 79afa43..1a54f14 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -71,8 +71,6 @@ namespace CodexTests.BasicTests .AsStorageNode() .AsValidator())); - AssertBalance(contracts, seller, Is.EqualTo(sellerInitialBalance)); - var availability = new StorageAvailability( totalSpace: 10.GB(), maxDuration: TimeSpan.FromMinutes(30), @@ -90,8 +88,6 @@ namespace CodexTests.BasicTests .WithAccount(myAccount) .WithInitial(10.Eth(), buyerInitialBalance))); - AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance)); - var contentId = buyer.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) diff --git a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs index 5653b45..1c51084 100644 --- a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs +++ b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs @@ -117,6 +117,41 @@ namespace FrameworkTests.NethereumWorkflow Assert.That(notFound, Is.Null); } + + [Test] + public void FailsToFindBlockBeforeFrontOfChain_history() + { + var first = blocks.First().Value; + + var notFound = finder.GetHighestBlockNumberBefore(first.JustBefore); + + Assert.That(notFound, Is.Null); + } + + [Test] + public void FailsToFindBlockAfterTailOfChain_future() + { + var last = blocks.Last().Value; + + var notFound = finder.GetLowestBlockNumberAfter(last.JustAfter); + + Assert.That(notFound, Is.Null); + } + + [Test] + public void RunThrough() + { + foreach (var pair in blocks) + { + finder.GetHighestBlockNumberBefore(pair.Value.JustBefore); + finder.GetHighestBlockNumberBefore(pair.Value.Time); + finder.GetHighestBlockNumberBefore(pair.Value.JustAfter); + + finder.GetLowestBlockNumberAfter(pair.Value.JustBefore); + finder.GetLowestBlockNumberAfter(pair.Value.Time); + finder.GetLowestBlockNumberAfter(pair.Value.JustAfter); + } + } } public class Block From d7c7d47a61748009850c23fb38a402b52d4a24f8 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 13:42:07 +0200 Subject: [PATCH 15/19] Fixes crash in blocktimefinder --- Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs | 4 ++-- .../FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs index 851a8bf..f2a89c2 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs @@ -70,7 +70,7 @@ namespace NethereumWorkflow.BlockUtils { var next = GetBlock(entry.BlockNumber + 1); return - entry.Utc < target && + entry.Utc <= target && next.Utc > target; } @@ -78,7 +78,7 @@ namespace NethereumWorkflow.BlockUtils { var previous = GetBlock(entry.BlockNumber - 1); return - entry.Utc > target && + entry.Utc >= target && previous.Utc < target; } diff --git a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs index 1c51084..d2be5da 100644 --- a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs +++ b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs @@ -166,5 +166,10 @@ namespace FrameworkTests.NethereumWorkflow public DateTime Time { get; } public DateTime JustBefore { get { return Time.AddSeconds(-1); } } public DateTime JustAfter { get { return Time.AddSeconds(1); } } + + public override string ToString() + { + return $"[{Number}]"; + } } } From 48e7f9895644ef375dc433e850e70bd679c92579 Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 13:46:30 +0200 Subject: [PATCH 16/19] Adds some logging --- .../NethereumWorkflow/BlockUtils/BlockCache.cs | 2 ++ .../NethereumWorkflow/BlockUtils/BlockTimeFinder.cs | 13 +++++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs b/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs index 1954963..c902eda 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs @@ -35,5 +35,7 @@ if (!entries.TryGetValue(number, out BlockTimeEntry? value)) return null; return value; } + + public int Size { get { return entries.Count; } } } } diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs index f2a89c2..3678cea 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs @@ -24,7 +24,7 @@ namespace NethereumWorkflow.BlockUtils if (moment <= bounds.Genesis.Utc) return null; if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber; - return Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector); + return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector)); } public ulong? GetLowestBlockNumberAfter(DateTime moment) @@ -33,7 +33,16 @@ namespace NethereumWorkflow.BlockUtils if (moment >= bounds.Current.Utc) return null; if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber; - return Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector); + return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ; + } + + private ulong Log(Func operation) + { + var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder)); + var result = operation(); + sw.End($"(Cache size: {cache.Size})"); + + return result; } private ulong Search(BlockTimeEntry lower, BlockTimeEntry upper, DateTime target, Func isWhatIwant) From 2ab84e2a61ff3e93f31a4201d3060f8221e25e0c Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 13:56:07 +0200 Subject: [PATCH 17/19] Time handling for rewarder --- Tools/TestNetRewarder/Processor.cs | 15 ++++++++------- Tools/TestNetRewarder/TimeSegmenter.cs | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 425c0e3..032b0ef 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -20,11 +20,11 @@ namespace TestNetRewarder public async Task ProcessTimeSegment(TimeRange timeRange) { + var connector = GethConnector.GethConnector.Initialize(log); + if (connector == null) throw new Exception("Invalid Geth information"); + try { - var connector = GethConnector.GethConnector.Initialize(log); - if (connector == null) return; - var blockRange = connector.GethNode.ConvertTimeRangeToBlockRange(timeRange); if (!IsNewBlockRange(blockRange)) { @@ -33,12 +33,12 @@ namespace TestNetRewarder } var chainState = new ChainState(historicState, connector.CodexContracts, blockRange); - await ProcessTimeSegment(chainState); - + await ProcessChainState(chainState); } catch (Exception ex) { log.Error("Exception processing time segment: " + ex); + throw; } } @@ -55,7 +55,7 @@ namespace TestNetRewarder return false; } - private async Task ProcessTimeSegment(ChainState chainState) + private async Task ProcessChainState(ChainState chainState) { var outgoingRewards = new List(); foreach (var reward in rewardRepo.Rewards) @@ -63,6 +63,7 @@ namespace TestNetRewarder ProcessReward(outgoingRewards, reward, chainState); } + log.Log($"Found {outgoingRewards.Count} rewards to send."); if (outgoingRewards.Any()) { if (!await SendRewardsCommand(outgoingRewards)) @@ -79,7 +80,7 @@ namespace TestNetRewarder Rewards = outgoingRewards.ToArray() }; - log.Log("Sending rewards: " + JsonConvert.SerializeObject(cmd)); + log.Debug("Sending rewards: " + JsonConvert.SerializeObject(cmd)); return await Program.BotClient.SendRewards(cmd); } diff --git a/Tools/TestNetRewarder/TimeSegmenter.cs b/Tools/TestNetRewarder/TimeSegmenter.cs index 5a555f7..4bdced2 100644 --- a/Tools/TestNetRewarder/TimeSegmenter.cs +++ b/Tools/TestNetRewarder/TimeSegmenter.cs @@ -31,11 +31,12 @@ namespace TestNetRewarder if (end > now) { // Wait for the entire time segment to be in the past. - var delay = (end - now).Add(TimeSpan.FromSeconds(3)); + var delay = end - now; waited = true; log.Log($"Waiting till time segment is in the past... {Time.FormatDuration(delay)}"); await Task.Delay(delay, Program.CancellationToken); } + await Task.Delay(TimeSpan.FromSeconds(3), Program.CancellationToken); if (Program.CancellationToken.IsCancellationRequested) return; From 6c4b9345cb28515e0a02f4cdca0b0baf5da6960c Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 14:08:30 +0200 Subject: [PATCH 18/19] sensible time range for bot test --- Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs | 2 +- .../CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs | 2 +- Tests/CodexTests/BasicTests/MarketplaceTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs index 3678cea..e4c73d5 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs @@ -40,7 +40,7 @@ namespace NethereumWorkflow.BlockUtils { var sw = Stopwatch.Begin(log, nameof(BlockTimeFinder)); var result = operation(); - sw.End($"(Cache size: {cache.Size})"); + sw.End($"(Bounds: [{bounds.Genesis.BlockNumber}-{bounds.Current.BlockNumber}] Cache: {cache.Size})"); return result; } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs index bfe15c5..3be2f7f 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexDiscordBotPlugin public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-8c64352"; + public override string Image => "codexstorage/codex-rewarderbot:sha-2ab84e2"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index 1a54f14..c113ac1 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -49,7 +49,7 @@ namespace CodexTests.BasicTests discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, interval: "60", - historyStartUtc: DateTime.UtcNow.AddHours(-1), + historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3), gethInfo: gethInfo, dataPath: null )); From ceed913143e8e13914b1cde69e217a6a091546da Mon Sep 17 00:00:00 2001 From: benbierens Date: Mon, 1 Apr 2024 15:31:54 +0200 Subject: [PATCH 19/19] moves bot tests --- .../CodexTests/BasicTests/DiscordBotTests.cs | 110 +++++++++++++++ .../CodexTests/BasicTests/MarketplaceTests.cs | 129 +++++------------- 2 files changed, 146 insertions(+), 93 deletions(-) create mode 100644 Tests/CodexTests/BasicTests/DiscordBotTests.cs diff --git a/Tests/CodexTests/BasicTests/DiscordBotTests.cs b/Tests/CodexTests/BasicTests/DiscordBotTests.cs new file mode 100644 index 0000000..eed7f48 --- /dev/null +++ b/Tests/CodexTests/BasicTests/DiscordBotTests.cs @@ -0,0 +1,110 @@ +using CodexContractsPlugin; +using CodexDiscordBotPlugin; +using CodexPlugin; +using GethPlugin; +using NUnit.Framework; +using Utils; + +namespace CodexTests.BasicTests +{ + [TestFixture] + public class DiscordBotTests : AutoBootstrapDistTest + { + [Test] + public void BotRewardTest() + { + var myAccount = EthAccount.GenerateNew(); + + var sellerInitialBalance = 234.TestTokens(); + var buyerInitialBalance = 100000.TestTokens(); + var fileSize = 11.MB(); + + var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); + var contracts = Ci.StartCodexContracts(geth); + + // start bot and rewarder + var gethInfo = new DiscordBotGethInfo( + host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, + port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, + privKey: geth.StartResult.Account.PrivateKey, + marketplaceAddress: contracts.Deployment.MarketplaceAddress, + tokenAddress: contracts.Deployment.TokenAddress, + abi: contracts.Deployment.Abi + ); + var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( + name: "bot", + token: "aaa", + serverName: "ThatBen's server", + adminRoleName: "bottest-admins", + adminChannelName: "admin-channel", + rewardChannelName: "rewards-channel", + kubeNamespace: "notneeded", + gethInfo: gethInfo + )); + var botContainer = bot.Containers.Single(); + Ci.DeployRewarderBot(new RewarderBotStartupConfig( + //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, + //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, + discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, + discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, + interval: "60", + historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3), + gethInfo: gethInfo, + dataPath: null + )); + + var numberOfHosts = 3; + + for (var i = 0; i < numberOfHosts; i++) + { + var seller = AddCodex(s => s + .WithName("Seller") + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) + .WithStorageQuota(11.GB()) + .EnableMarketplace(geth, contracts, m => m + .WithAccount(myAccount) + .WithInitial(10.Eth(), sellerInitialBalance) + .AsStorageNode() + .AsValidator())); + + var availability = new StorageAvailability( + totalSpace: 10.GB(), + maxDuration: TimeSpan.FromMinutes(30), + minPriceForTotalSpace: 1.TestTokens(), + maxCollateral: 20.TestTokens() + ); + seller.Marketplace.MakeStorageAvailable(availability); + } + + var testFile = GenerateTestFile(fileSize); + + var buyer = AddCodex(s => s + .WithName("Buyer") + .EnableMarketplace(geth, contracts, m => m + .WithAccount(myAccount) + .WithInitial(10.Eth(), buyerInitialBalance))); + + var contentId = buyer.UploadFile(testFile); + + var purchase = new StoragePurchaseRequest(contentId) + { + PricePerSlotPerSecond = 2.TestTokens(), + RequiredCollateral = 10.TestTokens(), + MinRequiredNumberOfNodes = 5, + NodeFailureTolerance = 2, + ProofProbability = 5, + Duration = TimeSpan.FromMinutes(6), + Expiry = TimeSpan.FromMinutes(5) + }; + + var purchaseContract = buyer.Marketplace.RequestStorage(purchase); + + purchaseContract.WaitForStorageContractStarted(); + + purchaseContract.WaitForStorageContractFinished(); + } + } +} diff --git a/Tests/CodexTests/BasicTests/MarketplaceTests.cs b/Tests/CodexTests/BasicTests/MarketplaceTests.cs index c113ac1..7474696 100644 --- a/Tests/CodexTests/BasicTests/MarketplaceTests.cs +++ b/Tests/CodexTests/BasicTests/MarketplaceTests.cs @@ -1,6 +1,5 @@ using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; -using CodexDiscordBotPlugin; using CodexPlugin; using GethPlugin; using NUnit.Framework; @@ -12,83 +11,51 @@ namespace CodexTests.BasicTests public class MarketplaceTests : AutoBootstrapDistTest { [Test] - public void BotRewardTest() + public void MarketplaceExample() { - var myAccount = EthAccount.GenerateNew(); - - var sellerInitialBalance = 234.TestTokens(); - var buyerInitialBalance = 100000.TestTokens(); - var fileSize = 11.MB(); + var hostInitialBalance = 234.TestTokens(); + var clientInitialBalance = 100000.TestTokens(); + var fileSize = 10.MB(); var geth = Ci.StartGethNode(s => s.IsMiner().WithName("disttest-geth")); var contracts = Ci.StartCodexContracts(geth); - // start bot and rewarder - var gethInfo = new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "bot", - token: "aaa", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - var botContainer = bot.Containers.Single(); - Ci.DeployRewarderBot(new RewarderBotStartupConfig( - //discordBotHost: "http://" + botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Host, - //discordBotPort: botContainer.GetAddress(GetTestLog(), DiscordBotContainerRecipe.RewardsPort).Port, - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - interval: "60", - historyStartUtc: GetTestRunTimeRange().From - TimeSpan.FromMinutes(3), - gethInfo: gethInfo, - dataPath: null - )); - var numberOfHosts = 3; - for (var i = 0; i < numberOfHosts; i++) { - var seller = AddCodex(s => s - .WithName("Seller") + var host = AddCodex(s => s + .WithName("Host") .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) { ContractClock = CodexLogLevel.Trace, }) .WithStorageQuota(11.GB()) .EnableMarketplace(geth, contracts, m => m - .WithAccount(myAccount) - .WithInitial(10.Eth(), sellerInitialBalance) + .WithInitial(10.Eth(), hostInitialBalance) .AsStorageNode() .AsValidator())); + AssertBalance(contracts, host, Is.EqualTo(hostInitialBalance)); + var availability = new StorageAvailability( totalSpace: 10.GB(), maxDuration: TimeSpan.FromMinutes(30), minPriceForTotalSpace: 1.TestTokens(), maxCollateral: 20.TestTokens() ); - seller.Marketplace.MakeStorageAvailable(availability); + host.Marketplace.MakeStorageAvailable(availability); } var testFile = GenerateTestFile(fileSize); - var buyer = AddCodex(s => s - .WithName("Buyer") + var client = AddCodex(s => s + .WithName("Client") .EnableMarketplace(geth, contracts, m => m - .WithAccount(myAccount) - .WithInitial(10.Eth(), buyerInitialBalance))); + .WithInitial(10.Eth(), clientInitialBalance))); - var contentId = buyer.UploadFile(testFile); + AssertBalance(contracts, client, Is.EqualTo(clientInitialBalance)); + + var contentId = client.UploadFile(testFile); var purchase = new StoragePurchaseRequest(contentId) { @@ -97,37 +64,31 @@ namespace CodexTests.BasicTests MinRequiredNumberOfNodes = 5, NodeFailureTolerance = 2, ProofProbability = 5, - Duration = TimeSpan.FromMinutes(6), - Expiry = TimeSpan.FromMinutes(5) + Duration = TimeSpan.FromMinutes(5), + Expiry = TimeSpan.FromMinutes(4) }; - var purchaseContract = buyer.Marketplace.RequestStorage(purchase); + var purchaseContract = client.Marketplace.RequestStorage(purchase); + + WaitForAllSlotFilledEvents(contracts, purchase, geth); purchaseContract.WaitForStorageContractStarted(); - //AssertBalance(contracts, seller, Is.LessThan(sellerInitialBalance), "Collateral was not placed."); - - //var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); - - //var request = GetOnChainStorageRequest(contracts, blockRange); - //AssertStorageRequest(request, purchase, contracts, buyer); - //AssertSlotFilledEvents(contracts, purchase, request, seller, blockRange); - //AssertContractSlot(contracts, request, 0, seller); + var request = GetOnChainStorageRequest(contracts, geth); + AssertStorageRequest(request, purchase, contracts, client); + AssertContractSlot(contracts, request, 0); purchaseContract.WaitForStorageContractFinished(); - var hold = 0; - - //AssertBalance(contracts, seller, Is.GreaterThan(sellerInitialBalance), "Seller was not paid for storage."); - //AssertBalance(contracts, buyer, Is.LessThan(buyerInitialBalance), "Buyer was not charged for storage."); - //Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished)); + AssertBalance(contracts, client, Is.LessThan(clientInitialBalance), "Buyer was not charged for storage."); + Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Finished)); } - private void WaitForAllSlotFilledEvents(IGethNode gethNode, ICodexContracts contracts, StoragePurchaseRequest purchase) + private void WaitForAllSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, IGethNode geth) { Time.Retry(() => { - var blockRange = gethNode.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); + var blockRange = geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange()); var slotFilledEvents = contracts.GetSlotFilledEvents(blockRange); Log($"SlotFilledEvents: {slotFilledEvents.Length} - NumSlots: {purchase.MinRequiredNumberOfNodes}"); @@ -136,24 +97,6 @@ namespace CodexTests.BasicTests }, Convert.ToInt32(purchase.Duration.TotalSeconds / 5) + 10, TimeSpan.FromSeconds(5), "Checking SlotFilled events"); } - //private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller) - //{ - // // Expect 1 fulfilled event for the purchase. - // var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); - // Assert.That(requestFulfilledEvents.Length, Is.EqualTo(1)); - // CollectionAssert.AreEqual(request.RequestId, requestFulfilledEvents[0].RequestId); - - // // Expect 1 filled-slot event for each slot in the purchase. - // var filledSlotEvents = contracts.GetSlotFilledEvents(GetTestRunTimeRange()); - // Assert.That(filledSlotEvents.Length, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); - // for (var i = 0; i < purchase.MinRequiredNumberOfNodes; i++) - // { - // var filledSlotEvent = filledSlotEvents.Single(e => e.SlotIndex == i); - // Assert.That(filledSlotEvent.RequestId.ToHex(), Is.EqualTo(request.RequestId.ToHex())); - // Assert.That(filledSlotEvent.Host, Is.EqualTo(seller.EthAddress)); - // } - //} - private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer) { Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started)); @@ -161,17 +104,17 @@ namespace CodexTests.BasicTests Assert.That(request.Ask.Slots, Is.EqualTo(purchase.MinRequiredNumberOfNodes)); } - //private Request GetOnChainStorageRequest(ICodexContracts contracts) - //{ - // var requests = contracts.GetStorageRequests(GetTestRunTimeRange()); - // Assert.That(requests.Length, Is.EqualTo(1)); - // return requests.Single(); - //} + private Request GetOnChainStorageRequest(ICodexContracts contracts, IGethNode geth) + { + var requests = contracts.GetStorageRequests(geth.ConvertTimeRangeToBlockRange(GetTestRunTimeRange())); + Assert.That(requests.Length, Is.EqualTo(1)); + return requests.Single(); + } - private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex, ICodexNode expectedSeller) + private void AssertContractSlot(ICodexContracts contracts, Request request, int contractSlotIndex) { var slotHost = contracts.GetSlotHost(request, contractSlotIndex); - Assert.That(slotHost, Is.EqualTo(expectedSeller.EthAddress)); + Assert.That(slotHost?.Address, Is.Not.Null); } } }