diff --git a/Framework/DiscordRewards/CheckConfig.cs b/Framework/DiscordRewards/CheckConfig.cs deleted file mode 100644 index 34425cea..00000000 --- a/Framework/DiscordRewards/CheckConfig.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Utils; - -namespace DiscordRewards -{ - public class CheckConfig - { - public CheckType Type { get; set; } - public ulong MinNumberOfHosts { get; set; } - public ByteSize MinSlotSize { get; set; } = 0.Bytes(); - public TimeSpan MinDuration { get; set; } = TimeSpan.Zero; - } - - public enum CheckType - { - Uninitialized, - HostFilledSlot, - HostFinishedSlot, - ClientPostedContract, - ClientStartedContract, - } -} diff --git a/Framework/DiscordRewards/EventsAndErrors.cs b/Framework/DiscordRewards/EventsAndErrors.cs new file mode 100644 index 00000000..7e18f07f --- /dev/null +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -0,0 +1,34 @@ +namespace DiscordRewards +{ + public class EventsAndErrors + { + public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); + public string[] Errors { get; set; } = Array.Empty(); + public ActiveChainAddresses ActiveChainAddresses { get; set; } = new ActiveChainAddresses(); + + public bool HasAny() + { + return + Errors.Length > 0 || + EventsOverview.Length > 0 || + ActiveChainAddresses.HasAny(); + } + } + + public class ChainEventMessage + { + public ulong BlockNumber { get; set; } + public string Message { get; set; } = string.Empty; + } + + public class ActiveChainAddresses + { + public string[] Hosts { get; set; } = Array.Empty(); + public string[] Clients { get; set; } = Array.Empty(); + + public bool HasAny() + { + return Hosts.Length > 0 || Clients.Length > 0; + } + } +} diff --git a/Framework/DiscordRewards/GiveRewardsCommand.cs b/Framework/DiscordRewards/GiveRewardsCommand.cs deleted file mode 100644 index 3aae088b..00000000 --- a/Framework/DiscordRewards/GiveRewardsCommand.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace DiscordRewards -{ - public class GiveRewardsCommand - { - public RewardUsersCommand[] Rewards { get; set; } = Array.Empty(); - public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); - public string[] Errors { get; set; } = Array.Empty(); - - public bool HasAny() - { - return Rewards.Any() || EventsOverview.Any(); - } - } - - public class RewardUsersCommand - { - public ulong RewardId { get; set; } - public string[] UserAddresses { get; set; } = Array.Empty(); - } - - public class ChainEventMessage - { - public ulong BlockNumber { get; set; } - public string Message { get; set; } = string.Empty; - } -} diff --git a/Framework/DiscordRewards/RewardConfig.cs b/Framework/DiscordRewards/RewardConfig.cs deleted file mode 100644 index dda0dfe1..00000000 --- a/Framework/DiscordRewards/RewardConfig.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace DiscordRewards -{ - public class RewardConfig - { - public const string UsernameTag = ""; - - public RewardConfig(ulong roleId, string message, CheckConfig checkConfig) - { - RoleId = roleId; - Message = message; - CheckConfig = checkConfig; - } - - public ulong RoleId { get; } - public string Message { get; } - public CheckConfig CheckConfig { get; } - } -} diff --git a/Framework/DiscordRewards/RewardRepo.cs b/Framework/DiscordRewards/RewardRepo.cs deleted file mode 100644 index 1c97caf9..00000000 --- a/Framework/DiscordRewards/RewardRepo.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace DiscordRewards -{ - public class RewardRepo - { - private static string Tag => RewardConfig.UsernameTag; - - public RewardConfig[] Rewards { get; } = new RewardConfig[0]; - - // Example configuration, from test server: - //{ - // // Filled any slot - // new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig - // { - // Type = CheckType.HostFilledSlot - // }), - - // // Finished any slot - // new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot - // }), - - // // Finished a sizable slot - // new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }), - - // // Posted any contract - // new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig - // { - // Type = CheckType.ClientPostedContract - // }), - - // // Started any contract - // new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig - // { - // Type = CheckType.ClientStartedContract - // }), - - // // Started a sizable contract - // new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.ClientStartedContract, - // MinNumberOfHosts = 4, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }) - //}; - } -} diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index f79eb199..3b2c609c 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -62,7 +62,6 @@ namespace NethereumWorkflow log.Debug(typeof(TFunction).ToString()); var handler = web3.Eth.GetContractQueryHandler(); var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber))); - var aaaa = 0; } public string SendTransaction(string contractAddress, TFunction function) where TFunction : FunctionMessage, new() diff --git a/Framework/Utils/RandomUtils.cs b/Framework/Utils/RandomUtils.cs index f4f28dd4..9f127167 100644 --- a/Framework/Utils/RandomUtils.cs +++ b/Framework/Utils/RandomUtils.cs @@ -34,10 +34,28 @@ var source = items.ToList(); while (source.Any()) { - result.Add(RandomUtils.PickOneRandom(source)); + result.Add(PickOneRandom(source)); } return result.ToArray(); } } + + public static string GenerateRandomString(long requiredLength) + { + lock (@lock) + { + var result = ""; + while (result.Length < requiredLength) + { + var remaining = requiredLength - result.Length; + var len = Math.Min(1024, remaining); + var bytes = new byte[len]; + random.NextBytes(bytes); + result += string.Join("", bytes.Select(b => b.ToString())); + } + + return result.Substring(0, Convert.ToInt32(requiredLength)); + } + } } } diff --git a/Framework/WebUtils/Http.cs b/Framework/WebUtils/Http.cs index 7c9a91ad..e4931996 100644 --- a/Framework/WebUtils/Http.cs +++ b/Framework/WebUtils/Http.cs @@ -20,11 +20,6 @@ namespace WebUtils private readonly Action onClientCreated; private readonly string id; - internal Http(string id, ILog log, IWebCallTimeSet timeSet) - : this(id, log, timeSet, DoNothing) - { - } - internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action onClientCreated) { this.id = id; @@ -89,9 +84,5 @@ namespace WebUtils onClientCreated(client); return client; } - - private static void DoNothing(HttpClient client) - { - } } } diff --git a/Framework/WebUtils/HttpFactory.cs b/Framework/WebUtils/HttpFactory.cs index 4d5d7c4c..8120527c 100644 --- a/Framework/WebUtils/HttpFactory.cs +++ b/Framework/WebUtils/HttpFactory.cs @@ -13,16 +13,28 @@ namespace WebUtils { private readonly ILog log; private readonly IWebCallTimeSet defaultTimeSet; + private readonly Action factoryOnClientCreated; public HttpFactory(ILog log) : this (log, new DefaultWebCallTimeSet()) { } + public HttpFactory(ILog log, Action onClientCreated) + : this(log, new DefaultWebCallTimeSet(), onClientCreated) + { + } + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet) + : this(log, defaultTimeSet, DoNothing) + { + } + + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet, Action onClientCreated) { this.log = log; this.defaultTimeSet = defaultTimeSet; + this.factoryOnClientCreated = onClientCreated; } public IHttp CreateHttp(string id, Action onClientCreated) @@ -32,12 +44,20 @@ namespace WebUtils public IHttp CreateHttp(string id, Action onClientCreated, IWebCallTimeSet ts) { - return new Http(id, log, ts, onClientCreated); + return new Http(id, log, ts, (c) => + { + factoryOnClientCreated(c); + onClientCreated(c); + }); } public IHttp CreateHttp(string id) { - return new Http(id, log, defaultTimeSet); + return new Http(id, log, defaultTimeSet, factoryOnClientCreated); + } + + private static void DoNothing(HttpClient client) + { } } } diff --git a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs deleted file mode 100644 index 8fdb79ec..00000000 --- a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs +++ /dev/null @@ -1,370 +0,0 @@ -using CodexClient; -using CodexContractsPlugin; -using CodexDiscordBotPlugin; -using CodexPlugin; -using CodexTests; -using Core; -using DiscordRewards; -using DistTestCore; -using GethPlugin; -using KubernetesWorkflow.Types; -using Logging; -using Newtonsoft.Json; -using NUnit.Framework; -using Utils; - -namespace ExperimentalTests.UtilityTests -{ - [TestFixture] - public class DiscordBotTests : AutoBootstrapDistTest - { - private readonly RewardRepo repo = new RewardRepo(); - private readonly TestToken hostInitialBalance = 3000000.TstWei(); - private readonly TestToken clientInitialBalance = 1000000000.TstWei(); - private readonly EthAccount clientAccount = EthAccountGenerator.GenerateNew(); - private readonly List hostAccounts = new List(); - private readonly List rewardsSeen = new List(); - private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); - private readonly List receivedEvents = new List(); - - [Test] - [DontDownloadLogs] - [Ignore("Used to debug testnet bots.")] - public void BotRewardTest() - { - var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth")); - var contracts = Ci.StartCodexContracts(geth); - var gethInfo = CreateGethInfo(geth, contracts); - - var botContainer = StartDiscordBot(gethInfo); - var rewarderContainer = StartRewarderBot(gethInfo, botContainer); - - StartHosts(geth, contracts); - var client = StartClient(geth, contracts); - - var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer); - apiCalls.Start(OnCommand); - - var purchaseContract = ClientPurchasesStorage(client); - purchaseContract.WaitForStorageContractStarted(); - purchaseContract.WaitForStorageContractFinished(); - - // todo: removed from codexclient: - //contracts.WaitUntilNextPeriod(); - //contracts.WaitUntilNextPeriod(); - - //var blocks = 3; - //Log($"Waiting {blocks} blocks for nodes to process payouts..."); - //Thread.Sleep(GethContainerRecipe.BlockInterval * blocks); - - Thread.Sleep(rewarderInterval * 3); - - apiCalls.Stop(); - - AssertEventOccurance("Created as New.", 1); - AssertEventOccurance("SlotFilled", Convert.ToInt32(GetNumberOfRequiredHosts())); - AssertEventOccurance("Transit: New -> Started", 1); - AssertEventOccurance("Transit: Started -> Finished", 1); - - foreach (var r in repo.Rewards) - { - var seen = rewardsSeen.Any(s => r.RoleId == s); - - Log($"{Lookup(r.RoleId)} = {seen}"); - } - - Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId))); - } - - private string Lookup(ulong rewardId) - { - var reward = repo.Rewards.Single(r => r.RoleId == rewardId); - return $"({rewardId})'{reward.Message}'"; - } - - private void AssertEventOccurance(string msg, int expectedCount) - { - Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount), - $"Event '{msg}' did not occure correct number of times."); - } - - private void OnCommand(string timestamp, GiveRewardsCommand call) - { - Log($""); - foreach (var e in call.EventsOverview) - { - Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order."); - } - - receivedEvents.AddRange(call.EventsOverview); - foreach (var e in call.EventsOverview) - { - Log("\tEvent: " + e); - } - foreach (var r in call.Rewards) - { - var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); - if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId); - foreach (var address in r.UserAddresses) - { - var user = IdentifyAccount(address); - Log("\tReward: " + user + ": " + reward.Message); - } - } - Log($""); - } - - private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client) - { - var testFile = GenerateTestFile(GetMinFileSize()); - var contentId = client.UploadFile(testFile); - var purchase = new StoragePurchaseRequest(contentId) - { - PricePerBytePerSecond = 2.TstWei(), - CollateralPerByte = 10.TstWei(), - MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(), - NodeFailureTolerance = 2, - ProofProbability = 5, - Duration = GetMinRequiredRequestDuration(), - Expiry = GetMinRequiredRequestDuration() - TimeSpan.FromMinutes(1) - }; - - return client.Marketplace.RequestStorage(purchase); - } - - private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts) - { - var node = StartCodex(s => s - .WithName("Client") - .EnableMarketplace(geth, contracts, m => m - .WithAccount(clientAccount) - .WithInitial(10.Eth(), clientInitialBalance))); - - Log($"Client {node.EthAccount.EthAddress}"); - return node; - } - - private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) - { - return Ci.DeployRewarderBot(new RewarderBotStartupConfig( - name: "rewarder-bot", - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)), - historyStartUtc: DateTime.UtcNow, - gethInfo: gethInfo, - dataPath: null - )); - } - - private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts) - { - return new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - } - - private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo) - { - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "discord-bot", - token: "aaa", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - return bot.Containers.Single(); - } - - private void StartHosts(IGethNode geth, ICodexContracts contracts) - { - var hosts = StartCodex(GetNumberOfLiveHosts(), s => s - .WithName("Host") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts())) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), hostInitialBalance) - .AsStorageNode() - .AsValidator())); - - var availability = new CreateStorageAvailability( - totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()), - maxDuration: TimeSpan.FromMinutes(30), - minPricePerBytePerSecond: 1.TstWei(), - totalCollateral: hostInitialBalance - ); - - foreach (var host in hosts) - { - hostAccounts.Add(host.EthAccount); - host.Marketplace.MakeStorageAvailable(availability); - } - } - - private int GetNumberOfLiveHosts() - { - return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3; - } - - private ByteSize Mult(ByteSize size, int mult) - { - return new ByteSize(size.SizeInBytes * mult); - } - - private ByteSize GetMinFileSizePlus(int plusMb) - { - return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes); - } - - private ByteSize GetMinFileSize() - { - ulong minSlotSize = 0; - ulong minNumHosts = 0; - foreach (var r in repo.Rewards) - { - var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes); - var h = r.CheckConfig.MinNumberOfHosts; - if (s > minSlotSize) minSlotSize = s; - if (h > minNumHosts) minNumHosts = h; - } - - var minFileSize = (minSlotSize + 1024) * minNumHosts; - return new ByteSize(Convert.ToInt64(minFileSize)); - } - - private uint GetNumberOfRequiredHosts() - { - return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); - } - - private TimeSpan GetMinRequiredRequestDuration() - { - return repo.Rewards.Max(r => r.CheckConfig.MinDuration) + TimeSpan.FromSeconds(10); - } - - private string IdentifyAccount(string address) - { - if (address == clientAccount.EthAddress.Address) return "Client"; - try - { - var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address); - return "Host" + index; - } - catch - { - return "UNKNOWN"; - } - } - - public class RewardApiCalls - { - private readonly ContainerFileMonitor monitor; - - public RewardApiCalls(ILog log, CoreInterface ci, RunningContainer botContainer) - { - monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log"); - } - - public void Start(Action onCommand) - { - monitor.Start(line => ParseLine(line, onCommand)); - } - - public void Stop() - { - monitor.Stop(); - } - - private void ParseLine(string line, Action onCommand) - { - try - { - var timestamp = line.Substring(0, 30); - var json = line.Substring(31); - - var cmd = JsonConvert.DeserializeObject(json); - if (cmd != null) - { - onCommand(timestamp, cmd); - } - } - catch - { - } - } - } - - public class ContainerFileMonitor - { - private readonly ILog log; - private readonly CoreInterface ci; - private readonly RunningContainer botContainer; - private readonly string filePath; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); - private readonly List seenLines = new List(); - private Task worker = Task.CompletedTask; - private Action onNewLine = c => { }; - - public ContainerFileMonitor(ILog log, CoreInterface ci, RunningContainer botContainer, string filePath) - { - this.log = log; - this.ci = ci; - this.botContainer = botContainer; - this.filePath = filePath; - } - - public void Start(Action onNewLine) - { - this.onNewLine = onNewLine; - worker = Task.Run(Worker); - } - - public void Stop() - { - cts.Cancel(); - worker.Wait(); - } - - // did any container crash? that's why it repeats? - - - private void Worker() - { - while (!cts.IsCancellationRequested) - { - Update(); - } - } - - private void Update() - { - Thread.Sleep(TimeSpan.FromSeconds(10)); - if (cts.IsCancellationRequested) return; - - var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath); - var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - foreach (var line in lines) - { - // log.Log("line: " + line); - - if (!seenLines.Contains(line)) - { - seenLines.Add(line); - onNewLine(line); - } - } - } - } - } -} diff --git a/Tests/FrameworkTests/Utils/RandomUtilsTests.cs b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs new file mode 100644 index 00000000..3859f0d3 --- /dev/null +++ b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs @@ -0,0 +1,19 @@ +using NUnit.Framework; +using Utils; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class RandomUtilsTests + { + [Test] + [Combinatorial] + public void TestRandomStringLength( + [Values(1, 5, 10, 1023, 1024, 1025, 2222)] int length) + { + var str = RandomUtils.GenerateRandomString(length); + + Assert.That(str.Length, Is.EqualTo(length)); + } + } +} diff --git a/Tools/AutoClient/App.cs b/Tools/AutoClient/App.cs index a581b6db..1a7e33a4 100644 --- a/Tools/AutoClient/App.cs +++ b/Tools/AutoClient/App.cs @@ -20,10 +20,6 @@ namespace AutoClient new FileLog(Path.Combine(config.LogPath, "performance")), new ConsoleLog() )); - - var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet()); - - CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath); } public Configuration Config { get; } @@ -31,7 +27,6 @@ namespace AutoClient public IFileGenerator Generator { get; } public CancellationTokenSource Cts { get; } = new CancellationTokenSource(); public Performance Performance { get; } - public CodexNodeFactory CodexNodeFactory { get; } private IFileGenerator CreateGenerator() { diff --git a/Tools/AutoClient/AutomaticPurchaser.cs b/Tools/AutoClient/AutomaticPurchaser.cs deleted file mode 100644 index 1886f321..00000000 --- a/Tools/AutoClient/AutomaticPurchaser.cs +++ /dev/null @@ -1,145 +0,0 @@ -using Logging; - -namespace AutoClient -{ - public class AutomaticPurchaser - { - private readonly App app; - private readonly ILog log; - private readonly CodexWrapper node; - private Task workerTask = Task.CompletedTask; - - public AutomaticPurchaser(App app, ILog log, CodexWrapper node) - { - this.app = app; - this.log = log; - this.node = node; - } - - public void Start() - { - workerTask = Task.Run(Worker); - } - - public void Stop() - { - workerTask.Wait(); - } - - private async Task Worker() - { - log.Log("Worker started."); - while (!app.Cts.Token.IsCancellationRequested) - { - try - { - var pid = await StartNewPurchase(); - await WaitTillFinished(pid); - } - catch (Exception ex) - { - log.Error("Worker failed with: " + ex); - await Task.Delay(TimeSpan.FromHours(6)); - } - } - } - - private async Task StartNewPurchase() - { - var file = await CreateFile(); - try - { - var cid = node.UploadFile(file); - var response = node.RequestStorage(cid); - return response.PurchaseId; - } - finally - { - DeleteFile(file); - } - } - - private async Task CreateFile() - { - return await app.Generator.Generate(); - } - - private void DeleteFile(string file) - { - try - { - File.Delete(file); - } - catch (Exception exc) - { - log.Error($"Failed to delete file '{file}': {exc}"); - } - } - - private async Task WaitTillFinished(string pid) - { - try - { - var emptyResponseTolerance = 10; - while (!app.Cts.Token.IsCancellationRequested) - { - var purchase = node.GetStoragePurchase(pid); - if (purchase == null) - { - await FixedShortDelay(); - emptyResponseTolerance--; - if (emptyResponseTolerance == 0) - { - log.Log("Received 10 empty responses. Stop tracking this purchase."); - await ExpiryTimeDelay(); - return; - } - continue; - } - if (purchase.IsCancelled) - { - app.Performance.StorageContractCancelled(); - return; - } - if (purchase.IsError) - { - app.Performance.StorageContractErrored(purchase.Error); - return; - } - if (purchase.IsFinished) - { - app.Performance.StorageContractFinished(); - return; - } - if (purchase.IsStarted) - { - app.Performance.StorageContractStarted(); - await FixedDurationDelay(); - } - - await FixedShortDelay(); - } - } - catch (Exception ex) - { - log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time."); - await ExpiryTimeDelay(); - } - } - - private async Task FixedDurationDelay() - { - await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token); - } - - private async Task ExpiryTimeDelay() - { - await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token); - } - - private async Task FixedShortDelay() - { - await Task.Delay(15 * 1000, app.Cts.Token); - } - } -} diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index edb0c658..13aea6fd 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -6,7 +6,14 @@ namespace AutoClient { [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")] public string CodexEndpoints { get; set; } = - "http://localhost:8080"; + "http://localhost:8080" + ";" + + "http://localhost:8081" + ";" + + "http://localhost:8082" + ";" + + "http://localhost:8083" + ";" + + "http://localhost:8084" + ";" + + "http://localhost:8085" + ";" + + "http://localhost:8086" + ";" + + "http://localhost:8087"; [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] public string DataPath { get; set; } = "datapath"; @@ -21,7 +28,7 @@ namespace AutoClient // Cluster nodes configured for max 7-day storage. [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")] - public int ContractExpiryMinutes { get; set; } = 60; + public int ContractExpiryMinutes { get; set; } = 15; [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")] public int NumHosts { get; set; } = 5; @@ -41,8 +48,16 @@ namespace AutoClient [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")] public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles"; - [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File with eth address used by codex node. Used for balance checking if geth/contracts information is provided.")] - public string EthAddressFile { get; set; } = "/root/codex-testnet-starter/scripts/eth.address"; + [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File(s) with eth address used by codex node. Used for balance checking if geth/contracts information is provided. Semi-colon separated.")] + public string EthAddressFile { get; set; } = + "/root/codex-testnet-starter/scripts/eth.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_2.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_3.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_4.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_5.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_6.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_7.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_8.address"; public string LogPath { diff --git a/Tools/AutoClient/LoadBalancer.cs b/Tools/AutoClient/LoadBalancer.cs new file mode 100644 index 00000000..407db704 --- /dev/null +++ b/Tools/AutoClient/LoadBalancer.cs @@ -0,0 +1,131 @@ +using Logging; + +namespace AutoClient +{ + public class LoadBalancer + { + private readonly List instances; + private readonly object instanceLock = new object(); + + private class Cdx + { + private readonly ILog log; + private readonly CodexWrapper instance; + private readonly List> queue = new List>(); + private readonly object queueLock = new object(); + private bool running = true; + private Task worker = Task.CompletedTask; + + public Cdx(App app, CodexWrapper instance) + { + Id = instance.Node.GetName(); + log = new LogPrefixer(app.Log, $"[Queue-{Id}]"); + this.instance = instance; + } + + public string Id { get; } + + public void Start() + { + worker = Task.Run(Worker); + } + + public void Stop() + { + running = false; + worker.Wait(); + } + + public void CheckErrors() + { + if (worker.IsFaulted) throw worker.Exception; + } + + public void Queue(Action action) + { + if (queue.Count > 2) log.Log("Queue full. Waiting..."); + while (queue.Count > 2) + { + Thread.Sleep(TimeSpan.FromSeconds(5.0)); + } + + lock (queueLock) + { + queue.Add(action); + } + } + + private void Worker() + { + try + { + while (running) + { + while (queue.Count == 0) Thread.Sleep(TimeSpan.FromSeconds(5.0)); + + Action action = w => { }; + lock (queueLock) + { + action = queue[0]; + queue.RemoveAt(0); + } + + action(instance); + } + } + catch (Exception ex) + { + log.Error("Exception in worker: " + ex); + throw; + } + } + } + + public LoadBalancer(App app, CodexWrapper[] instances) + { + this.instances = instances.Select(i => new Cdx(app, i)).ToList(); + } + + public void Start() + { + foreach (var i in instances) i.Start(); + } + + public void Stop() + { + foreach (var i in instances) i.Stop(); + } + + public void DispatchOnCodex(Action action) + { + lock (instanceLock) + { + var i = instances.First(); + instances.RemoveAt(0); + instances.Add(i); + + i.Queue(action); + } + } + + public void DispatchOnSpecificCodex(Action action, string id) + { + lock (instanceLock) + { + var i = instances.Single(a => a.Id == id); + instances.Remove(i); + instances.Add(i); + + i.Queue(action); + } + } + + public void CheckErrors() + { + lock (instanceLock) + { + foreach (var i in instances) i.CheckErrors(); + } + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs index 895651ae..7b9e8857 100644 --- a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs +++ b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs @@ -1,5 +1,4 @@ -using GethConnector; -using Logging; +using Logging; using Utils; namespace AutoClient.Modes.FolderStore @@ -8,37 +7,45 @@ namespace AutoClient.Modes.FolderStore { private readonly LogPrefixer log; private readonly GethConnector.GethConnector? connector; - private readonly EthAddress? address; + private readonly EthAddress[] addresses; public BalanceChecker(App app) { log = new LogPrefixer(app.Log, "(Balance) "); connector = GethConnector.GethConnector.Initialize(app.Log); - address = LoadAddress(app); + addresses = LoadAddresses(app); + + log.Log($"Loaded Eth-addresses for checking: {addresses.Length}"); + foreach (var addr in addresses) log.Log(" - " + addr); } - private EthAddress? LoadAddress(App app) + private EthAddress[] LoadAddresses(App app) { try { - if (string.IsNullOrEmpty(app.Config.EthAddressFile)) return null; - if (!File.Exists(app.Config.EthAddressFile)) return null; + if (string.IsNullOrEmpty(app.Config.EthAddressFile)) return Array.Empty(); - return new EthAddress( - File.ReadAllText(app.Config.EthAddressFile) - .Trim() - .Replace("\n", "") - .Replace(Environment.NewLine, "") - ); + var tokens = app.Config.EthAddressFile.Split(";", StringSplitOptions.RemoveEmptyEntries); + return tokens.Select(ConvertToAddress).Where(a => a != null).Cast().ToArray(); } catch (Exception exc) { log.Error($"Failed to load eth address from file: {exc}"); - return null; + return Array.Empty(); } } + private EthAddress? ConvertToAddress(string t) + { + if (!File.Exists(t)) return null; + return new EthAddress( + File.ReadAllText(t) + .Trim() + .Replace("\n", "") + .Replace(Environment.NewLine, "")); + } + public void Check() { if (connector == null) @@ -46,35 +53,32 @@ namespace AutoClient.Modes.FolderStore Log("Connector not configured. Can't check balances."); return; } - if (address == null) - { - Log("EthAddress not found. Can't check balances."); - return; - } - try + foreach (var address in addresses) { - PerformCheck(); - } - catch (Exception exc) - { - Log($"Exception while checking balances: {exc}"); + try + { + PerformCheck(address); + } + catch (Exception exc) + { + Log($"Exception while checking balances: {exc}"); + } } } - private void PerformCheck() + private void PerformCheck(EthAddress address) { var geth = connector!.GethNode; var contracts = connector!.CodexContracts; - var addr = address!; - var eth = geth.GetEthBalance(addr); - var tst = contracts.GetTestTokenBalance(addr); + var eth = geth.GetEthBalance(address); + var tst = contracts.GetTestTokenBalance(address); Log($"Balances: [{eth}] - [{tst}]"); - if (eth.Eth < 1) TryAddEth(geth, addr); - if (tst.Tst < 1) TryAddTst(contracts, addr); + if (eth.Eth < 1) TryAddEth(geth, address); + if (tst.Tst < 1) TryAddTst(contracts, address); } private void TryAddEth(GethPlugin.IGethNode geth, EthAddress addr) diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index 8963d378..327eb00b 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -4,32 +4,88 @@ using Utils; namespace AutoClient.Modes.FolderStore { + public interface IFileSaverEventHandler + { + void SaveChanges(); + void OnFailure(); + } + public class FileSaver + { + private readonly ILog log; + private readonly LoadBalancer loadBalancer; + private readonly Stats stats; + private readonly string folderFile; + private readonly FileStatus entry; + private readonly IFileSaverEventHandler handler; + + public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + { + this.log = log; + this.loadBalancer = loadBalancer; + this.stats = stats; + this.folderFile = folderFile; + this.entry = entry; + this.handler = handler; + } + + public void Process() + { + if (string.IsNullOrEmpty(entry.CodexNodeId)) + { + DispatchToAny(); + } + else + { + DispatchToSpecific(); + } + } + + private void DispatchToAny() + { + loadBalancer.DispatchOnCodex(instance => + { + entry.CodexNodeId = instance.Node.GetName(); + handler.SaveChanges(); + + var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + run.Process(); + }); + } + + private void DispatchToSpecific() + { + loadBalancer.DispatchOnSpecificCodex(instance => + { + var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + run.Process(); + }, entry.CodexNodeId); + } + } + + public class FileSaverRun { private readonly ILog log; private readonly CodexWrapper instance; private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly Action saveChanges; + private readonly IFileSaverEventHandler handler; private readonly QuotaCheck quotaCheck; - public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, Action saveChanges) + public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) { this.log = log; this.instance = instance; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.saveChanges = saveChanges; + this.handler = handler; quotaCheck = new QuotaCheck(log, folderFile, instance); } - public bool HasFailed { get; private set; } - public void Process() { - HasFailed = false; if (HasRecentPurchase()) { Log($"Purchase running: '{entry.PurchaseId}'"); @@ -71,7 +127,7 @@ namespace AutoClient.Modes.FolderStore Thread.Sleep(TimeSpan.FromMinutes(1.0)); } Log("Could not upload: Insufficient local storage quota."); - HasFailed = true; + handler.OnFailure(); return false; } @@ -108,7 +164,7 @@ namespace AutoClient.Modes.FolderStore var result = instance.Node.LocalFiles(); if (result == null) return false; if (result.Content == null) return false; - + var localCids = result.Content.Where(c => !string.IsNullOrEmpty(c.Cid.Id)).Select(c => c.Cid.Id).ToArray(); var isFound = localCids.Any(c => c.ToLowerInvariant() == entry.BasicCid.ToLowerInvariant()); if (isFound) @@ -150,9 +206,9 @@ namespace AutoClient.Modes.FolderStore entry.BasicCid = string.Empty; stats.FailedUploads++; log.Error("Failed to upload: " + exc); - HasFailed = true; + handler.OnFailure(); } - saveChanges(); + handler.SaveChanges(); } private void CreateNewPurchase() @@ -168,16 +224,17 @@ namespace AutoClient.Modes.FolderStore WaitForStarted(request); stats.StorageRequestStats.SuccessfullyStarted++; - saveChanges(); + handler.SaveChanges(); Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); } catch (Exception exc) { entry.ClearPurchase(); - saveChanges(); + handler.SaveChanges(); + log.Error("Failed to start new purchase: " + exc); - HasFailed = true; + handler.OnFailure(); } } @@ -196,7 +253,7 @@ namespace AutoClient.Modes.FolderStore throw new Exception("CID received from storage request was not protected."); } - saveChanges(); + handler.SaveChanges(); Log("Saved new purchaseId: " + entry.PurchaseId); return request; } @@ -230,8 +287,9 @@ namespace AutoClient.Modes.FolderStore else if (!update.IsSubmitted) { Log("Request failed to start. State: " + update.State); + entry.ClearPurchase(); - saveChanges(); + handler.SaveChanges(); return; } } @@ -239,10 +297,10 @@ namespace AutoClient.Modes.FolderStore } catch (Exception exc) { - HasFailed = true; + handler.OnFailure(); Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}"); throw; - } + } } private void WaitForSubmitted(IStoragePurchaseContract request) diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index ed358ccc..4d090e6d 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -1,29 +1,30 @@ using Logging; +using Utils; namespace AutoClient.Modes.FolderStore { - public class FolderSaver + public class FolderSaver : IFileSaverEventHandler { private const string FolderSaverFilename = "foldersaver.json"; private readonly App app; - private readonly CodexWrapper instance; + private readonly LoadBalancer loadBalancer; private readonly JsonFile statusFile; private readonly FolderStatus status; private readonly BalanceChecker balanceChecker; private int changeCounter = 0; private int failureCount = 0; - public FolderSaver(App app, CodexWrapper instance) + public FolderSaver(App app, LoadBalancer loadBalancer) { this.app = app; - this.instance = instance; + this.loadBalancer = loadBalancer; balanceChecker = new BalanceChecker(app); statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); status = statusFile.Load(); } - public void Run(CancellationTokenSource cts) + public void Run() { var folderFiles = Directory.GetFiles(app.Config.FolderToStore); if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); @@ -32,7 +33,8 @@ namespace AutoClient.Modes.FolderStore balanceChecker.Check(); foreach (var folderFile in folderFiles) { - if (cts.IsCancellationRequested) return; + if (app.Cts.IsCancellationRequested) return; + loadBalancer.CheckErrors(); if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename)) { @@ -42,7 +44,7 @@ namespace AutoClient.Modes.FolderStore if (failureCount > 3) { app.Log.Error("Failure count reached threshold. Stopping..."); - cts.Cancel(); + app.Cts.Cancel(); return; } @@ -87,7 +89,6 @@ namespace AutoClient.Modes.FolderStore { var fileSaver = CreateFileSaver(folderFile, entry); fileSaver.Process(); - if (fileSaver.HasFailed) failureCount++; } private void SaveFolderSaverJsonFile() @@ -101,7 +102,6 @@ namespace AutoClient.Modes.FolderStore ApplyPadding(folderFile); var fileSaver = CreateFileSaver(folderFile, entry); fileSaver.Process(); - if (fileSaver.HasFailed) failureCount++; if (!string.IsNullOrEmpty(entry.EncodedCid)) { @@ -126,33 +126,27 @@ namespace AutoClient.Modes.FolderStore if (info.Length < min) { var required = Math.Max(1024, min - info.Length); - status.Padding = paddingMessage + GenerateRandomString(required); + status.Padding = paddingMessage + RandomUtils.GenerateRandomString(required); statusFile.Save(status); } } - private string GenerateRandomString(long required) - { - var result = ""; - while (result.Length < required) - { - var bytes = new byte[1024]; - random.NextBytes(bytes); - result += string.Join("", bytes.Select(b => b.ToString())); - } - - return result; - } - private FileSaver CreateFileSaver(string folderFile, FileStatus entry) { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry, saveChanges: () => - { - statusFile.Save(status); - changeCounter++; - }); + return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this); + } + + public void SaveChanges() + { + statusFile.Save(status); + changeCounter++; + } + + public void OnFailure() + { + failureCount++; } } } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs index abb9c52a..3ea0c0c3 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs @@ -11,6 +11,7 @@ [Serializable] public class FileStatus { + public string CodexNodeId { get; set; } = string.Empty; public string Filename { get; set; } = string.Empty; public string BasicCid { get; set; } = string.Empty; public string EncodedCid { get; set; } = string.Empty; diff --git a/Tools/AutoClient/Modes/FolderStore/JsonFile.cs b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs index 6d0f972d..365e77c3 100644 --- a/Tools/AutoClient/Modes/FolderStore/JsonFile.cs +++ b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs @@ -6,6 +6,7 @@ namespace AutoClient.Modes.FolderStore { private readonly App app; private readonly string filePath; + private readonly object fileLock = new object(); public JsonFile(App app, string filePath) { @@ -15,35 +16,41 @@ namespace AutoClient.Modes.FolderStore public T Load() { - try + lock (fileLock) { - if (!File.Exists(filePath)) + try { - var state = new T(); - Save(state); - return state; + if (!File.Exists(filePath)) + { + var state = new T(); + Save(state); + return state; + } + var text = File.ReadAllText(filePath); + return JsonConvert.DeserializeObject(text)!; + } + catch (Exception exc) + { + app.Log.Error("Failed to load state: " + exc); + throw; } - var text = File.ReadAllText(filePath); - return JsonConvert.DeserializeObject(text)!; - } - catch (Exception exc) - { - app.Log.Error("Failed to load state: " + exc); - throw; } } public void Save(T state) { - try + lock (fileLock) { - var json = JsonConvert.SerializeObject(state, Formatting.Indented); - File.WriteAllText(filePath, json); - } - catch (Exception exc) - { - app.Log.Error("Failed to save state: " + exc); - throw; + try + { + var json = JsonConvert.SerializeObject(state, Formatting.Indented); + File.WriteAllText(filePath, json); + } + catch (Exception exc) + { + app.Log.Error("Failed to save state: " + exc); + throw; + } } } } diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 477ea9bf..1d368706 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -2,31 +2,28 @@ namespace AutoClient.Modes { - public class FolderStoreMode : IMode + public class FolderStoreMode { private readonly App app; - private readonly string folder; - private readonly PurchaseInfo purchaseInfo; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); private Task checkTask = Task.CompletedTask; + private readonly LoadBalancer loadBalancer; - public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo) + public FolderStoreMode(App app, LoadBalancer loadBalancer) { this.app = app; - this.folder = folder; - this.purchaseInfo = purchaseInfo; + this.loadBalancer = loadBalancer; } - public void Start(CodexWrapper instance, int index) + public void Start() { checkTask = Task.Run(() => { try { - var saver = new FolderSaver(app, instance); - while (!cts.IsCancellationRequested) + var saver = new FolderSaver(app, loadBalancer); + while (!app.Cts.IsCancellationRequested) { - saver.Run(cts); + saver.Run(); } } catch (Exception ex) @@ -39,7 +36,7 @@ namespace AutoClient.Modes public void Stop() { - cts.Cancel(); + app.Cts.Cancel(); checkTask.Wait(); } } diff --git a/Tools/AutoClient/Modes/Mode.cs b/Tools/AutoClient/Modes/Mode.cs deleted file mode 100644 index afc76f7a..00000000 --- a/Tools/AutoClient/Modes/Mode.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace AutoClient.Modes -{ - public interface IMode - { - void Start(CodexWrapper node, int index); - void Stop(); - } -} diff --git a/Tools/AutoClient/Modes/PurchasingMode.cs b/Tools/AutoClient/Modes/PurchasingMode.cs deleted file mode 100644 index 206ab365..00000000 --- a/Tools/AutoClient/Modes/PurchasingMode.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Logging; - -namespace AutoClient.Modes -{ - public class PurchasingMode : IMode - { - private readonly List purchasers = new List(); - private readonly App app; - private Task starterTask = Task.CompletedTask; - - public PurchasingMode(App app) - { - this.app = app; - } - - public void Start(CodexWrapper node, int index) - { - for (var i = 0; i < app.Config.NumConcurrentPurchases; i++) - { - purchasers.Add(new AutomaticPurchaser(app, new LogPrefixer(app.Log, $"({i}) "), node)); - } - - var delayPerPurchaser = - TimeSpan.FromSeconds(10 * index) + - TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases; - - starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser)); - } - - private async Task StartPurchasers(TimeSpan delayPerPurchaser) - { - foreach (var purchaser in purchasers) - { - purchaser.Start(); - await Task.Delay(delayPerPurchaser); - } - } - - public void Stop() - { - starterTask.Wait(); - foreach (var purchaser in purchasers) - { - purchaser.Stop(); - } - } - } -} diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index 6a35ccf5..36eb361a 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -1,22 +1,22 @@ using ArgsUniform; using AutoClient; using AutoClient.Modes; -using AutoClient.Modes.FolderStore; using CodexClient; using GethPlugin; using Utils; +using WebUtils; +using Logging; public class Program { private readonly App app; - private readonly List modes = new List(); public Program(Configuration config) { app = new App(config); } - public static async Task Main(string[] args) + public static void Main(string[] args) { var cts = new CancellationTokenSource(); Console.CancelKeyPress += (sender, args) => cts.Cancel(); @@ -30,60 +30,37 @@ public class Program } var p = new Program(config); - await p.Run(); + p.Run(); } - public async Task Run() + public void Run() { - await Task.CompletedTask; + if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!"); var codexNodes = CreateCodexWrappers(); + var loadBalancer = new LoadBalancer(app, codexNodes); + loadBalancer.Start(); - var i = 0; - foreach (var cdx in codexNodes) - { - var mode = CreateMode(); - modes.Add(mode); - - mode.Start(cdx, i); - i++; - } + var folderStore = new FolderStoreMode(app, loadBalancer); + folderStore.Start(); app.Cts.Token.WaitHandle.WaitOne(); - foreach (var mode in modes) mode.Stop(); - modes.Clear(); + folderStore.Stop(); + loadBalancer.Stop(); app.Log.Log("Done"); } - private IMode CreateMode() - { - if (!string.IsNullOrEmpty(app.Config.FolderToStore)) - { - return CreateFolderStoreMode(); - } - - return new PurchasingMode(app); - } - - private IMode CreateFolderStoreMode() - { - if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!"); - - return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo( - purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes), - purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120) - )); - } - private CodexWrapper[] CreateCodexWrappers() { var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries); var result = new List(); + var i = 1; foreach (var e in endpointStrs) { - result.Add(CreateCodexWrapper(e)); + result.Add(CreateCodexWrapper(e, i)); + i++; } return result.ToArray(); @@ -91,7 +68,7 @@ public class Program private readonly string LogLevel = "TRACE;info:discv5,providers,routingtable,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,websock,ws-session,muxedupgrade,upgrade,identify,contracts,clock,serde,json,serialization,JSONRPC-WS-CLIENT,JSONRPC-HTTP-CLIENT,repostore"; - private CodexWrapper CreateCodexWrapper(string endpoint) + private CodexWrapper CreateCodexWrapper(string endpoint, int number) { var splitIndex = endpoint.LastIndexOf(':'); var host = endpoint.Substring(0, splitIndex); @@ -103,11 +80,14 @@ public class Program port: port ); - var instance = CodexInstance.CreateFromApiEndpoint("[AutoClient]", address, EthAccountGenerator.GenerateNew()); - var node = app.CodexNodeFactory.CreateCodexNode(instance); + var numberStr = number.ToString().PadLeft(3, '0'); + var log = new LogPrefixer(app.Log, $"[{numberStr}] "); + var httpFactory = new HttpFactory(log, new AutoClientWebTimeSet()); + var codexNodeFactory = new CodexNodeFactory(log: log, httpFactory: httpFactory, dataDir: app.Config.DataPath); + var instance = CodexInstance.CreateFromApiEndpoint($"[AC-{numberStr}]", address, EthAccountGenerator.GenerateNew()); + var node = codexNodeFactory.CreateCodexNode(instance); node.SetLogLevel(LogLevel); - return new CodexWrapper(app, node); } diff --git a/Tools/BiblioTech/AdminChecker.cs b/Tools/BiblioTech/AdminChecker.cs index 532220de..96f89691 100644 --- a/Tools/BiblioTech/AdminChecker.cs +++ b/Tools/BiblioTech/AdminChecker.cs @@ -1,7 +1,6 @@ using BiblioTech.Options; using Discord; using Discord.WebSocket; -using Org.BouncyCastle.Utilities; namespace BiblioTech { diff --git a/Tools/BiblioTech/CallDispatcher.cs b/Tools/BiblioTech/CallDispatcher.cs new file mode 100644 index 00000000..b24ea830 --- /dev/null +++ b/Tools/BiblioTech/CallDispatcher.cs @@ -0,0 +1,66 @@ +using Logging; + +namespace BiblioTech +{ + public class CallDispatcher + { + private readonly ILog log; + private readonly object _lock = new object(); + private readonly List queue = new List(); + private readonly AutoResetEvent autoResetEvent = new AutoResetEvent(false); + + public CallDispatcher(ILog log) + { + this.log = log; + } + + public void Add(Action call) + { + lock (_lock) + { + queue.Add(call); + autoResetEvent.Set(); + if (queue.Count > 100) + { + log.Error("Queue overflow!"); + queue.Clear(); + } + } + } + + public void Start() + { + Task.Run(() => + { + while (true) + { + try + { + Worker(); + } + catch (Exception ex) + { + log.Error("Exception in CallDispatcher: " + ex); + } + } + }); + } + + private void Worker() + { + autoResetEvent.WaitOne(); + var tasks = Array.Empty(); + + lock (_lock) + { + tasks = queue.ToArray(); + queue.Clear(); + } + + foreach (var task in tasks) + { + task(); + } + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs new file mode 100644 index 00000000..2d8ec602 --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs @@ -0,0 +1,78 @@ +using BiblioTech.Rewards; +using Discord; +using Logging; +using System.Threading.Tasks; + +namespace BiblioTech.CodexChecking +{ + public class ActiveP2pRoleRemover + { + private readonly Configuration config; + private readonly ILog log; + private readonly CheckRepo repo; + + public ActiveP2pRoleRemover(Configuration config, ILog log, CheckRepo repo) + { + this.config = config; + this.log = log; + this.repo = repo; + } + + public void Start() + { + if (config.ActiveP2pRoleDurationMinutes > 0) + { + Task.Run(Worker); + } + } + + private void Worker() + { + var loopDelay = TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes) / 60; + var min = TimeSpan.FromMinutes(10.0); + if (loopDelay < min) loopDelay = min; + + try + { + while (true) + { + Thread.Sleep(loopDelay); + CheckP2pRoleRemoval(); + } + } + catch (Exception ex) + { + log.Error($"Exception in {nameof(ActiveP2pRoleRemover)}: {ex}"); + Environment.Exit(1); + } + } + + private void CheckP2pRoleRemoval() + { + var expiryMoment = DateTime.UtcNow - TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes); + + Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, expiryMoment), + Program.Config.ActiveP2pParticipantRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + if (report.UploadCheck.CompletedUtc > expiryMoment) return; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return; + + await giver.RemoveActiveP2pParticipant(user.Id); + } + + private bool ShouldRemoveRole(IUser user, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + + if (report.UploadCheck.CompletedUtc > expiryMoment) return false; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return false; + + return true; + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CheckRepo.cs b/Tools/BiblioTech/CodexChecking/CheckRepo.cs new file mode 100644 index 00000000..a8b2860b --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CheckRepo.cs @@ -0,0 +1,78 @@ +using Newtonsoft.Json; + +namespace BiblioTech.CodexChecking +{ + public class CheckRepo + { + private const string modelFilename = "model.json"; + private readonly Configuration config; + private readonly object _lock = new object(); + private CheckRepoModel? model = null; + + public CheckRepo(Configuration config) + { + this.config = config; + } + + public CheckReport GetOrCreate(ulong userId) + { + lock (_lock) + { + if (model == null) LoadModel(); + + var existing = model.Reports.SingleOrDefault(r => r.UserId == userId); + if (existing == null) + { + var newEntry = new CheckReport + { + UserId = userId, + }; + model.Reports.Add(newEntry); + SaveChanges(); + return newEntry; + } + return existing; + } + } + + public void SaveChanges() + { + File.WriteAllText(GetModelFilepath(), JsonConvert.SerializeObject(model, Formatting.Indented)); + } + + private void LoadModel() + { + if (!File.Exists(GetModelFilepath())) + { + model = new CheckRepoModel(); + SaveChanges(); + return; + } + + model = JsonConvert.DeserializeObject(File.ReadAllText(GetModelFilepath())); + } + + private string GetModelFilepath() + { + return Path.Combine(config.ChecksDataPath, modelFilename); + } + } + + public class CheckRepoModel + { + public List Reports { get; set; } = new List(); + } + + public class CheckReport + { + public ulong UserId { get; set; } + public TransferCheck UploadCheck { get; set; } = new TransferCheck(); + public TransferCheck DownloadCheck { get; set; } = new TransferCheck(); + } + + public class TransferCheck + { + public DateTime CompletedUtc { get; set; } = DateTime.MinValue; + public string UniqueData { get; set; } = string.Empty; + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs new file mode 100644 index 00000000..c5b9361a --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -0,0 +1,219 @@ +using CodexClient; +using FileUtils; +using Logging; +using Utils; + +namespace BiblioTech.CodexChecking +{ + public interface ICheckResponseHandler + { + Task CheckNotStarted(); + Task NowCompleted(ulong userId, string checkName); + Task GiveRoleReward(); + + Task InvalidData(); + Task CouldNotDownloadCid(); + Task GiveCidToUser(string cid); + Task GiveDataFileToUser(string fileContent); + + Task ToAdminChannel(string msg); + } + + public class CodexTwoWayChecker + { + private readonly ILog log; + private readonly Configuration config; + private readonly CheckRepo repo; + private readonly CodexWrapper codexWrapper; + + public CodexTwoWayChecker(ILog log, Configuration config, CheckRepo repo, CodexWrapper codexWrapper) + { + this.log = log; + this.config = config; + this.repo = repo; + this.codexWrapper = codexWrapper; + } + + public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + var cid = UploadData(check.UniqueData); + await handler.GiveCidToUser(cid); + } + + public async Task VerifyDownloadCheck(ICheckResponseHandler handler, ulong userId, string receivedData) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + await handler.CheckNotStarted(); + return; + } + + Log($"Verifying for downloadCheck: received: '{receivedData}' check: '{check.UniqueData}'"); + if (string.IsNullOrEmpty(receivedData) || receivedData != check.UniqueData) + { + await handler.InvalidData(); + return; + } + + await CheckNowCompleted(handler, check, userId, "DownloadCheck"); + } + + public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + await handler.GiveDataFileToUser(check.UniqueData); + } + + public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId, string receivedCid) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (string.IsNullOrEmpty(receivedCid)) + { + await handler.InvalidData(); + return; + } + + var manifest = GetManifest(receivedCid); + if (manifest == null) + { + await handler.CouldNotDownloadCid(); + return; + } + + if (IsManifestLengthCompatible(handler, check, manifest)) + { + if (IsContentCorrect(handler, check, receivedCid)) + { + await CheckNowCompleted(handler, check, userId, "UploadCheck"); + return; + } + } + + await handler.InvalidData(); + } + + private string GenerateUniqueData() + { + return $"{RandomBusyMessage.Get().Substring(5)}{RandomUtils.GenerateRandomString(12)}"; + } + + private string UploadData(string uniqueData) + { + var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString()); + + try + { + File.WriteAllText(filePath, uniqueData); + var file = new TrackedFile(log, filePath, "checkData"); + + return codexWrapper.OnCodex(node => + { + return node.UploadFile(file).Id; + }); + } + catch (Exception ex) + { + log.Error("Exception when uploading data: " + ex); + throw; + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + private Manifest? GetManifest(string receivedCid) + { + try + { + return codexWrapper.OnCodex(node => + { + return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest; + }); + } + catch + { + return null; + } + } + + private bool IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) + { + var dataLength = check.UniqueData.Length; + var manifestLength = manifest.OriginalBytes.SizeInBytes; + + Log($"Checking manifest length: dataLength={dataLength},manifestLength={manifestLength}"); + + return + manifestLength > (dataLength - 1) && + manifestLength < (dataLength + 1); + } + + private bool IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) + { + try + { + var content = codexWrapper.OnCodex(node => + { + var file = node.DownloadContent(new ContentId(receivedCid)); + if (file == null) return string.Empty; + try + { + return File.ReadAllText(file.Filename).Trim(); + } + finally + { + if (File.Exists(file.Filename)) File.Delete(file.Filename); + } + }); + + Log($"Checking content: content={content},check={check.UniqueData}"); + return content == check.UniqueData; + } + catch + { + return false; + } + } + + private async Task CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId, string checkName) + { + await handler.NowCompleted(userId, checkName); + + check.CompletedUtc = DateTime.UtcNow; + repo.SaveChanges(); + + await CheckUserForRoleRewards(handler, userId); + } + + private async Task CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId); + + if (check.UploadCheck.CompletedUtc != DateTime.MinValue && + check.DownloadCheck.CompletedUtc != DateTime.MinValue) + { + await handler.GiveRoleReward(); + } + } + + private void Log(string msg) + { + log.Log(msg); + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs new file mode 100644 index 00000000..3c295d7f --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs @@ -0,0 +1,85 @@ +using CodexClient; +using IdentityModel.Client; +using Logging; +using Utils; +using WebUtils; + +namespace BiblioTech.CodexChecking +{ + public class CodexWrapper + { + private readonly CodexNodeFactory factory; + private readonly ILog log; + private readonly Configuration config; + private readonly object codexLock = new object(); + private ICodexNode? currentCodexNode; + + public CodexWrapper(ILog log, Configuration config) + { + this.log = log; + this.config = config; + + var httpFactory = CreateHttpFactory(); + factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); + } + + public void OnCodex(Action action) + { + lock (codexLock) + { + action(Get()); + } + } + + public T OnCodex(Func func) + { + lock (codexLock) + { + return func(Get()); + } + } + + private ICodexNode Get() + { + if (currentCodexNode == null) + { + currentCodexNode = CreateCodex(); + } + + return currentCodexNode; + } + + private ICodexNode CreateCodex() + { + var endpoint = config.CodexEndpoint; + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); + + var address = new Address( + logName: $"cdx@{host}:{port}", + host: host, + port: port + ); + + var instance = CodexInstance.CreateFromApiEndpoint("ac", address); + return factory.CreateCodexNode(instance); + } + + private HttpFactory CreateHttpFactory() + { + if (string.IsNullOrEmpty(config.CodexEndpointAuth) || !config.CodexEndpointAuth.Contains(":")) + { + return new HttpFactory(log); + } + + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + + return new HttpFactory(log, onClientCreated: client => + { + client.SetBasicAuthentication(tokens[0], tokens[1]); + }); + } + } +} diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs deleted file mode 100644 index 15728d0d..00000000 --- a/Tools/BiblioTech/CodexCidChecker.cs +++ /dev/null @@ -1,204 +0,0 @@ -using CodexClient; -using Logging; -using Utils; - -namespace BiblioTech -{ - public class CodexCidChecker - { - private static readonly string nl = Environment.NewLine; - private readonly Configuration config; - private readonly ILog log; - private readonly Mutex checkMutex = new Mutex(); - private readonly CodexNodeFactory factory; - private ICodexNode? currentCodexNode; - - public CodexCidChecker(Configuration config, ILog log) - { - this.config = config; - this.log = log; - - factory = new CodexNodeFactory(log, dataDir: config.DataPath); - - if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) - { - throw new Exception("Todo: codexnodefactory httpfactory support basicauth!"); - //var tokens = config.CodexEndpointAuth.Split(':'); - //if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); - //client.SetBasicAuthentication(tokens[0], tokens[1]); - } - } - - public CheckResponse PerformCheck(string cid) - { - if (string.IsNullOrEmpty(config.CodexEndpoint)) - { - return new CheckResponse(false, "Codex CID checker is not (yet) available.", ""); - } - - try - { - checkMutex.WaitOne(); - var codex = GetCodex(); - var nodeCheck = CheckCodex(codex); - if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); - - return PerformCheck(codex, cid); - } - catch (Exception ex) - { - return new CheckResponse(false, "Internal server error", ex.ToString()); - } - finally - { - checkMutex.ReleaseMutex(); - } - } - - private CheckResponse PerformCheck(ICodexNode codex, string cid) - { - try - { - var manifest = codex.DownloadManifestOnly(new ContentId(cid)); - return SuccessMessage(manifest); - } - catch (Exception ex) - { - return UnexpectedException(ex); - } - } - - #region Response formatting - - private CheckResponse SuccessMessage(LocalDataset content) - { - return FormatResponse( - success: true, - title: $"Success: '{content.Cid}'", - error: "", - $"size: {content.Manifest.OriginalBytes} bytes", - $"blockSize: {content.Manifest.BlockSize} bytes", - $"protected: {content.Manifest.Protected}" - ); - } - - private CheckResponse UnexpectedException(Exception ex) - { - return FormatResponse( - success: false, - title: "Unexpected error", - error: ex.ToString(), - content: "Details will be sent to the bot-admin channel." - ); - } - - private CheckResponse UnexpectedReturnCode(string response) - { - var msg = "Unexpected return code. Response: " + response; - return FormatResponse( - success: false, - title: "Unexpected return code", - error: msg, - content: msg - ); - } - - private CheckResponse FailedToFetch(string response) - { - var msg = "Failed to download content. Response: " + response; - return FormatResponse( - success: false, - title: "Could not download content", - error: msg, - msg, - $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" - ); - } - - private CheckResponse CidFormatInvalid(string response) - { - return FormatResponse( - success: false, - title: "Invalid format", - error: "", - content: "Provided CID is not formatted correctly." - ); - } - - private CheckResponse FormatResponse(bool success, string title, string error, params string[] content) - { - var msg = string.Join(nl, - new string[] - { - title, - "```" - } - .Concat(content) - .Concat(new string[] - { - "```" - }) - ) + nl + nl; - - return new CheckResponse(success, msg, error); - } - - #endregion - - #region Codex Node API - - private ICodexNode GetCodex() - { - if (currentCodexNode == null) currentCodexNode = CreateCodex(); - return currentCodexNode; - } - - private bool CheckCodex(ICodexNode node) - { - try - { - var info = node.GetDebugInfo(); - if (info == null || string.IsNullOrEmpty(info.Id)) return false; - return true; - } - catch (Exception e) - { - log.Error(e.ToString()); - return false; - } - } - - private ICodexNode CreateCodex() - { - var endpoint = config.CodexEndpoint; - var splitIndex = endpoint.LastIndexOf(':'); - var host = endpoint.Substring(0, splitIndex); - var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); - - var address = new Address( - logName: $"cdx@{host}:{port}", - host: host, - port: port - ); - - var instance = CodexInstance.CreateFromApiEndpoint("ac", address); - return factory.CreateCodexNode(instance); - } - - #endregion - } - - public class CheckResponse - { - public CheckResponse(bool success, string message, string error) - { - Success = success; - Message = message; - Error = error; - } - - public bool Success { get; } - public string Message { get; } - public string Error { get; } - } -} diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index b6f2f7ca..10e1e24b 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -4,6 +4,9 @@ using Discord; using Newtonsoft.Json; using BiblioTech.Rewards; using Logging; +using BiblioTech.CodexChecking; +using Nethereum.Model; +using static Org.BouncyCastle.Math.EC.ECCurve; namespace BiblioTech { @@ -11,13 +14,15 @@ namespace BiblioTech { private readonly DiscordSocketClient client; private readonly CustomReplacement replacement; + private readonly ActiveP2pRoleRemover roleRemover; private readonly BaseCommand[] commands; private readonly ILog log; - public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, params BaseCommand[] commands) + public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, ActiveP2pRoleRemover roleRemover, params BaseCommand[] commands) { this.client = client; this.replacement = replacement; + this.roleRemover = roleRemover; this.commands = commands; this.log = log; client.Ready += Client_Ready; @@ -30,10 +35,15 @@ namespace BiblioTech Program.AdminChecker.SetGuild(guild); log.Log($"Initializing for guild: '{guild.Name}'"); - var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray(); - if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel"); - Program.AdminChecker.SetAdminChannel(adminChannels.First()); - Program.RoleDriver = new RoleDriver(client, log, replacement); + var adminChannel = GetChannel(guild, Program.Config.AdminChannelId); + if (adminChannel == null) throw new Exception("No admin message channel"); + var chainEventsChannel = GetChannel(guild, Program.Config.ChainEventsChannelId); + var rewardsChannel = GetChannel(guild, Program.Config.RewardsChannelId); + + Program.AdminChecker.SetAdminChannel(adminChannel); + Program.RoleDriver = new RoleDriver(client, Program.UserRepo, log, rewardsChannel); + Program.ChainActivityHandler = new ChainActivityHandler(log, Program.UserRepo); + Program.EventsSender = new ChainEventsSender(log, replacement, chainEventsChannel); var builders = commands.Select(c => { @@ -65,6 +75,8 @@ namespace BiblioTech { log.Log($"{cmd.Name} ({cmd.Description}) [{DescribOptions(cmd.Options)}]"); } + + roleRemover.Start(); } catch (HttpException exception) { @@ -72,9 +84,16 @@ namespace BiblioTech log.Error(json); throw; } + Program.Dispatcher.Start(); log.Log("Initialized."); } + private SocketTextChannel? GetChannel(SocketGuild guild, ulong id) + { + if (id == 0) return null; + return guild.TextChannels.SingleOrDefault(c => c.Id == id); + } + private string DescribOptions(IReadOnlyCollection options) { return string.Join(",", options.Select(DescribeOption).ToArray()); diff --git a/Tools/BiblioTech/Commands/AdminCommand.cs b/Tools/BiblioTech/Commands/AdminCommand.cs index 45b3fc29..f9877181 100644 --- a/Tools/BiblioTech/Commands/AdminCommand.cs +++ b/Tools/BiblioTech/Commands/AdminCommand.cs @@ -8,16 +8,10 @@ namespace BiblioTech.Commands private readonly ClearUserAssociationCommand clearCommand = new ClearUserAssociationCommand(); private readonly ReportCommand reportCommand = new ReportCommand(); private readonly WhoIsCommand whoIsCommand = new WhoIsCommand(); - private readonly AddSprCommand addSprCommand; - private readonly ClearSprsCommand clearSprsCommand; - private readonly GetSprCommand getSprCommand; private readonly LogReplaceCommand logReplaceCommand; - public AdminCommand(SprCommand sprCommand, CustomReplacement replacement) + public AdminCommand(CustomReplacement replacement) { - addSprCommand = new AddSprCommand(sprCommand); - clearSprsCommand = new ClearSprsCommand(sprCommand); - getSprCommand = new GetSprCommand(sprCommand); logReplaceCommand = new LogReplaceCommand(replacement); } @@ -30,9 +24,6 @@ namespace BiblioTech.Commands clearCommand, reportCommand, whoIsCommand, - addSprCommand, - clearSprsCommand, - getSprCommand, logReplaceCommand }; @@ -53,9 +44,6 @@ namespace BiblioTech.Commands await clearCommand.CommandHandler(context); await reportCommand.CommandHandler(context); await whoIsCommand.CommandHandler(context); - await addSprCommand.CommandHandler(context); - await clearSprsCommand.CommandHandler(context); - await getSprCommand.CommandHandler(context); await logReplaceCommand.CommandHandler(context); } @@ -144,78 +132,6 @@ namespace BiblioTech.Commands } } - public class AddSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("spr", "Codex SPR", true); - - public AddSprCommand(SprCommand sprCommand) - : base(name: "addspr", - description: "Adds a Codex SPR, to be given to users with '/boot'.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var spr = await stringOption.Parse(context); - - if (!string.IsNullOrEmpty(spr) ) - { - sprCommand.Add(spr); - await context.Followup("A-OK!"); - } - else - { - await context.Followup("SPR is null or empty."); - } - } - } - - public class ClearSprsCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("areyousure", "set to 'true' if you are.", true); - - public ClearSprsCommand(SprCommand sprCommand) - : base(name: "clearsprs", - description: "Clears all Codex SPRs in the bot. Users won't be able to use '/boot' till new ones are added.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var areyousure = await stringOption.Parse(context); - - if (areyousure != "true") return; - - sprCommand.Clear(); - await context.Followup("Cleared all SPRs."); - } - } - - public class GetSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - - public GetSprCommand(SprCommand sprCommand) - : base(name: "getsprs", - description: "Shows all Codex SPRs in the bot.") - { - this.sprCommand = sprCommand; - } - - protected override async Task onSubCommand(CommandContext context) - { - await context.Followup("SPRs: " + string.Join(", ", sprCommand.Get().Select(s => $"'{s}'"))); - } - } - public class LogReplaceCommand : SubCommandOption { private readonly CustomReplacement replacement; diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs deleted file mode 100644 index 1e77ce26..00000000 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ /dev/null @@ -1,111 +0,0 @@ -using BiblioTech.Options; -using Discord; - -namespace BiblioTech.Commands -{ - public class CheckCidCommand : BaseCommand - { - private readonly StringOption cidOption = new StringOption( - name: "cid", - description: "Codex Content-Identifier", - isRequired: true); - private readonly CodexCidChecker checker; - private readonly CidStorage cidStorage; - - public CheckCidCommand(CodexCidChecker checker) - { - this.checker = checker; - this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt")); - } - - public override string Name => "check"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Checks if content is available in the testnet."; - public override CommandOption[] Options => new[] { cidOption }; - - protected override async Task Invoke(CommandContext context) - { - var user = context.Command.User; - var cid = await cidOption.Parse(context); - if (string.IsNullOrEmpty(cid)) - { - await context.Followup("Option 'cid' was not received."); - return; - } - - var response = checker.PerformCheck(cid); - await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); - - if (response.Success) - { - await CheckAltruisticRole(context, user, cid, response.Message); - return; - } - - await context.Followup(response.Message); - } - - private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage) - { - if (cidStorage.TryAddCid(cid, user.Id)) - { - if (await GiveAltruisticRole(context, user, responseMessage)) - { - return; - } - } - else - { - await context.Followup($"{responseMessage}\n\nThis CID has already been used by another user. No role will be granted."); - return; - } - - await context.Followup(responseMessage); - } - - private async Task GiveAltruisticRole(CommandContext context, IUser user, string responseMessage) - { - try - { - await Program.RoleDriver.GiveAltruisticRole(user); - await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!"); - return true; - } - catch (Exception ex) - { - await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}"); - return false; - } - } - } - - public class CidStorage - { - private readonly string filePath; - private static readonly object _lock = new object(); - - public CidStorage(string filePath) - { - this.filePath = filePath; - if (!File.Exists(filePath)) - { - File.WriteAllText(filePath, string.Empty); - } - } - - public bool TryAddCid(string cid, ulong userId) - { - lock (_lock) - { - var existingEntries = File.ReadAllLines(filePath); - if (existingEntries.Any(line => line.Split(',')[0] == cid)) - { - return false; - } - - File.AppendAllLines(filePath, new[] { $"{cid},{userId}" }); - return true; - } - } - } -} diff --git a/Tools/BiblioTech/Commands/CheckDownloadCommand.cs b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs new file mode 100644 index 00000000..0e1aa563 --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs @@ -0,0 +1,58 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckDownloadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption contentOption = new StringOption( + name: "content", + description: "Content of the downloaded file", + isRequired: false); + + public CheckDownloadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkdownload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the download connectivity of your Codex node."; + public override CommandOption[] Options => [contentOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var content = await contentOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(content)) + { + await checker.StartDownloadCheck(handler, user.Id); + } + else + { + if (content.Length > 1024) + { + await context.Followup("Provided content is too long!"); + return; + } + await checker.VerifyDownloadCheck(handler, user.Id, content); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckDownloadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs new file mode 100644 index 00000000..5428e86b --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -0,0 +1,88 @@ +using System.Linq; +using BiblioTech.CodexChecking; +using BiblioTech.Options; +using Discord; + +namespace BiblioTech.Commands +{ + public class CheckResponseHandler : ICheckResponseHandler + { + private CommandContext context; + private readonly IUser user; + + public CheckResponseHandler(CommandContext context, IUser user) + { + this.context = context; + this.user = user; + } + + public async Task CheckNotStarted() + { + await context.Followup("Run this command without any arguments first, to begin the check process."); + } + + public async Task CouldNotDownloadCid() + { + await context.Followup("Could not download the CID."); + } + + public async Task GiveCidToUser(string cid) + { + await context.Followup( + FormatCatchyMessage("[💾] Please download this CID using your Codex node.", + $"👉 `{cid}`.", + "👉 Then provide the *content of the downloaded file* as argument to this command.")); + } + + public async Task GiveDataFileToUser(string fileContent) + { + await context.SendFile(fileContent, + FormatCatchyMessage("[💿] Please download the attached file.", + "👉 Upload it to your Codex node.", + "👉 Then provide the *CID* as argument to this command.")); + } + + private string FormatCatchyMessage(string title, params string[] content) + { + var entries = new List(); + entries.Add(title); + entries.Add("```"); + entries.AddRange(content); + entries.Add("```"); + return string.Join(Environment.NewLine, entries.ToArray()); + } + + public async Task GiveRoleReward() + { + try + { + await Program.RoleDriver.RunRoleGiver(async r => + { + await r.GiveAltruisticRole(user.Id); + await r.GiveActiveP2pParticipant(user.Id); + }); + await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!"); + } + catch (Exception ex) + { + await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user <@{user.Id}>: {ex.Message}"); + } + } + + public async Task InvalidData() + { + await context.Followup("The received data didn't match. Check has failed."); + } + + public async Task NowCompleted(ulong userId, string checkName) + { + await context.Followup("Successfully completed the check!"); + await Program.AdminChecker.SendInAdminChannel($"User <@{userId}> has completed check: {checkName}"); + } + + public async Task ToAdminChannel(string msg) + { + await Program.AdminChecker.SendInAdminChannel(msg); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckUploadCommand.cs b/Tools/BiblioTech/Commands/CheckUploadCommand.cs new file mode 100644 index 00000000..b1589b5c --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckUploadCommand.cs @@ -0,0 +1,53 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckUploadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption cidOption = new StringOption( + name: "cid", + description: "Codex Content-Identifier", + isRequired: false); + + public CheckUploadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkupload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the upload connectivity of your Codex node."; + public override CommandOption[] Options => [cidOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var cid = await cidOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(cid)) + { + await checker.StartUploadCheck(handler, user.Id); + } + else + { + await checker.VerifyUploadCheck(handler, user.Id, cid); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckUploadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Commands/SprCommand.cs b/Tools/BiblioTech/Commands/SprCommand.cs deleted file mode 100644 index 4b3235de..00000000 --- a/Tools/BiblioTech/Commands/SprCommand.cs +++ /dev/null @@ -1,48 +0,0 @@ -using BiblioTech.Options; - -namespace BiblioTech.Commands -{ - public class SprCommand : BaseCommand - { - private readonly Random random = new Random(); - private readonly List knownSprs = new List(); - - public override string Name => "boot"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Gets an SPR. (Signed peer record, used for bootstrapping.)"; - - protected override async Task Invoke(CommandContext context) - { - await ReplyWithRandomSpr(context); - } - - public void Add(string spr) - { - if (knownSprs.Contains(spr)) return; - knownSprs.Add(spr); - } - - public void Clear() - { - knownSprs.Clear(); - } - - public string[] Get() - { - return knownSprs.ToArray(); - } - - private async Task ReplyWithRandomSpr(CommandContext context) - { - if (!knownSprs.Any()) - { - await context.Followup("I'm sorry, no SPRs are available... :c"); - return; - } - - var i = random.Next(0, knownSprs.Count); - var spr = knownSprs[i]; - await context.Followup($"Your SPR: `{spr}`"); - } - } -} diff --git a/Tools/BiblioTech/Commands/UserAssociateCommand.cs b/Tools/BiblioTech/Commands/UserAssociateCommand.cs index 61d55fb9..dbe7dced 100644 --- a/Tools/BiblioTech/Commands/UserAssociateCommand.cs +++ b/Tools/BiblioTech/Commands/UserAssociateCommand.cs @@ -69,8 +69,8 @@ namespace BiblioTech.Commands { await context.Followup(new string[] { - "Done! Thank you for joining the test net!", - "By default, the bot will @-mention you with test-net related notifications.", + "Done! Thank you for joining!", + "By default, the bot will @-mention you with discord role notifications.", $"You can enable/disable this behavior with the '/{notifyCommand.Name}' command." }); diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 342fbb08..a8187379 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -26,9 +26,6 @@ namespace BiblioTech [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")] public ulong ChainEventsChannelId { get; set; } - [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] - public ulong AltruisticRoleId { get; set; } - [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")] public int RewardApiPort { get; set; } = 31080; @@ -47,8 +44,40 @@ namespace BiblioTech [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")] public string CodexEndpointAuth { get; set; } = ""; + #region Role Rewards + + /// + /// Awarded when both checkupload and checkdownload have been completed. + /// + [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] + public ulong AltruisticRoleId { get; set; } + + /// + /// Awarded as long as either checkupload or checkdownload were completed within the last ActiveP2pRoleDuration minutes. + /// + [Uniform("active-p2p-role-id", "apri", "ACTIVEP2PROLEID", false, "ID of discord server role for active p2p participants.")] + public ulong ActiveP2pParticipantRoleId { get; set; } + + [Uniform("active-p2p-role-duration", "aprd", "ACTIVEP2PROLEDURATION", false, "Duration in minutes for the active p2p participant role from the last successful check command.")] + public int ActiveP2pRoleDurationMinutes { get; set; } + + /// + /// Awarded as long as the user is hosting at least 1 slot. + /// + [Uniform("active-host-role-id", "ahri", "ACTIVEHOSTROLEID", false, "Id of discord server role for active slot hosters.")] + public ulong ActiveHostRoleId { get; set; } + + /// + /// Awarded as long as the user has at least 1 active storage purchase contract. + /// + [Uniform("active-client-role-id", "acri", "ACTIVECLIENTROLEID", false, "Id of discord server role for users with at least 1 active purchase contract.")] + public ulong ActiveClientRoleId { get; set; } + + #endregion + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); + public string ChecksDataPath => Path.Combine(DataPath, "checks"); public string LogPath => Path.Combine(DataPath, "logs"); public bool DebugNoDiscord => NoDiscord == 1; } diff --git a/Tools/BiblioTech/LoggingRoleDriver.cs b/Tools/BiblioTech/LoggingRoleDriver.cs index f5435523..b1759af9 100644 --- a/Tools/BiblioTech/LoggingRoleDriver.cs +++ b/Tools/BiblioTech/LoggingRoleDriver.cs @@ -15,18 +15,72 @@ namespace BiblioTech this.log = log; } - public async Task GiveAltruisticRole(IUser user) + public async Task RunRoleGiver(Func action) { await Task.CompletedTask; - - log.Log($"Give altruistic role to {user.Id}"); + await action(new LoggingRoleGiver(log)); } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) { await Task.CompletedTask; + } - log.Log(JsonConvert.SerializeObject(rewards, Formatting.None)); + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) + { + await Task.CompletedTask; + } + + private class LoggingRoleGiver : IRoleGiver + { + private readonly ILog log; + + public LoggingRoleGiver(ILog log) + { + this.log = log; + } + + public async Task GiveActiveClient(ulong userId) + { + log.Log($"Giving ActiveClient role to " + userId); + await Task.CompletedTask; + } + + public async Task GiveActiveHost(ulong userId) + { + log.Log($"Giving ActiveHost role to " + userId); + await Task.CompletedTask; + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + log.Log($"Giving ActiveP2p role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + log.Log($"Removing ActiveP2p role from " + userId); + await Task.CompletedTask; + } + + public async Task GiveAltruisticRole(ulong userId) + { + log.Log($"Giving Altruistic role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveClient(ulong userId) + { + log.Log($"Removing ActiveClient role from " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveHost(ulong userId) + { + log.Log($"Removing ActiveHost role from " + userId); + await Task.CompletedTask; + } } } } diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 42a065ae..fbd45b36 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -49,6 +49,23 @@ namespace BiblioTech.Options } } + public async Task SendFile(string fileContent, string message) + { + if (fileContent.Length < 1) throw new Exception("File content is empty."); + + var filename = Guid.NewGuid().ToString() + ".tmp"; + File.WriteAllText(filename, fileContent); + + await Command.FollowupWithFileAsync(filename, "Codex_UploadCheckFile.txt", text: message, ephemeral: true); + + // Detached task for cleaning up the stream resources. + _ = Task.Run(() => + { + Thread.Sleep(TimeSpan.FromMinutes(2)); + File.Delete(filename); + }); + } + private string FormatChunk(string[] chunk) { return string.Join(Environment.NewLine, chunk); diff --git a/Tools/BiblioTech/Options/StringOption.cs b/Tools/BiblioTech/Options/StringOption.cs index 15e04c9a..047bf15a 100644 --- a/Tools/BiblioTech/Options/StringOption.cs +++ b/Tools/BiblioTech/Options/StringOption.cs @@ -12,11 +12,12 @@ namespace BiblioTech.Options public async Task Parse(CommandContext context) { var strData = context.Options.SingleOrDefault(o => o.Name == Name); - if (strData == null) + if (strData == null && IsRequired) { await context.Followup("String option not received."); return null; } + if (strData == null) return null; return strData.Value as string; } } diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 46dee29c..0ce84bde 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -1,9 +1,13 @@ using ArgsUniform; +using BiblioTech.CodexChecking; using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; using Discord.WebSocket; +using DiscordRewards; using Logging; +using Nethereum.Model; +using Newtonsoft.Json; namespace BiblioTech { @@ -12,16 +16,17 @@ namespace BiblioTech private DiscordSocketClient client = null!; private CustomReplacement replacement = null!; + public static CallDispatcher Dispatcher { get; private set; } = null!; public static Configuration Config { get; private set; } = null!; public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; public static IDiscordRoleDriver RoleDriver { get; set; } = null!; + public static ChainActivityHandler ChainActivityHandler { get; set; } = null!; + public static ChainEventsSender EventsSender { get; set; } = null!; public static ILog Log { get; private set; } = null!; public static Task Main(string[] args) { - Log = new ConsoleLog(); - var uniformArgs = new ArgsUniform(PrintHelp, args); Config = uniformArgs.Parse(); @@ -30,9 +35,12 @@ namespace BiblioTech new ConsoleLog() ); + Dispatcher = new CallDispatcher(Log); + EnsurePath(Config.DataPath); EnsurePath(Config.UserDataPath); EnsurePath(Config.EndpointsPath); + EnsurePath(Config.ChecksDataPath); return new Program().MainAsync(args); } @@ -80,18 +88,20 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; - var checker = new CodexCidChecker(Config, Log); + var checkRepo = new CheckRepo(Config); + var codexWrapper = new CodexWrapper(Log, Config); + var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); - var sprCommand = new SprCommand(); - var handler = new CommandHandler(Log, client, replacement, + var roleRemover = new ActiveP2pRoleRemover(Config, Log, checkRepo); + var handler = new CommandHandler(Log, client, replacement, roleRemover, new GetBalanceCommand(associateCommand), new MintCommand(associateCommand), - sprCommand, associateCommand, notifyCommand, - new CheckCidCommand(checker), - new AdminCommand(sprCommand, replacement) + new CheckUploadCommand(checker), + new CheckDownloadCommand(checker), + new AdminCommand(replacement) ); await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); diff --git a/Tools/BiblioTech/RandomBusyMessage.cs b/Tools/BiblioTech/RandomBusyMessage.cs index 290c9273..8f0477a7 100644 --- a/Tools/BiblioTech/RandomBusyMessage.cs +++ b/Tools/BiblioTech/RandomBusyMessage.cs @@ -14,7 +14,9 @@ "Analyzing the wavelengths...", "Charging the flux-capacitor...", "Jumping to hyperspace...", - "Computing the ultimate answer..." + "Computing the ultimate answer...", + "Turning it off and on again...", + "Compiling from sources..." }; public static string Get() diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs new file mode 100644 index 00000000..d35d9fe9 --- /dev/null +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -0,0 +1,134 @@ +using Discord; +using DiscordRewards; +using Logging; + +namespace BiblioTech.Rewards +{ + public class ChainActivityHandler + { + private readonly ILog log; + private readonly UserRepo repo; + private ActiveUserIds? previousIds = null; + + public ChainActivityHandler(ILog log, UserRepo repo) + { + this.log = log; + this.repo = repo; + } + + public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) + { + var activeUserIds = ConvertToUserIds(activeChainAddresses); + if (!HasChanged(activeUserIds)) return; + + await GiveAndRemoveRoles(activeUserIds); + } + + private async Task GiveAndRemoveRoles(ActiveUserIds activeUserIds) + { + await Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, activeUserIds), + whenDone: g => GiveRolesToRemaining(g, activeUserIds), + Program.Config.ActiveClientRoleId, + Program.Config.ActiveHostRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, ActiveUserIds activeIds) + { + if (roleId == Program.Config.ActiveClientRoleId) + { + await CheckUserWithRole(user, activeIds.Clients, giver.RemoveActiveClient); + } + else if (roleId == Program.Config.ActiveHostRoleId) + { + await CheckUserWithRole(user, activeIds.Hosts, giver.RemoveActiveHost); + } + else + { + throw new Exception("Unknown roleId received!"); + } + } + + private async Task CheckUserWithRole(IUser user, List activeUsers, Func removeActiveRole) + { + if (ShouldUserHaveRole(user, activeUsers)) + { + activeUsers.Remove(user.Id); + } + else + { + await removeActiveRole(user.Id); + } + } + + private bool ShouldUserHaveRole(IUser user, List activeUsers) + { + return activeUsers.Any(id => id == user.Id); + } + + private async Task GiveRolesToRemaining(IRoleGiver giver, ActiveUserIds ids) + { + foreach (var client in ids.Clients) await giver.GiveActiveClient(client); + foreach (var host in ids.Hosts) await giver.GiveActiveHost(host); + } + + private bool HasChanged(ActiveUserIds activeUserIds) + { + if (previousIds == null) + { + previousIds = activeUserIds; + return true; + } + + if (!IsEquivalent(previousIds.Hosts, activeUserIds.Hosts)) return true; + if (!IsEquivalent(previousIds.Clients, activeUserIds.Clients)) return true; + return false; + } + + private static bool IsEquivalent(IEnumerable a, IEnumerable b) + { + return a.SequenceEqual(b); + } + + private ActiveUserIds ConvertToUserIds(ActiveChainAddresses activeChainAddresses) + { + return new ActiveUserIds + ( + hosts: Map(activeChainAddresses.Hosts), + clients: Map(activeChainAddresses.Clients) + ); + } + + private ulong[] Map(string[] ethAddresses) + { + var result = new List(); + foreach (var ethAddress in ethAddresses) + { + var userMaybe = repo.GetUserDataForAddressMaybe(new Utils.EthAddress(ethAddress)); + if (userMaybe != null) + { + result.Add(userMaybe.DiscordId); + } + } + + return result.Order().ToArray(); + } + + private void Log(string msg) + { + log.Log(msg); + } + + private class ActiveUserIds + { + public ActiveUserIds(IEnumerable hosts, IEnumerable clients) + { + Hosts = hosts.ToList(); + Clients = clients.ToList(); + } + + public List Hosts { get; } + public List Clients { get; } + } + } +} diff --git a/Tools/BiblioTech/Rewards/CustomReplacement.cs b/Tools/BiblioTech/Rewards/CustomReplacement.cs index dbd8eaf3..912aa789 100644 --- a/Tools/BiblioTech/Rewards/CustomReplacement.cs +++ b/Tools/BiblioTech/Rewards/CustomReplacement.cs @@ -28,14 +28,14 @@ namespace BiblioTech.Rewards public void Add(string from, string to) { - if (replacements.ContainsKey(from)) + AddOrUpdate(from, to); + + var lower = from.ToLowerInvariant(); + if (lower != from) { - replacements[from] = to; - } - else - { - replacements.Add(from, to); + AddOrUpdate(lower, to); } + Save(); } @@ -55,6 +55,18 @@ namespace BiblioTech.Rewards return result; } + private void AddOrUpdate(string from, string to) + { + if (replacements.ContainsKey(from)) + { + replacements[from] = to; + } + else + { + replacements.Add(from, to); + } + } + private void Save() { ReplaceJson[] replaces = replacements.Select(pair => diff --git a/Tools/BiblioTech/Rewards/RewardContext.cs b/Tools/BiblioTech/Rewards/RewardContext.cs deleted file mode 100644 index d05d5f6d..00000000 --- a/Tools/BiblioTech/Rewards/RewardContext.cs +++ /dev/null @@ -1,102 +0,0 @@ -using Discord.WebSocket; -using Discord; -using DiscordRewards; - -namespace BiblioTech.Rewards -{ - public class RewardContext - { - private readonly Dictionary users; - private readonly Dictionary roles; - private readonly SocketTextChannel? rewardsChannel; - - public RewardContext(Dictionary users, Dictionary roles, SocketTextChannel? rewardsChannel) - { - this.users = users; - this.roles = roles; - this.rewardsChannel = rewardsChannel; - } - - public async Task ProcessGiveRewardsCommand(UserReward[] rewards) - { - foreach (var rewardCommand in rewards) - { - if (roles.ContainsKey(rewardCommand.RewardCommand.RewardId)) - { - var role = roles[rewardCommand.RewardCommand.RewardId]; - await ProcessRewardCommand(role, rewardCommand); - } - else - { - Program.Log.Error($"RoleID not found on guild: {rewardCommand.RewardCommand.RewardId}"); - } - } - } - - private async Task ProcessRewardCommand(RoleReward role, UserReward reward) - { - foreach (var user in reward.Users) - { - await GiveReward(role, user); - } - } - - private async Task GiveReward(RoleReward role, UserData user) - { - if (!users.ContainsKey(user.DiscordId)) - { - Program.Log.Log($"User by id '{user.DiscordId}' not found."); - return; - } - - var guildUser = users[user.DiscordId]; - - var alreadyHas = guildUser.RoleIds.ToArray(); - var logMessage = $"Giving reward '{role.SocketRole.Id}' to user '{user.DiscordId}'({user.Name})[" + - $"alreadyHas:{string.Join(",", alreadyHas.Select(a => a.ToString()))}]: "; - - - if (alreadyHas.Any(r => r == role.Reward.RoleId)) - { - logMessage += "Already has role"; - Program.Log.Log(logMessage); - return; - } - - await GiveRole(guildUser, role.SocketRole); - await SendNotification(role, user, guildUser); - await Task.Delay(1000); - logMessage += "Role given. Notification sent."; - Program.Log.Log(logMessage); - } - - private async Task GiveRole(IGuildUser user, SocketRole role) - { - try - { - Program.Log.Log($"Giving role {role.Name}={role.Id} to user {user.DisplayName}"); - await user.AddRoleAsync(role); - } - catch (Exception ex) - { - Program.Log.Error($"Failed to give role '{role.Name}' to user '{user.DisplayName}': {ex}"); - } - } - - private async Task SendNotification(RoleReward reward, UserData userData, IGuildUser user) - { - try - { - if (userData.NotificationsEnabled && rewardsChannel != null) - { - var msg = reward.Reward.Message.Replace(RewardConfig.UsernameTag, $"<@{user.Id}>"); - await rewardsChannel.SendMessageAsync(msg); - } - } - catch (Exception ex) - { - Program.Log.Error($"Failed to notify user '{user.DisplayName}' about role '{reward.SocketRole.Name}': {ex}"); - } - } - } -} diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 3a20a084..43a01ad5 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -4,10 +4,26 @@ using Microsoft.AspNetCore.Mvc; namespace BiblioTech.Rewards { + /// + /// We like callbacks in this interface because we're trying to batch role-modifying operations, + /// So that we're not poking the server lots of times very quickly. + /// public interface IDiscordRoleDriver { - Task GiveRewards(GiveRewardsCommand rewards); - Task GiveAltruisticRole(IUser user); + Task RunRoleGiver(Func action); + Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate); + Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate); + } + + public interface IRoleGiver + { + Task GiveAltruisticRole(ulong userId); + Task GiveActiveP2pParticipant(ulong userId); + Task RemoveActiveP2pParticipant(ulong userId); + Task GiveActiveHost(ulong userId); + Task RemoveActiveHost(ulong userId); + Task GiveActiveClient(ulong userId); + Task RemoveActiveClient(ulong userId); } [Route("api/[controller]")] @@ -21,16 +37,19 @@ namespace BiblioTech.Rewards } [HttpPost] - public async Task Give(GiveRewardsCommand cmd) + public async Task Give(EventsAndErrors cmd) { - try + Program.Dispatcher.Add(() => { - await Program.RoleDriver.GiveRewards(cmd); - } - catch (Exception ex) + Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses).Wait(); + }); + + Program.Dispatcher.Add(() => { - Program.Log.Error("Exception: " + ex); - } + Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors).Wait(); + }); + + await Task.CompletedTask; return "OK"; } } diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index 1964da8e..1cc9f3f2 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -1,6 +1,7 @@ using Discord; using Discord.WebSocket; using DiscordRewards; +using k8s.KubeConfigModels; using Logging; using Newtonsoft.Json; using Utils; @@ -10,145 +11,52 @@ namespace BiblioTech.Rewards public class RoleDriver : IDiscordRoleDriver { private readonly DiscordSocketClient client; + private readonly UserRepo userRepo; private readonly ILog log; private readonly SocketTextChannel? rewardsChannel; - private readonly ChainEventsSender eventsSender; - private readonly RewardRepo repo = new RewardRepo(); - public RoleDriver(DiscordSocketClient client, ILog log, CustomReplacement replacement) + public RoleDriver(DiscordSocketClient client, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) { this.client = client; + this.userRepo = userRepo; this.log = log; - rewardsChannel = GetChannel(Program.Config.RewardsChannelId); - eventsSender = new ChainEventsSender(log, replacement, GetChannel(Program.Config.ChainEventsChannelId)); + this.rewardsChannel = rewardsChannel; } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task RunRoleGiver(Func action) { - log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'"); + var context = OpenRoleModifyContext(); + var mapper = new RoleMapper(context); + await action(mapper); + } - if (rewards.Rewards.Any()) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) + { + await IterateUsersWithRoles(onUserWithRole, g => Task.CompletedTask, rolesToIterate); + } + + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) + { + var context = OpenRoleModifyContext(); + var mapper = new RoleMapper(context); + foreach (var user in context.Users) { - await ProcessRewards(rewards); - } - - await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors); - } - - public async Task GiveAltruisticRole(IUser user) - { - var guild = GetGuild(); - var role = guild.Roles.SingleOrDefault(r => r.Id == Program.Config.AltruisticRoleId); - if (role == null) return; - - var guildUser = guild.Users.SingleOrDefault(u => u.Id == user.Id); - if (guildUser == null) return; - - await guildUser.AddRoleAsync(role); - } - - private async Task ProcessRewards(GiveRewardsCommand rewards) - { - try - { - var guild = GetGuild(); - // We load all role and user information first, - // so we don't ask the server for the same info multiple times. - var context = new RewardContext( - await LoadAllUsers(guild), - LookUpAllRoles(guild, rewards), - rewardsChannel); - - await context.ProcessGiveRewardsCommand(LookUpUsers(rewards)); - } - catch (Exception ex) - { - log.Error("Failed to process rewards: " + ex); - } - } - - private SocketTextChannel? GetChannel(ulong id) - { - if (id == 0) return null; - return GetGuild().TextChannels.SingleOrDefault(c => c.Id == id); - } - - private async Task> LoadAllUsers(SocketGuild guild) - { - log.Log("Loading all users.."); - var result = new Dictionary(); - var users = guild.GetUsersAsync(); - await foreach (var ulist in users) - { - foreach (var u in ulist) + foreach (var role in rolesToIterate) { - result.Add(u.Id, u); - //var roleIds = string.Join(",", u.RoleIds.Select(r => r.ToString()).ToArray()); - //log.Log($" > {u.Id}({u.DisplayName}) has [{roleIds}]"); - } - } - return result; - } - - private Dictionary LookUpAllRoles(SocketGuild guild, GiveRewardsCommand rewards) - { - var result = new Dictionary(); - foreach (var r in rewards.Rewards) - { - if (!result.ContainsKey(r.RewardId)) - { - var rewardConfig = repo.Rewards.SingleOrDefault(rr => rr.RoleId == r.RewardId); - if (rewardConfig == null) + if (user.RoleIds.Contains(role)) { - log.Log($"No Reward is configured for id '{r.RewardId}'."); - } - else - { - var socketRole = guild.GetRole(r.RewardId); - if (socketRole == null) - { - log.Log($"Guild Role by id '{r.RewardId}' not found."); - } - else - { - result.Add(r.RewardId, new RoleReward(socketRole, rewardConfig)); - } + await onUserWithRole(mapper, user, role); } } } - - return result; + await whenDone(mapper); } - private UserReward[] LookUpUsers(GiveRewardsCommand rewards) + private RoleModifyContext OpenRoleModifyContext() { - return rewards.Rewards.Select(LookUpUserData).ToArray(); - } - - private UserReward LookUpUserData(RewardUsersCommand command) - { - return new UserReward(command, - command.UserAddresses - .Select(LookUpUserDataForAddress) - .Where(d => d != null) - .Cast() - .ToArray()); - } - - private UserData? LookUpUserDataForAddress(string address) - { - try - { - var userData = Program.UserRepo.GetUserDataForAddress(new EthAddress(address)); - if (userData != null) log.Log($"User '{userData.Name}' was looked up."); - else log.Log($"Lookup for user was unsuccessful. EthAddress: '{address}'"); - return userData; - } - catch (Exception ex) - { - log.Error("Error during UserData lookup: " + ex); - return null; - } + var context = new RoleModifyContext(GetGuild(), userRepo, log, rewardsChannel); + context.Initialize(); + return context; } private SocketGuild GetGuild() @@ -163,27 +71,48 @@ namespace BiblioTech.Rewards } } - public class RoleReward + public class RoleMapper : IRoleGiver { - public RoleReward(SocketRole socketRole, RewardConfig reward) + private readonly RoleModifyContext context; + + public RoleMapper(RoleModifyContext context) { - SocketRole = socketRole; - Reward = reward; + this.context = context; } - public SocketRole SocketRole { get; } - public RewardConfig Reward { get; } - } - - public class UserReward - { - public UserReward(RewardUsersCommand rewardCommand, UserData[] users) + public async Task GiveActiveClient(ulong userId) { - RewardCommand = rewardCommand; - Users = users; + await context.GiveRole(userId, Program.Config.ActiveClientRoleId); } - public RewardUsersCommand RewardCommand { get; } - public UserData[] Users { get; } + public async Task GiveActiveHost(ulong userId) + { + await context.GiveRole(userId, Program.Config.ActiveHostRoleId); + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + await context.GiveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task GiveAltruisticRole(ulong userId) + { + await context.GiveRole(userId, Program.Config.AltruisticRoleId); + } + + public async Task RemoveActiveClient(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveClientRoleId); + } + + public async Task RemoveActiveHost(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveHostRoleId); + } } } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs new file mode 100644 index 00000000..6fd4fb48 --- /dev/null +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -0,0 +1,135 @@ +using Discord.WebSocket; +using Discord; +using DiscordRewards; +using Nethereum.Model; +using Logging; + +namespace BiblioTech.Rewards +{ + public class RoleModifyContext + { + private Dictionary users = new(); + private Dictionary roles = new(); + private DateTime lastLoad = DateTime.MinValue; + private readonly object _lock = new object(); + + private readonly SocketGuild guild; + private readonly UserRepo userRepo; + private readonly ILog log; + private readonly SocketTextChannel? rewardsChannel; + + public RoleModifyContext(SocketGuild guild, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) + { + this.guild = guild; + this.userRepo = userRepo; + this.log = log; + this.rewardsChannel = rewardsChannel; + } + + public void Initialize() + { + lock (_lock) + { + var span = DateTime.UtcNow - lastLoad; + if (span > TimeSpan.FromMinutes(10)) + { + lastLoad = DateTime.UtcNow; + log.Log("Loading all users and roles..."); + var task = LoadAllUsers(guild); + task.Wait(); + this.users = task.Result; + this.roles = LoadAllRoles(guild); + } + } + } + + public IGuildUser[] Users => users.Values.ToArray(); + + public async Task GiveRole(ulong userId, ulong roleId) + { + Log($"Giving role {roleId} to user {userId}"); + var role = GetRole(roleId); + var guildUser = GetUser(userId); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.AddRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Added role '{role.Name}' for user <@{userId}>."); + + await SendNotification(guildUser, role); + } + + public async Task RemoveRole(ulong userId, ulong roleId) + { + Log($"Removing role {roleId} from user {userId}"); + var role = GetRole(roleId); + var guildUser = GetUser(userId); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.RemoveRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Removed role '{role.Name}' for user <@{userId}>."); + } + + private SocketRole? GetRole(ulong roleId) + { + if (roles.ContainsKey(roleId)) return roles[roleId]; + return null; + } + + private IGuildUser? GetUser(ulong userId) + { + if (users.ContainsKey(userId)) return users[userId]; + return null; + } + + private void Log(string msg) + { + log.Log(msg); + } + + private async Task> LoadAllUsers(SocketGuild guild) + { + var result = new Dictionary(); + var users = guild.GetUsersAsync(); + await foreach (var ulist in users) + { + foreach (var u in ulist) + { + result.Add(u.Id, u); + } + } + return result; + } + + private Dictionary LoadAllRoles(SocketGuild guild) + { + var result = new Dictionary(); + var roles = guild.Roles.ToArray(); + foreach (var role in roles) + { + result.Add(role.Id, role); + } + return result; + } + + private async Task SendNotification(IGuildUser user, SocketRole role) + { + try + { + var userData = userRepo.GetUser(user); + if (userData == null) return; + + if (userData.NotificationsEnabled && rewardsChannel != null) + { + var msg = $"<@{user.Id}> has received '{role.Name}'."; + await rewardsChannel.SendMessageAsync(msg); + } + } + catch (Exception ex) + { + log.Error($"Failed to notify user '{user.DisplayName}' about role '{role.Name}': {ex}"); + } + } + } +} diff --git a/Tools/BiblioTech/UserRepo.cs b/Tools/BiblioTech/UserRepo.cs index d1a766ab..701fdfe9 100644 --- a/Tools/BiblioTech/UserRepo.cs +++ b/Tools/BiblioTech/UserRepo.cs @@ -41,6 +41,12 @@ namespace BiblioTech return cache.Values.ToArray(); } + public UserData GetUser(IUser user) + { + if (cache.Count == 0) LoadAllUserData(); + return GetOrCreate(user); + } + public void AddMintEventForUser(IUser user, EthAddress usedAddress, Transaction? eth, Transaction? tokens) { lock (repoLock) @@ -68,10 +74,10 @@ namespace BiblioTech lock (repoLock) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { - result.Add("User has not joined the test net."); + result.Add("User has not interacted with bot."); } else { @@ -100,36 +106,33 @@ namespace BiblioTech public string[] GetUserReport(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) return new[] { "User has not joined the test net." }; return userData.CreateOverview(); } public string[] GetUserReport(EthAddress ethAddress) { - var userData = GetUserDataForAddress(ethAddress); + var userData = GetUserDataForAddressMaybe(ethAddress); if (userData == null) return new[] { "No user is using this eth address." }; return userData.CreateOverview(); } - public UserData? GetUserDataForAddress(EthAddress? address) + public UserData? GetUserDataForAddressMaybe(EthAddress? address) { if (address == null) return null; - // If this becomes a performance problem, switch to in-memory cached list. - var files = Directory.GetFiles(Program.Config.UserDataPath); - foreach (var file in files) + var lower = address.Address.ToLowerInvariant(); + if (string.IsNullOrEmpty(lower)) return null; + + if (cache.Count == 0) LoadAllUserData(); + foreach (var item in cache.Values) { - try + if (item.CurrentAddress != null && + item.CurrentAddress.Address.ToLowerInvariant() == lower) { - var user = JsonConvert.DeserializeObject(File.ReadAllText(file))!; - if (user.CurrentAddress != null && - user.CurrentAddress.Address == address.Address) - { - return user; - } + return item; } - catch { } } return null; @@ -137,7 +140,7 @@ namespace BiblioTech private SetAddressResponse SetUserAddress(IUser user, EthAddress? address) { - if (GetUserDataForAddress(address) != null) + if (GetUserDataForAddressMaybe(address) != null) { return SetAddressResponse.AddressAlreadyInUse; } @@ -152,13 +155,12 @@ namespace BiblioTech private void SetUserNotification(IUser user, bool notifyEnabled) { - var userData = GetUserData(user); - if (userData == null) return; + var userData = GetOrCreate(user); userData.NotificationsEnabled = notifyEnabled; SaveUserData(userData); } - private UserData? GetUserData(IUser user) + private UserData? GetUserDataMaybe(IUser user) { if (cache.ContainsKey(user.Id)) { @@ -177,7 +179,7 @@ namespace BiblioTech private UserData GetOrCreate(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { return CreateAndSaveNewUserData(user); diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index 6a5c5759..c5ddd9a0 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -21,7 +21,7 @@ namespace TestNetRewarder return result == "Pong"; } - public async Task SendRewards(GiveRewardsCommand command) + public async Task SendRewards(EventsAndErrors command) { if (command == null) return false; var result = await HttpPostJson(command); diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 2b49ef11..eed26962 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -8,7 +8,6 @@ namespace TestNetRewarder public class Processor : ITimeSegmentHandler { private readonly RequestBuilder builder; - private readonly RewardChecker rewardChecker; private readonly EventsFormatter eventsFormatter; private readonly ChainState chainState; private readonly Configuration config; @@ -24,22 +23,16 @@ namespace TestNetRewarder lastPeriodUpdateUtc = DateTime.UtcNow; builder = new RequestBuilder(); - rewardChecker = new RewardChecker(builder); eventsFormatter = new EventsFormatter(config); - var handler = new ChainStateChangeHandlerMux( - rewardChecker.Handler, - eventsFormatter - ); - - chainState = new ChainState(log, contracts, handler, config.HistoryStartUtc, + chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc, doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); } public async Task Initialize() { var events = eventsFormatter.GetInitializationEvents(config); - var request = builder.Build(events, Array.Empty()); + var request = builder.Build(chainState, events, Array.Empty()); if (request.HasAny()) { await client.SendRewards(request); @@ -54,9 +47,8 @@ namespace TestNetRewarder var numberOfChainEvents = await ProcessEvents(timeRange); var duration = sw.Elapsed; - if (numberOfChainEvents == 0) return TimeSegmentResponse.Underload; - if (numberOfChainEvents > 10) return TimeSegmentResponse.Overload; - if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Overload; + if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Underload; + if (duration > TimeSpan.FromSeconds(3)) return TimeSegmentResponse.Overload; return TimeSegmentResponse.OK; } catch (Exception ex) @@ -76,7 +68,7 @@ namespace TestNetRewarder var events = eventsFormatter.GetEvents(); var errors = eventsFormatter.GetErrors(); - var request = builder.Build(events, errors); + var request = builder.Build(chainState, events, errors); if (request.HasAny()) { await client.SendRewards(request); diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index ab21f61c..e3ed24e7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -45,6 +45,7 @@ namespace TestNetRewarder Log.Log("Starting TestNet Rewarder..."); var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor); + await EnsureBotOnline(); await processor.Initialize(); while (!CancellationToken.IsCancellationRequested) diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs index b1f641b8..5b859269 100644 --- a/Tools/TestNetRewarder/RequestBuilder.cs +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -1,40 +1,55 @@ -using DiscordRewards; +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; using Utils; namespace TestNetRewarder { - public class RequestBuilder : IRewardGiver + public class RequestBuilder { - private readonly Dictionary> rewards = new Dictionary>(); - - public void Give(RewardConfig reward, EthAddress receiver) + public EventsAndErrors Build(ChainState chainState, ChainEventMessage[] lines, string[] errors) { - if (rewards.ContainsKey(reward.RoleId)) + var activeChainAddresses = CollectActiveAddresses(chainState); + + return new EventsAndErrors { - rewards[reward.RoleId].Add(receiver); + EventsOverview = lines, + Errors = errors, + ActiveChainAddresses = activeChainAddresses + }; + } + + private ActiveChainAddresses CollectActiveAddresses(ChainState chainState) + { + var hosts = new List(); + var clients = new List(); + + foreach (var request in chainState.Requests) + { + CollectAddresses(request, hosts, clients); } - else + + return new ActiveChainAddresses { - rewards.Add(reward.RoleId, new List { receiver }); + Hosts = hosts.ToArray(), + Clients = clients.ToArray() + }; + } + + private void CollectAddresses(IChainStateRequest request, List hosts, List clients) + { + if (request.State != CodexContractsPlugin.RequestState.Started) return; + + AddIfNew(clients, request.Client); + foreach (var host in request.Hosts.GetHosts()) + { + AddIfNew(hosts, host); } } - public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors) + private void AddIfNew(List list, EthAddress address) { - var result = new GiveRewardsCommand - { - Rewards = rewards.Select(p => new RewardUsersCommand - { - RewardId = p.Key, - UserAddresses = p.Value.Select(v => v.Address).ToArray() - }).ToArray(), - EventsOverview = lines, - Errors = errors - }; - - rewards.Clear(); - - return result; + var addr = address.Address; + if (!list.Contains(addr)) list.Add(addr); } } } diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs deleted file mode 100644 index 2f947d8d..00000000 --- a/Tools/TestNetRewarder/RewardCheck.cs +++ /dev/null @@ -1,113 +0,0 @@ -using BlockchainUtils; -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; -using System.Numerics; -using Utils; - -namespace TestNetRewarder -{ - public interface IRewardGiver - { - void Give(RewardConfig reward, EthAddress receiver); - } - - public class RewardCheck : IChainStateChangeHandler - { - private readonly RewardConfig reward; - private readonly IRewardGiver giver; - - public RewardCheck(RewardConfig reward, IRewardGiver giver) - { - this.reward = reward; - this.giver = giver; - } - - public void OnNewRequest(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientPostedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnRequestCancelled(RequestEvent requestEvent) - { - } - - public void OnRequestFailed(RequestEvent requestEvent) - { - } - - public void OnRequestFinished(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.HostFinishedSlot, requestEvent)) - { - foreach (var host in requestEvent.Request.Hosts.GetHosts()) - { - GiveReward(reward, host); - } - } - } - - public void OnRequestFulfilled(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientStartedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex) - { - if (MeetsRequirements(CheckType.HostFilledSlot, requestEvent)) - { - if (host != null) - { - GiveReward(reward, host); - } - } - } - - public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnError(string msg) - { - } - - public void OnProofSubmitted(BlockTimeEntry block, string id) - { - } - - private void GiveReward(RewardConfig reward, EthAddress receiver) - { - giver.Give(reward, receiver); - } - - private bool MeetsRequirements(CheckType type, RequestEvent requestEvent) - { - return - reward.CheckConfig.Type == type && - MeetsDurationRequirement(requestEvent.Request) && - MeetsSizeRequirement(requestEvent.Request); - } - - private bool MeetsSizeRequirement(IChainStateRequest r) - { - var slotSize = r.Request.Ask.SlotSize; - ulong min = Convert.ToUInt64(reward.CheckConfig.MinSlotSize.SizeInBytes); - return slotSize >= min; - } - - private bool MeetsDurationRequirement(IChainStateRequest r) - { - var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); - return duration >= reward.CheckConfig.MinDuration; - } - } -} diff --git a/Tools/TestNetRewarder/RewardChecker.cs b/Tools/TestNetRewarder/RewardChecker.cs deleted file mode 100644 index f8e500cc..00000000 --- a/Tools/TestNetRewarder/RewardChecker.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; - -namespace TestNetRewarder -{ - public class RewardChecker - { - public RewardChecker(IRewardGiver giver) - { - var repo = new RewardRepo(); - var checks = repo.Rewards.Select(r => new RewardCheck(r, giver)).ToArray(); - Handler = new ChainStateChangeHandlerMux(checks); - } - - public IChainStateChangeHandler Handler { get; } - } -}