From 1240575f934617eec0dadb8d57dfc5957b710877 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 13:10:01 +0200 Subject: [PATCH 01/36] implements loadbalancer --- Tools/AutoClient/App.cs | 5 - Tools/AutoClient/AutomaticPurchaser.cs | 145 ------------------ Tools/AutoClient/LoadBalancer.cs | 112 ++++++++++++++ .../Modes/FolderStore/BalanceChecker.cs | 3 +- .../AutoClient/Modes/FolderStore/FileSaver.cs | 66 ++++++-- .../Modes/FolderStore/FolderSaver.cs | 27 ++-- Tools/AutoClient/Modes/FolderStoreMode.cs | 14 +- Tools/AutoClient/Modes/Mode.cs | 8 - Tools/AutoClient/Modes/PurchasingMode.cs | 48 ------ Tools/AutoClient/Program.cs | 59 +++---- 10 files changed, 202 insertions(+), 285 deletions(-) delete mode 100644 Tools/AutoClient/AutomaticPurchaser.cs create mode 100644 Tools/AutoClient/LoadBalancer.cs delete mode 100644 Tools/AutoClient/Modes/Mode.cs delete mode 100644 Tools/AutoClient/Modes/PurchasingMode.cs diff --git a/Tools/AutoClient/App.cs b/Tools/AutoClient/App.cs index a581b6db..1a7e33a4 100644 --- a/Tools/AutoClient/App.cs +++ b/Tools/AutoClient/App.cs @@ -20,10 +20,6 @@ namespace AutoClient new FileLog(Path.Combine(config.LogPath, "performance")), new ConsoleLog() )); - - var httpFactory = new HttpFactory(Log, new AutoClientWebTimeSet()); - - CodexNodeFactory = new CodexNodeFactory(log: Log, httpFactory: httpFactory, dataDir: Config.DataPath); } public Configuration Config { get; } @@ -31,7 +27,6 @@ namespace AutoClient public IFileGenerator Generator { get; } public CancellationTokenSource Cts { get; } = new CancellationTokenSource(); public Performance Performance { get; } - public CodexNodeFactory CodexNodeFactory { get; } private IFileGenerator CreateGenerator() { diff --git a/Tools/AutoClient/AutomaticPurchaser.cs b/Tools/AutoClient/AutomaticPurchaser.cs deleted file mode 100644 index 1886f321..00000000 --- a/Tools/AutoClient/AutomaticPurchaser.cs +++ /dev/null @@ -1,145 +0,0 @@ -using Logging; - -namespace AutoClient -{ - public class AutomaticPurchaser - { - private readonly App app; - private readonly ILog log; - private readonly CodexWrapper node; - private Task workerTask = Task.CompletedTask; - - public AutomaticPurchaser(App app, ILog log, CodexWrapper node) - { - this.app = app; - this.log = log; - this.node = node; - } - - public void Start() - { - workerTask = Task.Run(Worker); - } - - public void Stop() - { - workerTask.Wait(); - } - - private async Task Worker() - { - log.Log("Worker started."); - while (!app.Cts.Token.IsCancellationRequested) - { - try - { - var pid = await StartNewPurchase(); - await WaitTillFinished(pid); - } - catch (Exception ex) - { - log.Error("Worker failed with: " + ex); - await Task.Delay(TimeSpan.FromHours(6)); - } - } - } - - private async Task StartNewPurchase() - { - var file = await CreateFile(); - try - { - var cid = node.UploadFile(file); - var response = node.RequestStorage(cid); - return response.PurchaseId; - } - finally - { - DeleteFile(file); - } - } - - private async Task CreateFile() - { - return await app.Generator.Generate(); - } - - private void DeleteFile(string file) - { - try - { - File.Delete(file); - } - catch (Exception exc) - { - log.Error($"Failed to delete file '{file}': {exc}"); - } - } - - private async Task WaitTillFinished(string pid) - { - try - { - var emptyResponseTolerance = 10; - while (!app.Cts.Token.IsCancellationRequested) - { - var purchase = node.GetStoragePurchase(pid); - if (purchase == null) - { - await FixedShortDelay(); - emptyResponseTolerance--; - if (emptyResponseTolerance == 0) - { - log.Log("Received 10 empty responses. Stop tracking this purchase."); - await ExpiryTimeDelay(); - return; - } - continue; - } - if (purchase.IsCancelled) - { - app.Performance.StorageContractCancelled(); - return; - } - if (purchase.IsError) - { - app.Performance.StorageContractErrored(purchase.Error); - return; - } - if (purchase.IsFinished) - { - app.Performance.StorageContractFinished(); - return; - } - if (purchase.IsStarted) - { - app.Performance.StorageContractStarted(); - await FixedDurationDelay(); - } - - await FixedShortDelay(); - } - } - catch (Exception ex) - { - log.Log($"Wait failed with exception: {ex}. Assume contract will expire: Wait expiry time."); - await ExpiryTimeDelay(); - } - } - - private async Task FixedDurationDelay() - { - await Task.Delay(app.Config.ContractDurationMinutes * 60 * 1000, app.Cts.Token); - } - - private async Task ExpiryTimeDelay() - { - await Task.Delay(app.Config.ContractExpiryMinutes * 60 * 1000, app.Cts.Token); - } - - private async Task FixedShortDelay() - { - await Task.Delay(15 * 1000, app.Cts.Token); - } - } -} diff --git a/Tools/AutoClient/LoadBalancer.cs b/Tools/AutoClient/LoadBalancer.cs new file mode 100644 index 00000000..64627adc --- /dev/null +++ b/Tools/AutoClient/LoadBalancer.cs @@ -0,0 +1,112 @@ +namespace AutoClient +{ + public class LoadBalancer + { + private readonly App app; + private readonly List instances; + private readonly object instanceLock = new object(); + private readonly List tasks = new List(); + private readonly object taskLock = new object(); + + private class Cdx + { + public Cdx(CodexWrapper instance) + { + Instance = instance; + } + + public CodexWrapper Instance { get; } + public bool IsBusy { get; set; } = false; + } + + public LoadBalancer(App app, CodexWrapper[] instances) + { + this.app = app; + this.instances = instances.Select(i => new Cdx(i)).ToList(); + } + + public void DispatchOnCodex(Action action) + { + lock (taskLock) + { + WaitUntilNotAllBusy(); + + tasks.Add(Task.Run(() => RunTask(action))); + } + } + + public void CleanUpTasks() + { + lock (taskLock) + { + foreach (var task in tasks) + { + if (task.IsFaulted) throw task.Exception; + } + + tasks.RemoveAll(t => t.IsCompleted); + } + } + + private void RunTask(Action action) + { + var instance = GetAndSetFreeInstance(); + try + { + action(instance.Instance); + } + finally + { + ReleaseInstance(instance); + } + } + + private Cdx GetAndSetFreeInstance() + { + lock (instanceLock) + { + return GetSetInstance(); + } + } + + private Cdx GetSetInstance() + { + var i = instances.First(); + instances.RemoveAt(0); + instances.Add(i); + + if (i.IsBusy) return GetSetInstance(); + + i.IsBusy = true; + return i; + } + + private void ReleaseInstance(Cdx instance) + { + lock (instanceLock) + { + instance.IsBusy = false; + } + } + + private void WaitUntilNotAllBusy() + { + if (AllBusy()) + { + app.Log.Log("[LoadBalancer] All instances are busy. Waiting..."); + while (AllBusy()) + { + Thread.Sleep(TimeSpan.FromSeconds(5.0)); + } + } + } + + private bool AllBusy() + { + lock (instanceLock) + { + return instances.All(i => i.IsBusy); + } + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs index 895651ae..502d48cd 100644 --- a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs +++ b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs @@ -1,5 +1,4 @@ -using GethConnector; -using Logging; +using Logging; using Utils; namespace AutoClient.Modes.FolderStore diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index ec5299f1..35630307 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -4,32 +4,64 @@ using Utils; namespace AutoClient.Modes.FolderStore { + public interface IFileSaverEventHandler + { + void SaveChanges(); + void OnFailure(); + } + public class FileSaver + { + private readonly ILog log; + private readonly LoadBalancer loadBalancer; + private readonly Stats stats; + private readonly string folderFile; + private readonly FileStatus entry; + private readonly IFileSaverEventHandler handler; + + public FileSaver(ILog log, LoadBalancer loadBalancer, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) + { + this.log = log; + this.loadBalancer = loadBalancer; + this.stats = stats; + this.folderFile = folderFile; + this.entry = entry; + this.handler = handler; + } + + public void Process() + { + loadBalancer.DispatchOnCodex(instance => + { + var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + run.Process(); + }); + } + } + + public class FileSaverRun { private readonly ILog log; private readonly CodexWrapper instance; private readonly Stats stats; private readonly string folderFile; private readonly FileStatus entry; - private readonly Action saveChanges; + private readonly IFileSaverEventHandler handler; private readonly QuotaCheck quotaCheck; - public FileSaver(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, Action saveChanges) + public FileSaverRun(ILog log, CodexWrapper instance, Stats stats, string folderFile, FileStatus entry, IFileSaverEventHandler handler) { this.log = log; this.instance = instance; this.stats = stats; this.folderFile = folderFile; this.entry = entry; - this.saveChanges = saveChanges; + this.handler = handler; quotaCheck = new QuotaCheck(log, folderFile, instance); } - public bool HasFailed { get; private set; } - public void Process() { - HasFailed = false; if (HasRecentPurchase()) { Log($"Purchase running: '{entry.PurchaseId}'"); @@ -71,7 +103,7 @@ namespace AutoClient.Modes.FolderStore Thread.Sleep(TimeSpan.FromMinutes(1.0)); } Log("Could not upload: Insufficient local storage quota."); - HasFailed = true; + handler.OnFailure(); return false; } @@ -108,7 +140,7 @@ namespace AutoClient.Modes.FolderStore var result = instance.Node.LocalFiles(); if (result == null) return false; if (result.Content == null) return false; - + var localCids = result.Content.Where(c => !string.IsNullOrEmpty(c.Cid.Id)).Select(c => c.Cid.Id).ToArray(); var isFound = localCids.Any(c => c.ToLowerInvariant() == entry.BasicCid.ToLowerInvariant()); if (isFound) @@ -150,9 +182,9 @@ namespace AutoClient.Modes.FolderStore entry.BasicCid = string.Empty; stats.FailedUploads++; log.Error("Failed to upload: " + exc); - HasFailed = true; + handler.OnFailure(); } - saveChanges(); + handler.SaveChanges(); } private void CreateNewPurchase() @@ -168,7 +200,7 @@ namespace AutoClient.Modes.FolderStore WaitForStarted(request); stats.StorageRequestStats.SuccessfullyStarted++; - saveChanges(); + handler.SaveChanges(); Log($"Successfully started new purchase: '{entry.PurchaseId}' for {Time.FormatDuration(request.Purchase.Duration)}"); } @@ -176,9 +208,9 @@ namespace AutoClient.Modes.FolderStore { entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - saveChanges(); + handler.SaveChanges(); log.Error("Failed to start new purchase: " + exc); - HasFailed = true; + handler.OnFailure(); } } @@ -197,7 +229,7 @@ namespace AutoClient.Modes.FolderStore throw new Exception("CID received from storage request was not protected."); } - saveChanges(); + handler.SaveChanges(); Log("Saved new purchaseId: " + entry.PurchaseId); return request; } @@ -233,7 +265,7 @@ namespace AutoClient.Modes.FolderStore Log("Request failed to start. State: " + update.State); entry.EncodedCid = string.Empty; entry.PurchaseId = string.Empty; - saveChanges(); + handler.SaveChanges(); return; } } @@ -241,10 +273,10 @@ namespace AutoClient.Modes.FolderStore } catch (Exception exc) { - HasFailed = true; + handler.OnFailure(); Log($"Exception in {nameof(WaitForSubmittedToStarted)}: {exc}"); throw; - } + } } private void WaitForSubmitted(IStoragePurchaseContract request) diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index ed358ccc..0605c150 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -2,21 +2,21 @@ namespace AutoClient.Modes.FolderStore { - public class FolderSaver + public class FolderSaver : IFileSaverEventHandler { private const string FolderSaverFilename = "foldersaver.json"; private readonly App app; - private readonly CodexWrapper instance; + private readonly LoadBalancer loadBalancer; private readonly JsonFile statusFile; private readonly FolderStatus status; private readonly BalanceChecker balanceChecker; private int changeCounter = 0; private int failureCount = 0; - public FolderSaver(App app, CodexWrapper instance) + public FolderSaver(App app, LoadBalancer loadBalancer) { this.app = app; - this.instance = instance; + this.loadBalancer = loadBalancer; balanceChecker = new BalanceChecker(app); statusFile = new JsonFile(app, Path.Combine(app.Config.FolderToStore, FolderSaverFilename)); @@ -87,7 +87,6 @@ namespace AutoClient.Modes.FolderStore { var fileSaver = CreateFileSaver(folderFile, entry); fileSaver.Process(); - if (fileSaver.HasFailed) failureCount++; } private void SaveFolderSaverJsonFile() @@ -101,7 +100,6 @@ namespace AutoClient.Modes.FolderStore ApplyPadding(folderFile); var fileSaver = CreateFileSaver(folderFile, entry); fileSaver.Process(); - if (fileSaver.HasFailed) failureCount++; if (!string.IsNullOrEmpty(entry.EncodedCid)) { @@ -148,11 +146,18 @@ namespace AutoClient.Modes.FolderStore { var fixedLength = entry.Filename.PadRight(35); var prefix = $"[{fixedLength}] "; - return new FileSaver(new LogPrefixer(app.Log, prefix), instance, status.Stats, folderFile, entry, saveChanges: () => - { - statusFile.Save(status); - changeCounter++; - }); + return new FileSaver(new LogPrefixer(app.Log, prefix), loadBalancer, status.Stats, folderFile, entry, this); + } + + public void SaveChanges() + { + statusFile.Save(status); + changeCounter++; + } + + public void OnFailure() + { + failureCount++; } } } diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 477ea9bf..64a29065 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -2,28 +2,26 @@ namespace AutoClient.Modes { - public class FolderStoreMode : IMode + public class FolderStoreMode { private readonly App app; - private readonly string folder; - private readonly PurchaseInfo purchaseInfo; private readonly CancellationTokenSource cts = new CancellationTokenSource(); private Task checkTask = Task.CompletedTask; + private readonly LoadBalancer loadBalancer; - public FolderStoreMode(App app, string folder, PurchaseInfo purchaseInfo) + public FolderStoreMode(App app, LoadBalancer loadBalancer) { this.app = app; - this.folder = folder; - this.purchaseInfo = purchaseInfo; + this.loadBalancer = loadBalancer; } - public void Start(CodexWrapper instance, int index) + public void Start() { checkTask = Task.Run(() => { try { - var saver = new FolderSaver(app, instance); + var saver = new FolderSaver(app, loadBalancer); while (!cts.IsCancellationRequested) { saver.Run(cts); diff --git a/Tools/AutoClient/Modes/Mode.cs b/Tools/AutoClient/Modes/Mode.cs deleted file mode 100644 index afc76f7a..00000000 --- a/Tools/AutoClient/Modes/Mode.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace AutoClient.Modes -{ - public interface IMode - { - void Start(CodexWrapper node, int index); - void Stop(); - } -} diff --git a/Tools/AutoClient/Modes/PurchasingMode.cs b/Tools/AutoClient/Modes/PurchasingMode.cs deleted file mode 100644 index 206ab365..00000000 --- a/Tools/AutoClient/Modes/PurchasingMode.cs +++ /dev/null @@ -1,48 +0,0 @@ -using Logging; - -namespace AutoClient.Modes -{ - public class PurchasingMode : IMode - { - private readonly List purchasers = new List(); - private readonly App app; - private Task starterTask = Task.CompletedTask; - - public PurchasingMode(App app) - { - this.app = app; - } - - public void Start(CodexWrapper node, int index) - { - for (var i = 0; i < app.Config.NumConcurrentPurchases; i++) - { - purchasers.Add(new AutomaticPurchaser(app, new LogPrefixer(app.Log, $"({i}) "), node)); - } - - var delayPerPurchaser = - TimeSpan.FromSeconds(10 * index) + - TimeSpan.FromMinutes(app.Config.ContractDurationMinutes) / app.Config.NumConcurrentPurchases; - - starterTask = Task.Run(() => StartPurchasers(delayPerPurchaser)); - } - - private async Task StartPurchasers(TimeSpan delayPerPurchaser) - { - foreach (var purchaser in purchasers) - { - purchaser.Start(); - await Task.Delay(delayPerPurchaser); - } - } - - public void Stop() - { - starterTask.Wait(); - foreach (var purchaser in purchasers) - { - purchaser.Stop(); - } - } - } -} diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index 6a35ccf5..7bda2d3e 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -1,22 +1,22 @@ using ArgsUniform; using AutoClient; using AutoClient.Modes; -using AutoClient.Modes.FolderStore; using CodexClient; using GethPlugin; using Utils; +using WebUtils; +using Logging; public class Program { private readonly App app; - private readonly List modes = new List(); public Program(Configuration config) { app = new App(config); } - public static async Task Main(string[] args) + public static void Main(string[] args) { var cts = new CancellationTokenSource(); Console.CancelKeyPress += (sender, args) => cts.Cancel(); @@ -30,60 +30,35 @@ public class Program } var p = new Program(config); - await p.Run(); + p.Run(); } - public async Task Run() + public void Run() { - await Task.CompletedTask; + if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!"); var codexNodes = CreateCodexWrappers(); + var loadBalancer = new LoadBalancer(app, codexNodes); - var i = 0; - foreach (var cdx in codexNodes) - { - var mode = CreateMode(); - modes.Add(mode); - - mode.Start(cdx, i); - i++; - } + var folderStore = new FolderStoreMode(app, loadBalancer); + folderStore.Start(); app.Cts.Token.WaitHandle.WaitOne(); - foreach (var mode in modes) mode.Stop(); - modes.Clear(); + folderStore.Stop(); app.Log.Log("Done"); } - private IMode CreateMode() - { - if (!string.IsNullOrEmpty(app.Config.FolderToStore)) - { - return CreateFolderStoreMode(); - } - - return new PurchasingMode(app); - } - - private IMode CreateFolderStoreMode() - { - if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!"); - - return new FolderStoreMode(app, app.Config.FolderToStore, new PurchaseInfo( - purchaseDurationTotal: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes), - purchaseDurationSafe: TimeSpan.FromMinutes(app.Config.ContractDurationMinutes - 120) - )); - } - private CodexWrapper[] CreateCodexWrappers() { var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries); var result = new List(); + var i = 1; foreach (var e in endpointStrs) { - result.Add(CreateCodexWrapper(e)); + result.Add(CreateCodexWrapper(e, i)); + i++; } return result.ToArray(); @@ -91,7 +66,7 @@ public class Program private readonly string LogLevel = "TRACE;info:discv5,providers,routingtable,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,websock,ws-session,muxedupgrade,upgrade,identify,contracts,clock,serde,json,serialization,JSONRPC-WS-CLIENT,JSONRPC-HTTP-CLIENT,repostore"; - private CodexWrapper CreateCodexWrapper(string endpoint) + private CodexWrapper CreateCodexWrapper(string endpoint, int number) { var splitIndex = endpoint.LastIndexOf(':'); var host = endpoint.Substring(0, splitIndex); @@ -103,11 +78,13 @@ public class Program port: port ); + var log = new LogPrefixer(app.Log, $"[{number.ToString().PadLeft(3, '0')}] "); + var httpFactory = new HttpFactory(log, new AutoClientWebTimeSet()); + var codexNodeFactory = new CodexNodeFactory(log: log, httpFactory: httpFactory, dataDir: app.Config.DataPath); var instance = CodexInstance.CreateFromApiEndpoint("[AutoClient]", address, EthAccountGenerator.GenerateNew()); - var node = app.CodexNodeFactory.CreateCodexNode(instance); + var node = codexNodeFactory.CreateCodexNode(instance); node.SetLogLevel(LogLevel); - return new CodexWrapper(app, node); } From 592bedc3238e50a04600f76484b1a34211b2ea67 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 13:30:20 +0200 Subject: [PATCH 02/36] adds node name to autoclient log prefix --- Tools/AutoClient/Program.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index 7bda2d3e..ba23e961 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -78,10 +78,11 @@ public class Program port: port ); - var log = new LogPrefixer(app.Log, $"[{number.ToString().PadLeft(3, '0')}] "); + var numberStr = number.ToString().PadLeft(3, '0'); + var log = new LogPrefixer(app.Log, $"[{numberStr}] "); var httpFactory = new HttpFactory(log, new AutoClientWebTimeSet()); var codexNodeFactory = new CodexNodeFactory(log: log, httpFactory: httpFactory, dataDir: app.Config.DataPath); - var instance = CodexInstance.CreateFromApiEndpoint("[AutoClient]", address, EthAccountGenerator.GenerateNew()); + var instance = CodexInstance.CreateFromApiEndpoint($"[AC-{numberStr}]", address, EthAccountGenerator.GenerateNew()); var node = codexNodeFactory.CreateCodexNode(instance); node.SetLogLevel(LogLevel); From b5f3dfe0346d91428c4472f6746335f76a846d05 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 14:18:27 +0200 Subject: [PATCH 03/36] Assigning files to nodes --- Tools/AutoClient/LoadBalancer.cs | 161 ++++++++++-------- .../AutoClient/Modes/FolderStore/FileSaver.cs | 24 +++ .../Modes/FolderStore/FolderSaver.cs | 1 + .../Modes/FolderStore/FolderStatus.cs | 1 + Tools/AutoClient/Program.cs | 2 + 5 files changed, 114 insertions(+), 75 deletions(-) diff --git a/Tools/AutoClient/LoadBalancer.cs b/Tools/AutoClient/LoadBalancer.cs index 64627adc..aef4801a 100644 --- a/Tools/AutoClient/LoadBalancer.cs +++ b/Tools/AutoClient/LoadBalancer.cs @@ -1,111 +1,122 @@ -namespace AutoClient +using Logging; + +namespace AutoClient { public class LoadBalancer { - private readonly App app; private readonly List instances; private readonly object instanceLock = new object(); - private readonly List tasks = new List(); - private readonly object taskLock = new object(); private class Cdx { - public Cdx(CodexWrapper instance) + private readonly ILog log; + private readonly CodexWrapper instance; + private readonly List> queue = new List>(); + private readonly object queueLock = new object(); + private bool running = true; + private Task worker = Task.CompletedTask; + + public Cdx(App app, CodexWrapper instance) { - Instance = instance; + Id = instance.Node.GetName(); + log = new LogPrefixer(app.Log, $"[Queue-{Id}]"); + this.instance = instance; } - public CodexWrapper Instance { get; } - public bool IsBusy { get; set; } = false; + public string Id { get; } + + public void Start() + { + worker = Task.Run(Worker); + } + + public void Stop() + { + running = false; + worker.Wait(); + } + + public void CheckErrors() + { + if (worker.IsFaulted) throw worker.Exception; + } + + public void Queue(Action action) + { + while (queue.Count > 2) + { + log.Log("Queue full. Waiting..."); + Thread.Sleep(TimeSpan.FromSeconds(5.0)); + } + + lock (queueLock) + { + queue.Add(action); + } + } + + private void Worker() + { + while (running) + { + while (queue.Count == 0) Thread.Sleep(TimeSpan.FromSeconds(5.0)); + + Action action = w => { }; + lock (queueLock) + { + action = queue[0]; + queue.RemoveAt(0); + } + + action(instance); + } + } } public LoadBalancer(App app, CodexWrapper[] instances) { - this.app = app; - this.instances = instances.Select(i => new Cdx(i)).ToList(); + this.instances = instances.Select(i => new Cdx(app, i)).ToList(); + } + + public void Start() + { + foreach (var i in instances) i.Start(); + } + + public void Stop() + { + foreach (var i in instances) i.Stop(); } public void DispatchOnCodex(Action action) { - lock (taskLock) + lock (instanceLock) { - WaitUntilNotAllBusy(); + var i = instances.First(); + instances.RemoveAt(0); + instances.Add(i); - tasks.Add(Task.Run(() => RunTask(action))); + i.Queue(action); } } - public void CleanUpTasks() - { - lock (taskLock) - { - foreach (var task in tasks) - { - if (task.IsFaulted) throw task.Exception; - } - - tasks.RemoveAll(t => t.IsCompleted); - } - } - - private void RunTask(Action action) - { - var instance = GetAndSetFreeInstance(); - try - { - action(instance.Instance); - } - finally - { - ReleaseInstance(instance); - } - } - - private Cdx GetAndSetFreeInstance() + public void DispatchOnSpecificCodex(Action action, string id) { lock (instanceLock) { - return GetSetInstance(); + var i = instances.Single(a => a.Id == id); + instances.Remove(i); + instances.Add(i); + + i.Queue(action); } } - private Cdx GetSetInstance() - { - var i = instances.First(); - instances.RemoveAt(0); - instances.Add(i); - - if (i.IsBusy) return GetSetInstance(); - - i.IsBusy = true; - return i; - } - - private void ReleaseInstance(Cdx instance) + public void CheckErrors() { lock (instanceLock) { - instance.IsBusy = false; - } - } - - private void WaitUntilNotAllBusy() - { - if (AllBusy()) - { - app.Log.Log("[LoadBalancer] All instances are busy. Waiting..."); - while (AllBusy()) - { - Thread.Sleep(TimeSpan.FromSeconds(5.0)); - } - } - } - - private bool AllBusy() - { - lock (instanceLock) - { - return instances.All(i => i.IsBusy); + foreach (var i in instances) i.CheckErrors(); } } } diff --git a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs index 35630307..027065e4 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileSaver.cs @@ -30,13 +30,37 @@ namespace AutoClient.Modes.FolderStore } public void Process() + { + if (string.IsNullOrEmpty(entry.CodexNodeId)) + { + DispatchToAny(); + } + else + { + DispatchToSpecific(); + } + } + + private void DispatchToAny() { loadBalancer.DispatchOnCodex(instance => { + entry.CodexNodeId = instance.Node.GetName(); + handler.SaveChanges(); + var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); run.Process(); }); } + + private void DispatchToSpecific() + { + loadBalancer.DispatchOnSpecificCodex(instance => + { + var run = new FileSaverRun(log, instance, stats, folderFile, entry, handler); + run.Process(); + }, entry.CodexNodeId); + } } public class FileSaverRun diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index 0605c150..0b139470 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -33,6 +33,7 @@ namespace AutoClient.Modes.FolderStore foreach (var folderFile in folderFiles) { if (cts.IsCancellationRequested) return; + loadBalancer.CheckErrors(); if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename)) { diff --git a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs index e9b7ec14..18357037 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderStatus.cs @@ -11,6 +11,7 @@ [Serializable] public class FileStatus { + public string CodexNodeId { get; set; } = string.Empty; public string Filename { get; set; } = string.Empty; public string BasicCid { get; set; } = string.Empty; public string EncodedCid { get; set; } = string.Empty; diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index ba23e961..36eb361a 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -38,6 +38,7 @@ public class Program if (app.Config.ContractDurationMinutes - 1 < 5) throw new Exception("Contract duration config option not long enough!"); var codexNodes = CreateCodexWrappers(); var loadBalancer = new LoadBalancer(app, codexNodes); + loadBalancer.Start(); var folderStore = new FolderStoreMode(app, loadBalancer); folderStore.Start(); @@ -45,6 +46,7 @@ public class Program app.Cts.Token.WaitHandle.WaitOne(); folderStore.Stop(); + loadBalancer.Stop(); app.Log.Log("Done"); } From 2c9f9d100852bcb4a2ef7e073dd580de15199bf5 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 14:28:23 +0200 Subject: [PATCH 04/36] logs errors in worker loop --- Tools/AutoClient/LoadBalancer.cs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Tools/AutoClient/LoadBalancer.cs b/Tools/AutoClient/LoadBalancer.cs index aef4801a..407db704 100644 --- a/Tools/AutoClient/LoadBalancer.cs +++ b/Tools/AutoClient/LoadBalancer.cs @@ -43,9 +43,9 @@ namespace AutoClient public void Queue(Action action) { + if (queue.Count > 2) log.Log("Queue full. Waiting..."); while (queue.Count > 2) { - log.Log("Queue full. Waiting..."); Thread.Sleep(TimeSpan.FromSeconds(5.0)); } @@ -57,18 +57,26 @@ namespace AutoClient private void Worker() { - while (running) + try { - while (queue.Count == 0) Thread.Sleep(TimeSpan.FromSeconds(5.0)); - - Action action = w => { }; - lock (queueLock) + while (running) { - action = queue[0]; - queue.RemoveAt(0); - } + while (queue.Count == 0) Thread.Sleep(TimeSpan.FromSeconds(5.0)); - action(instance); + Action action = w => { }; + lock (queueLock) + { + action = queue[0]; + queue.RemoveAt(0); + } + + action(instance); + } + } + catch (Exception ex) + { + log.Error("Exception in worker: " + ex); + throw; } } } From d53597717505482e69265a7c8657909c5e76ea4d Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 14:42:34 +0200 Subject: [PATCH 05/36] pluralizes balance checker --- Tools/AutoClient/Configuration.cs | 2 +- .../Modes/FolderStore/BalanceChecker.cs | 60 ++++++++++--------- .../AutoClient/Modes/FolderStore/JsonFile.cs | 47 ++++++++------- 3 files changed, 59 insertions(+), 50 deletions(-) diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index edb0c658..5b5da689 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -41,7 +41,7 @@ namespace AutoClient [Uniform("folderToStore", "fts", "FOLDERTOSTORE", false, "When set, autoclient will attempt to upload and purchase storage for every non-JSON file in the provided folder.")] public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles"; - [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File with eth address used by codex node. Used for balance checking if geth/contracts information is provided.")] + [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File(s) with eth address used by codex node. Used for balance checking if geth/contracts information is provided. Semi-colon separated.")] public string EthAddressFile { get; set; } = "/root/codex-testnet-starter/scripts/eth.address"; public string LogPath diff --git a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs index 502d48cd..102aedbc 100644 --- a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs +++ b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs @@ -7,37 +7,42 @@ namespace AutoClient.Modes.FolderStore { private readonly LogPrefixer log; private readonly GethConnector.GethConnector? connector; - private readonly EthAddress? address; + private readonly EthAddress[] addresses; public BalanceChecker(App app) { log = new LogPrefixer(app.Log, "(Balance) "); connector = GethConnector.GethConnector.Initialize(app.Log); - address = LoadAddress(app); + addresses = LoadAddresses(app); } - private EthAddress? LoadAddress(App app) + private EthAddress[] LoadAddresses(App app) { try { - if (string.IsNullOrEmpty(app.Config.EthAddressFile)) return null; - if (!File.Exists(app.Config.EthAddressFile)) return null; + if (string.IsNullOrEmpty(app.Config.EthAddressFile)) return Array.Empty(); + if (!File.Exists(app.Config.EthAddressFile)) return Array.Empty(); - return new EthAddress( - File.ReadAllText(app.Config.EthAddressFile) - .Trim() - .Replace("\n", "") - .Replace(Environment.NewLine, "") - ); + var tokens = app.Config.EthAddressFile.Split(";", StringSplitOptions.RemoveEmptyEntries); + return tokens.Select(ConvertToAddress).Where(a => a != null).ToArray(); } catch (Exception exc) { log.Error($"Failed to load eth address from file: {exc}"); - return null; + return Array.Empty(); } } + private EthAddress ConvertToAddress(string t) + { + return new EthAddress( + File.ReadAllText(t) + .Trim() + .Replace("\n", "") + .Replace(Environment.NewLine, "")); + } + public void Check() { if (connector == null) @@ -45,35 +50,32 @@ namespace AutoClient.Modes.FolderStore Log("Connector not configured. Can't check balances."); return; } - if (address == null) - { - Log("EthAddress not found. Can't check balances."); - return; - } - try + foreach (var address in addresses) { - PerformCheck(); - } - catch (Exception exc) - { - Log($"Exception while checking balances: {exc}"); + try + { + PerformCheck(address); + } + catch (Exception exc) + { + Log($"Exception while checking balances: {exc}"); + } } } - private void PerformCheck() + private void PerformCheck(EthAddress address) { var geth = connector!.GethNode; var contracts = connector!.CodexContracts; - var addr = address!; - var eth = geth.GetEthBalance(addr); - var tst = contracts.GetTestTokenBalance(addr); + var eth = geth.GetEthBalance(address); + var tst = contracts.GetTestTokenBalance(address); Log($"Balances: [{eth}] - [{tst}]"); - if (eth.Eth < 1) TryAddEth(geth, addr); - if (tst.Tst < 1) TryAddTst(contracts, addr); + if (eth.Eth < 1) TryAddEth(geth, address); + if (tst.Tst < 1) TryAddTst(contracts, address); } private void TryAddEth(GethPlugin.IGethNode geth, EthAddress addr) diff --git a/Tools/AutoClient/Modes/FolderStore/JsonFile.cs b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs index 6d0f972d..365e77c3 100644 --- a/Tools/AutoClient/Modes/FolderStore/JsonFile.cs +++ b/Tools/AutoClient/Modes/FolderStore/JsonFile.cs @@ -6,6 +6,7 @@ namespace AutoClient.Modes.FolderStore { private readonly App app; private readonly string filePath; + private readonly object fileLock = new object(); public JsonFile(App app, string filePath) { @@ -15,35 +16,41 @@ namespace AutoClient.Modes.FolderStore public T Load() { - try + lock (fileLock) { - if (!File.Exists(filePath)) + try { - var state = new T(); - Save(state); - return state; + if (!File.Exists(filePath)) + { + var state = new T(); + Save(state); + return state; + } + var text = File.ReadAllText(filePath); + return JsonConvert.DeserializeObject(text)!; + } + catch (Exception exc) + { + app.Log.Error("Failed to load state: " + exc); + throw; } - var text = File.ReadAllText(filePath); - return JsonConvert.DeserializeObject(text)!; - } - catch (Exception exc) - { - app.Log.Error("Failed to load state: " + exc); - throw; } } public void Save(T state) { - try + lock (fileLock) { - var json = JsonConvert.SerializeObject(state, Formatting.Indented); - File.WriteAllText(filePath, json); - } - catch (Exception exc) - { - app.Log.Error("Failed to save state: " + exc); - throw; + try + { + var json = JsonConvert.SerializeObject(state, Formatting.Indented); + File.WriteAllText(filePath, json); + } + catch (Exception exc) + { + app.Log.Error("Failed to save state: " + exc); + throw; + } } } } From accbe5139210b5d783606ca5171544fdfda26620 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 14:52:30 +0200 Subject: [PATCH 06/36] logging for balance checker --- Tools/AutoClient/Configuration.cs | 8 ++++++-- Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs | 3 +++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 5b5da689..56d472bf 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -6,7 +6,7 @@ namespace AutoClient { [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")] public string CodexEndpoints { get; set; } = - "http://localhost:8080"; + "http://localhost:8080;http://localhost:8081;http://localhost:8082;http://localhost:8083"; [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] public string DataPath { get; set; } = "datapath"; @@ -42,7 +42,11 @@ namespace AutoClient public string FolderToStore { get; set; } = "/data/EthereumMainnetPreMergeEraFiles"; [Uniform("ethAddressFile", "eaf", "ETHADDRESSFILE", false, "File(s) with eth address used by codex node. Used for balance checking if geth/contracts information is provided. Semi-colon separated.")] - public string EthAddressFile { get; set; } = "/root/codex-testnet-starter/scripts/eth.address"; + public string EthAddressFile { get; set; } = + "/root/codex-testnet-starter/scripts/eth.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_2.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_3.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_4.address"; public string LogPath { diff --git a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs index 102aedbc..83a290a2 100644 --- a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs +++ b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs @@ -15,6 +15,9 @@ namespace AutoClient.Modes.FolderStore connector = GethConnector.GethConnector.Initialize(app.Log); addresses = LoadAddresses(app); + + log.Log($"Loaded Eth-addresses for checking: {addresses.Length}"); + foreach (var addr in addresses) log.Log(" - " + addr); } private EthAddress[] LoadAddresses(App app) From 7167252e8b32e7939bb5530ab9ed16fa8ac18efb Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 15:04:58 +0200 Subject: [PATCH 07/36] Fixes check for eth address files --- Framework/NethereumWorkflow/NethereumInteraction.cs | 1 - Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Framework/NethereumWorkflow/NethereumInteraction.cs b/Framework/NethereumWorkflow/NethereumInteraction.cs index f79eb199..3b2c609c 100644 --- a/Framework/NethereumWorkflow/NethereumInteraction.cs +++ b/Framework/NethereumWorkflow/NethereumInteraction.cs @@ -62,7 +62,6 @@ namespace NethereumWorkflow log.Debug(typeof(TFunction).ToString()); var handler = web3.Eth.GetContractQueryHandler(); var result = Time.Wait(handler.QueryRawAsync(contractAddress, function, new BlockParameter(blockNumber))); - var aaaa = 0; } public string SendTransaction(string contractAddress, TFunction function) where TFunction : FunctionMessage, new() diff --git a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs index 83a290a2..7b9e8857 100644 --- a/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs +++ b/Tools/AutoClient/Modes/FolderStore/BalanceChecker.cs @@ -25,10 +25,9 @@ namespace AutoClient.Modes.FolderStore try { if (string.IsNullOrEmpty(app.Config.EthAddressFile)) return Array.Empty(); - if (!File.Exists(app.Config.EthAddressFile)) return Array.Empty(); var tokens = app.Config.EthAddressFile.Split(";", StringSplitOptions.RemoveEmptyEntries); - return tokens.Select(ConvertToAddress).Where(a => a != null).ToArray(); + return tokens.Select(ConvertToAddress).Where(a => a != null).Cast().ToArray(); } catch (Exception exc) { @@ -37,8 +36,9 @@ namespace AutoClient.Modes.FolderStore } } - private EthAddress ConvertToAddress(string t) + private EthAddress? ConvertToAddress(string t) { + if (!File.Exists(t)) return null; return new EthAddress( File.ReadAllText(t) .Trim() From 3064445af9315e2260b7da23469c34a4ace23c19 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 3 Apr 2025 15:59:48 +0200 Subject: [PATCH 08/36] scale up to 8 archiver --- Tools/AutoClient/Configuration.cs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/Tools/AutoClient/Configuration.cs b/Tools/AutoClient/Configuration.cs index 56d472bf..13aea6fd 100644 --- a/Tools/AutoClient/Configuration.cs +++ b/Tools/AutoClient/Configuration.cs @@ -6,7 +6,14 @@ namespace AutoClient { [Uniform("codex-endpoints", "ce", "CODEXENDPOINTS", false, "Codex endpoints. Semi-colon separated. (default 'http://localhost:8080')")] public string CodexEndpoints { get; set; } = - "http://localhost:8080;http://localhost:8081;http://localhost:8082;http://localhost:8083"; + "http://localhost:8080" + ";" + + "http://localhost:8081" + ";" + + "http://localhost:8082" + ";" + + "http://localhost:8083" + ";" + + "http://localhost:8084" + ";" + + "http://localhost:8085" + ";" + + "http://localhost:8086" + ";" + + "http://localhost:8087"; [Uniform("datapath", "dp", "DATAPATH", false, "Root path where all data files will be saved.")] public string DataPath { get; set; } = "datapath"; @@ -21,7 +28,7 @@ namespace AutoClient // Cluster nodes configured for max 7-day storage. [Uniform("contract-expiry", "ce", "CONTRACTEXPIRY", false, "contract expiry in minutes. (default 15 minutes)")] - public int ContractExpiryMinutes { get; set; } = 60; + public int ContractExpiryMinutes { get; set; } = 15; [Uniform("num-hosts", "nh", "NUMHOSTS", false, "Number of hosts for contract. (default 10)")] public int NumHosts { get; set; } = 5; @@ -46,7 +53,11 @@ namespace AutoClient "/root/codex-testnet-starter/scripts/eth.address" + ";" + "/root/codex-testnet-starter/scripts/eth_2.address" + ";" + "/root/codex-testnet-starter/scripts/eth_3.address" + ";" + - "/root/codex-testnet-starter/scripts/eth_4.address"; + "/root/codex-testnet-starter/scripts/eth_4.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_5.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_6.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_7.address" + ";" + + "/root/codex-testnet-starter/scripts/eth_8.address"; public string LogPath { From d2a48e94cb659f2dba579a05b1e23db5976cddbc Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 4 Apr 2025 08:31:27 +0200 Subject: [PATCH 09/36] replaced redundant CTS --- Tools/AutoClient/Modes/FolderStore/FolderSaver.cs | 6 +++--- Tools/AutoClient/Modes/FolderStoreMode.cs | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index 0b139470..2f71b23b 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -23,7 +23,7 @@ namespace AutoClient.Modes.FolderStore status = statusFile.Load(); } - public void Run(CancellationTokenSource cts) + public void Run() { var folderFiles = Directory.GetFiles(app.Config.FolderToStore); if (!folderFiles.Any()) throw new Exception("No files found in " + app.Config.FolderToStore); @@ -32,7 +32,7 @@ namespace AutoClient.Modes.FolderStore balanceChecker.Check(); foreach (var folderFile in folderFiles) { - if (cts.IsCancellationRequested) return; + if (app.Cts.IsCancellationRequested) return; loadBalancer.CheckErrors(); if (!folderFile.ToLowerInvariant().EndsWith(FolderSaverFilename)) @@ -43,7 +43,7 @@ namespace AutoClient.Modes.FolderStore if (failureCount > 3) { app.Log.Error("Failure count reached threshold. Stopping..."); - cts.Cancel(); + app.Cts.Cancel(); return; } diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 64a29065..1d368706 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -5,7 +5,6 @@ namespace AutoClient.Modes public class FolderStoreMode { private readonly App app; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); private Task checkTask = Task.CompletedTask; private readonly LoadBalancer loadBalancer; @@ -22,9 +21,9 @@ namespace AutoClient.Modes try { var saver = new FolderSaver(app, loadBalancer); - while (!cts.IsCancellationRequested) + while (!app.Cts.IsCancellationRequested) { - saver.Run(cts); + saver.Run(); } } catch (Exception ex) @@ -37,7 +36,7 @@ namespace AutoClient.Modes public void Stop() { - cts.Cancel(); + app.Cts.Cancel(); checkTask.Wait(); } } From 89f1f74ffad8936cc2e63ce5c467638d87de0b54 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 9 Apr 2025 14:39:35 +0200 Subject: [PATCH 10/36] Fix for exceptionally long error messages in admin channel on checkCID fail --- Tools/BiblioTech/CodexCidChecker.cs | 61 ++++---------------- Tools/BiblioTech/Commands/CheckCidCommand.cs | 22 ++++++- 2 files changed, 32 insertions(+), 51 deletions(-) diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs index 15728d0d..dd3b30c7 100644 --- a/Tools/BiblioTech/CodexCidChecker.cs +++ b/Tools/BiblioTech/CodexCidChecker.cs @@ -33,7 +33,7 @@ namespace BiblioTech { if (string.IsNullOrEmpty(config.CodexEndpoint)) { - return new CheckResponse(false, "Codex CID checker is not (yet) available.", ""); + return new CheckResponse(false, "Codex CID checker is not (yet) available."); } try @@ -41,14 +41,10 @@ namespace BiblioTech checkMutex.WaitOne(); var codex = GetCodex(); var nodeCheck = CheckCodex(codex); - if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); + if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check."); return PerformCheck(codex, cid); } - catch (Exception ex) - { - return new CheckResponse(false, "Internal server error", ex.ToString()); - } finally { checkMutex.ReleaseMutex(); @@ -62,9 +58,9 @@ namespace BiblioTech var manifest = codex.DownloadManifestOnly(new ContentId(cid)); return SuccessMessage(manifest); } - catch (Exception ex) + catch { - return UnexpectedException(ex); + return FailedMessage(); } } @@ -74,58 +70,27 @@ namespace BiblioTech { return FormatResponse( success: true, - title: $"Success: '{content.Cid}'", - error: "", + title: $"Success", + $"cid: '{content.Cid}'", $"size: {content.Manifest.OriginalBytes} bytes", $"blockSize: {content.Manifest.BlockSize} bytes", $"protected: {content.Manifest.Protected}" ); } - private CheckResponse UnexpectedException(Exception ex) + private CheckResponse FailedMessage() { - return FormatResponse( - success: false, - title: "Unexpected error", - error: ex.ToString(), - content: "Details will be sent to the bot-admin channel." - ); - } + var msg = "Could not download content."; - private CheckResponse UnexpectedReturnCode(string response) - { - var msg = "Unexpected return code. Response: " + response; return FormatResponse( success: false, - title: "Unexpected return code", - error: msg, - content: msg - ); - } - - private CheckResponse FailedToFetch(string response) - { - var msg = "Failed to download content. Response: " + response; - return FormatResponse( - success: false, - title: "Could not download content", - error: msg, + title: "Failed", msg, $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" ); } - private CheckResponse CidFormatInvalid(string response) - { - return FormatResponse( - success: false, - title: "Invalid format", - error: "", - content: "Provided CID is not formatted correctly." - ); - } - - private CheckResponse FormatResponse(bool success, string title, string error, params string[] content) + private CheckResponse FormatResponse(bool success, string title, params string[] content) { var msg = string.Join(nl, new string[] @@ -140,7 +105,7 @@ namespace BiblioTech }) ) + nl + nl; - return new CheckResponse(success, msg, error); + return new CheckResponse(success, msg); } #endregion @@ -190,15 +155,13 @@ namespace BiblioTech public class CheckResponse { - public CheckResponse(bool success, string message, string error) + public CheckResponse(bool success, string message) { Success = success; Message = message; - Error = error; } public bool Success { get; } public string Message { get; } - public string Error { get; } } } diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs index 1e77ce26..1c5d6138 100644 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ b/Tools/BiblioTech/Commands/CheckCidCommand.cs @@ -1,5 +1,6 @@ using BiblioTech.Options; using Discord; +using Discord.WebSocket; namespace BiblioTech.Commands { @@ -33,18 +34,35 @@ namespace BiblioTech.Commands return; } - var response = checker.PerformCheck(cid); - await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); + try + { + await PerformCheck(context, user, cid); + } + catch (Exception ex) + { + await RespondeWithError(context, ex); + } + } + private async Task PerformCheck(CommandContext context, SocketUser user, string cid) + { + var response = checker.PerformCheck(cid); if (response.Success) { await CheckAltruisticRole(context, user, cid, response.Message); return; } + await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}'"); await context.Followup(response.Message); } + private async Task RespondeWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckCidCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage) { if (cidStorage.TryAddCid(cid, user.Id)) From 7ccdbd3c268a5921799784218bef48a1faed05f0 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 9 Apr 2025 15:25:31 +0200 Subject: [PATCH 11/36] wip --- Framework/WebUtils/Http.cs | 9 - Framework/WebUtils/HttpFactory.cs | 24 ++- Tools/BiblioTech/CodexCidChecker.cs | 167 ------------------- Tools/BiblioTech/CodexTwoWayChecker.cs | 59 +++++++ Tools/BiblioTech/Commands/CheckCidCommand.cs | 4 +- Tools/BiblioTech/Program.cs | 2 +- 6 files changed, 84 insertions(+), 181 deletions(-) delete mode 100644 Tools/BiblioTech/CodexCidChecker.cs create mode 100644 Tools/BiblioTech/CodexTwoWayChecker.cs diff --git a/Framework/WebUtils/Http.cs b/Framework/WebUtils/Http.cs index 7c9a91ad..e4931996 100644 --- a/Framework/WebUtils/Http.cs +++ b/Framework/WebUtils/Http.cs @@ -20,11 +20,6 @@ namespace WebUtils private readonly Action onClientCreated; private readonly string id; - internal Http(string id, ILog log, IWebCallTimeSet timeSet) - : this(id, log, timeSet, DoNothing) - { - } - internal Http(string id, ILog log, IWebCallTimeSet timeSet, Action onClientCreated) { this.id = id; @@ -89,9 +84,5 @@ namespace WebUtils onClientCreated(client); return client; } - - private static void DoNothing(HttpClient client) - { - } } } diff --git a/Framework/WebUtils/HttpFactory.cs b/Framework/WebUtils/HttpFactory.cs index 4d5d7c4c..8120527c 100644 --- a/Framework/WebUtils/HttpFactory.cs +++ b/Framework/WebUtils/HttpFactory.cs @@ -13,16 +13,28 @@ namespace WebUtils { private readonly ILog log; private readonly IWebCallTimeSet defaultTimeSet; + private readonly Action factoryOnClientCreated; public HttpFactory(ILog log) : this (log, new DefaultWebCallTimeSet()) { } + public HttpFactory(ILog log, Action onClientCreated) + : this(log, new DefaultWebCallTimeSet(), onClientCreated) + { + } + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet) + : this(log, defaultTimeSet, DoNothing) + { + } + + public HttpFactory(ILog log, IWebCallTimeSet defaultTimeSet, Action onClientCreated) { this.log = log; this.defaultTimeSet = defaultTimeSet; + this.factoryOnClientCreated = onClientCreated; } public IHttp CreateHttp(string id, Action onClientCreated) @@ -32,12 +44,20 @@ namespace WebUtils public IHttp CreateHttp(string id, Action onClientCreated, IWebCallTimeSet ts) { - return new Http(id, log, ts, onClientCreated); + return new Http(id, log, ts, (c) => + { + factoryOnClientCreated(c); + onClientCreated(c); + }); } public IHttp CreateHttp(string id) { - return new Http(id, log, defaultTimeSet); + return new Http(id, log, defaultTimeSet, factoryOnClientCreated); + } + + private static void DoNothing(HttpClient client) + { } } } diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs deleted file mode 100644 index dd3b30c7..00000000 --- a/Tools/BiblioTech/CodexCidChecker.cs +++ /dev/null @@ -1,167 +0,0 @@ -using CodexClient; -using Logging; -using Utils; - -namespace BiblioTech -{ - public class CodexCidChecker - { - private static readonly string nl = Environment.NewLine; - private readonly Configuration config; - private readonly ILog log; - private readonly Mutex checkMutex = new Mutex(); - private readonly CodexNodeFactory factory; - private ICodexNode? currentCodexNode; - - public CodexCidChecker(Configuration config, ILog log) - { - this.config = config; - this.log = log; - - factory = new CodexNodeFactory(log, dataDir: config.DataPath); - - if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) - { - throw new Exception("Todo: codexnodefactory httpfactory support basicauth!"); - //var tokens = config.CodexEndpointAuth.Split(':'); - //if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); - //client.SetBasicAuthentication(tokens[0], tokens[1]); - } - } - - public CheckResponse PerformCheck(string cid) - { - if (string.IsNullOrEmpty(config.CodexEndpoint)) - { - return new CheckResponse(false, "Codex CID checker is not (yet) available."); - } - - try - { - checkMutex.WaitOne(); - var codex = GetCodex(); - var nodeCheck = CheckCodex(codex); - if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check."); - - return PerformCheck(codex, cid); - } - finally - { - checkMutex.ReleaseMutex(); - } - } - - private CheckResponse PerformCheck(ICodexNode codex, string cid) - { - try - { - var manifest = codex.DownloadManifestOnly(new ContentId(cid)); - return SuccessMessage(manifest); - } - catch - { - return FailedMessage(); - } - } - - #region Response formatting - - private CheckResponse SuccessMessage(LocalDataset content) - { - return FormatResponse( - success: true, - title: $"Success", - $"cid: '{content.Cid}'", - $"size: {content.Manifest.OriginalBytes} bytes", - $"blockSize: {content.Manifest.BlockSize} bytes", - $"protected: {content.Manifest.Protected}" - ); - } - - private CheckResponse FailedMessage() - { - var msg = "Could not download content."; - - return FormatResponse( - success: false, - title: "Failed", - msg, - $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" - ); - } - - private CheckResponse FormatResponse(bool success, string title, params string[] content) - { - var msg = string.Join(nl, - new string[] - { - title, - "```" - } - .Concat(content) - .Concat(new string[] - { - "```" - }) - ) + nl + nl; - - return new CheckResponse(success, msg); - } - - #endregion - - #region Codex Node API - - private ICodexNode GetCodex() - { - if (currentCodexNode == null) currentCodexNode = CreateCodex(); - return currentCodexNode; - } - - private bool CheckCodex(ICodexNode node) - { - try - { - var info = node.GetDebugInfo(); - if (info == null || string.IsNullOrEmpty(info.Id)) return false; - return true; - } - catch (Exception e) - { - log.Error(e.ToString()); - return false; - } - } - - private ICodexNode CreateCodex() - { - var endpoint = config.CodexEndpoint; - var splitIndex = endpoint.LastIndexOf(':'); - var host = endpoint.Substring(0, splitIndex); - var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); - - var address = new Address( - logName: $"cdx@{host}:{port}", - host: host, - port: port - ); - - var instance = CodexInstance.CreateFromApiEndpoint("ac", address); - return factory.CreateCodexNode(instance); - } - - #endregion - } - - public class CheckResponse - { - public CheckResponse(bool success, string message) - { - Success = success; - Message = message; - } - - public bool Success { get; } - public string Message { get; } - } -} diff --git a/Tools/BiblioTech/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexTwoWayChecker.cs new file mode 100644 index 00000000..8aa27f5b --- /dev/null +++ b/Tools/BiblioTech/CodexTwoWayChecker.cs @@ -0,0 +1,59 @@ +using CodexClient; +using IdentityModel.Client; +using Logging; +using Utils; +using WebUtils; + +namespace BiblioTech +{ + public class CodexTwoWayChecker + { + private static readonly string nl = Environment.NewLine; + private readonly Configuration config; + private readonly ILog log; + private readonly CodexNodeFactory factory; + private ICodexNode? currentCodexNode; + + public CodexTwoWayChecker(Configuration config, ILog log) + { + this.config = config; + this.log = log; + + var httpFactory = CreateHttpFactory(); + + factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); + } + + // down check: + // generate unique data + // upload to cloud node + // give CID to user to download + // user inputs unique data into command to clear this check + + // up check: + // generate unique data + // create file and send it to user via discord api + // user uploads and gives CID via command + // download manifest: file is not larger than expected + // download file: contents is unique data -> clear this check + + // both checks: altruistic role + + private HttpFactory CreateHttpFactory() + { + if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + { + return new HttpFactory(log); + } + + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + + return new HttpFactory(log, onClientCreated: client => + { + client.SetBasicAuthentication(tokens[0], tokens[1]); + }); + } + + } +} diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs index 1c5d6138..f19a33f5 100644 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ b/Tools/BiblioTech/Commands/CheckCidCommand.cs @@ -10,10 +10,10 @@ namespace BiblioTech.Commands name: "cid", description: "Codex Content-Identifier", isRequired: true); - private readonly CodexCidChecker checker; + private readonly CodexTwoWayChecker checker; private readonly CidStorage cidStorage; - public CheckCidCommand(CodexCidChecker checker) + public CheckCidCommand(CodexTwoWayChecker checker) { this.checker = checker; this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt")); diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 46dee29c..ffc4f86d 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -80,7 +80,7 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; - var checker = new CodexCidChecker(Config, Log); + var checker = new CodexTwoWayChecker(Config, Log); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); var sprCommand = new SprCommand(); From 9ccc4c559cf025d5603786b04f2eb332892d228c Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 14:54:49 +0200 Subject: [PATCH 12/36] sets up upload and download checking commands --- Framework/Utils/RandomUtils.cs | 18 +- .../Modes/FolderStore/FolderSaver.cs | 16 +- Tools/BiblioTech/CodexChecking/CheckRepo.cs | 78 +++++++ .../CodexChecking/CodexTwoWayChecker.cs | 209 ++++++++++++++++++ .../BiblioTech/CodexChecking/CodexWrapper.cs | 85 +++++++ Tools/BiblioTech/CodexTwoWayChecker.cs | 59 ----- Tools/BiblioTech/Commands/CheckCidCommand.cs | 129 ----------- .../Commands/CheckDownloadCommand.cs | 53 +++++ .../Commands/CheckResponseHandler.cs | 66 ++++++ .../BiblioTech/Commands/CheckUploadCommand.cs | 53 +++++ Tools/BiblioTech/Configuration.cs | 1 + Tools/BiblioTech/Options/CommandContext.cs | 10 + Tools/BiblioTech/Program.cs | 8 +- Tools/BiblioTech/RandomBusyMessage.cs | 4 +- 14 files changed, 583 insertions(+), 206 deletions(-) create mode 100644 Tools/BiblioTech/CodexChecking/CheckRepo.cs create mode 100644 Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs create mode 100644 Tools/BiblioTech/CodexChecking/CodexWrapper.cs delete mode 100644 Tools/BiblioTech/CodexTwoWayChecker.cs delete mode 100644 Tools/BiblioTech/Commands/CheckCidCommand.cs create mode 100644 Tools/BiblioTech/Commands/CheckDownloadCommand.cs create mode 100644 Tools/BiblioTech/Commands/CheckResponseHandler.cs create mode 100644 Tools/BiblioTech/Commands/CheckUploadCommand.cs diff --git a/Framework/Utils/RandomUtils.cs b/Framework/Utils/RandomUtils.cs index f4f28dd4..b5d2580b 100644 --- a/Framework/Utils/RandomUtils.cs +++ b/Framework/Utils/RandomUtils.cs @@ -34,10 +34,26 @@ var source = items.ToList(); while (source.Any()) { - result.Add(RandomUtils.PickOneRandom(source)); + result.Add(PickOneRandom(source)); } return result.ToArray(); } } + + public static string GenerateRandomString(long requiredLength) + { + lock (@lock) + { + var result = ""; + while (result.Length < requiredLength) + { + var bytes = new byte[1024]; + random.NextBytes(bytes); + result += string.Join("", bytes.Select(b => b.ToString())); + } + + return result; + } + } } } diff --git a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs index ed358ccc..77251080 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderSaver.cs @@ -1,4 +1,5 @@ using Logging; +using Utils; namespace AutoClient.Modes.FolderStore { @@ -126,24 +127,11 @@ namespace AutoClient.Modes.FolderStore if (info.Length < min) { var required = Math.Max(1024, min - info.Length); - status.Padding = paddingMessage + GenerateRandomString(required); + status.Padding = paddingMessage + RandomUtils.GenerateRandomString(required); statusFile.Save(status); } } - private string GenerateRandomString(long required) - { - var result = ""; - while (result.Length < required) - { - var bytes = new byte[1024]; - random.NextBytes(bytes); - result += string.Join("", bytes.Select(b => b.ToString())); - } - - return result; - } - private FileSaver CreateFileSaver(string folderFile, FileStatus entry) { var fixedLength = entry.Filename.PadRight(35); diff --git a/Tools/BiblioTech/CodexChecking/CheckRepo.cs b/Tools/BiblioTech/CodexChecking/CheckRepo.cs new file mode 100644 index 00000000..a8b2860b --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CheckRepo.cs @@ -0,0 +1,78 @@ +using Newtonsoft.Json; + +namespace BiblioTech.CodexChecking +{ + public class CheckRepo + { + private const string modelFilename = "model.json"; + private readonly Configuration config; + private readonly object _lock = new object(); + private CheckRepoModel? model = null; + + public CheckRepo(Configuration config) + { + this.config = config; + } + + public CheckReport GetOrCreate(ulong userId) + { + lock (_lock) + { + if (model == null) LoadModel(); + + var existing = model.Reports.SingleOrDefault(r => r.UserId == userId); + if (existing == null) + { + var newEntry = new CheckReport + { + UserId = userId, + }; + model.Reports.Add(newEntry); + SaveChanges(); + return newEntry; + } + return existing; + } + } + + public void SaveChanges() + { + File.WriteAllText(GetModelFilepath(), JsonConvert.SerializeObject(model, Formatting.Indented)); + } + + private void LoadModel() + { + if (!File.Exists(GetModelFilepath())) + { + model = new CheckRepoModel(); + SaveChanges(); + return; + } + + model = JsonConvert.DeserializeObject(File.ReadAllText(GetModelFilepath())); + } + + private string GetModelFilepath() + { + return Path.Combine(config.ChecksDataPath, modelFilename); + } + } + + public class CheckRepoModel + { + public List Reports { get; set; } = new List(); + } + + public class CheckReport + { + public ulong UserId { get; set; } + public TransferCheck UploadCheck { get; set; } = new TransferCheck(); + public TransferCheck DownloadCheck { get; set; } = new TransferCheck(); + } + + public class TransferCheck + { + public DateTime CompletedUtc { get; set; } = DateTime.MinValue; + public string UniqueData { get; set; } = string.Empty; + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs new file mode 100644 index 00000000..8dadc97b --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -0,0 +1,209 @@ +using CodexClient; +using FileUtils; +using Logging; +using Utils; + +namespace BiblioTech.CodexChecking +{ + public interface ICheckResponseHandler + { + Task CheckNotStarted(); + Task NowCompleted(); + Task GiveRoleReward(); + + Task InvalidData(); + Task CouldNotDownloadCid(); + Task GiveCidToUser(string cid); + Task GiveDataFileToUser(string fileContent); + } + + public class CodexTwoWayChecker + { + private readonly ILog log; + private readonly Configuration config; + private readonly CheckRepo repo; + private readonly CodexWrapper codexWrapper; + + public CodexTwoWayChecker(ILog log, Configuration config, CheckRepo repo, CodexWrapper codexWrapper) + { + this.log = log; + this.config = config; + this.repo = repo; + this.codexWrapper = codexWrapper; + } + + public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + var cid = UploadData(check.UniqueData); + await handler.GiveCidToUser(cid); + } + + public async Task VerifyDownloadCheck(ICheckResponseHandler handler, ulong userId, string receivedData) + { + var check = repo.GetOrCreate(userId).DownloadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + await handler.CheckNotStarted(); + return; + } + + if (string.IsNullOrEmpty(receivedData) || receivedData != check.UniqueData) + { + await handler.InvalidData(); + return; + } + + CheckNowCompleted(handler, check, userId); + } + + public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (string.IsNullOrEmpty(check.UniqueData)) + { + check.UniqueData = GenerateUniqueData(); + repo.SaveChanges(); + } + + await handler.GiveDataFileToUser(check.UniqueData); + } + + public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId, string receivedCid) + { + var check = repo.GetOrCreate(userId).UploadCheck; + if (string.IsNullOrEmpty(receivedCid)) + { + await handler.InvalidData(); + return; + } + + var manifest = GetManifest(receivedCid); + if (manifest == null) + { + await handler.CouldNotDownloadCid(); + return; + } + + if (IsManifestLengthCompatible(check, manifest)) + { + if (IsContentCorrect(check, receivedCid)) + { + CheckNowCompleted(handler, check, userId); + return; + } + } + + await handler.InvalidData(); + } + + private string GenerateUniqueData() + { + return $"'{RandomBusyMessage.Get()}'{RandomUtils.GenerateRandomString(12)}"; + } + + private string UploadData(string uniqueData) + { + var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString()); + + try + { + File.WriteAllText(filePath, uniqueData); + var file = new TrackedFile(log, filePath, "checkData"); + + return codexWrapper.OnCodex(node => + { + return node.UploadFile(file).Id; + }); + } + catch (Exception ex) + { + log.Error("Exception when uploading data: " + ex); + throw; + } + finally + { + if (File.Exists(filePath)) File.Delete(filePath); + } + } + + private Manifest? GetManifest(string receivedCid) + { + try + { + return codexWrapper.OnCodex(node => + { + return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest; + }); + } + catch + { + return null; + } + } + + private bool IsManifestLengthCompatible(TransferCheck check, Manifest manifest) + { + var dataLength = check.UniqueData.Length; + var manifestLength = manifest.OriginalBytes.SizeInBytes; + + return + manifestLength > (dataLength - 1) && + manifestLength < (dataLength + 1); + } + + private bool IsContentCorrect(TransferCheck check, string receivedCid) + { + try + { + return codexWrapper.OnCodex(node => + { + var file = node.DownloadContent(new ContentId(receivedCid)); + if (file == null) return false; + try + { + var content = File.ReadAllText(file.Filename).Trim(); + return content == check.UniqueData; + } + finally + { + if (File.Exists(file.Filename)) File.Delete(file.Filename); + } + }); + } + catch + { + return false; + } + } + + private void CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId) + { + if (check.CompletedUtc != DateTime.MinValue) return; + + check.CompletedUtc = DateTime.UtcNow; + repo.SaveChanges(); + + handler.NowCompleted(); + CheckUserForRoleRewards(handler, userId); + } + + private void CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId) + { + var check = repo.GetOrCreate(userId); + + if ( + check.UploadCheck.CompletedUtc != DateTime.MinValue && + check.DownloadCheck.CompletedUtc != DateTime.MinValue) + { + handler.GiveRoleReward(); + } + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs new file mode 100644 index 00000000..a8cbe785 --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs @@ -0,0 +1,85 @@ +using CodexClient; +using IdentityModel.Client; +using Logging; +using Utils; +using WebUtils; + +namespace BiblioTech.CodexChecking +{ + public class CodexWrapper + { + private readonly CodexNodeFactory factory; + private readonly ILog log; + private readonly Configuration config; + private readonly object codexLock = new object(); + private ICodexNode? currentCodexNode; + + public CodexWrapper(ILog log, Configuration config) + { + this.log = log; + this.config = config; + + var httpFactory = CreateHttpFactory(); + factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); + } + + public void OnCodex(Action action) + { + lock (codexLock) + { + action(Get()); + } + } + + public T OnCodex(Func func) + { + lock (codexLock) + { + return func(Get()); + } + } + + private ICodexNode Get() + { + if (currentCodexNode == null) + { + currentCodexNode = CreateCodex(); + } + + return currentCodexNode; + } + + private ICodexNode CreateCodex() + { + var endpoint = config.CodexEndpoint; + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); + + var address = new Address( + logName: $"cdx@{host}:{port}", + host: host, + port: port + ); + + var instance = CodexInstance.CreateFromApiEndpoint("ac", address); + return factory.CreateCodexNode(instance); + } + + private HttpFactory CreateHttpFactory() + { + if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + { + return new HttpFactory(log); + } + + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + + return new HttpFactory(log, onClientCreated: client => + { + client.SetBasicAuthentication(tokens[0], tokens[1]); + }); + } + } +} diff --git a/Tools/BiblioTech/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexTwoWayChecker.cs deleted file mode 100644 index 8aa27f5b..00000000 --- a/Tools/BiblioTech/CodexTwoWayChecker.cs +++ /dev/null @@ -1,59 +0,0 @@ -using CodexClient; -using IdentityModel.Client; -using Logging; -using Utils; -using WebUtils; - -namespace BiblioTech -{ - public class CodexTwoWayChecker - { - private static readonly string nl = Environment.NewLine; - private readonly Configuration config; - private readonly ILog log; - private readonly CodexNodeFactory factory; - private ICodexNode? currentCodexNode; - - public CodexTwoWayChecker(Configuration config, ILog log) - { - this.config = config; - this.log = log; - - var httpFactory = CreateHttpFactory(); - - factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath); - } - - // down check: - // generate unique data - // upload to cloud node - // give CID to user to download - // user inputs unique data into command to clear this check - - // up check: - // generate unique data - // create file and send it to user via discord api - // user uploads and gives CID via command - // download manifest: file is not larger than expected - // download file: contents is unique data -> clear this check - - // both checks: altruistic role - - private HttpFactory CreateHttpFactory() - { - if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) - { - return new HttpFactory(log); - } - - var tokens = config.CodexEndpointAuth.Split(':'); - if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); - - return new HttpFactory(log, onClientCreated: client => - { - client.SetBasicAuthentication(tokens[0], tokens[1]); - }); - } - - } -} diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs deleted file mode 100644 index f19a33f5..00000000 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ /dev/null @@ -1,129 +0,0 @@ -using BiblioTech.Options; -using Discord; -using Discord.WebSocket; - -namespace BiblioTech.Commands -{ - public class CheckCidCommand : BaseCommand - { - private readonly StringOption cidOption = new StringOption( - name: "cid", - description: "Codex Content-Identifier", - isRequired: true); - private readonly CodexTwoWayChecker checker; - private readonly CidStorage cidStorage; - - public CheckCidCommand(CodexTwoWayChecker checker) - { - this.checker = checker; - this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt")); - } - - public override string Name => "check"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Checks if content is available in the testnet."; - public override CommandOption[] Options => new[] { cidOption }; - - protected override async Task Invoke(CommandContext context) - { - var user = context.Command.User; - var cid = await cidOption.Parse(context); - if (string.IsNullOrEmpty(cid)) - { - await context.Followup("Option 'cid' was not received."); - return; - } - - try - { - await PerformCheck(context, user, cid); - } - catch (Exception ex) - { - await RespondeWithError(context, ex); - } - } - - private async Task PerformCheck(CommandContext context, SocketUser user, string cid) - { - var response = checker.PerformCheck(cid); - if (response.Success) - { - await CheckAltruisticRole(context, user, cid, response.Message); - return; - } - - await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}'"); - await context.Followup(response.Message); - } - - private async Task RespondeWithError(CommandContext context, Exception ex) - { - await Program.AdminChecker.SendInAdminChannel("Exception during CheckCidCommand: " + ex); - await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); - } - - private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage) - { - if (cidStorage.TryAddCid(cid, user.Id)) - { - if (await GiveAltruisticRole(context, user, responseMessage)) - { - return; - } - } - else - { - await context.Followup($"{responseMessage}\n\nThis CID has already been used by another user. No role will be granted."); - return; - } - - await context.Followup(responseMessage); - } - - private async Task GiveAltruisticRole(CommandContext context, IUser user, string responseMessage) - { - try - { - await Program.RoleDriver.GiveAltruisticRole(user); - await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!"); - return true; - } - catch (Exception ex) - { - await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}"); - return false; - } - } - } - - public class CidStorage - { - private readonly string filePath; - private static readonly object _lock = new object(); - - public CidStorage(string filePath) - { - this.filePath = filePath; - if (!File.Exists(filePath)) - { - File.WriteAllText(filePath, string.Empty); - } - } - - public bool TryAddCid(string cid, ulong userId) - { - lock (_lock) - { - var existingEntries = File.ReadAllLines(filePath); - if (existingEntries.Any(line => line.Split(',')[0] == cid)) - { - return false; - } - - File.AppendAllLines(filePath, new[] { $"{cid},{userId}" }); - return true; - } - } - } -} diff --git a/Tools/BiblioTech/Commands/CheckDownloadCommand.cs b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs new file mode 100644 index 00000000..32cff02d --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs @@ -0,0 +1,53 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckDownloadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption contentOption = new StringOption( + name: "content", + description: "Content of the downloaded file", + isRequired: false); + + public CheckDownloadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkdownload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the download connectivity of your Codex node."; + public override CommandOption[] Options => [contentOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var content = await contentOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(content)) + { + await checker.StartDownloadCheck(handler, user.Id); + } + else + { + await checker.VerifyDownloadCheck(handler, user.Id, content); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckDownloadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs new file mode 100644 index 00000000..be6dd2be --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -0,0 +1,66 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; +using Discord; + +namespace BiblioTech.Commands +{ + public class CheckResponseHandler : ICheckResponseHandler + { + private CommandContext context; + private readonly IUser user; + + public CheckResponseHandler(CommandContext context, IUser user) + { + this.context = context; + this.user = user; + } + + public async Task CheckNotStarted() + { + await context.Followup("Run this command without any arguments first, to begin the check process."); + } + + public async Task CouldNotDownloadCid() + { + await context.Followup("Could not download the CID."); + } + + public async Task GiveCidToUser(string cid) + { + await context.Followup("Please download this CID using your Codex node. " + + "Then provide the content of the downloaded file as argument to this command. " + + $"`{cid}`"); + } + + public async Task GiveDataFileToUser(string fileContent) + { + await context.Followup("Please download the attached file. Upload it to your Codex node, " + + "then provide the CID as argument to this command."); + + await context.SendFile(fileContent); + } + + public async Task GiveRoleReward() + { + try + { + await Program.RoleDriver.GiveAltruisticRole(user); + await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!"); + } + catch (Exception ex) + { + await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user <@{user.Id}>: {ex.Message}"); + } + } + + public async Task InvalidData() + { + await context.Followup("The received data didn't match. Check has failed."); + } + + public async Task NowCompleted() + { + await context.Followup("Successfully completed the check!"); + } + } +} diff --git a/Tools/BiblioTech/Commands/CheckUploadCommand.cs b/Tools/BiblioTech/Commands/CheckUploadCommand.cs new file mode 100644 index 00000000..b1589b5c --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckUploadCommand.cs @@ -0,0 +1,53 @@ +using BiblioTech.CodexChecking; +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckUploadCommand : BaseCommand + { + private readonly CodexTwoWayChecker checker; + + private readonly StringOption cidOption = new StringOption( + name: "cid", + description: "Codex Content-Identifier", + isRequired: false); + + public CheckUploadCommand(CodexTwoWayChecker checker) + { + this.checker = checker; + } + + public override string Name => "checkupload"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks the upload connectivity of your Codex node."; + public override CommandOption[] Options => [cidOption]; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var cid = await cidOption.Parse(context); + try + { + var handler = new CheckResponseHandler(context, user); + if (string.IsNullOrEmpty(cid)) + { + await checker.StartUploadCheck(handler, user.Id); + } + else + { + await checker.VerifyUploadCheck(handler, user.Id, cid); + } + } + catch (Exception ex) + { + await RespondWithError(context, ex); + } + } + + private async Task RespondWithError(CommandContext context, Exception ex) + { + await Program.AdminChecker.SendInAdminChannel("Exception during CheckUploadCommand: " + ex); + await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel."); + } + } +} diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 342fbb08..b732e090 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -49,6 +49,7 @@ namespace BiblioTech public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); + public string ChecksDataPath => Path.Combine(DataPath, "checks"); public string LogPath => Path.Combine(DataPath, "logs"); public bool DebugNoDiscord => NoDiscord == 1; } diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 42a065ae..884ef8b7 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -49,6 +49,16 @@ namespace BiblioTech.Options } } + public async Task SendFile(string fileContent) + { + using var stream = new MemoryStream(); + using var writer = new StreamWriter(stream); + writer.Write(fileContent); + stream.Position = 0; + + await Command.RespondWithFileAsync(stream, "CheckFile.txt", ephemeral: true); + } + private string FormatChunk(string[] chunk) { return string.Join(Environment.NewLine, chunk); diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index ffc4f86d..b3d62338 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -1,4 +1,5 @@ using ArgsUniform; +using BiblioTech.CodexChecking; using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; @@ -80,7 +81,9 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; - var checker = new CodexTwoWayChecker(Config, Log); + var checkRepo = new CheckRepo(Config); + var codexWrapper = new CodexWrapper(Log, Config); + var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); var sprCommand = new SprCommand(); @@ -90,7 +93,8 @@ namespace BiblioTech sprCommand, associateCommand, notifyCommand, - new CheckCidCommand(checker), + new CheckUploadCommand(checker), + new CheckDownloadCommand(checker), new AdminCommand(sprCommand, replacement) ); diff --git a/Tools/BiblioTech/RandomBusyMessage.cs b/Tools/BiblioTech/RandomBusyMessage.cs index 290c9273..8f0477a7 100644 --- a/Tools/BiblioTech/RandomBusyMessage.cs +++ b/Tools/BiblioTech/RandomBusyMessage.cs @@ -14,7 +14,9 @@ "Analyzing the wavelengths...", "Charging the flux-capacitor...", "Jumping to hyperspace...", - "Computing the ultimate answer..." + "Computing the ultimate answer...", + "Turning it off and on again...", + "Compiling from sources..." }; public static string Get() From 6e6b9a6bfe603dd199eaf9ad0cae72ebc24b35d5 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 15:08:50 +0200 Subject: [PATCH 13/36] fixes auth string parsing --- Tools/BiblioTech/CodexChecking/CodexWrapper.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs index a8cbe785..3c295d7f 100644 --- a/Tools/BiblioTech/CodexChecking/CodexWrapper.cs +++ b/Tools/BiblioTech/CodexChecking/CodexWrapper.cs @@ -68,7 +68,7 @@ namespace BiblioTech.CodexChecking private HttpFactory CreateHttpFactory() { - if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + if (string.IsNullOrEmpty(config.CodexEndpointAuth) || !config.CodexEndpointAuth.Contains(":")) { return new HttpFactory(log); } From 8326578006ce91076f81ab8bf97333457cc941ea Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 15:18:29 +0200 Subject: [PATCH 14/36] forgot to ensure checkdata path --- Tools/BiblioTech/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index b3d62338..af2c9ca3 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -34,6 +34,7 @@ namespace BiblioTech EnsurePath(Config.DataPath); EnsurePath(Config.UserDataPath); EnsurePath(Config.EndpointsPath); + EnsurePath(Config.ChecksDataPath); return new Program().MainAsync(args); } From 2ea5bf1c5d8127ae320b5badd817265bb429ac1a Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 17:49:26 +0200 Subject: [PATCH 15/36] attempt to fix discord file sending --- Tools/BiblioTech/Options/CommandContext.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 884ef8b7..8f350c89 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -51,12 +51,19 @@ namespace BiblioTech.Options public async Task SendFile(string fileContent) { - using var stream = new MemoryStream(); - using var writer = new StreamWriter(stream); + var stream = new MemoryStream(); + var writer = new StreamWriter(stream); writer.Write(fileContent); - stream.Position = 0; await Command.RespondWithFileAsync(stream, "CheckFile.txt", ephemeral: true); + + // Detached task for cleaning up the stream resources. + _ = Task.Run(() => + { + Thread.Sleep(TimeSpan.FromSeconds(30)); + writer.Dispose(); + stream.Dispose(); + }); } private string FormatChunk(string[] chunk) From 99f7e25f52994f0faaf7c51efdea1ac9c60790ce Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 18:04:06 +0200 Subject: [PATCH 16/36] still trying to fix the discord file send --- Framework/Utils/RandomUtils.cs | 3 ++- Tools/BiblioTech/Commands/CheckResponseHandler.cs | 4 +--- Tools/BiblioTech/Options/CommandContext.cs | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Framework/Utils/RandomUtils.cs b/Framework/Utils/RandomUtils.cs index b5d2580b..f3792123 100644 --- a/Framework/Utils/RandomUtils.cs +++ b/Framework/Utils/RandomUtils.cs @@ -47,7 +47,8 @@ var result = ""; while (result.Length < requiredLength) { - var bytes = new byte[1024]; + var len = Math.Min(1024, requiredLength - result.Length); + var bytes = new byte[len]; random.NextBytes(bytes); result += string.Join("", bytes.Select(b => b.ToString())); } diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index be6dd2be..9a32a7c3 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -34,10 +34,8 @@ namespace BiblioTech.Commands public async Task GiveDataFileToUser(string fileContent) { - await context.Followup("Please download the attached file. Upload it to your Codex node, " + + await context.SendFile(fileContent, "Please download the attached file. Upload it to your Codex node, " + "then provide the CID as argument to this command."); - - await context.SendFile(fileContent); } public async Task GiveRoleReward() diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 8f350c89..73ca344d 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -49,13 +49,13 @@ namespace BiblioTech.Options } } - public async Task SendFile(string fileContent) + public async Task SendFile(string fileContent, string message) { var stream = new MemoryStream(); var writer = new StreamWriter(stream); writer.Write(fileContent); - await Command.RespondWithFileAsync(stream, "CheckFile.txt", ephemeral: true); + await Command.RespondWithFileAsync(stream, "CheckFile.txt", text: message, ephemeral: true); // Detached task for cleaning up the stream resources. _ = Task.Run(() => From 9540e97e2d3edf43ac0f4a139fb126083e04bd67 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Thu, 10 Apr 2025 18:11:02 +0200 Subject: [PATCH 17/36] still trying to fix it --- Tools/BiblioTech/Options/CommandContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index 73ca344d..c454be0f 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -55,7 +55,7 @@ namespace BiblioTech.Options var writer = new StreamWriter(stream); writer.Write(fileContent); - await Command.RespondWithFileAsync(stream, "CheckFile.txt", text: message, ephemeral: true); + await Command.FollowupWithFileAsync(stream, "CheckFile.txt", text: message, ephemeral: true); // Detached task for cleaning up the stream resources. _ = Task.Run(() => From fc9aa0726c3cebe82fa21170ee43a111de24c2f4 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 08:32:18 +0200 Subject: [PATCH 18/36] Adds debug messages for upload check --- Framework/Utils/RandomUtils.cs | 5 +++-- .../CodexChecking/CodexTwoWayChecker.cs | 22 ++++++++++++------- .../Commands/CheckResponseHandler.cs | 5 +++++ Tools/BiblioTech/Options/StringOption.cs | 3 ++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/Framework/Utils/RandomUtils.cs b/Framework/Utils/RandomUtils.cs index f3792123..9f127167 100644 --- a/Framework/Utils/RandomUtils.cs +++ b/Framework/Utils/RandomUtils.cs @@ -47,13 +47,14 @@ var result = ""; while (result.Length < requiredLength) { - var len = Math.Min(1024, requiredLength - result.Length); + var remaining = requiredLength - result.Length; + var len = Math.Min(1024, remaining); var bytes = new byte[len]; random.NextBytes(bytes); result += string.Join("", bytes.Select(b => b.ToString())); } - return result; + return result.Substring(0, Convert.ToInt32(requiredLength)); } } } diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index 8dadc97b..216ab7f0 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -15,6 +15,8 @@ namespace BiblioTech.CodexChecking Task CouldNotDownloadCid(); Task GiveCidToUser(string cid); Task GiveDataFileToUser(string fileContent); + + Task ToAdminChannel(string msg); } public class CodexTwoWayChecker @@ -91,9 +93,9 @@ namespace BiblioTech.CodexChecking return; } - if (IsManifestLengthCompatible(check, manifest)) + if (await IsManifestLengthCompatible(handler, check, manifest)) { - if (IsContentCorrect(check, receivedCid)) + if (await IsContentCorrect(handler, check, receivedCid)) { CheckNowCompleted(handler, check, userId); return; @@ -148,34 +150,38 @@ namespace BiblioTech.CodexChecking } } - private bool IsManifestLengthCompatible(TransferCheck check, Manifest manifest) + private async Task IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) { var dataLength = check.UniqueData.Length; var manifestLength = manifest.OriginalBytes.SizeInBytes; + await handler.ToAdminChannel($"Debug:dataLength={dataLength},manifestLength={manifestLength}"); + return manifestLength > (dataLength - 1) && manifestLength < (dataLength + 1); } - private bool IsContentCorrect(TransferCheck check, string receivedCid) + private async Task IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) { try { - return codexWrapper.OnCodex(node => + var content = codexWrapper.OnCodex(node => { var file = node.DownloadContent(new ContentId(receivedCid)); - if (file == null) return false; + if (file == null) return string.Empty; try { - var content = File.ReadAllText(file.Filename).Trim(); - return content == check.UniqueData; + return File.ReadAllText(file.Filename).Trim(); } finally { if (File.Exists(file.Filename)) File.Delete(file.Filename); } }); + + await handler.ToAdminChannel($"Debug:content=`{content}`,check=`{check.UniqueData}`"); + return content == check.UniqueData; } catch { diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index 9a32a7c3..8777003e 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -60,5 +60,10 @@ namespace BiblioTech.Commands { await context.Followup("Successfully completed the check!"); } + + public async Task ToAdminChannel(string msg) + { + await Program.AdminChecker.SendInAdminChannel(msg); + } } } diff --git a/Tools/BiblioTech/Options/StringOption.cs b/Tools/BiblioTech/Options/StringOption.cs index 15e04c9a..047bf15a 100644 --- a/Tools/BiblioTech/Options/StringOption.cs +++ b/Tools/BiblioTech/Options/StringOption.cs @@ -12,11 +12,12 @@ namespace BiblioTech.Options public async Task Parse(CommandContext context) { var strData = context.Options.SingleOrDefault(o => o.Name == Name); - if (strData == null) + if (strData == null && IsRequired) { await context.Followup("String option not received."); return null; } + if (strData == null) return null; return strData.Value as string; } } From a9cba76de506b27ac97d445c6eb156882e093171 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 08:58:39 +0200 Subject: [PATCH 19/36] debugging check commands. --- .../FrameworkTests/Utils/RandomUtilsTests.cs | 19 ++++++++++++ .../CodexChecking/CodexTwoWayChecker.cs | 29 ++++++++++++------- .../Commands/CheckDownloadCommand.cs | 5 ++++ .../Commands/CheckResponseHandler.cs | 3 +- 4 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 Tests/FrameworkTests/Utils/RandomUtilsTests.cs diff --git a/Tests/FrameworkTests/Utils/RandomUtilsTests.cs b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs new file mode 100644 index 00000000..3859f0d3 --- /dev/null +++ b/Tests/FrameworkTests/Utils/RandomUtilsTests.cs @@ -0,0 +1,19 @@ +using NUnit.Framework; +using Utils; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class RandomUtilsTests + { + [Test] + [Combinatorial] + public void TestRandomStringLength( + [Values(1, 5, 10, 1023, 1024, 1025, 2222)] int length) + { + var str = RandomUtils.GenerateRandomString(length); + + Assert.That(str.Length, Is.EqualTo(length)); + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index 216ab7f0..4381198a 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -8,7 +8,7 @@ namespace BiblioTech.CodexChecking public interface ICheckResponseHandler { Task CheckNotStarted(); - Task NowCompleted(); + Task NowCompleted(ulong userId, string checkName); Task GiveRoleReward(); Task InvalidData(); @@ -56,13 +56,14 @@ namespace BiblioTech.CodexChecking return; } + Log($"Verifying for downloadCheck: received: '{receivedData}' check: '{check.UniqueData}'"); if (string.IsNullOrEmpty(receivedData) || receivedData != check.UniqueData) { await handler.InvalidData(); return; } - CheckNowCompleted(handler, check, userId); + await CheckNowCompleted(handler, check, userId, "DownloadCheck"); } public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId) @@ -93,11 +94,12 @@ namespace BiblioTech.CodexChecking return; } - if (await IsManifestLengthCompatible(handler, check, manifest)) + await IsManifestLengthCompatible(handler, check, manifest); + if (true) // debugging, always pass the length check { if (await IsContentCorrect(handler, check, receivedCid)) { - CheckNowCompleted(handler, check, userId); + await CheckNowCompleted(handler, check, userId, "UploadCheck"); return; } } @@ -107,7 +109,7 @@ namespace BiblioTech.CodexChecking private string GenerateUniqueData() { - return $"'{RandomBusyMessage.Get()}'{RandomUtils.GenerateRandomString(12)}"; + return $"{RandomBusyMessage.Get().Substring(5)}{RandomUtils.GenerateRandomString(12)}"; } private string UploadData(string uniqueData) @@ -155,6 +157,7 @@ namespace BiblioTech.CodexChecking var dataLength = check.UniqueData.Length; var manifestLength = manifest.OriginalBytes.SizeInBytes; + Log($"Checking manifest length: dataLength={dataLength},manifestLength={manifestLength}"); await handler.ToAdminChannel($"Debug:dataLength={dataLength},manifestLength={manifestLength}"); return @@ -180,6 +183,7 @@ namespace BiblioTech.CodexChecking } }); + Log($"Checking content: content={content},check={check.UniqueData}"); await handler.ToAdminChannel($"Debug:content=`{content}`,check=`{check.UniqueData}`"); return content == check.UniqueData; } @@ -189,18 +193,18 @@ namespace BiblioTech.CodexChecking } } - private void CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId) + private async Task CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId, string checkName) { if (check.CompletedUtc != DateTime.MinValue) return; check.CompletedUtc = DateTime.UtcNow; repo.SaveChanges(); - handler.NowCompleted(); - CheckUserForRoleRewards(handler, userId); + await handler.NowCompleted(userId, checkName); + await CheckUserForRoleRewards(handler, userId); } - private void CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId) + private async Task CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId) { var check = repo.GetOrCreate(userId); @@ -208,8 +212,13 @@ namespace BiblioTech.CodexChecking check.UploadCheck.CompletedUtc != DateTime.MinValue && check.DownloadCheck.CompletedUtc != DateTime.MinValue) { - handler.GiveRoleReward(); + await handler.GiveRoleReward(); } } + + private void Log(string msg) + { + log.Log(msg); + } } } diff --git a/Tools/BiblioTech/Commands/CheckDownloadCommand.cs b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs index 32cff02d..0e1aa563 100644 --- a/Tools/BiblioTech/Commands/CheckDownloadCommand.cs +++ b/Tools/BiblioTech/Commands/CheckDownloadCommand.cs @@ -35,6 +35,11 @@ namespace BiblioTech.Commands } else { + if (content.Length > 1024) + { + await context.Followup("Provided content is too long!"); + return; + } await checker.VerifyDownloadCheck(handler, user.Id, content); } } diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index 8777003e..c3cff1b1 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -56,9 +56,10 @@ namespace BiblioTech.Commands await context.Followup("The received data didn't match. Check has failed."); } - public async Task NowCompleted() + public async Task NowCompleted(ulong userId, string checkName) { await context.Followup("Successfully completed the check!"); + await Program.AdminChecker.SendInAdminChannel($"User <@{userId}> has completed check: {checkName}"); } public async Task ToAdminChannel(string msg) From 1b1e1020498a6ec9326b593f5897ae5cc2b0ce57 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 09:11:30 +0200 Subject: [PATCH 20/36] Replacing streams with temp file on discord file upload --- Tools/BiblioTech/Options/CommandContext.cs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Tools/BiblioTech/Options/CommandContext.cs b/Tools/BiblioTech/Options/CommandContext.cs index c454be0f..fbd45b36 100644 --- a/Tools/BiblioTech/Options/CommandContext.cs +++ b/Tools/BiblioTech/Options/CommandContext.cs @@ -51,18 +51,18 @@ namespace BiblioTech.Options public async Task SendFile(string fileContent, string message) { - var stream = new MemoryStream(); - var writer = new StreamWriter(stream); - writer.Write(fileContent); + if (fileContent.Length < 1) throw new Exception("File content is empty."); - await Command.FollowupWithFileAsync(stream, "CheckFile.txt", text: message, ephemeral: true); + var filename = Guid.NewGuid().ToString() + ".tmp"; + File.WriteAllText(filename, fileContent); + + await Command.FollowupWithFileAsync(filename, "Codex_UploadCheckFile.txt", text: message, ephemeral: true); // Detached task for cleaning up the stream resources. _ = Task.Run(() => { - Thread.Sleep(TimeSpan.FromSeconds(30)); - writer.Dispose(); - stream.Dispose(); + Thread.Sleep(TimeSpan.FromMinutes(2)); + File.Delete(filename); }); } From edc091e2c9404fabacdd315a663454defac1a5f6 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 14:52:55 +0200 Subject: [PATCH 21/36] fixes issue where checks dont end if they were previously completed --- Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index 4381198a..f54bea55 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -195,12 +195,12 @@ namespace BiblioTech.CodexChecking private async Task CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId, string checkName) { - if (check.CompletedUtc != DateTime.MinValue) return; + await handler.NowCompleted(userId, checkName); + if (check.CompletedUtc != DateTime.MinValue) return; check.CompletedUtc = DateTime.UtcNow; repo.SaveChanges(); - await handler.NowCompleted(userId, checkName); await CheckUserForRoleRewards(handler, userId); } From c88b9d920e94c04405010ae7d60997fcf751f293 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 15:01:20 +0200 Subject: [PATCH 22/36] removes debug messages in admin channel --- Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index f54bea55..5f37cfb2 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -94,10 +94,9 @@ namespace BiblioTech.CodexChecking return; } - await IsManifestLengthCompatible(handler, check, manifest); - if (true) // debugging, always pass the length check + if (IsManifestLengthCompatible(handler, check, manifest)) { - if (await IsContentCorrect(handler, check, receivedCid)) + if (IsContentCorrect(handler, check, receivedCid)) { await CheckNowCompleted(handler, check, userId, "UploadCheck"); return; @@ -152,20 +151,19 @@ namespace BiblioTech.CodexChecking } } - private async Task IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) + private bool IsManifestLengthCompatible(ICheckResponseHandler handler, TransferCheck check, Manifest manifest) { var dataLength = check.UniqueData.Length; var manifestLength = manifest.OriginalBytes.SizeInBytes; Log($"Checking manifest length: dataLength={dataLength},manifestLength={manifestLength}"); - await handler.ToAdminChannel($"Debug:dataLength={dataLength},manifestLength={manifestLength}"); return manifestLength > (dataLength - 1) && manifestLength < (dataLength + 1); } - private async Task IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) + private bool IsContentCorrect(ICheckResponseHandler handler, TransferCheck check, string receivedCid) { try { @@ -184,7 +182,6 @@ namespace BiblioTech.CodexChecking }); Log($"Checking content: content={content},check={check.UniqueData}"); - await handler.ToAdminChannel($"Debug:content=`{content}`,check=`{check.UniqueData}`"); return content == check.UniqueData; } catch From 112c1f37c18c6ae949fbfe0607b733b814ca58d8 Mon Sep 17 00:00:00 2001 From: ThatBen Date: Fri, 11 Apr 2025 15:19:03 +0200 Subject: [PATCH 23/36] Making the instruction messages more catchy --- .../Commands/CheckResponseHandler.cs | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index c3cff1b1..8d20e5bf 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -1,4 +1,5 @@ -using BiblioTech.CodexChecking; +using System.Linq; +using BiblioTech.CodexChecking; using BiblioTech.Options; using Discord; @@ -27,15 +28,28 @@ namespace BiblioTech.Commands public async Task GiveCidToUser(string cid) { - await context.Followup("Please download this CID using your Codex node. " + - "Then provide the content of the downloaded file as argument to this command. " + - $"`{cid}`"); + await context.Followup( + FormatCatchyMessage("[💾] Please download this CID using your Codex node.", + $"👉 `{cid}`.", + "👉 Then provide the *content of the downloaded file* as argument to this command.")); } public async Task GiveDataFileToUser(string fileContent) { - await context.SendFile(fileContent, "Please download the attached file. Upload it to your Codex node, " + - "then provide the CID as argument to this command."); + await context.SendFile(fileContent, + FormatCatchyMessage("[💿] Please download the attached file.", + "👉 Upload it to your Codex node.", + "👉 Then provide the *CID* as argument to this command.")); + } + + private string FormatCatchyMessage(string title, params string[] content) + { + var entries = new List(); + entries.Add(title); + entries.Add("```"); + entries.AddRange(content); + entries.Add("```"); + return string.Join(Environment.NewLine, entries.ToArray()); } public async Task GiveRoleReward() From 9e4e56205a1e987d594e3bf97d41f625ea095095 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 16 Apr 2025 15:17:40 +0200 Subject: [PATCH 24/36] Cleans up a lot of old reward system code. Adds periodic role check for p2p participant role --- Framework/DiscordRewards/CheckConfig.cs | 21 --- ...veRewardsCommand.cs => EventsAndErrors.cs} | 11 +- Framework/DiscordRewards/RewardConfig.cs | 18 -- Framework/DiscordRewards/RewardRepo.cs | 53 ------ .../UtilityTests/DiscordBotTests.cs | 8 +- Tools/BiblioTech/AdminChecker.cs | 1 - .../CodexChecking/ActiveP2pRoleRemover.cs | 66 +++++++ .../CodexChecking/CodexTwoWayChecker.cs | 4 +- Tools/BiblioTech/CommandHandler.cs | 27 ++- .../Commands/CheckResponseHandler.cs | 6 +- Tools/BiblioTech/Configuration.cs | 34 +++- Tools/BiblioTech/LoggingRoleDriver.cs | 29 ++- Tools/BiblioTech/Program.cs | 5 +- Tools/BiblioTech/Rewards/RewardContext.cs | 102 ----------- Tools/BiblioTech/Rewards/RewardController.cs | 16 +- Tools/BiblioTech/Rewards/RoleDriver.cs | 168 ++++-------------- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 115 ++++++++++++ Tools/BiblioTech/UserRepo.cs | 7 + Tools/TestNetRewarder/BotClient.cs | 2 +- Tools/TestNetRewarder/Processor.cs | 9 +- Tools/TestNetRewarder/RequestBuilder.cs | 29 +-- Tools/TestNetRewarder/RewardCheck.cs | 113 ------------ Tools/TestNetRewarder/RewardChecker.cs | 17 -- 23 files changed, 332 insertions(+), 529 deletions(-) delete mode 100644 Framework/DiscordRewards/CheckConfig.cs rename Framework/DiscordRewards/{GiveRewardsCommand.cs => EventsAndErrors.cs} (53%) delete mode 100644 Framework/DiscordRewards/RewardConfig.cs delete mode 100644 Framework/DiscordRewards/RewardRepo.cs create mode 100644 Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs delete mode 100644 Tools/BiblioTech/Rewards/RewardContext.cs create mode 100644 Tools/BiblioTech/Rewards/RoleModifyContext.cs delete mode 100644 Tools/TestNetRewarder/RewardCheck.cs delete mode 100644 Tools/TestNetRewarder/RewardChecker.cs diff --git a/Framework/DiscordRewards/CheckConfig.cs b/Framework/DiscordRewards/CheckConfig.cs deleted file mode 100644 index 34425cea..00000000 --- a/Framework/DiscordRewards/CheckConfig.cs +++ /dev/null @@ -1,21 +0,0 @@ -using Utils; - -namespace DiscordRewards -{ - public class CheckConfig - { - public CheckType Type { get; set; } - public ulong MinNumberOfHosts { get; set; } - public ByteSize MinSlotSize { get; set; } = 0.Bytes(); - public TimeSpan MinDuration { get; set; } = TimeSpan.Zero; - } - - public enum CheckType - { - Uninitialized, - HostFilledSlot, - HostFinishedSlot, - ClientPostedContract, - ClientStartedContract, - } -} diff --git a/Framework/DiscordRewards/GiveRewardsCommand.cs b/Framework/DiscordRewards/EventsAndErrors.cs similarity index 53% rename from Framework/DiscordRewards/GiveRewardsCommand.cs rename to Framework/DiscordRewards/EventsAndErrors.cs index 3aae088b..53e4b3cc 100644 --- a/Framework/DiscordRewards/GiveRewardsCommand.cs +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -1,23 +1,16 @@ namespace DiscordRewards { - public class GiveRewardsCommand + public class EventsAndErrors { - public RewardUsersCommand[] Rewards { get; set; } = Array.Empty(); public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); public string[] Errors { get; set; } = Array.Empty(); public bool HasAny() { - return Rewards.Any() || EventsOverview.Any(); + return Errors.Length > 0 || EventsOverview.Length > 0; } } - public class RewardUsersCommand - { - public ulong RewardId { get; set; } - public string[] UserAddresses { get; set; } = Array.Empty(); - } - public class ChainEventMessage { public ulong BlockNumber { get; set; } diff --git a/Framework/DiscordRewards/RewardConfig.cs b/Framework/DiscordRewards/RewardConfig.cs deleted file mode 100644 index dda0dfe1..00000000 --- a/Framework/DiscordRewards/RewardConfig.cs +++ /dev/null @@ -1,18 +0,0 @@ -namespace DiscordRewards -{ - public class RewardConfig - { - public const string UsernameTag = ""; - - public RewardConfig(ulong roleId, string message, CheckConfig checkConfig) - { - RoleId = roleId; - Message = message; - CheckConfig = checkConfig; - } - - public ulong RoleId { get; } - public string Message { get; } - public CheckConfig CheckConfig { get; } - } -} diff --git a/Framework/DiscordRewards/RewardRepo.cs b/Framework/DiscordRewards/RewardRepo.cs deleted file mode 100644 index 1c97caf9..00000000 --- a/Framework/DiscordRewards/RewardRepo.cs +++ /dev/null @@ -1,53 +0,0 @@ -namespace DiscordRewards -{ - public class RewardRepo - { - private static string Tag => RewardConfig.UsernameTag; - - public RewardConfig[] Rewards { get; } = new RewardConfig[0]; - - // Example configuration, from test server: - //{ - // // Filled any slot - // new RewardConfig(1187039439558541498, $"{Tag} successfully filled their first slot!", new CheckConfig - // { - // Type = CheckType.HostFilledSlot - // }), - - // // Finished any slot - // new RewardConfig(1202286165630390339, $"{Tag} successfully finished their first slot!", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot - // }), - - // // Finished a sizable slot - // new RewardConfig(1202286218738405418, $"{Tag} finished their first 1GB-24h slot! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.HostFinishedSlot, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }), - - // // Posted any contract - // new RewardConfig(1202286258370383913, $"{Tag} posted their first contract!", new CheckConfig - // { - // Type = CheckType.ClientPostedContract - // }), - - // // Started any contract - // new RewardConfig(1202286330873126992, $"A contract created by {Tag} reached Started state for the first time!", new CheckConfig - // { - // Type = CheckType.ClientStartedContract - // }), - - // // Started a sizable contract - // new RewardConfig(1202286381670608909, $"A large contract created by {Tag} reached Started state for the first time! (10mb/5mins for test)", new CheckConfig - // { - // Type = CheckType.ClientStartedContract, - // MinNumberOfHosts = 4, - // MinSlotSize = 10.MB(), - // MinDuration = TimeSpan.FromMinutes(5.0), - // }) - //}; - } -} diff --git a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs index abebac0c..22819e1b 100644 --- a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs @@ -88,7 +88,7 @@ namespace ExperimentalTests.UtilityTests $"Event '{msg}' did not occure correct number of times."); } - private void OnCommand(string timestamp, GiveRewardsCommand call) + private void OnCommand(string timestamp, EventsAndErrors call) { Log($""); foreach (var e in call.EventsOverview) @@ -276,7 +276,7 @@ namespace ExperimentalTests.UtilityTests monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log"); } - public void Start(Action onCommand) + public void Start(Action onCommand) { monitor.Start(line => ParseLine(line, onCommand)); } @@ -286,14 +286,14 @@ namespace ExperimentalTests.UtilityTests monitor.Stop(); } - private void ParseLine(string line, Action onCommand) + private void ParseLine(string line, Action onCommand) { try { var timestamp = line.Substring(0, 30); var json = line.Substring(31); - var cmd = JsonConvert.DeserializeObject(json); + var cmd = JsonConvert.DeserializeObject(json); if (cmd != null) { onCommand(timestamp, cmd); diff --git a/Tools/BiblioTech/AdminChecker.cs b/Tools/BiblioTech/AdminChecker.cs index 532220de..96f89691 100644 --- a/Tools/BiblioTech/AdminChecker.cs +++ b/Tools/BiblioTech/AdminChecker.cs @@ -1,7 +1,6 @@ using BiblioTech.Options; using Discord; using Discord.WebSocket; -using Org.BouncyCastle.Utilities; namespace BiblioTech { diff --git a/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs new file mode 100644 index 00000000..4a448792 --- /dev/null +++ b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs @@ -0,0 +1,66 @@ +using Discord; +using Logging; +using System.Threading.Tasks; + +namespace BiblioTech.CodexChecking +{ + public class ActiveP2pRoleRemover + { + private readonly Configuration config; + private readonly ILog log; + private readonly CheckRepo repo; + + public ActiveP2pRoleRemover(Configuration config, ILog log, CheckRepo repo) + { + this.config = config; + this.log = log; + this.repo = repo; + } + + public void Start() + { + if (config.ActiveP2pRoleDurationMinutes > 0) + { + Task.Run(Worker); + } + } + + private void Worker() + { + var loopDelay = TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes) / 60; + var min = TimeSpan.FromMinutes(10.0); + if (loopDelay < min) loopDelay = min; + + try + { + while (true) + { + Thread.Sleep(loopDelay); + CheckP2pRoleRemoval(); + } + } + catch (Exception ex) + { + log.Error($"Exception in {nameof(ActiveP2pRoleRemover)}: {ex}"); + Environment.Exit(1); + } + } + + private void CheckP2pRoleRemoval() + { + var expiryMoment = DateTime.UtcNow - TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes); + + Program.RoleDriver.IterateRemoveActiveP2pParticipants(p => ShouldRemoveRole(p, expiryMoment)); + } + + private bool ShouldRemoveRole(IUser user, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + + if (report.UploadCheck.CompletedUtc > expiryMoment) return false; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return false; + + return true; + } + } +} diff --git a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs index 5f37cfb2..c5b9361a 100644 --- a/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs +++ b/Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs @@ -194,7 +194,6 @@ namespace BiblioTech.CodexChecking { await handler.NowCompleted(userId, checkName); - if (check.CompletedUtc != DateTime.MinValue) return; check.CompletedUtc = DateTime.UtcNow; repo.SaveChanges(); @@ -205,8 +204,7 @@ namespace BiblioTech.CodexChecking { var check = repo.GetOrCreate(userId); - if ( - check.UploadCheck.CompletedUtc != DateTime.MinValue && + if (check.UploadCheck.CompletedUtc != DateTime.MinValue && check.DownloadCheck.CompletedUtc != DateTime.MinValue) { await handler.GiveRoleReward(); diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index b6f2f7ca..8c888ea7 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -4,6 +4,9 @@ using Discord; using Newtonsoft.Json; using BiblioTech.Rewards; using Logging; +using BiblioTech.CodexChecking; +using Nethereum.Model; +using static Org.BouncyCastle.Math.EC.ECCurve; namespace BiblioTech { @@ -11,13 +14,15 @@ namespace BiblioTech { private readonly DiscordSocketClient client; private readonly CustomReplacement replacement; + private readonly ActiveP2pRoleRemover roleRemover; private readonly BaseCommand[] commands; private readonly ILog log; - public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, params BaseCommand[] commands) + public CommandHandler(ILog log, DiscordSocketClient client, CustomReplacement replacement, ActiveP2pRoleRemover roleRemover, params BaseCommand[] commands) { this.client = client; this.replacement = replacement; + this.roleRemover = roleRemover; this.commands = commands; this.log = log; client.Ready += Client_Ready; @@ -30,10 +35,14 @@ namespace BiblioTech Program.AdminChecker.SetGuild(guild); log.Log($"Initializing for guild: '{guild.Name}'"); - var adminChannels = guild.TextChannels.Where(Program.AdminChecker.IsAdminChannel).ToArray(); - if (adminChannels == null || !adminChannels.Any()) throw new Exception("No admin message channel"); - Program.AdminChecker.SetAdminChannel(adminChannels.First()); - Program.RoleDriver = new RoleDriver(client, log, replacement); + var adminChannel = GetChannel(guild, Program.Config.AdminChannelId); + if (adminChannel == null) throw new Exception("No admin message channel"); + var chainEventsChannel = GetChannel(guild, Program.Config.ChainEventsChannelId); + var rewardsChannel = GetChannel(guild, Program.Config.RewardsChannelId); + + Program.AdminChecker.SetAdminChannel(adminChannel); + Program.RoleDriver = new RoleDriver(client, Program.UserRepo, log, rewardsChannel); + Program.EventsSender = new ChainEventsSender(log, replacement, chainEventsChannel); var builders = commands.Select(c => { @@ -65,6 +74,8 @@ namespace BiblioTech { log.Log($"{cmd.Name} ({cmd.Description}) [{DescribOptions(cmd.Options)}]"); } + + roleRemover.Start(); } catch (HttpException exception) { @@ -75,6 +86,12 @@ namespace BiblioTech log.Log("Initialized."); } + private SocketTextChannel? GetChannel(SocketGuild guild, ulong id) + { + if (id == 0) return null; + return guild.TextChannels.SingleOrDefault(c => c.Id == id); + } + private string DescribOptions(IReadOnlyCollection options) { return string.Join(",", options.Select(DescribeOption).ToArray()); diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index 8d20e5bf..ca9dedd4 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -56,7 +56,11 @@ namespace BiblioTech.Commands { try { - await Program.RoleDriver.GiveAltruisticRole(user); + await Program.RoleDriver.RunRoleGiver(async r => + { + await r.GiveAltruisticRole(user); + await r.GiveActiveP2pParticipant(user); + }); await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!"); } catch (Exception ex) diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index b732e090..a8187379 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -26,9 +26,6 @@ namespace BiblioTech [Uniform("chain-events-channel-id", "cc", "CHAINEVENTSCHANNELID", false, "ID of the Discord server channel where chain events will be posted.")] public ulong ChainEventsChannelId { get; set; } - [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] - public ulong AltruisticRoleId { get; set; } - [Uniform("reward-api-port", "rp", "REWARDAPIPORT", true, "TCP listen port for the reward API.")] public int RewardApiPort { get; set; } = 31080; @@ -47,6 +44,37 @@ namespace BiblioTech [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")] public string CodexEndpointAuth { get; set; } = ""; + #region Role Rewards + + /// + /// Awarded when both checkupload and checkdownload have been completed. + /// + [Uniform("altruistic-role-id", "ar", "ALTRUISTICROLE", true, "ID of the Discord server role for Altruistic Mode.")] + public ulong AltruisticRoleId { get; set; } + + /// + /// Awarded as long as either checkupload or checkdownload were completed within the last ActiveP2pRoleDuration minutes. + /// + [Uniform("active-p2p-role-id", "apri", "ACTIVEP2PROLEID", false, "ID of discord server role for active p2p participants.")] + public ulong ActiveP2pParticipantRoleId { get; set; } + + [Uniform("active-p2p-role-duration", "aprd", "ACTIVEP2PROLEDURATION", false, "Duration in minutes for the active p2p participant role from the last successful check command.")] + public int ActiveP2pRoleDurationMinutes { get; set; } + + /// + /// Awarded as long as the user is hosting at least 1 slot. + /// + [Uniform("active-host-role-id", "ahri", "ACTIVEHOSTROLEID", false, "Id of discord server role for active slot hosters.")] + public ulong ActiveHostRoleId { get; set; } + + /// + /// Awarded as long as the user has at least 1 active storage purchase contract. + /// + [Uniform("active-client-role-id", "acri", "ACTIVECLIENTROLEID", false, "Id of discord server role for users with at least 1 active purchase contract.")] + public ulong ActiveClientRoleId { get; set; } + + #endregion + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); public string ChecksDataPath => Path.Combine(DataPath, "checks"); diff --git a/Tools/BiblioTech/LoggingRoleDriver.cs b/Tools/BiblioTech/LoggingRoleDriver.cs index f5435523..a5631271 100644 --- a/Tools/BiblioTech/LoggingRoleDriver.cs +++ b/Tools/BiblioTech/LoggingRoleDriver.cs @@ -15,18 +15,37 @@ namespace BiblioTech this.log = log; } - public async Task GiveAltruisticRole(IUser user) + public async Task RunRoleGiver(Func action) { await Task.CompletedTask; - - log.Log($"Give altruistic role to {user.Id}"); + await action(new LoggingRoleGiver(log)); } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task IterateRemoveActiveP2pParticipants(Func predicate) { await Task.CompletedTask; + } - log.Log(JsonConvert.SerializeObject(rewards, Formatting.None)); + private class LoggingRoleGiver : IRoleGiver + { + private readonly ILog log; + + public LoggingRoleGiver(ILog log) + { + this.log = log; + } + + public async Task GiveActiveP2pParticipant(IUser user) + { + log.Log($"Giving ActiveP2p role to " + user.Id); + await Task.CompletedTask; + } + + public async Task GiveAltruisticRole(IUser user) + { + log.Log($"Giving Altruistic role to " + user.Id); + await Task.CompletedTask; + } } } } diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index af2c9ca3..df1ce6bc 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -5,6 +5,7 @@ using BiblioTech.Rewards; using Discord; using Discord.WebSocket; using Logging; +using Nethereum.Model; namespace BiblioTech { @@ -17,6 +18,7 @@ namespace BiblioTech public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; public static IDiscordRoleDriver RoleDriver { get; set; } = null!; + public static ChainEventsSender EventsSender { get; set; } = null!; public static ILog Log { get; private set; } = null!; public static Task Main(string[] args) @@ -88,7 +90,8 @@ namespace BiblioTech var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); var sprCommand = new SprCommand(); - var handler = new CommandHandler(Log, client, replacement, + var roleRemover = new ActiveP2pRoleRemover(Config, Log, checkRepo); + var handler = new CommandHandler(Log, client, replacement, roleRemover, new GetBalanceCommand(associateCommand), new MintCommand(associateCommand), sprCommand, diff --git a/Tools/BiblioTech/Rewards/RewardContext.cs b/Tools/BiblioTech/Rewards/RewardContext.cs deleted file mode 100644 index d05d5f6d..00000000 --- a/Tools/BiblioTech/Rewards/RewardContext.cs +++ /dev/null @@ -1,102 +0,0 @@ -using Discord.WebSocket; -using Discord; -using DiscordRewards; - -namespace BiblioTech.Rewards -{ - public class RewardContext - { - private readonly Dictionary users; - private readonly Dictionary roles; - private readonly SocketTextChannel? rewardsChannel; - - public RewardContext(Dictionary users, Dictionary roles, SocketTextChannel? rewardsChannel) - { - this.users = users; - this.roles = roles; - this.rewardsChannel = rewardsChannel; - } - - public async Task ProcessGiveRewardsCommand(UserReward[] rewards) - { - foreach (var rewardCommand in rewards) - { - if (roles.ContainsKey(rewardCommand.RewardCommand.RewardId)) - { - var role = roles[rewardCommand.RewardCommand.RewardId]; - await ProcessRewardCommand(role, rewardCommand); - } - else - { - Program.Log.Error($"RoleID not found on guild: {rewardCommand.RewardCommand.RewardId}"); - } - } - } - - private async Task ProcessRewardCommand(RoleReward role, UserReward reward) - { - foreach (var user in reward.Users) - { - await GiveReward(role, user); - } - } - - private async Task GiveReward(RoleReward role, UserData user) - { - if (!users.ContainsKey(user.DiscordId)) - { - Program.Log.Log($"User by id '{user.DiscordId}' not found."); - return; - } - - var guildUser = users[user.DiscordId]; - - var alreadyHas = guildUser.RoleIds.ToArray(); - var logMessage = $"Giving reward '{role.SocketRole.Id}' to user '{user.DiscordId}'({user.Name})[" + - $"alreadyHas:{string.Join(",", alreadyHas.Select(a => a.ToString()))}]: "; - - - if (alreadyHas.Any(r => r == role.Reward.RoleId)) - { - logMessage += "Already has role"; - Program.Log.Log(logMessage); - return; - } - - await GiveRole(guildUser, role.SocketRole); - await SendNotification(role, user, guildUser); - await Task.Delay(1000); - logMessage += "Role given. Notification sent."; - Program.Log.Log(logMessage); - } - - private async Task GiveRole(IGuildUser user, SocketRole role) - { - try - { - Program.Log.Log($"Giving role {role.Name}={role.Id} to user {user.DisplayName}"); - await user.AddRoleAsync(role); - } - catch (Exception ex) - { - Program.Log.Error($"Failed to give role '{role.Name}' to user '{user.DisplayName}': {ex}"); - } - } - - private async Task SendNotification(RoleReward reward, UserData userData, IGuildUser user) - { - try - { - if (userData.NotificationsEnabled && rewardsChannel != null) - { - var msg = reward.Reward.Message.Replace(RewardConfig.UsernameTag, $"<@{user.Id}>"); - await rewardsChannel.SendMessageAsync(msg); - } - } - catch (Exception ex) - { - Program.Log.Error($"Failed to notify user '{user.DisplayName}' about role '{reward.SocketRole.Name}': {ex}"); - } - } - } -} diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 3a20a084..7cbed562 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -4,10 +4,20 @@ using Microsoft.AspNetCore.Mvc; namespace BiblioTech.Rewards { + /// + /// We like callbacks in this interface because we're trying to batch role-modifying operations, + /// So that we're not poking the server lots of times very quickly. + /// public interface IDiscordRoleDriver { - Task GiveRewards(GiveRewardsCommand rewards); + Task RunRoleGiver(Func action); + Task IterateRemoveActiveP2pParticipants(Func predicate); + } + + public interface IRoleGiver + { Task GiveAltruisticRole(IUser user); + Task GiveActiveP2pParticipant(IUser user); } [Route("api/[controller]")] @@ -21,11 +31,11 @@ namespace BiblioTech.Rewards } [HttpPost] - public async Task Give(GiveRewardsCommand cmd) + public async Task Give(EventsAndErrors cmd) { try { - await Program.RoleDriver.GiveRewards(cmd); + await Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors); } catch (Exception ex) { diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index 1964da8e..c716c011 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -1,6 +1,7 @@ using Discord; using Discord.WebSocket; using DiscordRewards; +using k8s.KubeConfigModels; using Logging; using Newtonsoft.Json; using Utils; @@ -10,145 +11,46 @@ namespace BiblioTech.Rewards public class RoleDriver : IDiscordRoleDriver { private readonly DiscordSocketClient client; + private readonly UserRepo userRepo; private readonly ILog log; private readonly SocketTextChannel? rewardsChannel; - private readonly ChainEventsSender eventsSender; - private readonly RewardRepo repo = new RewardRepo(); - public RoleDriver(DiscordSocketClient client, ILog log, CustomReplacement replacement) + public RoleDriver(DiscordSocketClient client, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) { this.client = client; + this.userRepo = userRepo; this.log = log; - rewardsChannel = GetChannel(Program.Config.RewardsChannelId); - eventsSender = new ChainEventsSender(log, replacement, GetChannel(Program.Config.ChainEventsChannelId)); + this.rewardsChannel = rewardsChannel; } - public async Task GiveRewards(GiveRewardsCommand rewards) + public async Task RunRoleGiver(Func action) { - log.Log($"Processing rewards command: '{JsonConvert.SerializeObject(rewards)}'"); + var context = await OpenRoleModifyContext(); + var mapper = new RoleMapper(context); + await action(mapper); + } - if (rewards.Rewards.Any()) + public async Task IterateRemoveActiveP2pParticipants(Func shouldRemove) + { + var context = await OpenRoleModifyContext(); + foreach (var user in context.Users) { - await ProcessRewards(rewards); - } - - await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors); - } - - public async Task GiveAltruisticRole(IUser user) - { - var guild = GetGuild(); - var role = guild.Roles.SingleOrDefault(r => r.Id == Program.Config.AltruisticRoleId); - if (role == null) return; - - var guildUser = guild.Users.SingleOrDefault(u => u.Id == user.Id); - if (guildUser == null) return; - - await guildUser.AddRoleAsync(role); - } - - private async Task ProcessRewards(GiveRewardsCommand rewards) - { - try - { - var guild = GetGuild(); - // We load all role and user information first, - // so we don't ask the server for the same info multiple times. - var context = new RewardContext( - await LoadAllUsers(guild), - LookUpAllRoles(guild, rewards), - rewardsChannel); - - await context.ProcessGiveRewardsCommand(LookUpUsers(rewards)); - } - catch (Exception ex) - { - log.Error("Failed to process rewards: " + ex); - } - } - - private SocketTextChannel? GetChannel(ulong id) - { - if (id == 0) return null; - return GetGuild().TextChannels.SingleOrDefault(c => c.Id == id); - } - - private async Task> LoadAllUsers(SocketGuild guild) - { - log.Log("Loading all users.."); - var result = new Dictionary(); - var users = guild.GetUsersAsync(); - await foreach (var ulist in users) - { - foreach (var u in ulist) + if (user.RoleIds.Any(r => r == Program.Config.ActiveP2pParticipantRoleId)) { - result.Add(u.Id, u); - //var roleIds = string.Join(",", u.RoleIds.Select(r => r.ToString()).ToArray()); - //log.Log($" > {u.Id}({u.DisplayName}) has [{roleIds}]"); - } - } - return result; - } - - private Dictionary LookUpAllRoles(SocketGuild guild, GiveRewardsCommand rewards) - { - var result = new Dictionary(); - foreach (var r in rewards.Rewards) - { - if (!result.ContainsKey(r.RewardId)) - { - var rewardConfig = repo.Rewards.SingleOrDefault(rr => rr.RoleId == r.RewardId); - if (rewardConfig == null) + // This user has the role. Should it be removed? + if (shouldRemove(user)) { - log.Log($"No Reward is configured for id '{r.RewardId}'."); - } - else - { - var socketRole = guild.GetRole(r.RewardId); - if (socketRole == null) - { - log.Log($"Guild Role by id '{r.RewardId}' not found."); - } - else - { - result.Add(r.RewardId, new RoleReward(socketRole, rewardConfig)); - } + await context.RemoveRole(user, Program.Config.ActiveP2pParticipantRoleId); } } } - - return result; } - private UserReward[] LookUpUsers(GiveRewardsCommand rewards) + private async Task OpenRoleModifyContext() { - return rewards.Rewards.Select(LookUpUserData).ToArray(); - } - - private UserReward LookUpUserData(RewardUsersCommand command) - { - return new UserReward(command, - command.UserAddresses - .Select(LookUpUserDataForAddress) - .Where(d => d != null) - .Cast() - .ToArray()); - } - - private UserData? LookUpUserDataForAddress(string address) - { - try - { - var userData = Program.UserRepo.GetUserDataForAddress(new EthAddress(address)); - if (userData != null) log.Log($"User '{userData.Name}' was looked up."); - else log.Log($"Lookup for user was unsuccessful. EthAddress: '{address}'"); - return userData; - } - catch (Exception ex) - { - log.Error("Error during UserData lookup: " + ex); - return null; - } + var context = new RoleModifyContext(GetGuild(), userRepo, log, rewardsChannel); + await context.Initialize(); + return context; } private SocketGuild GetGuild() @@ -163,27 +65,23 @@ namespace BiblioTech.Rewards } } - public class RoleReward + public class RoleMapper : IRoleGiver { - public RoleReward(SocketRole socketRole, RewardConfig reward) + private readonly RoleModifyContext context; + + public RoleMapper(RoleModifyContext context) { - SocketRole = socketRole; - Reward = reward; + this.context = context; } - public SocketRole SocketRole { get; } - public RewardConfig Reward { get; } - } - - public class UserReward - { - public UserReward(RewardUsersCommand rewardCommand, UserData[] users) + public async Task GiveActiveP2pParticipant(IUser user) { - RewardCommand = rewardCommand; - Users = users; + await context.GiveRole(user, Program.Config.ActiveP2pParticipantRoleId); } - public RewardUsersCommand RewardCommand { get; } - public UserData[] Users { get; } + public async Task GiveAltruisticRole(IUser user) + { + await context.GiveRole(user, Program.Config.AltruisticRoleId); + } } } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs new file mode 100644 index 00000000..124d687d --- /dev/null +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -0,0 +1,115 @@ +using Discord.WebSocket; +using Discord; +using DiscordRewards; +using Nethereum.Model; +using Logging; + +namespace BiblioTech.Rewards +{ + public class RoleModifyContext + { + private Dictionary users = new(); + private Dictionary roles = new(); + private readonly SocketGuild guild; + private readonly UserRepo userRepo; + private readonly ILog log; + private readonly SocketTextChannel? rewardsChannel; + + public RoleModifyContext(SocketGuild guild, UserRepo userRepo, ILog log, SocketTextChannel? rewardsChannel) + { + this.guild = guild; + this.userRepo = userRepo; + this.log = log; + this.rewardsChannel = rewardsChannel; + } + + public async Task Initialize() + { + this.users = await LoadAllUsers(guild); + this.roles = LoadAllRoles(guild); + } + + public IGuildUser[] Users => users.Values.ToArray(); + + public async Task GiveRole(IUser user, ulong roleId) + { + var role = GetRole(roleId); + var guildUser = GetUser(user.Id); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.AddRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Added role '{role.Name}' for user <@{user.Id}>."); + + await SendNotification(guildUser, role); + } + + public async Task RemoveRole(IUser user, ulong roleId) + { + var role = GetRole(roleId); + var guildUser = GetUser(user.Id); + if (role == null) return; + if (guildUser == null) return; + + await guildUser.RemoveRoleAsync(role); + await Program.AdminChecker.SendInAdminChannel($"Removed role '{role.Name}' for user <@{user.Id}>."); + } + + private SocketRole? GetRole(ulong roleId) + { + if (roles.ContainsKey(roleId)) return roles[roleId]; + return null; + } + + private IGuildUser? GetUser(ulong userId) + { + if (users.ContainsKey(userId)) return users[userId]; + return null; + } + + private async Task> LoadAllUsers(SocketGuild guild) + { + log.Log("Loading all users.."); + var result = new Dictionary(); + var users = guild.GetUsersAsync(); + await foreach (var ulist in users) + { + foreach (var u in ulist) + { + result.Add(u.Id, u); + } + } + return result; + } + + private Dictionary LoadAllRoles(SocketGuild guild) + { + var result = new Dictionary(); + var roles = guild.Roles.ToArray(); + foreach (var role in roles) + { + result.Add(role.Id, role); + } + return result; + } + + private async Task SendNotification(IGuildUser user, SocketRole role) + { + try + { + var userData = userRepo.GetUserById(user.Id); + if (userData == null) return; + + if (userData.NotificationsEnabled && rewardsChannel != null) + { + var msg = $"<@{user.Id}> has received '{role.Name}'."; + await rewardsChannel.SendMessageAsync(msg); + } + } + catch (Exception ex) + { + log.Error($"Failed to notify user '{user.DisplayName}' about role '{role.Name}': {ex}"); + } + } + } +} diff --git a/Tools/BiblioTech/UserRepo.cs b/Tools/BiblioTech/UserRepo.cs index d1a766ab..b6c02500 100644 --- a/Tools/BiblioTech/UserRepo.cs +++ b/Tools/BiblioTech/UserRepo.cs @@ -41,6 +41,13 @@ namespace BiblioTech return cache.Values.ToArray(); } + public UserData? GetUserById(ulong id) + { + if (cache.Count == 0) LoadAllUserData(); + if (cache.ContainsKey(id)) return cache[id]; + return null; + } + public void AddMintEventForUser(IUser user, EthAddress usedAddress, Transaction? eth, Transaction? tokens) { lock (repoLock) diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index 6a5c5759..c5ddd9a0 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -21,7 +21,7 @@ namespace TestNetRewarder return result == "Pong"; } - public async Task SendRewards(GiveRewardsCommand command) + public async Task SendRewards(EventsAndErrors command) { if (command == null) return false; var result = await HttpPostJson(command); diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 2b49ef11..7377a7da 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -8,7 +8,6 @@ namespace TestNetRewarder public class Processor : ITimeSegmentHandler { private readonly RequestBuilder builder; - private readonly RewardChecker rewardChecker; private readonly EventsFormatter eventsFormatter; private readonly ChainState chainState; private readonly Configuration config; @@ -24,15 +23,9 @@ namespace TestNetRewarder lastPeriodUpdateUtc = DateTime.UtcNow; builder = new RequestBuilder(); - rewardChecker = new RewardChecker(builder); eventsFormatter = new EventsFormatter(config); - var handler = new ChainStateChangeHandlerMux( - rewardChecker.Handler, - eventsFormatter - ); - - chainState = new ChainState(log, contracts, handler, config.HistoryStartUtc, + chainState = new ChainState(log, contracts, eventsFormatter, config.HistoryStartUtc, doProofPeriodMonitoring: config.ShowProofPeriodReports > 0); } diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs index b1f641b8..adc68032 100644 --- a/Tools/TestNetRewarder/RequestBuilder.cs +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -3,38 +3,15 @@ using Utils; namespace TestNetRewarder { - public class RequestBuilder : IRewardGiver + public class RequestBuilder { - private readonly Dictionary> rewards = new Dictionary>(); - - public void Give(RewardConfig reward, EthAddress receiver) + public EventsAndErrors Build(ChainEventMessage[] lines, string[] errors) { - if (rewards.ContainsKey(reward.RoleId)) + return new EventsAndErrors { - rewards[reward.RoleId].Add(receiver); - } - else - { - rewards.Add(reward.RoleId, new List { receiver }); - } - } - - public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors) - { - var result = new GiveRewardsCommand - { - Rewards = rewards.Select(p => new RewardUsersCommand - { - RewardId = p.Key, - UserAddresses = p.Value.Select(v => v.Address).ToArray() - }).ToArray(), EventsOverview = lines, Errors = errors }; - - rewards.Clear(); - - return result; } } } diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs deleted file mode 100644 index 2f947d8d..00000000 --- a/Tools/TestNetRewarder/RewardCheck.cs +++ /dev/null @@ -1,113 +0,0 @@ -using BlockchainUtils; -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; -using System.Numerics; -using Utils; - -namespace TestNetRewarder -{ - public interface IRewardGiver - { - void Give(RewardConfig reward, EthAddress receiver); - } - - public class RewardCheck : IChainStateChangeHandler - { - private readonly RewardConfig reward; - private readonly IRewardGiver giver; - - public RewardCheck(RewardConfig reward, IRewardGiver giver) - { - this.reward = reward; - this.giver = giver; - } - - public void OnNewRequest(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientPostedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnRequestCancelled(RequestEvent requestEvent) - { - } - - public void OnRequestFailed(RequestEvent requestEvent) - { - } - - public void OnRequestFinished(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.HostFinishedSlot, requestEvent)) - { - foreach (var host in requestEvent.Request.Hosts.GetHosts()) - { - GiveReward(reward, host); - } - } - } - - public void OnRequestFulfilled(RequestEvent requestEvent) - { - if (MeetsRequirements(CheckType.ClientStartedContract, requestEvent)) - { - GiveReward(reward, requestEvent.Request.Client); - } - } - - public void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex) - { - if (MeetsRequirements(CheckType.HostFilledSlot, requestEvent)) - { - if (host != null) - { - GiveReward(reward, host); - } - } - } - - public void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex) - { - } - - public void OnError(string msg) - { - } - - public void OnProofSubmitted(BlockTimeEntry block, string id) - { - } - - private void GiveReward(RewardConfig reward, EthAddress receiver) - { - giver.Give(reward, receiver); - } - - private bool MeetsRequirements(CheckType type, RequestEvent requestEvent) - { - return - reward.CheckConfig.Type == type && - MeetsDurationRequirement(requestEvent.Request) && - MeetsSizeRequirement(requestEvent.Request); - } - - private bool MeetsSizeRequirement(IChainStateRequest r) - { - var slotSize = r.Request.Ask.SlotSize; - ulong min = Convert.ToUInt64(reward.CheckConfig.MinSlotSize.SizeInBytes); - return slotSize >= min; - } - - private bool MeetsDurationRequirement(IChainStateRequest r) - { - var duration = TimeSpan.FromSeconds((double)r.Request.Ask.Duration); - return duration >= reward.CheckConfig.MinDuration; - } - } -} diff --git a/Tools/TestNetRewarder/RewardChecker.cs b/Tools/TestNetRewarder/RewardChecker.cs deleted file mode 100644 index f8e500cc..00000000 --- a/Tools/TestNetRewarder/RewardChecker.cs +++ /dev/null @@ -1,17 +0,0 @@ -using CodexContractsPlugin.ChainMonitor; -using DiscordRewards; - -namespace TestNetRewarder -{ - public class RewardChecker - { - public RewardChecker(IRewardGiver giver) - { - var repo = new RewardRepo(); - var checks = repo.Rewards.Select(r => new RewardCheck(r, giver)).ToArray(); - Handler = new ChainStateChangeHandlerMux(checks); - } - - public IChainStateChangeHandler Handler { get; } - } -} From 80cf288b9a117861b93959145517844c2e64326c Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 16 Apr 2025 15:39:30 +0200 Subject: [PATCH 25/36] More cleanup --- .../UtilityTests/DiscordBotTests.cs | 370 ------------------ Tools/BiblioTech/Commands/AdminCommand.cs | 86 +--- Tools/BiblioTech/Commands/SprCommand.cs | 48 --- .../Commands/UserAssociateCommand.cs | 4 +- Tools/BiblioTech/Program.cs | 4 +- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 2 +- Tools/BiblioTech/UserRepo.cs | 24 +- 7 files changed, 16 insertions(+), 522 deletions(-) delete mode 100644 Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs delete mode 100644 Tools/BiblioTech/Commands/SprCommand.cs diff --git a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs deleted file mode 100644 index 22819e1b..00000000 --- a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs +++ /dev/null @@ -1,370 +0,0 @@ -using CodexClient; -using CodexContractsPlugin; -using CodexDiscordBotPlugin; -using CodexPlugin; -using CodexTests; -using Core; -using DiscordRewards; -using DistTestCore; -using GethPlugin; -using KubernetesWorkflow.Types; -using Logging; -using Newtonsoft.Json; -using NUnit.Framework; -using Utils; - -namespace ExperimentalTests.UtilityTests -{ - [TestFixture] - public class DiscordBotTests : AutoBootstrapDistTest - { - private readonly RewardRepo repo = new RewardRepo(); - private readonly TestToken hostInitialBalance = 3000000.TstWei(); - private readonly TestToken clientInitialBalance = 1000000000.TstWei(); - private readonly EthAccount clientAccount = EthAccountGenerator.GenerateNew(); - private readonly List hostAccounts = new List(); - private readonly List rewardsSeen = new List(); - private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); - private readonly List receivedEvents = new List(); - - [Test] - [DontDownloadLogs] - [Ignore("Used to debug testnet bots.")] - public void BotRewardTest() - { - var geth = StartGethNode(s => s.IsMiner().WithName("disttest-geth")); - var contracts = Ci.StartCodexContracts(geth); - var gethInfo = CreateGethInfo(geth, contracts); - - var botContainer = StartDiscordBot(gethInfo); - var rewarderContainer = StartRewarderBot(gethInfo, botContainer); - - StartHosts(geth, contracts); - var client = StartClient(geth, contracts); - - var apiCalls = new RewardApiCalls(GetTestLog(), Ci, botContainer); - apiCalls.Start(OnCommand); - - var purchaseContract = ClientPurchasesStorage(client); - purchaseContract.WaitForStorageContractStarted(); - purchaseContract.WaitForStorageContractFinished(); - - // todo: removed from codexclient: - //contracts.WaitUntilNextPeriod(); - //contracts.WaitUntilNextPeriod(); - - //var blocks = 3; - //Log($"Waiting {blocks} blocks for nodes to process payouts..."); - //Thread.Sleep(GethContainerRecipe.BlockInterval * blocks); - - Thread.Sleep(rewarderInterval * 3); - - apiCalls.Stop(); - - AssertEventOccurance("Created as New.", 1); - AssertEventOccurance("SlotFilled", Convert.ToInt32(GetNumberOfRequiredHosts())); - AssertEventOccurance("Transit: New -> Started", 1); - AssertEventOccurance("Transit: Started -> Finished", 1); - - foreach (var r in repo.Rewards) - { - var seen = rewardsSeen.Any(s => r.RoleId == s); - - Log($"{Lookup(r.RoleId)} = {seen}"); - } - - Assert.That(repo.Rewards.All(r => rewardsSeen.Contains(r.RoleId))); - } - - private string Lookup(ulong rewardId) - { - var reward = repo.Rewards.Single(r => r.RoleId == rewardId); - return $"({rewardId})'{reward.Message}'"; - } - - private void AssertEventOccurance(string msg, int expectedCount) - { - Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount), - $"Event '{msg}' did not occure correct number of times."); - } - - private void OnCommand(string timestamp, EventsAndErrors call) - { - Log($""); - foreach (var e in call.EventsOverview) - { - Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order."); - } - - receivedEvents.AddRange(call.EventsOverview); - foreach (var e in call.EventsOverview) - { - Log("\tEvent: " + e); - } - foreach (var r in call.Rewards) - { - var reward = repo.Rewards.Single(a => a.RoleId == r.RewardId); - if (r.UserAddresses.Any()) rewardsSeen.Add(reward.RoleId); - foreach (var address in r.UserAddresses) - { - var user = IdentifyAccount(address); - Log("\tReward: " + user + ": " + reward.Message); - } - } - Log($""); - } - - private IStoragePurchaseContract ClientPurchasesStorage(ICodexNode client) - { - var testFile = GenerateTestFile(GetMinFileSize()); - var contentId = client.UploadFile(testFile); - var purchase = new StoragePurchaseRequest(contentId) - { - PricePerBytePerSecond = 2.TstWei(), - CollateralPerByte = 10.TstWei(), - MinRequiredNumberOfNodes = GetNumberOfRequiredHosts(), - NodeFailureTolerance = 2, - ProofProbability = 5, - Duration = GetMinRequiredRequestDuration(), - Expiry = GetMinRequiredRequestDuration() - TimeSpan.FromMinutes(1) - }; - - return client.Marketplace.RequestStorage(purchase); - } - - private ICodexNode StartClient(IGethNode geth, ICodexContracts contracts) - { - var node = StartCodex(s => s - .WithName("Client") - .EnableMarketplace(geth, contracts, m => m - .WithAccount(clientAccount) - .WithInitial(10.Eth(), clientInitialBalance))); - - Log($"Client {node.EthAccount.EthAddress}"); - return node; - } - - private RunningPod StartRewarderBot(DiscordBotGethInfo gethInfo, RunningContainer botContainer) - { - return Ci.DeployRewarderBot(new RewarderBotStartupConfig( - name: "rewarder-bot", - discordBotHost: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Host, - discordBotPort: botContainer.GetInternalAddress(DiscordBotContainerRecipe.RewardsPort).Port, - intervalMinutes: Convert.ToInt32(Math.Round(rewarderInterval.TotalMinutes)), - historyStartUtc: DateTime.UtcNow, - gethInfo: gethInfo, - dataPath: null - )); - } - - private DiscordBotGethInfo CreateGethInfo(IGethNode geth, ICodexContracts contracts) - { - return new DiscordBotGethInfo( - host: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Host, - port: geth.Container.GetInternalAddress(GethContainerRecipe.HttpPortTag).Port, - privKey: geth.StartResult.Account.PrivateKey, - marketplaceAddress: contracts.Deployment.MarketplaceAddress, - tokenAddress: contracts.Deployment.TokenAddress, - abi: contracts.Deployment.Abi - ); - } - - private RunningContainer StartDiscordBot(DiscordBotGethInfo gethInfo) - { - var bot = Ci.DeployCodexDiscordBot(new DiscordBotStartupConfig( - name: "discord-bot", - token: "aaa", - serverName: "ThatBen's server", - adminRoleName: "bottest-admins", - adminChannelName: "admin-channel", - rewardChannelName: "rewards-channel", - kubeNamespace: "notneeded", - gethInfo: gethInfo - )); - return bot.Containers.Single(); - } - - private void StartHosts(IGethNode geth, ICodexContracts contracts) - { - var hosts = StartCodex(GetNumberOfLiveHosts(), s => s - .WithName("Host") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) - { - ContractClock = CodexLogLevel.Trace, - }) - .WithStorageQuota(Mult(GetMinFileSizePlus(50), GetNumberOfLiveHosts())) - .EnableMarketplace(geth, contracts, m => m - .WithInitial(10.Eth(), hostInitialBalance) - .AsStorageNode() - .AsValidator())); - - var availability = new StorageAvailability( - totalSpace: Mult(GetMinFileSize(), GetNumberOfLiveHosts()), - maxDuration: TimeSpan.FromMinutes(30), - minPricePerBytePerSecond: 1.TstWei(), - totalCollateral: hostInitialBalance - ); - - foreach (var host in hosts) - { - hostAccounts.Add(host.EthAccount); - host.Marketplace.MakeStorageAvailable(availability); - } - } - - private int GetNumberOfLiveHosts() - { - return Convert.ToInt32(GetNumberOfRequiredHosts()) + 3; - } - - private ByteSize Mult(ByteSize size, int mult) - { - return new ByteSize(size.SizeInBytes * mult); - } - - private ByteSize GetMinFileSizePlus(int plusMb) - { - return new ByteSize(GetMinFileSize().SizeInBytes + plusMb.MB().SizeInBytes); - } - - private ByteSize GetMinFileSize() - { - ulong minSlotSize = 0; - ulong minNumHosts = 0; - foreach (var r in repo.Rewards) - { - var s = Convert.ToUInt64(r.CheckConfig.MinSlotSize.SizeInBytes); - var h = r.CheckConfig.MinNumberOfHosts; - if (s > minSlotSize) minSlotSize = s; - if (h > minNumHosts) minNumHosts = h; - } - - var minFileSize = (minSlotSize + 1024) * minNumHosts; - return new ByteSize(Convert.ToInt64(minFileSize)); - } - - private uint GetNumberOfRequiredHosts() - { - return Convert.ToUInt32(repo.Rewards.Max(r => r.CheckConfig.MinNumberOfHosts)); - } - - private TimeSpan GetMinRequiredRequestDuration() - { - return repo.Rewards.Max(r => r.CheckConfig.MinDuration) + TimeSpan.FromSeconds(10); - } - - private string IdentifyAccount(string address) - { - if (address == clientAccount.EthAddress.Address) return "Client"; - try - { - var index = hostAccounts.FindIndex(a => a.EthAddress.Address == address); - return "Host" + index; - } - catch - { - return "UNKNOWN"; - } - } - - public class RewardApiCalls - { - private readonly ContainerFileMonitor monitor; - - public RewardApiCalls(ILog log, CoreInterface ci, RunningContainer botContainer) - { - monitor = new ContainerFileMonitor(log, ci, botContainer, "/app/datapath/logs/discordbot.log"); - } - - public void Start(Action onCommand) - { - monitor.Start(line => ParseLine(line, onCommand)); - } - - public void Stop() - { - monitor.Stop(); - } - - private void ParseLine(string line, Action onCommand) - { - try - { - var timestamp = line.Substring(0, 30); - var json = line.Substring(31); - - var cmd = JsonConvert.DeserializeObject(json); - if (cmd != null) - { - onCommand(timestamp, cmd); - } - } - catch - { - } - } - } - - public class ContainerFileMonitor - { - private readonly ILog log; - private readonly CoreInterface ci; - private readonly RunningContainer botContainer; - private readonly string filePath; - private readonly CancellationTokenSource cts = new CancellationTokenSource(); - private readonly List seenLines = new List(); - private Task worker = Task.CompletedTask; - private Action onNewLine = c => { }; - - public ContainerFileMonitor(ILog log, CoreInterface ci, RunningContainer botContainer, string filePath) - { - this.log = log; - this.ci = ci; - this.botContainer = botContainer; - this.filePath = filePath; - } - - public void Start(Action onNewLine) - { - this.onNewLine = onNewLine; - worker = Task.Run(Worker); - } - - public void Stop() - { - cts.Cancel(); - worker.Wait(); - } - - // did any container crash? that's why it repeats? - - - private void Worker() - { - while (!cts.IsCancellationRequested) - { - Update(); - } - } - - private void Update() - { - Thread.Sleep(TimeSpan.FromSeconds(10)); - if (cts.IsCancellationRequested) return; - - var botLog = ci.ExecuteContainerCommand(botContainer, "cat", filePath); - var lines = botLog.Split(Environment.NewLine, StringSplitOptions.RemoveEmptyEntries); - foreach (var line in lines) - { - // log.Log("line: " + line); - - if (!seenLines.Contains(line)) - { - seenLines.Add(line); - onNewLine(line); - } - } - } - } - } -} diff --git a/Tools/BiblioTech/Commands/AdminCommand.cs b/Tools/BiblioTech/Commands/AdminCommand.cs index 45b3fc29..f9877181 100644 --- a/Tools/BiblioTech/Commands/AdminCommand.cs +++ b/Tools/BiblioTech/Commands/AdminCommand.cs @@ -8,16 +8,10 @@ namespace BiblioTech.Commands private readonly ClearUserAssociationCommand clearCommand = new ClearUserAssociationCommand(); private readonly ReportCommand reportCommand = new ReportCommand(); private readonly WhoIsCommand whoIsCommand = new WhoIsCommand(); - private readonly AddSprCommand addSprCommand; - private readonly ClearSprsCommand clearSprsCommand; - private readonly GetSprCommand getSprCommand; private readonly LogReplaceCommand logReplaceCommand; - public AdminCommand(SprCommand sprCommand, CustomReplacement replacement) + public AdminCommand(CustomReplacement replacement) { - addSprCommand = new AddSprCommand(sprCommand); - clearSprsCommand = new ClearSprsCommand(sprCommand); - getSprCommand = new GetSprCommand(sprCommand); logReplaceCommand = new LogReplaceCommand(replacement); } @@ -30,9 +24,6 @@ namespace BiblioTech.Commands clearCommand, reportCommand, whoIsCommand, - addSprCommand, - clearSprsCommand, - getSprCommand, logReplaceCommand }; @@ -53,9 +44,6 @@ namespace BiblioTech.Commands await clearCommand.CommandHandler(context); await reportCommand.CommandHandler(context); await whoIsCommand.CommandHandler(context); - await addSprCommand.CommandHandler(context); - await clearSprsCommand.CommandHandler(context); - await getSprCommand.CommandHandler(context); await logReplaceCommand.CommandHandler(context); } @@ -144,78 +132,6 @@ namespace BiblioTech.Commands } } - public class AddSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("spr", "Codex SPR", true); - - public AddSprCommand(SprCommand sprCommand) - : base(name: "addspr", - description: "Adds a Codex SPR, to be given to users with '/boot'.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var spr = await stringOption.Parse(context); - - if (!string.IsNullOrEmpty(spr) ) - { - sprCommand.Add(spr); - await context.Followup("A-OK!"); - } - else - { - await context.Followup("SPR is null or empty."); - } - } - } - - public class ClearSprsCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - private readonly StringOption stringOption = new StringOption("areyousure", "set to 'true' if you are.", true); - - public ClearSprsCommand(SprCommand sprCommand) - : base(name: "clearsprs", - description: "Clears all Codex SPRs in the bot. Users won't be able to use '/boot' till new ones are added.") - { - this.sprCommand = sprCommand; - } - - public override CommandOption[] Options => new[] { stringOption }; - - protected override async Task onSubCommand(CommandContext context) - { - var areyousure = await stringOption.Parse(context); - - if (areyousure != "true") return; - - sprCommand.Clear(); - await context.Followup("Cleared all SPRs."); - } - } - - public class GetSprCommand : SubCommandOption - { - private readonly SprCommand sprCommand; - - public GetSprCommand(SprCommand sprCommand) - : base(name: "getsprs", - description: "Shows all Codex SPRs in the bot.") - { - this.sprCommand = sprCommand; - } - - protected override async Task onSubCommand(CommandContext context) - { - await context.Followup("SPRs: " + string.Join(", ", sprCommand.Get().Select(s => $"'{s}'"))); - } - } - public class LogReplaceCommand : SubCommandOption { private readonly CustomReplacement replacement; diff --git a/Tools/BiblioTech/Commands/SprCommand.cs b/Tools/BiblioTech/Commands/SprCommand.cs deleted file mode 100644 index 4b3235de..00000000 --- a/Tools/BiblioTech/Commands/SprCommand.cs +++ /dev/null @@ -1,48 +0,0 @@ -using BiblioTech.Options; - -namespace BiblioTech.Commands -{ - public class SprCommand : BaseCommand - { - private readonly Random random = new Random(); - private readonly List knownSprs = new List(); - - public override string Name => "boot"; - public override string StartingMessage => RandomBusyMessage.Get(); - public override string Description => "Gets an SPR. (Signed peer record, used for bootstrapping.)"; - - protected override async Task Invoke(CommandContext context) - { - await ReplyWithRandomSpr(context); - } - - public void Add(string spr) - { - if (knownSprs.Contains(spr)) return; - knownSprs.Add(spr); - } - - public void Clear() - { - knownSprs.Clear(); - } - - public string[] Get() - { - return knownSprs.ToArray(); - } - - private async Task ReplyWithRandomSpr(CommandContext context) - { - if (!knownSprs.Any()) - { - await context.Followup("I'm sorry, no SPRs are available... :c"); - return; - } - - var i = random.Next(0, knownSprs.Count); - var spr = knownSprs[i]; - await context.Followup($"Your SPR: `{spr}`"); - } - } -} diff --git a/Tools/BiblioTech/Commands/UserAssociateCommand.cs b/Tools/BiblioTech/Commands/UserAssociateCommand.cs index 61d55fb9..dbe7dced 100644 --- a/Tools/BiblioTech/Commands/UserAssociateCommand.cs +++ b/Tools/BiblioTech/Commands/UserAssociateCommand.cs @@ -69,8 +69,8 @@ namespace BiblioTech.Commands { await context.Followup(new string[] { - "Done! Thank you for joining the test net!", - "By default, the bot will @-mention you with test-net related notifications.", + "Done! Thank you for joining!", + "By default, the bot will @-mention you with discord role notifications.", $"You can enable/disable this behavior with the '/{notifyCommand.Name}' command." }); diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index df1ce6bc..0a86a8aa 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -89,17 +89,15 @@ namespace BiblioTech var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); - var sprCommand = new SprCommand(); var roleRemover = new ActiveP2pRoleRemover(Config, Log, checkRepo); var handler = new CommandHandler(Log, client, replacement, roleRemover, new GetBalanceCommand(associateCommand), new MintCommand(associateCommand), - sprCommand, associateCommand, notifyCommand, new CheckUploadCommand(checker), new CheckDownloadCommand(checker), - new AdminCommand(sprCommand, replacement) + new AdminCommand(replacement) ); await client.LoginAsync(TokenType.Bot, Config.ApplicationToken); diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs index 124d687d..f65d4cf2 100644 --- a/Tools/BiblioTech/Rewards/RoleModifyContext.cs +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -97,7 +97,7 @@ namespace BiblioTech.Rewards { try { - var userData = userRepo.GetUserById(user.Id); + var userData = userRepo.GetUser(user); if (userData == null) return; if (userData.NotificationsEnabled && rewardsChannel != null) diff --git a/Tools/BiblioTech/UserRepo.cs b/Tools/BiblioTech/UserRepo.cs index b6c02500..07831dd6 100644 --- a/Tools/BiblioTech/UserRepo.cs +++ b/Tools/BiblioTech/UserRepo.cs @@ -41,11 +41,10 @@ namespace BiblioTech return cache.Values.ToArray(); } - public UserData? GetUserById(ulong id) + public UserData GetUser(IUser user) { if (cache.Count == 0) LoadAllUserData(); - if (cache.ContainsKey(id)) return cache[id]; - return null; + return GetOrCreate(user); } public void AddMintEventForUser(IUser user, EthAddress usedAddress, Transaction? eth, Transaction? tokens) @@ -75,10 +74,10 @@ namespace BiblioTech lock (repoLock) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { - result.Add("User has not joined the test net."); + result.Add("User has not interacted with bot."); } else { @@ -107,19 +106,19 @@ namespace BiblioTech public string[] GetUserReport(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) return new[] { "User has not joined the test net." }; return userData.CreateOverview(); } public string[] GetUserReport(EthAddress ethAddress) { - var userData = GetUserDataForAddress(ethAddress); + var userData = GetUserDataForAddressMaybe(ethAddress); if (userData == null) return new[] { "No user is using this eth address." }; return userData.CreateOverview(); } - public UserData? GetUserDataForAddress(EthAddress? address) + public UserData? GetUserDataForAddressMaybe(EthAddress? address) { if (address == null) return null; @@ -144,7 +143,7 @@ namespace BiblioTech private SetAddressResponse SetUserAddress(IUser user, EthAddress? address) { - if (GetUserDataForAddress(address) != null) + if (GetUserDataForAddressMaybe(address) != null) { return SetAddressResponse.AddressAlreadyInUse; } @@ -159,13 +158,12 @@ namespace BiblioTech private void SetUserNotification(IUser user, bool notifyEnabled) { - var userData = GetUserData(user); - if (userData == null) return; + var userData = GetOrCreate(user); userData.NotificationsEnabled = notifyEnabled; SaveUserData(userData); } - private UserData? GetUserData(IUser user) + private UserData? GetUserDataMaybe(IUser user) { if (cache.ContainsKey(user.Id)) { @@ -184,7 +182,7 @@ namespace BiblioTech private UserData GetOrCreate(IUser user) { - var userData = GetUserData(user); + var userData = GetUserDataMaybe(user); if (userData == null) { return CreateAndSaveNewUserData(user); From a2c8c18c5cda6f391f9d9913342f0f09291ed329 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 16 Apr 2025 16:25:31 +0200 Subject: [PATCH 26/36] wip --- Framework/DiscordRewards/EventsAndErrors.cs | 13 +++- Tools/BiblioTech/CommandHandler.cs | 1 + Tools/BiblioTech/Program.cs | 1 + .../Rewards/ChainActivityHandler.cs | 60 +++++++++++++++++++ Tools/TestNetRewarder/Processor.cs | 6 +- Tools/TestNetRewarder/RequestBuilder.cs | 44 +++++++++++++- 6 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 Tools/BiblioTech/Rewards/ChainActivityHandler.cs diff --git a/Framework/DiscordRewards/EventsAndErrors.cs b/Framework/DiscordRewards/EventsAndErrors.cs index 53e4b3cc..e26568c0 100644 --- a/Framework/DiscordRewards/EventsAndErrors.cs +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -4,10 +4,15 @@ { public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); public string[] Errors { get; set; } = Array.Empty(); + public ActiveChainAddresses ActiveChainAddresses { get; set; } = new ActiveChainAddresses(); public bool HasAny() { - return Errors.Length > 0 || EventsOverview.Length > 0; + return + Errors.Length > 0 || + EventsOverview.Length > 0 || + ActiveChainAddresses.Hosts.Length > 0 || + ActiveChainAddresses.Clients.Length > 0; } } @@ -16,4 +21,10 @@ public ulong BlockNumber { get; set; } public string Message { get; set; } = string.Empty; } + + public class ActiveChainAddresses + { + public string[] Hosts { get; set; } = Array.Empty(); + public string[] Clients { get; set; } = Array.Empty(); + } } diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index 8c888ea7..5a7a83e7 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -42,6 +42,7 @@ namespace BiblioTech Program.AdminChecker.SetAdminChannel(adminChannel); Program.RoleDriver = new RoleDriver(client, Program.UserRepo, log, rewardsChannel); + Program.ChainActivityHandler = new ChainActivityHandler(log); Program.EventsSender = new ChainEventsSender(log, replacement, chainEventsChannel); var builders = commands.Select(c => diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 0a86a8aa..cb3b13e5 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -18,6 +18,7 @@ namespace BiblioTech public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; public static IDiscordRoleDriver RoleDriver { get; set; } = null!; + public static ChainActivityHandler ChainActivityHandler { get; set; } = null!; public static ChainEventsSender EventsSender { get; set; } = null!; public static ILog Log { get; private set; } = null!; diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs new file mode 100644 index 00000000..2025725a --- /dev/null +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -0,0 +1,60 @@ +using DiscordRewards; +using Logging; + +namespace BiblioTech.Rewards +{ + public class ChainActivityHandler + { + private readonly ILog log; + private readonly UserRepo repo; + + public ChainActivityHandler(ILog log, UserRepo repo) + { + this.log = log; + this.repo = repo; + } + + public async Task Process(ActiveChainAddresses activeChainAddresses) + { + var activeUserIds = ConvertToUserIds(activeChainAddresses); + if (!activeUserIds.HasAny()) return; + + todo call role driver to add roles to new activeIds or remove them. + } + + private ActiveUserIds ConvertToUserIds(ActiveChainAddresses activeChainAddresses) + { + return new ActiveUserIds + { + Hosts = Map(activeChainAddresses.Hosts), + Clients = Map(activeChainAddresses.Clients) + }; + } + + private ulong[] Map(string[] ethAddresses) + { + var result = new List(); + foreach (var ethAddress in ethAddresses) + { + var userMaybe = repo.GetUserDataForAddressMaybe(new Utils.EthAddress(ethAddress)); + if (userMaybe != null) + { + result.Add(userMaybe.DiscordId); + } + } + + return result.ToArray(); + } + + private class ActiveUserIds + { + public ulong[] Hosts { get; set; } = Array.Empty(); + public ulong[] Clients { get; set; } = Array.Empty(); + + public bool HasAny() + { + return Hosts.Any() || Clients.Any(); + } + } + } +} diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index 7377a7da..b018d377 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -32,7 +32,7 @@ namespace TestNetRewarder public async Task Initialize() { var events = eventsFormatter.GetInitializationEvents(config); - var request = builder.Build(events, Array.Empty()); + var request = builder.Build(chainState, events, Array.Empty()); if (request.HasAny()) { await client.SendRewards(request); @@ -49,7 +49,7 @@ namespace TestNetRewarder if (numberOfChainEvents == 0) return TimeSegmentResponse.Underload; if (numberOfChainEvents > 10) return TimeSegmentResponse.Overload; - if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Overload; + if (duration > TimeSpan.FromSeconds(3)) return TimeSegmentResponse.Overload; return TimeSegmentResponse.OK; } catch (Exception ex) @@ -69,7 +69,7 @@ namespace TestNetRewarder var events = eventsFormatter.GetEvents(); var errors = eventsFormatter.GetErrors(); - var request = builder.Build(events, errors); + var request = builder.Build(chainState, events, errors); if (request.HasAny()) { await client.SendRewards(request); diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs index adc68032..5b859269 100644 --- a/Tools/TestNetRewarder/RequestBuilder.cs +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -1,17 +1,55 @@ -using DiscordRewards; +using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; using Utils; namespace TestNetRewarder { public class RequestBuilder { - public EventsAndErrors Build(ChainEventMessage[] lines, string[] errors) + public EventsAndErrors Build(ChainState chainState, ChainEventMessage[] lines, string[] errors) { + var activeChainAddresses = CollectActiveAddresses(chainState); + return new EventsAndErrors { EventsOverview = lines, - Errors = errors + Errors = errors, + ActiveChainAddresses = activeChainAddresses }; } + + private ActiveChainAddresses CollectActiveAddresses(ChainState chainState) + { + var hosts = new List(); + var clients = new List(); + + foreach (var request in chainState.Requests) + { + CollectAddresses(request, hosts, clients); + } + + return new ActiveChainAddresses + { + Hosts = hosts.ToArray(), + Clients = clients.ToArray() + }; + } + + private void CollectAddresses(IChainStateRequest request, List hosts, List clients) + { + if (request.State != CodexContractsPlugin.RequestState.Started) return; + + AddIfNew(clients, request.Client); + foreach (var host in request.Hosts.GetHosts()) + { + AddIfNew(hosts, host); + } + } + + private void AddIfNew(List list, EthAddress address) + { + var addr = address.Address; + if (!list.Contains(addr)) list.Add(addr); + } } } From 0eaaa625f10bbbc5879401f536f41664ca2b63f8 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 09:35:13 +0200 Subject: [PATCH 27/36] Implements giving and removing of activehost and activeclient roles --- .../CodexChecking/ActiveP2pRoleRemover.cs | 16 +++- Tools/BiblioTech/CommandHandler.cs | 2 +- .../Commands/CheckResponseHandler.cs | 4 +- Tools/BiblioTech/LoggingRoleDriver.cs | 45 ++++++++- .../Rewards/ChainActivityHandler.cs | 94 +++++++++++++++++-- Tools/BiblioTech/Rewards/RewardController.cs | 12 ++- Tools/BiblioTech/Rewards/RoleDriver.cs | 49 ++++++++-- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 12 +-- 8 files changed, 197 insertions(+), 37 deletions(-) diff --git a/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs index 4a448792..2d8ec602 100644 --- a/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs +++ b/Tools/BiblioTech/CodexChecking/ActiveP2pRoleRemover.cs @@ -1,4 +1,5 @@ -using Discord; +using BiblioTech.Rewards; +using Discord; using Logging; using System.Threading.Tasks; @@ -50,7 +51,18 @@ namespace BiblioTech.CodexChecking { var expiryMoment = DateTime.UtcNow - TimeSpan.FromMinutes(config.ActiveP2pRoleDurationMinutes); - Program.RoleDriver.IterateRemoveActiveP2pParticipants(p => ShouldRemoveRole(p, expiryMoment)); + Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, expiryMoment), + Program.Config.ActiveP2pParticipantRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, DateTime expiryMoment) + { + var report = repo.GetOrCreate(user.Id); + if (report.UploadCheck.CompletedUtc > expiryMoment) return; + if (report.DownloadCheck.CompletedUtc > expiryMoment) return; + + await giver.RemoveActiveP2pParticipant(user.Id); } private bool ShouldRemoveRole(IUser user, DateTime expiryMoment) diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index 5a7a83e7..822d4446 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -42,7 +42,7 @@ namespace BiblioTech Program.AdminChecker.SetAdminChannel(adminChannel); Program.RoleDriver = new RoleDriver(client, Program.UserRepo, log, rewardsChannel); - Program.ChainActivityHandler = new ChainActivityHandler(log); + Program.ChainActivityHandler = new ChainActivityHandler(log, Program.UserRepo); Program.EventsSender = new ChainEventsSender(log, replacement, chainEventsChannel); var builders = commands.Select(c => diff --git a/Tools/BiblioTech/Commands/CheckResponseHandler.cs b/Tools/BiblioTech/Commands/CheckResponseHandler.cs index ca9dedd4..5428e86b 100644 --- a/Tools/BiblioTech/Commands/CheckResponseHandler.cs +++ b/Tools/BiblioTech/Commands/CheckResponseHandler.cs @@ -58,8 +58,8 @@ namespace BiblioTech.Commands { await Program.RoleDriver.RunRoleGiver(async r => { - await r.GiveAltruisticRole(user); - await r.GiveActiveP2pParticipant(user); + await r.GiveAltruisticRole(user.Id); + await r.GiveActiveP2pParticipant(user.Id); }); await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!"); } diff --git a/Tools/BiblioTech/LoggingRoleDriver.cs b/Tools/BiblioTech/LoggingRoleDriver.cs index a5631271..b1759af9 100644 --- a/Tools/BiblioTech/LoggingRoleDriver.cs +++ b/Tools/BiblioTech/LoggingRoleDriver.cs @@ -21,7 +21,12 @@ namespace BiblioTech await action(new LoggingRoleGiver(log)); } - public async Task IterateRemoveActiveP2pParticipants(Func predicate) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) + { + await Task.CompletedTask; + } + + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) { await Task.CompletedTask; } @@ -35,15 +40,45 @@ namespace BiblioTech this.log = log; } - public async Task GiveActiveP2pParticipant(IUser user) + public async Task GiveActiveClient(ulong userId) { - log.Log($"Giving ActiveP2p role to " + user.Id); + log.Log($"Giving ActiveClient role to " + userId); await Task.CompletedTask; } - public async Task GiveAltruisticRole(IUser user) + public async Task GiveActiveHost(ulong userId) { - log.Log($"Giving Altruistic role to " + user.Id); + log.Log($"Giving ActiveHost role to " + userId); + await Task.CompletedTask; + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + log.Log($"Giving ActiveP2p role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + log.Log($"Removing ActiveP2p role from " + userId); + await Task.CompletedTask; + } + + public async Task GiveAltruisticRole(ulong userId) + { + log.Log($"Giving Altruistic role to " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveClient(ulong userId) + { + log.Log($"Removing ActiveClient role from " + userId); + await Task.CompletedTask; + } + + public async Task RemoveActiveHost(ulong userId) + { + log.Log($"Removing ActiveHost role from " + userId); await Task.CompletedTask; } } diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index 2025725a..a3bb17a8 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -1,4 +1,5 @@ -using DiscordRewards; +using Discord; +using DiscordRewards; using Logging; namespace BiblioTech.Rewards @@ -7,6 +8,7 @@ namespace BiblioTech.Rewards { private readonly ILog log; private readonly UserRepo repo; + private ActiveUserIds? previousIds = null; public ChainActivityHandler(ILog log, UserRepo repo) { @@ -18,17 +20,85 @@ namespace BiblioTech.Rewards { var activeUserIds = ConvertToUserIds(activeChainAddresses); if (!activeUserIds.HasAny()) return; + + if (!HasChanged(activeUserIds)) return; - todo call role driver to add roles to new activeIds or remove them. + await GiveAndRemoveRoles(activeUserIds); + } + + private async Task GiveAndRemoveRoles(ActiveUserIds activeUserIds) + { + await Program.RoleDriver.IterateUsersWithRoles( + (g, u, r) => OnUserWithRole(g, u, r, activeUserIds), + whenDone: g => GiveRolesToRemaining(g, activeUserIds), + Program.Config.ActiveClientRoleId, + Program.Config.ActiveHostRoleId); + } + + private async Task OnUserWithRole(IRoleGiver giver, IUser user, ulong roleId, ActiveUserIds activeIds) + { + if (roleId == Program.Config.ActiveClientRoleId) + { + await CheckUserWithRole(user, activeIds.Clients, giver.RemoveActiveClient); + } + else if (roleId == Program.Config.ActiveHostRoleId) + { + await CheckUserWithRole(user, activeIds.Hosts, giver.RemoveActiveHost); + } + else + { + throw new Exception("Unknown roleId received!"); + } + } + + private async Task CheckUserWithRole(IUser user, List activeUsers, Func removeActiveRole) + { + if (ShouldUserHaveRole(user, activeUsers)) + { + activeUsers.Remove(user.Id); + } + else + { + await removeActiveRole(user.Id); + } + } + + private bool ShouldUserHaveRole(IUser user, List activeUsers) + { + return activeUsers.Any(id => id == user.Id); + } + + private async Task GiveRolesToRemaining(IRoleGiver giver, ActiveUserIds ids) + { + foreach (var client in ids.Clients) await giver.GiveActiveClient(client); + foreach (var host in ids.Hosts) await giver.GiveActiveHost(host); + } + + private bool HasChanged(ActiveUserIds activeUserIds) + { + if (previousIds == null) + { + previousIds = activeUserIds; + return true; + } + + if (!IsEquivalent(previousIds.Hosts, activeUserIds.Hosts)) return true; + if (!IsEquivalent(previousIds.Clients, activeUserIds.Clients)) return true; + return false; + } + + private static bool IsEquivalent(IEnumerable a, IEnumerable b) + { + return a.SequenceEqual(b); } private ActiveUserIds ConvertToUserIds(ActiveChainAddresses activeChainAddresses) { return new ActiveUserIds - { - Hosts = Map(activeChainAddresses.Hosts), - Clients = Map(activeChainAddresses.Clients) - }; + ( + hosts: Map(activeChainAddresses.Hosts), + clients: Map(activeChainAddresses.Clients) + ); } private ulong[] Map(string[] ethAddresses) @@ -43,13 +113,19 @@ namespace BiblioTech.Rewards } } - return result.ToArray(); + return result.Order().ToArray(); } private class ActiveUserIds { - public ulong[] Hosts { get; set; } = Array.Empty(); - public ulong[] Clients { get; set; } = Array.Empty(); + public ActiveUserIds(IEnumerable hosts, IEnumerable clients) + { + Hosts = hosts.ToList(); + Clients = clients.ToList(); + } + + public List Hosts { get; } + public List Clients { get; } public bool HasAny() { diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 7cbed562..7a4b8aa3 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -11,13 +11,19 @@ namespace BiblioTech.Rewards public interface IDiscordRoleDriver { Task RunRoleGiver(Func action); - Task IterateRemoveActiveP2pParticipants(Func predicate); + Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate); + Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate); } public interface IRoleGiver { - Task GiveAltruisticRole(IUser user); - Task GiveActiveP2pParticipant(IUser user); + Task GiveAltruisticRole(ulong userId); + Task GiveActiveP2pParticipant(ulong userId); + Task RemoveActiveP2pParticipant(ulong userId); + Task GiveActiveHost(ulong userId); + Task RemoveActiveHost(ulong userId); + Task GiveActiveClient(ulong userId); + Task RemoveActiveClient(ulong userId); } [Route("api/[controller]")] diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index c716c011..41b0e44c 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -30,20 +30,26 @@ namespace BiblioTech.Rewards await action(mapper); } - public async Task IterateRemoveActiveP2pParticipants(Func shouldRemove) + public async Task IterateUsersWithRoles(Func onUserWithRole, params ulong[] rolesToIterate) + { + await IterateUsersWithRoles(onUserWithRole, g => Task.CompletedTask, rolesToIterate); + } + + public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) { var context = await OpenRoleModifyContext(); + var mapper = new RoleMapper(context); foreach (var user in context.Users) { - if (user.RoleIds.Any(r => r == Program.Config.ActiveP2pParticipantRoleId)) + foreach (var role in rolesToIterate) { - // This user has the role. Should it be removed? - if (shouldRemove(user)) + if (user.RoleIds.Contains(role)) { - await context.RemoveRole(user, Program.Config.ActiveP2pParticipantRoleId); + await onUserWithRole(mapper, user, role); } } } + await whenDone(mapper); } private async Task OpenRoleModifyContext() @@ -74,14 +80,39 @@ namespace BiblioTech.Rewards this.context = context; } - public async Task GiveActiveP2pParticipant(IUser user) + public async Task GiveActiveClient(ulong userId) { - await context.GiveRole(user, Program.Config.ActiveP2pParticipantRoleId); + await context.GiveRole(userId, Program.Config.ActiveClientRoleId); } - public async Task GiveAltruisticRole(IUser user) + public async Task GiveActiveHost(ulong userId) { - await context.GiveRole(user, Program.Config.AltruisticRoleId); + await context.GiveRole(userId, Program.Config.ActiveHostRoleId); + } + + public async Task GiveActiveP2pParticipant(ulong userId) + { + await context.GiveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task RemoveActiveP2pParticipant(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveP2pParticipantRoleId); + } + + public async Task GiveAltruisticRole(ulong userId) + { + await context.GiveRole(userId, Program.Config.AltruisticRoleId); + } + + public async Task RemoveActiveClient(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveClientRoleId); + } + + public async Task RemoveActiveHost(ulong userId) + { + await context.RemoveRole(userId, Program.Config.ActiveHostRoleId); } } } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs index f65d4cf2..58914f5e 100644 --- a/Tools/BiblioTech/Rewards/RoleModifyContext.cs +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -31,28 +31,28 @@ namespace BiblioTech.Rewards public IGuildUser[] Users => users.Values.ToArray(); - public async Task GiveRole(IUser user, ulong roleId) + public async Task GiveRole(ulong userId, ulong roleId) { var role = GetRole(roleId); - var guildUser = GetUser(user.Id); + var guildUser = GetUser(userId); if (role == null) return; if (guildUser == null) return; await guildUser.AddRoleAsync(role); - await Program.AdminChecker.SendInAdminChannel($"Added role '{role.Name}' for user <@{user.Id}>."); + await Program.AdminChecker.SendInAdminChannel($"Added role '{role.Name}' for user <@{userId}>."); await SendNotification(guildUser, role); } - public async Task RemoveRole(IUser user, ulong roleId) + public async Task RemoveRole(ulong userId, ulong roleId) { var role = GetRole(roleId); - var guildUser = GetUser(user.Id); + var guildUser = GetUser(userId); if (role == null) return; if (guildUser == null) return; await guildUser.RemoveRoleAsync(role); - await Program.AdminChecker.SendInAdminChannel($"Removed role '{role.Name}' for user <@{user.Id}>."); + await Program.AdminChecker.SendInAdminChannel($"Removed role '{role.Name}' for user <@{userId}>."); } private SocketRole? GetRole(ulong roleId) From e4d99932ddb6590525ec47420bbeaf4c43dbd507 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 13:53:21 +0200 Subject: [PATCH 28/36] Adds logging to debug active chain roles --- Framework/DiscordRewards/EventsAndErrors.cs | 13 +++++++-- .../Rewards/ChainActivityHandler.cs | 28 +++++++++++++++++-- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 7 +++++ Tools/TestNetRewarder/Program.cs | 1 + 4 files changed, 45 insertions(+), 4 deletions(-) diff --git a/Framework/DiscordRewards/EventsAndErrors.cs b/Framework/DiscordRewards/EventsAndErrors.cs index e26568c0..638299c6 100644 --- a/Framework/DiscordRewards/EventsAndErrors.cs +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -11,8 +11,7 @@ return Errors.Length > 0 || EventsOverview.Length > 0 || - ActiveChainAddresses.Hosts.Length > 0 || - ActiveChainAddresses.Clients.Length > 0; + ActiveChainAddresses.HasAny(); } } @@ -26,5 +25,15 @@ { public string[] Hosts { get; set; } = Array.Empty(); public string[] Clients { get; set; } = Array.Empty(); + + public bool HasAny() + { + return Hosts.Length > 0 || Clients.Length > 0; + } + + public override string ToString() + { + return "Hosts:" + string.Join(",", Hosts) + "Clients:" + string.Join(",", Clients); + } } } diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index a3bb17a8..1541bde8 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -18,10 +18,24 @@ namespace BiblioTech.Rewards public async Task Process(ActiveChainAddresses activeChainAddresses) { + if (!activeChainAddresses.HasAny()) + { + Log("Received empty activeChainAddresses."); + return; + } + var activeUserIds = ConvertToUserIds(activeChainAddresses); - if (!activeUserIds.HasAny()) return; + if (!activeUserIds.HasAny()) + { + Log("Empty userIds after lookup of addresses: " + activeChainAddresses); + return; + } - if (!HasChanged(activeUserIds)) return; + if (!HasChanged(activeUserIds)) + { + Log("Active userIds has not changed: " + activeUserIds); + return; + } await GiveAndRemoveRoles(activeUserIds); } @@ -116,6 +130,11 @@ namespace BiblioTech.Rewards return result.Order().ToArray(); } + private void Log(string msg) + { + log.Log(msg); + } + private class ActiveUserIds { public ActiveUserIds(IEnumerable hosts, IEnumerable clients) @@ -131,6 +150,11 @@ namespace BiblioTech.Rewards { return Hosts.Any() || Clients.Any(); } + + public override string ToString() + { + return "Hosts:" + string.Join(",", Hosts) + "Clients:" + string.Join(",", Clients); + } } } } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs index 58914f5e..3fbc5132 100644 --- a/Tools/BiblioTech/Rewards/RoleModifyContext.cs +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -33,6 +33,7 @@ namespace BiblioTech.Rewards public async Task GiveRole(ulong userId, ulong roleId) { + Log($"Giving role {roleId} to user {userId}"); var role = GetRole(roleId); var guildUser = GetUser(userId); if (role == null) return; @@ -46,6 +47,7 @@ namespace BiblioTech.Rewards public async Task RemoveRole(ulong userId, ulong roleId) { + Log($"Removing role {roleId} from user {userId}"); var role = GetRole(roleId); var guildUser = GetUser(userId); if (role == null) return; @@ -67,6 +69,11 @@ namespace BiblioTech.Rewards return null; } + private void Log(string msg) + { + log.Log(msg); + } + private async Task> LoadAllUsers(SocketGuild guild) { log.Log("Loading all users.."); diff --git a/Tools/TestNetRewarder/Program.cs b/Tools/TestNetRewarder/Program.cs index ab21f61c..e3ed24e7 100644 --- a/Tools/TestNetRewarder/Program.cs +++ b/Tools/TestNetRewarder/Program.cs @@ -45,6 +45,7 @@ namespace TestNetRewarder Log.Log("Starting TestNet Rewarder..."); var segmenter = new TimeSegmenter(Log, Config.Interval, Config.HistoryStartUtc, processor); + await EnsureBotOnline(); await processor.Initialize(); while (!CancellationToken.IsCancellationRequested) From abe03abff6b1888ed752404de1beb793fd0b06c4 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 14:05:10 +0200 Subject: [PATCH 29/36] Sometimes you just forget to call the function --- .../Rewards/ChainActivityHandler.cs | 23 ++++--------------- Tools/BiblioTech/Rewards/RewardController.cs | 10 ++++++-- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index 1541bde8..66fe5e63 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -16,27 +16,12 @@ namespace BiblioTech.Rewards this.repo = repo; } - public async Task Process(ActiveChainAddresses activeChainAddresses) + public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) { - if (!activeChainAddresses.HasAny()) - { - Log("Received empty activeChainAddresses."); - return; - } - + if (!activeChainAddresses.HasAny()) return; var activeUserIds = ConvertToUserIds(activeChainAddresses); - if (!activeUserIds.HasAny()) - { - Log("Empty userIds after lookup of addresses: " + activeChainAddresses); - return; - } - - if (!HasChanged(activeUserIds)) - { - Log("Active userIds has not changed: " + activeUserIds); - return; - } - + if (!activeUserIds.HasAny()) return; + if (!HasChanged(activeUserIds)) return; await GiveAndRemoveRoles(activeUserIds); } diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 7a4b8aa3..93e9b008 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -38,16 +38,22 @@ namespace BiblioTech.Rewards [HttpPost] public async Task Give(EventsAndErrors cmd) + { + await Safe(() => Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses)); + await Safe(() => Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors)); + return "OK"; + } + + private async Task Safe(Func action) { try { - await Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors); + await action(); } catch (Exception ex) { Program.Log.Error("Exception: " + ex); } - return "OK"; } } } From b118f7d103d458e46f1e4bd51b55ab26e71c0f2d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 14:19:52 +0200 Subject: [PATCH 30/36] debugging bot api call --- Tools/BiblioTech/Program.cs | 14 ++++++++++++++ Tools/BiblioTech/Rewards/RewardController.cs | 2 ++ 2 files changed, 16 insertions(+) diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index cb3b13e5..2e0427e6 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -4,8 +4,10 @@ using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; using Discord.WebSocket; +using DiscordRewards; using Logging; using Nethereum.Model; +using Newtonsoft.Json; namespace BiblioTech { @@ -42,6 +44,18 @@ namespace BiblioTech return new Program().MainAsync(args); } + public static void Write(EventsAndErrors cmd) + { + if (Log == null) return; + + if (cmd == null) + { + Log.Log("cmd is null!"); + return; + } + Log.Log(JsonConvert.SerializeObject(cmd)); + } + public async Task MainAsync(string[] args) { Log.Log("Starting Codex Discord Bot..."); diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 93e9b008..1879c6e7 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -39,6 +39,8 @@ namespace BiblioTech.Rewards [HttpPost] public async Task Give(EventsAndErrors cmd) { + Program.Write(cmd); + await Safe(() => Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses)); await Safe(() => Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors)); return "OK"; From 2b7b8161b3879e8445c598cff9a7d9826813484d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 14:55:32 +0200 Subject: [PATCH 31/36] Fixes issue where api calls are missed due to initialized components --- Tools/BiblioTech/CallDispatcher.cs | 66 +++++++++++++++++++ Tools/BiblioTech/CommandHandler.cs | 1 + Tools/BiblioTech/Program.cs | 17 +---- .../Rewards/ChainActivityHandler.cs | 21 +++++- Tools/BiblioTech/Rewards/RewardController.cs | 25 +++---- 5 files changed, 98 insertions(+), 32 deletions(-) create mode 100644 Tools/BiblioTech/CallDispatcher.cs diff --git a/Tools/BiblioTech/CallDispatcher.cs b/Tools/BiblioTech/CallDispatcher.cs new file mode 100644 index 00000000..b24ea830 --- /dev/null +++ b/Tools/BiblioTech/CallDispatcher.cs @@ -0,0 +1,66 @@ +using Logging; + +namespace BiblioTech +{ + public class CallDispatcher + { + private readonly ILog log; + private readonly object _lock = new object(); + private readonly List queue = new List(); + private readonly AutoResetEvent autoResetEvent = new AutoResetEvent(false); + + public CallDispatcher(ILog log) + { + this.log = log; + } + + public void Add(Action call) + { + lock (_lock) + { + queue.Add(call); + autoResetEvent.Set(); + if (queue.Count > 100) + { + log.Error("Queue overflow!"); + queue.Clear(); + } + } + } + + public void Start() + { + Task.Run(() => + { + while (true) + { + try + { + Worker(); + } + catch (Exception ex) + { + log.Error("Exception in CallDispatcher: " + ex); + } + } + }); + } + + private void Worker() + { + autoResetEvent.WaitOne(); + var tasks = Array.Empty(); + + lock (_lock) + { + tasks = queue.ToArray(); + queue.Clear(); + } + + foreach (var task in tasks) + { + task(); + } + } + } +} diff --git a/Tools/BiblioTech/CommandHandler.cs b/Tools/BiblioTech/CommandHandler.cs index 822d4446..10e1e24b 100644 --- a/Tools/BiblioTech/CommandHandler.cs +++ b/Tools/BiblioTech/CommandHandler.cs @@ -84,6 +84,7 @@ namespace BiblioTech log.Error(json); throw; } + Program.Dispatcher.Start(); log.Log("Initialized."); } diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 2e0427e6..0ce84bde 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -16,6 +16,7 @@ namespace BiblioTech private DiscordSocketClient client = null!; private CustomReplacement replacement = null!; + public static CallDispatcher Dispatcher { get; private set; } = null!; public static Configuration Config { get; private set; } = null!; public static UserRepo UserRepo { get; } = new UserRepo(); public static AdminChecker AdminChecker { get; private set; } = null!; @@ -26,8 +27,6 @@ namespace BiblioTech public static Task Main(string[] args) { - Log = new ConsoleLog(); - var uniformArgs = new ArgsUniform(PrintHelp, args); Config = uniformArgs.Parse(); @@ -36,6 +35,8 @@ namespace BiblioTech new ConsoleLog() ); + Dispatcher = new CallDispatcher(Log); + EnsurePath(Config.DataPath); EnsurePath(Config.UserDataPath); EnsurePath(Config.EndpointsPath); @@ -44,18 +45,6 @@ namespace BiblioTech return new Program().MainAsync(args); } - public static void Write(EventsAndErrors cmd) - { - if (Log == null) return; - - if (cmd == null) - { - Log.Log("cmd is null!"); - return; - } - Log.Log(JsonConvert.SerializeObject(cmd)); - } - public async Task MainAsync(string[] args) { Log.Log("Starting Codex Discord Bot..."); diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index 66fe5e63..64f08e38 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -18,10 +18,25 @@ namespace BiblioTech.Rewards public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) { - if (!activeChainAddresses.HasAny()) return; + if (!activeChainAddresses.HasAny()) + { + Log("Received empty activeChainAddresses."); + return; + } + var activeUserIds = ConvertToUserIds(activeChainAddresses); - if (!activeUserIds.HasAny()) return; - if (!HasChanged(activeUserIds)) return; + if (!activeUserIds.HasAny()) + { + Log("Empty userIds after lookup of addresses: " + activeChainAddresses); + return; + } + + if (!HasChanged(activeUserIds)) + { + Log("Active userIds has not changed: " + activeUserIds); + return; + } + await GiveAndRemoveRoles(activeUserIds); } diff --git a/Tools/BiblioTech/Rewards/RewardController.cs b/Tools/BiblioTech/Rewards/RewardController.cs index 1879c6e7..43a01ad5 100644 --- a/Tools/BiblioTech/Rewards/RewardController.cs +++ b/Tools/BiblioTech/Rewards/RewardController.cs @@ -39,23 +39,18 @@ namespace BiblioTech.Rewards [HttpPost] public async Task Give(EventsAndErrors cmd) { - Program.Write(cmd); + Program.Dispatcher.Add(() => + { + Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses).Wait(); + }); - await Safe(() => Program.ChainActivityHandler.ProcessChainActivity(cmd.ActiveChainAddresses)); - await Safe(() => Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors)); + Program.Dispatcher.Add(() => + { + Program.EventsSender.ProcessChainEvents(cmd.EventsOverview, cmd.Errors).Wait(); + }); + + await Task.CompletedTask; return "OK"; } - - private async Task Safe(Func action) - { - try - { - await action(); - } - catch (Exception ex) - { - Program.Log.Error("Exception: " + ex); - } - } } } From 9a58da26aa14065f53ce3b2e2a6bea81137a8fec Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 15:14:59 +0200 Subject: [PATCH 32/36] cleanup log messages --- .../Rewards/ChainActivityHandler.cs | 20 +++---------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index 64f08e38..c190e7b7 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -18,24 +18,10 @@ namespace BiblioTech.Rewards public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) { - if (!activeChainAddresses.HasAny()) - { - Log("Received empty activeChainAddresses."); - return; - } - + if (!activeChainAddresses.HasAny()) return; var activeUserIds = ConvertToUserIds(activeChainAddresses); - if (!activeUserIds.HasAny()) - { - Log("Empty userIds after lookup of addresses: " + activeChainAddresses); - return; - } - - if (!HasChanged(activeUserIds)) - { - Log("Active userIds has not changed: " + activeUserIds); - return; - } + if (!activeUserIds.HasAny()) return; + if (!HasChanged(activeUserIds)) return; await GiveAndRemoveRoles(activeUserIds); } From cd17b0c887817cfb757de137bcf0902974b58681 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 15:26:57 +0200 Subject: [PATCH 33/36] Speeds up looking for users by ethAddress. Reduces load on discord server. --- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 13 ++++++++++--- Tools/BiblioTech/UserRepo.cs | 19 ++++++++----------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs index 3fbc5132..9b38dcea 100644 --- a/Tools/BiblioTech/Rewards/RoleModifyContext.cs +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -10,6 +10,8 @@ namespace BiblioTech.Rewards { private Dictionary users = new(); private Dictionary roles = new(); + private DateTime lastLoad = DateTime.MinValue; + private readonly SocketGuild guild; private readonly UserRepo userRepo; private readonly ILog log; @@ -25,8 +27,14 @@ namespace BiblioTech.Rewards public async Task Initialize() { - this.users = await LoadAllUsers(guild); - this.roles = LoadAllRoles(guild); + var span = DateTime.UtcNow - lastLoad; + if (span > TimeSpan.FromMinutes(10)) + { + lastLoad = DateTime.UtcNow; + log.Log("Loading all users and roles..."); + this.users = await LoadAllUsers(guild); + this.roles = LoadAllRoles(guild); + } } public IGuildUser[] Users => users.Values.ToArray(); @@ -76,7 +84,6 @@ namespace BiblioTech.Rewards private async Task> LoadAllUsers(SocketGuild guild) { - log.Log("Loading all users.."); var result = new Dictionary(); var users = guild.GetUsersAsync(); await foreach (var ulist in users) diff --git a/Tools/BiblioTech/UserRepo.cs b/Tools/BiblioTech/UserRepo.cs index 07831dd6..701fdfe9 100644 --- a/Tools/BiblioTech/UserRepo.cs +++ b/Tools/BiblioTech/UserRepo.cs @@ -122,20 +122,17 @@ namespace BiblioTech { if (address == null) return null; - // If this becomes a performance problem, switch to in-memory cached list. - var files = Directory.GetFiles(Program.Config.UserDataPath); - foreach (var file in files) + var lower = address.Address.ToLowerInvariant(); + if (string.IsNullOrEmpty(lower)) return null; + + if (cache.Count == 0) LoadAllUserData(); + foreach (var item in cache.Values) { - try + if (item.CurrentAddress != null && + item.CurrentAddress.Address.ToLowerInvariant() == lower) { - var user = JsonConvert.DeserializeObject(File.ReadAllText(file))!; - if (user.CurrentAddress != null && - user.CurrentAddress.Address == address.Address) - { - return user; - } + return item; } - catch { } } return null; From 16b5ee4fd1a3c24d7e88d2ddcfda0b2b78d245ad Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 15:29:40 +0200 Subject: [PATCH 34/36] makes logreplace command automatically add lowercap versions of from-string --- Tools/BiblioTech/Rewards/CustomReplacement.cs | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/Tools/BiblioTech/Rewards/CustomReplacement.cs b/Tools/BiblioTech/Rewards/CustomReplacement.cs index dbd8eaf3..912aa789 100644 --- a/Tools/BiblioTech/Rewards/CustomReplacement.cs +++ b/Tools/BiblioTech/Rewards/CustomReplacement.cs @@ -28,14 +28,14 @@ namespace BiblioTech.Rewards public void Add(string from, string to) { - if (replacements.ContainsKey(from)) + AddOrUpdate(from, to); + + var lower = from.ToLowerInvariant(); + if (lower != from) { - replacements[from] = to; - } - else - { - replacements.Add(from, to); + AddOrUpdate(lower, to); } + Save(); } @@ -55,6 +55,18 @@ namespace BiblioTech.Rewards return result; } + private void AddOrUpdate(string from, string to) + { + if (replacements.ContainsKey(from)) + { + replacements[from] = to; + } + else + { + replacements.Add(from, to); + } + } + private void Save() { ReplaceJson[] replaces = replacements.Select(pair => From abdc7a0ab15d32676c00847b0057a749ecf16bd4 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 15:51:20 +0200 Subject: [PATCH 35/36] block and lock for loading of discord users --- Tools/BiblioTech/Rewards/RoleDriver.cs | 8 ++++---- Tools/BiblioTech/Rewards/RoleModifyContext.cs | 20 ++++++++++++------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index 41b0e44c..1cc9f3f2 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -25,7 +25,7 @@ namespace BiblioTech.Rewards public async Task RunRoleGiver(Func action) { - var context = await OpenRoleModifyContext(); + var context = OpenRoleModifyContext(); var mapper = new RoleMapper(context); await action(mapper); } @@ -37,7 +37,7 @@ namespace BiblioTech.Rewards public async Task IterateUsersWithRoles(Func onUserWithRole, Func whenDone, params ulong[] rolesToIterate) { - var context = await OpenRoleModifyContext(); + var context = OpenRoleModifyContext(); var mapper = new RoleMapper(context); foreach (var user in context.Users) { @@ -52,10 +52,10 @@ namespace BiblioTech.Rewards await whenDone(mapper); } - private async Task OpenRoleModifyContext() + private RoleModifyContext OpenRoleModifyContext() { var context = new RoleModifyContext(GetGuild(), userRepo, log, rewardsChannel); - await context.Initialize(); + context.Initialize(); return context; } diff --git a/Tools/BiblioTech/Rewards/RoleModifyContext.cs b/Tools/BiblioTech/Rewards/RoleModifyContext.cs index 9b38dcea..6fd4fb48 100644 --- a/Tools/BiblioTech/Rewards/RoleModifyContext.cs +++ b/Tools/BiblioTech/Rewards/RoleModifyContext.cs @@ -11,6 +11,7 @@ namespace BiblioTech.Rewards private Dictionary users = new(); private Dictionary roles = new(); private DateTime lastLoad = DateTime.MinValue; + private readonly object _lock = new object(); private readonly SocketGuild guild; private readonly UserRepo userRepo; @@ -25,15 +26,20 @@ namespace BiblioTech.Rewards this.rewardsChannel = rewardsChannel; } - public async Task Initialize() + public void Initialize() { - var span = DateTime.UtcNow - lastLoad; - if (span > TimeSpan.FromMinutes(10)) + lock (_lock) { - lastLoad = DateTime.UtcNow; - log.Log("Loading all users and roles..."); - this.users = await LoadAllUsers(guild); - this.roles = LoadAllRoles(guild); + var span = DateTime.UtcNow - lastLoad; + if (span > TimeSpan.FromMinutes(10)) + { + lastLoad = DateTime.UtcNow; + log.Log("Loading all users and roles..."); + var task = LoadAllUsers(guild); + task.Wait(); + this.users = task.Result; + this.roles = LoadAllRoles(guild); + } } } From e8ef65d641749a495b48cf909bdb419a6d8dde40 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 17 Apr 2025 16:06:34 +0200 Subject: [PATCH 36/36] Allows processing of empty chain addresses so roles can be cleaned up --- Framework/DiscordRewards/EventsAndErrors.cs | 5 ----- Tools/BiblioTech/Rewards/ChainActivityHandler.cs | 12 ------------ Tools/TestNetRewarder/Processor.cs | 3 +-- 3 files changed, 1 insertion(+), 19 deletions(-) diff --git a/Framework/DiscordRewards/EventsAndErrors.cs b/Framework/DiscordRewards/EventsAndErrors.cs index 638299c6..7e18f07f 100644 --- a/Framework/DiscordRewards/EventsAndErrors.cs +++ b/Framework/DiscordRewards/EventsAndErrors.cs @@ -30,10 +30,5 @@ { return Hosts.Length > 0 || Clients.Length > 0; } - - public override string ToString() - { - return "Hosts:" + string.Join(",", Hosts) + "Clients:" + string.Join(",", Clients); - } } } diff --git a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs index c190e7b7..d35d9fe9 100644 --- a/Tools/BiblioTech/Rewards/ChainActivityHandler.cs +++ b/Tools/BiblioTech/Rewards/ChainActivityHandler.cs @@ -18,9 +18,7 @@ namespace BiblioTech.Rewards public async Task ProcessChainActivity(ActiveChainAddresses activeChainAddresses) { - if (!activeChainAddresses.HasAny()) return; var activeUserIds = ConvertToUserIds(activeChainAddresses); - if (!activeUserIds.HasAny()) return; if (!HasChanged(activeUserIds)) return; await GiveAndRemoveRoles(activeUserIds); @@ -131,16 +129,6 @@ namespace BiblioTech.Rewards public List Hosts { get; } public List Clients { get; } - - public bool HasAny() - { - return Hosts.Any() || Clients.Any(); - } - - public override string ToString() - { - return "Hosts:" + string.Join(",", Hosts) + "Clients:" + string.Join(",", Clients); - } } } } diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index b018d377..eed26962 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -47,8 +47,7 @@ namespace TestNetRewarder var numberOfChainEvents = await ProcessEvents(timeRange); var duration = sw.Elapsed; - if (numberOfChainEvents == 0) return TimeSegmentResponse.Underload; - if (numberOfChainEvents > 10) return TimeSegmentResponse.Overload; + if (duration > TimeSpan.FromSeconds(1)) return TimeSegmentResponse.Underload; if (duration > TimeSpan.FromSeconds(3)) return TimeSegmentResponse.Overload; return TimeSegmentResponse.OK; }