diff --git a/Framework/DiscordRewards/GiveRewardsCommand.cs b/Framework/DiscordRewards/GiveRewardsCommand.cs index 67c611b2..3aae088b 100644 --- a/Framework/DiscordRewards/GiveRewardsCommand.cs +++ b/Framework/DiscordRewards/GiveRewardsCommand.cs @@ -3,7 +3,8 @@ public class GiveRewardsCommand { public RewardUsersCommand[] Rewards { get; set; } = Array.Empty(); - public string[] EventsOverview { get; set; } = Array.Empty(); + public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty(); + public string[] Errors { get; set; } = Array.Empty(); public bool HasAny() { @@ -16,4 +17,10 @@ public ulong RewardId { get; set; } public string[] UserAddresses { get; set; } = Array.Empty(); } + + public class ChainEventMessage + { + public ulong BlockNumber { get; set; } + public string Message { get; set; } = string.Empty; + } } diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs index d9a6ac29..c174de99 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs @@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils public ulong? GetHighestBlockNumberBefore(DateTime moment) { bounds.Initialize(); - if (moment <= bounds.Genesis.Utc) return null; + if (moment < bounds.Genesis.Utc) return null; + if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber; if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber; return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector)); @@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils public ulong? GetLowestBlockNumberAfter(DateTime moment) { bounds.Initialize(); - if (moment >= bounds.Current.Utc) return null; + if (moment > bounds.Current.Utc) return null; + if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber; if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber; return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ; diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs index 4377b22b..30756c4c 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs @@ -17,6 +17,8 @@ namespace CodexContractsPlugin.ChainMonitor void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex); void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex); void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex); + + void OnError(string msg); } public class RequestEvent @@ -67,7 +69,11 @@ namespace CodexContractsPlugin.ChainMonitor private void Apply(ChainEvents events) { if (events.BlockInterval.TimeRange.From < TotalSpan.From) - throw new Exception("Attempt to update ChainState with set of events from before its current record."); + { + var msg = "Attempt to update ChainState with set of events from before its current record."; + handler.OnError(msg); + throw new Exception(msg); + } log.Log($"ChainState updating: {events.BlockInterval}"); @@ -110,7 +116,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(RequestFulfilledEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.UpdateState(@event.Block.BlockNumber, RequestState.Started); handler.OnRequestFulfilled(new RequestEvent(@event.Block, r)); @@ -118,7 +124,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(RequestCancelledEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled); handler.OnRequestCancelled(new RequestEvent(@event.Block, r)); @@ -126,7 +132,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(RequestFailedEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.UpdateState(@event.Block.BlockNumber, RequestState.Failed); handler.OnRequestFailed(new RequestEvent(@event.Block, r)); @@ -134,7 +140,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(SlotFilledEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.Hosts.Add(@event.Host, (int)@event.SlotIndex); r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})"); @@ -143,7 +149,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(SlotFreedEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.Hosts.RemoveHost((int)@event.SlotIndex); r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})"); @@ -152,7 +158,7 @@ namespace CodexContractsPlugin.ChainMonitor private void ApplyEvent(SlotReservationsFullEventDTO @event) { - var r = FindRequest(@event.RequestId); + var r = FindRequest(@event); if (r == null) return; r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})"); handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex); @@ -171,10 +177,23 @@ namespace CodexContractsPlugin.ChainMonitor } } - private ChainStateRequest? FindRequest(byte[] requestId) + private ChainStateRequest? FindRequest(IHasRequestId request) { - var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId)); - if (r == null) log.Log("Unable to find request by ID!"); + var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId)); + if (r == null) + { + var blockNumber = "unknown"; + if (request is IHasBlock blk) + { + blockNumber = blk.Block.BlockNumber.ToString(); + } + + var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " + + $"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})"; + + log.Error(msg); + handler.OnError(msg); + } return r; } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs index 597cb864..799573f6 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs @@ -1,10 +1,5 @@ using GethPlugin; -using System; -using System.Collections.Generic; -using System.Linq; using System.Numerics; -using System.Text; -using System.Threading.Tasks; namespace CodexContractsPlugin.ChainMonitor { @@ -56,5 +51,10 @@ namespace CodexContractsPlugin.ChainMonitor { foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex); } + + public void OnError(string msg) + { + foreach (var handler in Handlers) handler.OnError(msg); + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs index 1b478ae8..749f30c4 100644 --- a/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs +++ b/ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs @@ -36,5 +36,9 @@ namespace CodexContractsPlugin.ChainMonitor public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex) { } + + public void OnError(string msg) + { + } } } diff --git a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs index ba1e1683..413d340d 100644 --- a/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs +++ b/ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs @@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace BlockTimeEntry Block { get; set; } } - public partial class Request : RequestBase, IHasBlock + public interface IHasRequestId + { + byte[] RequestId { get; set; } + } + + public partial class Request : RequestBase, IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } @@ -28,38 +33,38 @@ namespace CodexContractsPlugin.Marketplace } } - public partial class RequestFulfilledEventDTO : IHasBlock + public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class RequestCancelledEventDTO : IHasBlock + public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class RequestFailedEventDTO : IHasBlock + public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class SlotFilledEventDTO : IHasBlock + public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } public EthAddress Host { get; set; } } - public partial class SlotFreedEventDTO : IHasBlock + public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } } - public partial class SlotReservationsFullEventDTO : IHasBlock + public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId { [JsonIgnore] public BlockTimeEntry Block { get; set; } diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index c1df2d67..66d5963f 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -10,7 +10,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "2E-7C-A2-F3-67-D9-F2-A6-4E-D5-FF-A2-EC-65-ED-59-CE-89-A8-92-57-5E-CF-40-9A-83-49-0B-49-42-5D-EC"; + private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 1216ec04..9f3e33fa 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -73,7 +73,7 @@ namespace CodexPlugin public Stream DownloadFile(string contentId, Action onFailure) { var fileResponse = OnCodex( - api => api.DownloadNetworkAsync(contentId), + api => api.DownloadNetworkStreamAsync(contentId), CreateRetryConfig(nameof(DownloadFile), onFailure)); if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); @@ -88,25 +88,25 @@ namespace CodexPlugin public StorageAvailability SalesAvailability(StorageAvailability request) { var body = mapper.Map(request); - var read = OnCodex(api => api.OfferStorageAsync(body)); + var read = OnCodex(api => api.OfferStorageAsync(body)); return mapper.Map(read); } public StorageAvailability[] GetAvailabilities() { - var collection = OnCodex>(api => api.GetAvailabilitiesAsync()); + var collection = OnCodex(api => api.GetAvailabilitiesAsync()); return mapper.Map(collection); } public string RequestStorage(StoragePurchaseRequest request) { var body = mapper.Map(request); - return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); + return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); } public CodexSpace Space() { - var space = OnCodex(api => api.SpaceAsync()); + var space = OnCodex(api => api.SpaceAsync()); return mapper.Map(space); } diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index c57bddfa..9a202128 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -456,9 +456,35 @@ paths: "/data/{cid}/network": get: - summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able." + summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed." tags: [ Data ] operationId: downloadNetwork + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "File to be downloaded." + responses: + "200": + description: Manifest information for download that has been started. + content: + application/json: + schema: + $ref: "#/components/schemas/DataItem" + "400": + description: Invalid CID is specified + "404": + description: Failed to download dataset manifest + "500": + description: Well it was bad-bad + + "/data/{cid}/network/stream": + get: + summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able." + tags: [ Data ] + operationId: downloadNetworkStream parameters: - in: path name: cid @@ -481,6 +507,32 @@ paths: "500": description: Well it was bad-bad + "/data/{cid}/network/manifest": + get: + summary: "Download only the dataset manifest from the network to the local node if it's not available locally." + tags: [ Data ] + operationId: downloadNetworkManifest + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "File for which the manifest is to be downloaded." + responses: + "200": + description: Manifest information. + content: + application/json: + schema: + $ref: "#/components/schemas/DataItem" + "400": + description: Invalid CID is specified + "404": + description: Failed to download dataset manifest + "500": + description: Well it was bad-bad + "/space": get: summary: "Gets a summary of the storage space allocation of the node." @@ -792,4 +844,4 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DebugInfo" \ No newline at end of file + $ref: "#/components/schemas/DebugInfo" diff --git a/Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs b/Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs new file mode 100644 index 00000000..29618947 --- /dev/null +++ b/Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs @@ -0,0 +1,228 @@ +using CodexPlugin; +using FileUtils; +using Logging; +using NUnit.Framework; +using Utils; + +namespace CodexTests.DownloadConnectivityTests +{ + [TestFixture] + public class MultiswarmTests : AutoBootstrapDistTest + { + [Test] + [Combinatorial] + public void Multiswarm( + [Values(3, 5)] int numFiles, + [Values(5, 20)] int fileSizeMb, + [Values(1)] int uploadersPerFile, + [Values(3)] int downloadersPerFile, + [Values(1)] int maxUploadsPerNode, + [Values(2, 3)] int maxDownloadsPerNode + ) + { + var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode); + Assert.That(plan.NodePlans.Count, Is.LessThan(30)); + + RunThePlan(plan, fileSizeMb); + } + + private void RunThePlan(Plan plan, int fileSizeMb) + { + foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB()); + var nodes = StartCodex(plan.NodePlans.Count); + for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i]; + + // Upload all files to their nodes. + foreach (var filePlan in plan.FilePlans) + { + foreach (var uploader in filePlan.Uploaders) + { + filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!); + } + } + + Thread.Sleep(5000); // Everything is processed and announced. + + // Start all downloads (almost) simultaneously. + var tasks = new List(); + foreach (var filePlan in plan.FilePlans) + { + foreach (var downloader in filePlan.Downloaders) + { + tasks.Add(Task.Run(() => + { + var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!); + lock (filePlan.DownloadedFiles) + { + filePlan.DownloadedFiles.Add(downloadedFile); + } + })); + } + } + + Task.WaitAll(tasks.ToArray()); + + // Assert all files are correct. + foreach (var filePlan in plan.FilePlans) + { + foreach (var downloadedFile in filePlan.DownloadedFiles) + { + filePlan.File!.AssertIsEqual(downloadedFile); + } + } + } + + private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode) + { + var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode); + plan.Initialize(); + plan.LogPlan(GetTestLog()); + return plan; + } + } + + public class FilePlan + { + public FilePlan(int number) + { + Number = number; + } + + public int Number { get; } + public TrackedFile? File { get; set; } + public ContentId? Cid { get; set; } + public List DownloadedFiles { get; } = new List(); + public List Uploaders { get; } = new List(); + public List Downloaders { get; } = new List(); + + public override string ToString() + { + return $"FilePlan[{Number}] " + + $"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " + + $"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]"; + } + } + + public class NodePlan + { + public NodePlan(int number) + { + Number = number; + } + + public int Number { get; } + public ICodexNode? Node { get; set; } + public List Uploads { get; } = new List(); + public List Downloads { get; } = new List(); + + public bool Contains(FilePlan plan) + { + return Uploads.Contains(plan) || Downloads.Contains(plan); + } + + public override string ToString() + { + return $"NodePlan[{Number}] " + + $"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " + + $"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]"; + } + } + + public class Plan + { + private readonly int numFiles; + private readonly int uploadersPerFile; + private readonly int downloadersPerFile; + private readonly int maxUploadsPerNode; + private readonly int maxDownloadsPerNode; + + public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode) + { + this.numFiles = numFiles; + this.uploadersPerFile = uploadersPerFile; + this.downloadersPerFile = downloadersPerFile; + this.maxUploadsPerNode = maxUploadsPerNode; + this.maxDownloadsPerNode = maxDownloadsPerNode; + } + + public List FilePlans { get; } = new List(); + public List NodePlans { get; } = new List(); + + public void Initialize() + { + for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i)); + foreach (var filePlan in FilePlans) + { + while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan); + while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan); + } + + CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number)); + CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number)); + + foreach (var filePlan in FilePlans) + { + Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile)); + Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile)); + } + foreach (var nodePlan in NodePlans) + { + Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode)); + Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode)); + } + } + + public void LogPlan(ILog log) + { + log.Log("The plan:"); + log.Log("Input:"); + log.Log($"numFiles: {numFiles}"); + log.Log($"uploadersPerFile: {uploadersPerFile}"); + log.Log($"downloadersPerFile: {downloadersPerFile}"); + log.Log($"maxUploadsPerNode: {maxUploadsPerNode}"); + log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}"); + log.Log("Setup:"); + log.Log($"number of nodes: {NodePlans.Count}"); + foreach (var filePlan in FilePlans) log.Log(filePlan.ToString()); + foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString()); + } + + private void AddDownloader(FilePlan filePlan) + { + var nodePlan = GetOrCreateDownloaderNode(filePlan); + filePlan.Downloaders.Add(nodePlan); + nodePlan.Downloads.Add(filePlan); + } + + private void AddUploader(FilePlan filePlan) + { + var nodePlan = GetOrCreateUploaderNode(filePlan); + filePlan.Uploaders.Add(nodePlan); + nodePlan.Uploads.Add(filePlan); + } + + private NodePlan GetOrCreateDownloaderNode(FilePlan notIn) + { + var available = NodePlans.Where(n => + n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn) + ).ToArray(); + if (available.Any()) return RandomUtils.GetOneRandom(available); + + var newNodePlan = new NodePlan(NodePlans.Count); + NodePlans.Add(newNodePlan); + return newNodePlan; + } + + private NodePlan GetOrCreateUploaderNode(FilePlan notIn) + { + var available = NodePlans.Where(n => + n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn) + ).ToArray(); + if (available.Any()) return RandomUtils.GetOneRandom(available); + + var newNodePlan = new NodePlan(NodePlans.Count); + NodePlans.Add(newNodePlan); + return newNodePlan; + } + } +} diff --git a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs index c327febc..50094dcd 100644 --- a/Tests/CodexTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/CodexTests/UtilityTests/DiscordBotTests.cs @@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests private readonly List hostAccounts = new List(); private readonly List rewardsSeen = new List(); private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); - private readonly List receivedEvents = new List(); + private readonly List receivedEvents = new List(); [Test] [DontDownloadLogs] @@ -73,13 +73,18 @@ namespace CodexTests.UtilityTests private void AssertEventOccurance(string msg, int expectedCount) { - Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount), + Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount), $"Event '{msg}' did not occure correct number of times."); } private void OnCommand(string timestamp, GiveRewardsCommand call) { Log($""); + foreach (var e in call.EventsOverview) + { + Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order."); + } + receivedEvents.AddRange(call.EventsOverview); foreach (var e in call.EventsOverview) { diff --git a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs index d2be5daa..17f5ee68 100644 --- a/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs +++ b/Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs @@ -19,11 +19,20 @@ namespace FrameworkTests.NethereumWorkflow { var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30); blocks = new Dictionary(); - + + Block? prev = null; for (ulong i = 0; i < 30; i++) { ulong d = 100 + i; - blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2))); + var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2)); + blocks.Add(d, newBlock); + + if (prev != null) + { + prev.Next = newBlock; + newBlock.Previous = prev; + } + prev = newBlock; } } @@ -99,23 +108,23 @@ namespace FrameworkTests.NethereumWorkflow } [Test] - public void FailsToFindBlockBeforeFrontOfChain() + public void FindsGenesisBlockAtFrontOfChain() { var first = blocks.First().Value; - var notFound = finder.GetHighestBlockNumberBefore(first.Time); + var firstNumber = finder.GetHighestBlockNumberBefore(first.Time); - Assert.That(notFound, Is.Null); + Assert.That(firstNumber, Is.EqualTo(first.Number)); } [Test] - public void FailsToFindBlockAfterTailOfChain() + public void FindsCurrentBlockAtTailOfChain() { var last = blocks.Last().Value; - var notFound = finder.GetLowestBlockNumberAfter(last.Time); + var lastNumber = finder.GetLowestBlockNumberAfter(last.Time); - Assert.That(notFound, Is.Null); + Assert.That(lastNumber, Is.EqualTo(last.Number)); } [Test] @@ -143,13 +152,27 @@ namespace FrameworkTests.NethereumWorkflow { foreach (var pair in blocks) { - finder.GetHighestBlockNumberBefore(pair.Value.JustBefore); - finder.GetHighestBlockNumberBefore(pair.Value.Time); - finder.GetHighestBlockNumberBefore(pair.Value.JustAfter); + var block = pair.Value; - finder.GetLowestBlockNumberAfter(pair.Value.JustBefore); - finder.GetLowestBlockNumberAfter(pair.Value.Time); - finder.GetLowestBlockNumberAfter(pair.Value.JustAfter); + AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore)); + AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time)); + AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter)); + + AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore)); + AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time)); + AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter)); + } + } + + private void AssertLink(Block? expected, ulong? actual) + { + if (expected == null) + { + Assert.That(actual, Is.Null); + } + else + { + Assert.That(expected.Number, Is.EqualTo(actual!.Value)); } } } @@ -167,6 +190,9 @@ namespace FrameworkTests.NethereumWorkflow public DateTime JustBefore { get { return Time.AddSeconds(-1); } } public DateTime JustAfter { get { return Time.AddSeconds(1); } } + public Block? Next { get; set; } + public Block? Previous { get; set; } + public override string ToString() { return $"[{Number}]"; diff --git a/Tests/FrameworkTests/Utils/RunLengthEncoding.cs b/Tests/FrameworkTests/Utils/RunLengthEncoding.cs new file mode 100644 index 00000000..0f70cebd --- /dev/null +++ b/Tests/FrameworkTests/Utils/RunLengthEncoding.cs @@ -0,0 +1,334 @@ +namespace FrameworkTests.Utils +{ + public class Run + { + public Run(int start, int length) + { + Start = start; + Length = length; + } + + public int Start { get; } + public int Length { get; private set; } + + public bool Includes(int index) + { + return index >= Start && index < (Start + Length); + } + + public RunUpdate ExpandToInclude(int index) + { + if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}"); + if (index == (Start + Length)) + { + Length++; + return new RunUpdate(); + } + if (index == (Start - 1)) + { + return new RunUpdate( + newRuns: [new Run(Start - 1, Length + 1)], + removeRuns: [this] + ); + } + throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}"); + } + + public RunUpdate Unset(int index) + { + if (!Includes(index)) + { + return new RunUpdate(); + } + + if (index == Start) + { + // First index: Replace self with new run at next index, unless empty. + if (Length == 1) + { + return new RunUpdate( + newRuns: Array.Empty(), + removeRuns: [this] + ); + } + return new RunUpdate( + newRuns: [new Run(Start + 1, Length - 1)], + removeRuns: [this] + ); + } + + if (index == (Start + Length - 1)) + { + // Last index: Become one smaller. + Length--; + return new RunUpdate(); + } + + // Split: + var newRunLength = (Start + Length - 1) - index; + Length = index - Start; + return new RunUpdate( + newRuns: [new Run(index + 1, newRunLength)], + removeRuns: Array.Empty() + ); + } + + public void Iterate(Action action) + { + for (var i = 0; i < Length; i++) + { + action(Start + i); + } + } + + public override string ToString() + { + return $"[{Start},{Length}]"; + } + + public override bool Equals(object? obj) + { + return obj is Run run && + Start == run.Start && + Length == run.Length; + } + + public override int GetHashCode() + { + return HashCode.Combine(Start, Length); + } + + public static bool operator ==(Run? obj1, Run? obj2) + { + if (ReferenceEquals(obj1, obj2)) return true; + if (ReferenceEquals(obj1, null)) return false; + if (ReferenceEquals(obj2, null)) return false; + return obj1.Equals(obj2); + } + public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2); + } + + public class RunUpdate + { + public RunUpdate() + : this(Array.Empty(), Array.Empty()) + { + } + + public RunUpdate(Run[] newRuns, Run[] removeRuns) + { + NewRuns = newRuns; + RemoveRuns = removeRuns; + } + + public Run[] NewRuns { get; } + public Run[] RemoveRuns { get; } + } + + public partial class IndexSet + { + private readonly SortedList runs = new SortedList(); + + public IndexSet() + { + } + + public IndexSet(int[] indices) + { + foreach (var i in indices) Set(i); + } + + public static IndexSet FromRunLengthEncoded(int[] rle) + { + var set = new IndexSet(); + for (var i = 0; i < rle.Length; i += 2) + { + var start = rle[i]; + var length = rle[i + 1]; + set.runs.Add(start, new Run(start, length)); + } + + return set; + } + + public bool IsSet(int index) + { + if (runs.ContainsKey(index)) return true; + + var run = GetRunAt(index); + if (run == null) return false; + return true; + } + + public void Set(int index) + { + if (IsSet(index)) return; + + var runBefore = GetRunAt(index - 1); + var runAfter = GetRunExact(index + 1); + + if (runBefore == null) + { + if (runAfter == null) + { + CreateNewRun(index); + } + else + { + HandleUpdate(runAfter.ExpandToInclude(index)); + } + } + else + { + if (runAfter == null) + { + HandleUpdate(runBefore.ExpandToInclude(index)); + } + else + { + // new index will connect runBefore with runAfter. We merge! + HandleUpdate(new RunUpdate( + newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)], + removeRuns: [runBefore, runAfter] + )); + } + } + } + + public void Unset(int index) + { + if (runs.ContainsKey(index)) + { + HandleUpdate(runs[index].Unset(index)); + } + else + { + var run = GetRunAt(index); + if (run == null) return; + HandleUpdate(run.Unset(index)); + } + } + + public void Iterate(Action onIndex) + { + foreach (var run in runs.Values) + { + run.Iterate(onIndex); + } + } + + public int[] RunLengthEncoded() + { + return Encode().ToArray(); + } + + public override string ToString() + { + return string.Join("&", runs.Select(r => r.ToString()).ToArray()); + } + + private IEnumerable Encode() + { + foreach (var pair in runs) + { + yield return pair.Value.Start; + yield return pair.Value.Length; + } + } + + private Run? GetRunAt(int index) + { + foreach (var run in runs.Values) + { + if (run.Includes(index)) return run; + } + return null; + } + + private Run? GetRunExact(int index) + { + if (runs.ContainsKey(index)) return runs[index]; + return null; + } + + private void HandleUpdate(RunUpdate runUpdate) + { + foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start); + foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun); + } + + private void CreateNewRun(int index) + { + if (runs.ContainsKey(index + 1)) + { + var length = runs[index + 1].Length + 1; + runs.Add(index, new Run(index, length)); + runs.Remove(index + 1); + } + else + { + runs.Add(index, new Run(index, 1)); + } + } + } + + public partial class IndexSet + { + public IndexSet Overlap(IndexSet other) + { + var result = new IndexSet(); + Iterate(i => + { + if (other.IsSet(i)) result.Set(i); + }); + return result; + } + + public IndexSet Merge(IndexSet other) + { + var result = new IndexSet(); + Iterate(result.Set); + other.Iterate(result.Set); + return result; + } + + public IndexSet Without(IndexSet other) + { + var result = new IndexSet(); + Iterate(i => + { + if (!other.IsSet(i)) result.Set(i); + }); + return result; + } + + public override bool Equals(object? obj) + { + if (obj is IndexSet set) + { + if (set.runs.Count != runs.Count) return false; + foreach (var pair in runs) + { + if (!set.runs.ContainsKey(pair.Key)) return false; + if (set.runs[pair.Key] != pair.Value) return false; + } + return true; + } + return false; + } + + public override int GetHashCode() + { + return HashCode.Combine(runs); + } + + public static bool operator ==(IndexSet? obj1, IndexSet? obj2) + { + if (ReferenceEquals(obj1, obj2)) return true; + if (ReferenceEquals(obj1, null)) return false; + if (ReferenceEquals(obj2, null)) return false; + return obj1.Equals(obj2); + } + public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2); + } +} diff --git a/Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs b/Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs new file mode 100644 index 00000000..4ff63a65 --- /dev/null +++ b/Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs @@ -0,0 +1,94 @@ +using NUnit.Framework; + +namespace FrameworkTests.Utils +{ + [TestFixture] + public class RunLengthEncodingLogicalTests + { + [Test] + public void EqualityTest() + { + var setA = new IndexSet([1, 2, 3, 4]); + var setB = new IndexSet([1, 2, 3, 4]); + + Assert.That(setA, Is.EqualTo(setB)); + Assert.That(setA == setB); + } + + [Test] + public void InequalityTest1() + { + var setA = new IndexSet([1, 2, 4, 5]); + var setB = new IndexSet([1, 2, 3, 4]); + + Assert.That(setA, Is.Not.EqualTo(setB)); + Assert.That(setA != setB); + } + + [Test] + public void InequalityTest2() + { + var setA = new IndexSet([1, 2, 3]); + var setB = new IndexSet([1, 2, 3, 4]); + + Assert.That(setA, Is.Not.EqualTo(setB)); + Assert.That(setA != setB); + } + + [Test] + public void InequalityTest3() + { + var setA = new IndexSet([2, 3, 4, 5]); + var setB = new IndexSet([1, 2, 3, 4]); + + Assert.That(setA, Is.Not.EqualTo(setB)); + Assert.That(setA != setB); + } + + [Test] + public void InequalityTest() + { + var setA = new IndexSet([2, 3, 4]); + var setB = new IndexSet([1, 2, 3, 4]); + + Assert.That(setA, Is.Not.EqualTo(setB)); + Assert.That(setA != setB); + } + + [Test] + public void Overlap() + { + var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]); + var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]); + var expectedSet = new IndexSet([3, 4, 5, 11]); + + var set = setA.Overlap(setB); + + Assert.That(set, Is.EqualTo(expectedSet)); + } + + [Test] + public void Merge() + { + var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]); + var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]); + var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]); + + var set = setA.Merge(setB); + + Assert.That(set, Is.EqualTo(expectedSet)); + } + + [Test] + public void Without() + { + var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]); + var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]); + var expectedSet = new IndexSet([1, 2, 14]); + + var set = setA.Without(setB); + + Assert.That(set, Is.EqualTo(expectedSet)); + } + } +} diff --git a/Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs b/Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs index d3544c14..66e47f68 100644 --- a/Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs +++ b/Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs @@ -1,12 +1,40 @@ using NUnit.Framework; -using NUnit.Framework.Interfaces; -using static FrameworkTests.Utils.RunLengthEncodingTests; namespace FrameworkTests.Utils { [TestFixture] public class RunLengthEncodingRunTests { + [Test] + public void EqualityTest() + { + var runA = new Run(1, 4); + var runB = new Run(1, 4); + + Assert.That(runA, Is.EqualTo(runB)); + Assert.That(runA == runB); + } + + [Test] + public void InequalityTest1() + { + var runA = new Run(1, 4); + var runB = new Run(1, 5); + + Assert.That(runA, Is.Not.EqualTo(runB)); + Assert.That(runA != runB); + } + + [Test] + public void InequalityTest2() + { + var runA = new Run(1, 4); + var runB = new Run(2, 4); + + Assert.That(runA, Is.Not.EqualTo(runB)); + Assert.That(runA != runB); + } + [Test] [Combinatorial] public void RunIncludes( @@ -33,23 +61,58 @@ namespace FrameworkTests.Utils } [Test] - public void RunExpandToInclude() + public void RunExpandThrowsWhenIndexNotAdjacent() { var run = new Run(2, 3); + Assert.That(!run.Includes(1)); Assert.That(run.Includes(2)); Assert.That(run.Includes(4)); Assert.That(!run.Includes(5)); - Assert.That(run.ExpandToInclude(1), Is.False); - Assert.That(run.ExpandToInclude(2), Is.False); - Assert.That(run.ExpandToInclude(4), Is.False); - Assert.That(run.ExpandToInclude(6), Is.False); + Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf()); + Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf()); + } - Assert.That(run.ExpandToInclude(5), Is.True); + [Test] + public void RunExpandThrowsWhenIndexAlreadyIncluded() + { + var run = new Run(2, 3); + Assert.That(!run.Includes(1)); + Assert.That(run.Includes(2)); + Assert.That(run.Includes(4)); + Assert.That(!run.Includes(5)); + + Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf()); + Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf()); + } + + [Test] + public void RunExpandToIncludeAfter() + { + var run = new Run(2, 3); + var update = run.ExpandToInclude(5); + Assert.That(update, Is.Not.Null); + Assert.That(update.NewRuns.Length, Is.EqualTo(0)); + Assert.That(update.RemoveRuns.Length, Is.EqualTo(0)); Assert.That(run.Includes(5)); Assert.That(!run.Includes(6)); } + [Test] + public void RunExpandToIncludeBefore() + { + var run = new Run(2, 3); + var update = run.ExpandToInclude(1); + + Assert.That(update, Is.Not.Null); + Assert.That(update.NewRuns.Length, Is.EqualTo(1)); + Assert.That(update.RemoveRuns.Length, Is.EqualTo(1)); + + Assert.That(update.RemoveRuns[0], Is.SameAs(run)); + Assert.That(update.NewRuns[0].Start, Is.EqualTo(1)); + Assert.That(update.NewRuns[0].Length, Is.EqualTo(4)); + } + [Test] public void RunCanUnsetLastIndex() { @@ -100,94 +163,9 @@ namespace FrameworkTests.Utils { var run = new Run(2, 4); var seen = new List(); - run.Iterate(i => seen.Add(i)); + run.Iterate(seen.Add); CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen); } } - - public class Run - { - public Run(int start, int length) - { - Start = start; - Length = length; - } - - public int Start { get; } - public int Length { get; private set; } - - public bool Includes(int index) - { - return index >= Start && index < (Start + Length); - } - - public bool ExpandToInclude(int index) - { - if (index == (Start + Length)) - { - Length++; - return true; - } - return false; - } - - public RunUpdate Unset(int index) - { - if (!Includes(index)) - { - return new RunUpdate(); - } - - if (index == Start) - { - // First index: Replace self with new run at next index, unless empty. - if (Length == 1) - { - return new RunUpdate(Array.Empty(), new[] { this }); - } - return new RunUpdate( - newRuns: new[] { new Run(Start + 1, Length - 1) }, - removeRuns: new[] { this } - ); - } - - if (index == (Start + Length - 1)) - { - // Last index: Become one smaller. - Length--; - return new RunUpdate(); - } - - // Split: - var newRunLength = (Start + Length - 1) - index; - Length = index - Start; - return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty()); - } - - public void Iterate(Action action) - { - for (var i = 0; i < Length; i++) - { - action(Start + i); - } - } - } - - public class RunUpdate - { - public RunUpdate() - : this(Array.Empty(), Array.Empty()) - { - } - - public RunUpdate(Run[] newRuns, Run[] removeRuns) - { - NewRuns = newRuns; - RemoveRuns = removeRuns; - } - - public Run[] NewRuns { get; } - public Run[] RemoveRuns { get; } - } } diff --git a/Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs b/Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs index aaddbcb7..1ace8bf7 100644 --- a/Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs +++ b/Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs @@ -1,9 +1,4 @@ -using Logging; -using Microsoft.VisualStudio.TestPlatform.Common; -using NuGet.Frameworks; -using NUnit.Framework; -using System.Collections.Concurrent; -using System.Numerics; +using NUnit.Framework; using Utils; namespace FrameworkTests.Utils @@ -119,6 +114,19 @@ namespace FrameworkTests.Utils }, encoded); } + [Test] + public void SetIndexBetweenRuns() + { + var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 }); + set.Set(11); + var encoded = set.RunLengthEncoded(); + + CollectionAssert.AreEqual(new[] + { + 8, 7 + }, encoded); + } + [Test] public void SetIndexAfterRun() { @@ -201,120 +209,5 @@ namespace FrameworkTests.Utils all.Sort(); return all.ToArray(); } - - public class IndexSet - { - private readonly SortedList runs = new SortedList(); - - public IndexSet() - { - } - - public IndexSet(int[] indices) - { - foreach (var i in indices) Set(i); - } - - public static IndexSet FromRunLengthEncoded(int[] rle) - { - var set = new IndexSet(); - for (var i = 0; i < rle.Length; i += 2) - { - var start = rle[i]; - var length = rle[i + 1]; - set.runs.Add(start, new Run(start, length)); - } - - return set; - } - - public bool IsSet(int index) - { - if (runs.ContainsKey(index)) return true; - - var run = GetRunBefore(index); - if (run == null) return false; - - return run.Includes(index); - } - - public void Set(int index) - { - if (runs.ContainsKey(index)) return; - - var run = GetRunBefore(index); - if (run == null || !run.ExpandToInclude(index)) - { - CreateNewRun(index); - } - } - - public void Unset(int index) - { - if (runs.ContainsKey(index)) - { - HandleUpdate(runs[index].Unset(index)); - } - else - { - var run = GetRunBefore(index); - if (run == null) return; - HandleUpdate(run.Unset(index)); - } - } - - public void Iterate(Action onIndex) - { - foreach (var run in runs.Values) - { - run.Iterate(onIndex); - } - } - - public int[] RunLengthEncoded() - { - return Encode().ToArray(); - } - - private IEnumerable Encode() - { - foreach (var pair in runs) - { - yield return pair.Value.Start; - yield return pair.Value.Length; - } - } - - private Run? GetRunBefore(int index) - { - Run? result = null; - foreach (var pair in runs) - { - if (pair.Key < index) result = pair.Value; - else return result; - } - return result; - } - - private void HandleUpdate(RunUpdate runUpdate) - { - foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun); - foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start); - } - - private void CreateNewRun(int index) - { - if (runs.ContainsKey(index + 1)) - { - var length = runs[index + 1].Length + 1; - runs.Add(index, new Run(index, length)); - runs.Remove(index + 1); - } - else - { - runs.Add(index, new Run(index, 1)); - } - } - } } } diff --git a/Tools/AutoClient/Purchaser.cs b/Tools/AutoClient/Purchaser.cs index 34e19e40..97f90619 100644 --- a/Tools/AutoClient/Purchaser.cs +++ b/Tools/AutoClient/Purchaser.cs @@ -68,7 +68,7 @@ namespace AutoClient var filename = Guid.NewGuid().ToString().ToLowerInvariant(); { using var fileStream = File.OpenWrite(filename); - var fileResponse = await codex.DownloadNetworkAsync(cid); + var fileResponse = await codex.DownloadNetworkStreamAsync(cid); fileResponse.Stream.CopyTo(fileStream); } var time = sw.Elapsed; @@ -84,8 +84,15 @@ namespace AutoClient private async Task StartNewPurchase() { var file = await CreateFile(); - var cid = await UploadFile(file); - return await RequestStorage(cid); + try + { + var cid = await UploadFile(file); + return await RequestStorage(cid); + } + finally + { + DeleteFile(file); + } } private async Task CreateFile() @@ -93,6 +100,18 @@ namespace AutoClient return await app.Generator.Generate(); } + private void DeleteFile(string file) + { + try + { + File.Delete(file); + } + catch (Exception exc) + { + app.Log.Error($"Failed to delete file '{file}': {exc}"); + } + } + private async Task UploadFile(string filename) { using var fileStream = File.OpenRead(filename); diff --git a/Tools/BiblioTech/BaseCommand.cs b/Tools/BiblioTech/BaseCommand.cs index a5864712..eb411dd2 100644 --- a/Tools/BiblioTech/BaseCommand.cs +++ b/Tools/BiblioTech/BaseCommand.cs @@ -1,7 +1,6 @@ using Discord.WebSocket; using BiblioTech.Options; using Discord; -using k8s.KubeConfigModels; namespace BiblioTech { diff --git a/Tools/BiblioTech/BiblioTech.csproj b/Tools/BiblioTech/BiblioTech.csproj index c8960926..fe7e7f25 100644 --- a/Tools/BiblioTech/BiblioTech.csproj +++ b/Tools/BiblioTech/BiblioTech.csproj @@ -12,6 +12,7 @@ + diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs new file mode 100644 index 00000000..21ebc337 --- /dev/null +++ b/Tools/BiblioTech/CodexCidChecker.cs @@ -0,0 +1,198 @@ +using CodexOpenApi; +using IdentityModel.Client; +using Utils; + +namespace BiblioTech +{ + public class CodexCidChecker + { + private static readonly string nl = Environment.NewLine; + private readonly Configuration config; + private CodexApi? currentCodexNode; + + public CodexCidChecker(Configuration config) + { + this.config = config; + } + + public async Task PerformCheck(string cid) + { + if (string.IsNullOrEmpty(config.CodexEndpoint)) + { + return new CheckResponse(false, "Codex CID checker is not (yet) available.", ""); + } + + try + { + var codex = GetCodex(); + var nodeCheck = await CheckCodex(codex); + if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); + + return await PerformCheck(codex, cid); + } + catch (Exception ex) + { + return new CheckResponse(false, "Internal server error", ex.ToString()); + } + } + + private async Task PerformCheck(CodexApi codex, string cid) + { + try + { + var manifest = await codex.DownloadNetworkManifestAsync(cid); + return SuccessMessage(manifest); + } + catch (ApiException apiEx) + { + if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response); + if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response); + return UnexpectedReturnCode(apiEx.Response); + } + catch (Exception ex) + { + return UnexpectedException(ex); + } + } + + #region Response formatting + + private CheckResponse SuccessMessage(DataItem content) + { + return FormatResponse( + success: true, + title: $"Success: '{content.Cid}'", + error: "", + $"size: {content.Manifest.OriginalBytes} bytes", + $"blockSize: {content.Manifest.BlockSize} bytes", + $"protected: {content.Manifest.Protected}" + ); + } + + private CheckResponse UnexpectedException(Exception ex) + { + return FormatResponse( + success: false, + title: "Unexpected error", + error: ex.ToString(), + content: "Details will be sent to the bot-admin channel." + ); + } + + private CheckResponse UnexpectedReturnCode(string response) + { + var msg = "Unexpected return code. Response: " + response; + return FormatResponse( + success: false, + title: "Unexpected return code", + error: msg, + content: msg + ); + } + + private CheckResponse FailedToFetch(string response) + { + var msg = "Failed to download content. Response: " + response; + return FormatResponse( + success: false, + title: "Could not download content", + error: msg, + msg, + $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" + ); + } + + private CheckResponse CidFormatInvalid(string response) + { + return FormatResponse( + success: false, + title: "Invalid format", + error: "", + content: "Provided CID is not formatted correctly." + ); + } + + private CheckResponse FormatResponse(bool success, string title, string error, params string[] content) + { + var msg = string.Join(nl, + new string[] + { + title, + "```" + } + .Concat(content) + .Concat(new string[] + { + "```" + }) + ) + nl + nl; + + return new CheckResponse(success, msg, error); + } + + #endregion + + #region Codex Node API + + private CodexApi GetCodex() + { + if (currentCodexNode == null) currentCodexNode = CreateCodex(); + return currentCodexNode; + } + + private async Task CheckCodex(CodexApi codex) + { + try + { + var info = await currentCodexNode!.GetDebugInfoAsync(); + if (info == null || string.IsNullOrEmpty(info.Id)) return false; + return true; + } + catch (Exception e) + { + return false; + } + } + + private CodexApi CreateCodex() + { + var endpoint = config.CodexEndpoint; + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); + + var address = new Address( + host: host, + port: port + ); + + var client = new HttpClient(); + if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + { + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + client.SetBasicAuthentication(tokens[0], tokens[1]); + } + + var codex = new CodexApi(client); + codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + return codex; + } + + #endregion + } + + public class CheckResponse + { + public CheckResponse(bool success, string message, string error) + { + Success = success; + Message = message; + Error = error; + } + + public bool Success { get; } + public string Message { get; } + public string Error { get; } + } +} diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs new file mode 100644 index 00000000..01b3b796 --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckCidCommand.cs @@ -0,0 +1,38 @@ +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckCidCommand : BaseCommand + { + private readonly StringOption cidOption = new StringOption( + name: "cid", + description: "Codex Content-Identifier", + isRequired: true); + private readonly CodexCidChecker checker; + + public CheckCidCommand(CodexCidChecker checker) + { + this.checker = checker; + } + + public override string Name => "check"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks if content is available in the testnet."; + public override CommandOption[] Options => new[] { cidOption }; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var cid = await cidOption.Parse(context); + if (string.IsNullOrEmpty(cid)) + { + await context.Followup("Option 'cid' was not received."); + return; + } + + var response = await checker.PerformCheck(cid); + await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); + await context.Followup(response.Message); + } + } +} diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 6f27fd1f..7cacf9bc 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -38,6 +38,12 @@ namespace BiblioTech [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")] public int NoDiscord { get; set; } = 0; + [Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")] + public string CodexEndpoint { get; set; } = "http://localhost:8080"; + + [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")] + public string CodexEndpointAuth { get; set; } = ""; + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); public string LogPath => Path.Combine(DataPath, "logs"); diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 683ef215..ee258e33 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -3,7 +3,6 @@ using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; using Discord.WebSocket; -using DiscordRewards; using Logging; namespace BiblioTech @@ -81,6 +80,7 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; + var checker = new CodexCidChecker(Config); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); var sprCommand = new SprCommand(); @@ -90,6 +90,7 @@ namespace BiblioTech sprCommand, associateCommand, notifyCommand, + new CheckCidCommand(checker), new AdminCommand(sprCommand, replacement) ); diff --git a/Tools/BiblioTech/Rewards/ChainEventsSender.cs b/Tools/BiblioTech/Rewards/ChainEventsSender.cs index 07cb6112..7e273289 100644 --- a/Tools/BiblioTech/Rewards/ChainEventsSender.cs +++ b/Tools/BiblioTech/Rewards/ChainEventsSender.cs @@ -1,4 +1,5 @@ using Discord.WebSocket; +using DiscordRewards; using Logging; namespace BiblioTech.Rewards @@ -16,24 +17,17 @@ namespace BiblioTech.Rewards this.eventsChannel = eventsChannel; } - public async Task ProcessChainEvents(string[] eventsOverview) + public async Task ProcessChainEvents(ChainEventMessage[] eventsOverview, string[] errors) { + await SendErrorsToAdminChannel(errors); + if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return; try { await Task.Run(async () => { var users = Program.UserRepo.GetAllUserData(); - - foreach (var e in eventsOverview) - { - if (!string.IsNullOrEmpty(e)) - { - var @event = ApplyReplacements(users, e); - await eventsChannel.SendMessageAsync(@event); - await Task.Delay(1000); - } - } + await SendChainEventsInOrder(eventsOverview, eventsChannel, users); }); } catch (Exception ex) @@ -42,6 +36,37 @@ namespace BiblioTech.Rewards } } + private async Task SendErrorsToAdminChannel(string[] errors) + { + try + { + foreach (var error in errors) + { + await Program.AdminChecker.SendInAdminChannel(error); + } + } + catch (Exception exc) + { + log.Error("Failed to send error messages to admin channel. " + exc); + Environment.Exit(1); + } + } + + private async Task SendChainEventsInOrder(ChainEventMessage[] eventsOverview, SocketTextChannel eventsChannel, UserData[] users) + { + eventsOverview = eventsOverview.OrderBy(e => e.BlockNumber).ToArray(); + foreach (var e in eventsOverview) + { + var msg = e.Message; + if (!string.IsNullOrEmpty(msg)) + { + var @event = ApplyReplacements(users, msg); + await eventsChannel.SendMessageAsync(@event); + await Task.Delay(300); + } + } + } + private string ApplyReplacements(UserData[] users, string msg) { var result = ApplyUserAddressReplacements(users, msg); diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index 71019b78..b3bf7079 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -31,7 +31,7 @@ namespace BiblioTech.Rewards await ProcessRewards(rewards); } - await eventsSender.ProcessChainEvents(rewards.EventsOverview); + await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors); } private async Task ProcessRewards(GiveRewardsCommand rewards) diff --git a/Tools/MarketInsights/AverageHistory.cs b/Tools/MarketInsights/AverageHistory.cs index af752d1b..5af20104 100644 --- a/Tools/MarketInsights/AverageHistory.cs +++ b/Tools/MarketInsights/AverageHistory.cs @@ -1,6 +1,5 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; -using Nethereum.Model; using TestNetRewarder; using Utils; @@ -40,7 +39,7 @@ namespace MarketInsights private MarketTimeSegment BuildContribution(TimeRange timeRange) { - var builder = new ContributionBuilder(timeRange); + var builder = new ContributionBuilder(appState.Log, timeRange); mux.Handlers.Add(builder); chainState.Update(timeRange.To); mux.Handlers.Remove(builder); diff --git a/Tools/MarketInsights/ContributionBuilder.cs b/Tools/MarketInsights/ContributionBuilder.cs index 63ab4dae..5d1c4f77 100644 --- a/Tools/MarketInsights/ContributionBuilder.cs +++ b/Tools/MarketInsights/ContributionBuilder.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin.ChainMonitor; using GethPlugin; +using Logging; using System.Numerics; using Utils; @@ -8,14 +9,16 @@ namespace MarketInsights public class ContributionBuilder : IChainStateChangeHandler { private readonly MarketTimeSegment segment = new MarketTimeSegment(); + private readonly ILog log; - public ContributionBuilder(TimeRange timeRange) + public ContributionBuilder(ILog log, TimeRange timeRange) { segment = new MarketTimeSegment { FromUtc = timeRange.From, ToUtc = timeRange.To }; + this.log = log; } public void OnNewRequest(RequestEvent requestEvent) @@ -55,6 +58,11 @@ namespace MarketInsights { } + public void OnError(string msg) + { + log.Error(msg); + } + public MarketTimeSegment GetSegment() { return segment; diff --git a/Tools/TestNetRewarder/BotClient.cs b/Tools/TestNetRewarder/BotClient.cs index 3ee1fb99..6a5c5759 100644 --- a/Tools/TestNetRewarder/BotClient.cs +++ b/Tools/TestNetRewarder/BotClient.cs @@ -18,7 +18,6 @@ namespace TestNetRewarder public async Task IsOnline() { var result = await HttpGet(); - log.Log("Is DiscordBot online: " + result); return result == "Pong"; } diff --git a/Tools/TestNetRewarder/EventsFormatter.cs b/Tools/TestNetRewarder/EventsFormatter.cs index 43c8605e..924645d0 100644 --- a/Tools/TestNetRewarder/EventsFormatter.cs +++ b/Tools/TestNetRewarder/EventsFormatter.cs @@ -1,5 +1,6 @@ using CodexContractsPlugin; using CodexContractsPlugin.ChainMonitor; +using DiscordRewards; using GethPlugin; using System.Globalization; using System.Numerics; @@ -10,19 +11,22 @@ namespace TestNetRewarder public class EventsFormatter : IChainStateChangeHandler { private static readonly string nl = Environment.NewLine; - private readonly List events = new List(); + private readonly List events = new List(); + private readonly List errors = new List(); private readonly EmojiMaps emojiMaps = new EmojiMaps(); - public string[] GetEvents() + public ChainEventMessage[] GetEvents() { var result = events.ToArray(); events.Clear(); return result; } - public void AddError(string error) + public string[] GetErrors() { - AddBlock("📢 **Error**", error); + var result = errors.ToArray(); + errors.Clear(); + return result; } public void OnNewRequest(RequestEvent requestEvent) @@ -82,20 +86,35 @@ namespace TestNetRewarder $"Slot Index: {slotIndex}" ); } + + public void OnError(string msg) + { + errors.Add(msg); + } private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content) { var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]"; var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}"; - AddBlock(title, content); + AddBlock(requestEvent.Block.BlockNumber, title, content); } - private void AddBlock(string title, params string[] content) + private void AddBlock(ulong blockNumber, string title, params string[] content) { - events.Add(FormatBlock(title, content)); + events.Add(FormatBlock(blockNumber, title, content)); } - private string FormatBlock(string title, params string[] content) + private ChainEventMessage FormatBlock(ulong blockNumber, string title, params string[] content) + { + var msg = FormatBlockMessage(title, content); + return new ChainEventMessage + { + BlockNumber = blockNumber, + Message = msg + }; + } + + private string FormatBlockMessage(string title, string[] content) { if (content == null || !content.Any()) { diff --git a/Tools/TestNetRewarder/Processor.cs b/Tools/TestNetRewarder/Processor.cs index a6d2ccdf..40651bef 100644 --- a/Tools/TestNetRewarder/Processor.cs +++ b/Tools/TestNetRewarder/Processor.cs @@ -48,7 +48,7 @@ namespace TestNetRewarder { var msg = "Exception processing time segment: " + ex; log.Error(msg); - eventsFormatter.AddError(msg); + eventsFormatter.OnError(msg); throw; } } @@ -58,8 +58,9 @@ namespace TestNetRewarder var numberOfChainEvents = chainState.Update(timeRange.To); var events = eventsFormatter.GetEvents(); + var errors = eventsFormatter.GetErrors(); - var request = builder.Build(events); + var request = builder.Build(events, errors); if (request.HasAny()) { await client.SendRewards(request); diff --git a/Tools/TestNetRewarder/RequestBuilder.cs b/Tools/TestNetRewarder/RequestBuilder.cs index 32b48fba..6ea25f34 100644 --- a/Tools/TestNetRewarder/RequestBuilder.cs +++ b/Tools/TestNetRewarder/RequestBuilder.cs @@ -19,7 +19,7 @@ namespace TestNetRewarder } } - public GiveRewardsCommand Build(string[] lines) + public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors) { var result = new GiveRewardsCommand { @@ -28,7 +28,8 @@ namespace TestNetRewarder RewardId = p.Key, UserAddresses = p.Value.Select(v => v.Address).ToArray() }).ToArray(), - EventsOverview = lines + EventsOverview = lines, + Errors = errors }; rewards.Clear(); diff --git a/Tools/TestNetRewarder/RewardCheck.cs b/Tools/TestNetRewarder/RewardCheck.cs index e6662cba..abfafda8 100644 --- a/Tools/TestNetRewarder/RewardCheck.cs +++ b/Tools/TestNetRewarder/RewardCheck.cs @@ -76,6 +76,10 @@ namespace TestNetRewarder { } + public void OnError(string msg) + { + } + private void GiveReward(RewardConfig reward, EthAddress receiver) { giver.Give(reward, receiver);