status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into feature/block-retransmit 

# Conflicts:
#	ProjectPlugins/CodexPlugin/ApiChecker.cs
This commit is contained in:
Ben 2024-10-22 12:54:22 +02:00
parent 3e245b707c 2f39327db2
commit a0abea4432
No known key found for this signature in database
GPG Key ID: 0F16E812E736C24B
32 changed files with 1262 additions and 297 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Framework/DiscordRewards/GiveRewardsCommand.cs
View File

@ -3,7 +3,8 @@
public class GiveRewardsCommand
{
public RewardUsersCommand[] Rewards { get; set; } = Array.Empty<RewardUsersCommand>();
public string[] EventsOverview { get; set; } = Array.Empty<string>();
public ChainEventMessage[] EventsOverview { get; set; } = Array.Empty<ChainEventMessage>();
public string[] Errors { get; set; } = Array.Empty<string>();
public bool HasAny()
{
@ -16,4 +17,10 @@
public ulong RewardId { get; set; }
public string[] UserAddresses { get; set; } = Array.Empty<string>();
}
public class ChainEventMessage
{
public ulong BlockNumber { get; set; }
public string Message { get; set; } = string.Empty;
}
}

6
Framework/NethereumWorkflow/BlockUtils/BlockTimeFinder.cs
View File

@ -29,7 +29,8 @@ namespace NethereumWorkflow.BlockUtils
public ulong? GetHighestBlockNumberBefore(DateTime moment)
{
bounds.Initialize();
if (moment <= bounds.Genesis.Utc) return null;
if (moment < bounds.Genesis.Utc) return null;
if (moment == bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
if (moment >= bounds.Current.Utc) return bounds.Current.BlockNumber;
return Log(() => Search(bounds.Genesis, bounds.Current, moment, HighestBeforeSelector));
@ -38,7 +39,8 @@ namespace NethereumWorkflow.BlockUtils
public ulong? GetLowestBlockNumberAfter(DateTime moment)
{
bounds.Initialize();
if (moment >= bounds.Current.Utc) return null;
if (moment > bounds.Current.Utc) return null;
if (moment == bounds.Current.Utc) return bounds.Current.BlockNumber;
if (moment <= bounds.Genesis.Utc) return bounds.Genesis.BlockNumber;
return Log(()=> Search(bounds.Genesis, bounds.Current, moment, LowestAfterSelector)); ;

39
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainState.cs
View File

@ -17,6 +17,8 @@ namespace CodexContractsPlugin.ChainMonitor
void OnSlotFilled(RequestEvent requestEvent, EthAddress host, BigInteger slotIndex);
void OnSlotFreed(RequestEvent requestEvent, BigInteger slotIndex);
void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex);
void OnError(string msg);
}
public class RequestEvent
@ -67,7 +69,11 @@ namespace CodexContractsPlugin.ChainMonitor
private void Apply(ChainEvents events)
{
if (events.BlockInterval.TimeRange.From < TotalSpan.From)
throw new Exception("Attempt to update ChainState with set of events from before its current record.");
{
var msg = "Attempt to update ChainState with set of events from before its current record.";
handler.OnError(msg);
throw new Exception(msg);
}
log.Log($"ChainState updating: {events.BlockInterval}");
@ -110,7 +116,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(RequestFulfilledEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.UpdateState(@event.Block.BlockNumber, RequestState.Started);
handler.OnRequestFulfilled(new RequestEvent(@event.Block, r));
@ -118,7 +124,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(RequestCancelledEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.UpdateState(@event.Block.BlockNumber, RequestState.Cancelled);
handler.OnRequestCancelled(new RequestEvent(@event.Block, r));
@ -126,7 +132,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(RequestFailedEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.UpdateState(@event.Block.BlockNumber, RequestState.Failed);
handler.OnRequestFailed(new RequestEvent(@event.Block, r));
@ -134,7 +140,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(SlotFilledEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.Hosts.Add(@event.Host, (int)@event.SlotIndex);
r.Log($"[{@event.Block.BlockNumber}] SlotFilled (host:'{@event.Host}', slotIndex:{@event.SlotIndex})");
@ -143,7 +149,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(SlotFreedEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.Hosts.RemoveHost((int)@event.SlotIndex);
r.Log($"[{@event.Block.BlockNumber}] SlotFreed (slotIndex:{@event.SlotIndex})");
@ -152,7 +158,7 @@ namespace CodexContractsPlugin.ChainMonitor
private void ApplyEvent(SlotReservationsFullEventDTO @event)
{
var r = FindRequest(@event.RequestId);
var r = FindRequest(@event);
if (r == null) return;
r.Log($"[{@event.Block.BlockNumber}] SlotReservationsFull (slotIndex:{@event.SlotIndex})");
handler.OnSlotReservationsFull(new RequestEvent(@event.Block, r), @event.SlotIndex);
@ -171,10 +177,23 @@ namespace CodexContractsPlugin.ChainMonitor
}
}
private ChainStateRequest? FindRequest(byte[] requestId)
private ChainStateRequest? FindRequest(IHasRequestId request)
{
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, requestId));
if (r == null) log.Log("Unable to find request by ID!");
var r = requests.SingleOrDefault(r => Equal(r.Request.RequestId, request.RequestId));
if (r == null)
{
var blockNumber = "unknown";
if (request is IHasBlock blk)
{
blockNumber = blk.Block.BlockNumber.ToString();
}
var msg = $"Received event of type '{request.GetType()}' in block '{blockNumber}' for request by Id: '{request.RequestId}'. " +
$"Failed to find request. Request creation event not seen! (Tracker start time: {TotalSpan.From})";
log.Error(msg);
handler.OnError(msg);
}
return r;
}

10
ProjectPlugins/CodexContractsPlugin/ChainMonitor/ChainStateChangeHandlerMux.cs
View File

@ -1,10 +1,5 @@
using GethPlugin;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Numerics;
using System.Text;
using System.Threading.Tasks;
namespace CodexContractsPlugin.ChainMonitor
{
@ -56,5 +51,10 @@ namespace CodexContractsPlugin.ChainMonitor
{
foreach (var handler in Handlers) handler.OnSlotReservationsFull(requestEvent, slotIndex);
}
public void OnError(string msg)
{
foreach (var handler in Handlers) handler.OnError(msg);
}
}
}

4
ProjectPlugins/CodexContractsPlugin/ChainMonitor/DoNothingChainEventHandler.cs
View File

@ -36,5 +36,9 @@ namespace CodexContractsPlugin.ChainMonitor
public void OnSlotReservationsFull(RequestEvent requestEvent, BigInteger slotIndex)
{
}
public void OnError(string msg)
{
}
}
}

19
ProjectPlugins/CodexContractsPlugin/Marketplace/Customizations.cs
View File

@ -10,7 +10,12 @@ namespace CodexContractsPlugin.Marketplace
BlockTimeEntry Block { get; set; }
}
public partial class Request : RequestBase, IHasBlock
public interface IHasRequestId
{
byte[] RequestId { get; set; }
}
public partial class Request : RequestBase, IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
@ -28,38 +33,38 @@ namespace CodexContractsPlugin.Marketplace
}
}
public partial class RequestFulfilledEventDTO : IHasBlock
public partial class RequestFulfilledEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class RequestCancelledEventDTO : IHasBlock
public partial class RequestCancelledEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class RequestFailedEventDTO : IHasBlock
public partial class RequestFailedEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class SlotFilledEventDTO : IHasBlock
public partial class SlotFilledEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
public EthAddress Host { get; set; }
}
public partial class SlotFreedEventDTO : IHasBlock
public partial class SlotFreedEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }
}
public partial class SlotReservationsFullEventDTO : IHasBlock
public partial class SlotReservationsFullEventDTO : IHasBlock, IHasRequestId
{
[JsonIgnore]
public BlockTimeEntry Block { get; set; }

2
ProjectPlugins/CodexPlugin/ApiChecker.cs
View File

@ -10,7 +10,7 @@ namespace CodexPlugin
public class ApiChecker
{
// <INSERT-OPENAPI-YAML-HASH>
private const string OpenApiYamlHash = "2E-7C-A2-F3-67-D9-F2-A6-4E-D5-FF-A2-EC-65-ED-59-CE-89-A8-92-57-5E-CF-40-9A-83-49-0B-49-42-5D-EC";
private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";

10
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -73,7 +73,7 @@ namespace CodexPlugin
public Stream DownloadFile(string contentId, Action<Failure> onFailure)
{
var fileResponse = OnCodex(
api => api.DownloadNetworkAsync(contentId),
api => api.DownloadNetworkStreamAsync(contentId),
CreateRetryConfig(nameof(DownloadFile), onFailure));
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@ -88,25 +88,25 @@ namespace CodexPlugin
public StorageAvailability SalesAvailability(StorageAvailability request)
{
var body = mapper.Map(request);
var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
var read = OnCodex(api => api.OfferStorageAsync(body));
return mapper.Map(read);
}
public StorageAvailability[] GetAvailabilities()
{
var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
var collection = OnCodex(api => api.GetAvailabilitiesAsync());
return mapper.Map(collection);
}
public string RequestStorage(StoragePurchaseRequest request)
{
var body = mapper.Map(request);
return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
}
public CodexSpace Space()
{
var space = OnCodex<Space>(api => api.SpaceAsync());
var space = OnCodex(api => api.SpaceAsync());
return mapper.Map(space);
}

56
ProjectPlugins/CodexPlugin/openapi.yaml
View File

@ -456,9 +456,35 @@ paths:
"/data/{cid}/network":
get:
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
tags: [ Data ]
operationId: downloadNetwork
parameters:
- in: path
name: cid
required: true
schema:
$ref: "#/components/schemas/Cid"
description: "File to be downloaded."
responses:
"200":
description: Manifest information for download that has been started.
content:
application/json:
schema:
$ref: "#/components/schemas/DataItem"
"400":
description: Invalid CID is specified
"404":
description: Failed to download dataset manifest
"500":
description: Well it was bad-bad
"/data/{cid}/network/stream":
get:
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
tags: [ Data ]
operationId: downloadNetworkStream
parameters:
- in: path
name: cid
@ -481,6 +507,32 @@ paths:
"500":
description: Well it was bad-bad
"/data/{cid}/network/manifest":
get:
summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
tags: [ Data ]
operationId: downloadNetworkManifest
parameters:
- in: path
name: cid
required: true
schema:
$ref: "#/components/schemas/Cid"
description: "File for which the manifest is to be downloaded."
responses:
"200":
description: Manifest information.
content:
application/json:
schema:
$ref: "#/components/schemas/DataItem"
"400":
description: Invalid CID is specified
"404":
description: Failed to download dataset manifest
"500":
description: Well it was bad-bad
"/space":
get:
summary: "Gets a summary of the storage space allocation of the node."
@ -792,4 +844,4 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/DebugInfo"
$ref: "#/components/schemas/DebugInfo"

228
Tests/CodexTests/DownloadConnectivityTests/MultiswarmTests.cs Normal file
View File

@ -0,0 +1,228 @@
using CodexPlugin;
using FileUtils;
using Logging;
using NUnit.Framework;
using Utils;
namespace CodexTests.DownloadConnectivityTests
{
[TestFixture]
public class MultiswarmTests : AutoBootstrapDistTest
{
[Test]
[Combinatorial]
public void Multiswarm(
[Values(3, 5)] int numFiles,
[Values(5, 20)] int fileSizeMb,
[Values(1)] int uploadersPerFile,
[Values(3)] int downloadersPerFile,
[Values(1)] int maxUploadsPerNode,
[Values(2, 3)] int maxDownloadsPerNode
)
{
var plan = CreateThePlan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
Assert.That(plan.NodePlans.Count, Is.LessThan(30));
RunThePlan(plan, fileSizeMb);
}
private void RunThePlan(Plan plan, int fileSizeMb)
{
foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB());
var nodes = StartCodex(plan.NodePlans.Count);
for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i];
// Upload all files to their nodes.
foreach (var filePlan in plan.FilePlans)
{
foreach (var uploader in filePlan.Uploaders)
{
filePlan.Cid = uploader.Node!.UploadFile(filePlan.File!);
}
}
Thread.Sleep(5000); // Everything is processed and announced.
// Start all downloads (almost) simultaneously.
var tasks = new List<Task>();
foreach (var filePlan in plan.FilePlans)
{
foreach (var downloader in filePlan.Downloaders)
{
tasks.Add(Task.Run(() =>
{
var downloadedFile = downloader.Node!.DownloadContent(filePlan.Cid!);
lock (filePlan.DownloadedFiles)
{
filePlan.DownloadedFiles.Add(downloadedFile);
}
}));
}
}
Task.WaitAll(tasks.ToArray());
// Assert all files are correct.
foreach (var filePlan in plan.FilePlans)
{
foreach (var downloadedFile in filePlan.DownloadedFiles)
{
filePlan.File!.AssertIsEqual(downloadedFile);
}
}
}
private Plan CreateThePlan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
{
var plan = new Plan(numFiles, uploadersPerFile, downloadersPerFile, maxUploadsPerNode, maxDownloadsPerNode);
plan.Initialize();
plan.LogPlan(GetTestLog());
return plan;
}
}
public class FilePlan
{
public FilePlan(int number)
{
Number = number;
}
public int Number { get; }
public TrackedFile? File { get; set; }
public ContentId? Cid { get; set; }
public List<TrackedFile?> DownloadedFiles { get; } = new List<TrackedFile?>();
public List<NodePlan> Uploaders { get; } = new List<NodePlan>();
public List<NodePlan> Downloaders { get; } = new List<NodePlan>();
public override string ToString()
{
return $"FilePlan[{Number}] " +
$"Uploaders:[{string.Join(",", Uploaders.Select(u => u.Number.ToString()))}] " +
$"Downloaders:[{string.Join(",", Downloaders.Select(u => u.Number.ToString()))}]";
}
}
public class NodePlan
{
public NodePlan(int number)
{
Number = number;
}
public int Number { get; }
public ICodexNode? Node { get; set; }
public List<FilePlan> Uploads { get; } = new List<FilePlan>();
public List<FilePlan> Downloads { get; } = new List<FilePlan>();
public bool Contains(FilePlan plan)
{
return Uploads.Contains(plan) || Downloads.Contains(plan);
}
public override string ToString()
{
return $"NodePlan[{Number}] " +
$"Uploads:[{string.Join(",", Uploads.Select(u => u.Number.ToString()))}] " +
$"Downloads:[{string.Join(",", Downloads.Select(u => u.Number.ToString()))}]";
}
}
public class Plan
{
private readonly int numFiles;
private readonly int uploadersPerFile;
private readonly int downloadersPerFile;
private readonly int maxUploadsPerNode;
private readonly int maxDownloadsPerNode;
public Plan(int numFiles, int uploadersPerFile, int downloadersPerFile, int maxUploadsPerNode, int maxDownloadsPerNode)
{
this.numFiles = numFiles;
this.uploadersPerFile = uploadersPerFile;
this.downloadersPerFile = downloadersPerFile;
this.maxUploadsPerNode = maxUploadsPerNode;
this.maxDownloadsPerNode = maxDownloadsPerNode;
}
public List<FilePlan> FilePlans { get; } = new List<FilePlan>();
public List<NodePlan> NodePlans { get; } = new List<NodePlan>();
public void Initialize()
{
for (int i = 0; i < numFiles; i++) FilePlans.Add(new FilePlan(i));
foreach (var filePlan in FilePlans)
{
while (filePlan.Uploaders.Count < uploadersPerFile) AddUploader(filePlan);
while (filePlan.Downloaders.Count < downloadersPerFile) AddDownloader(filePlan);
}
CollectionAssert.AllItemsAreUnique(FilePlans.Select(f => f.Number));
CollectionAssert.AllItemsAreUnique(NodePlans.Select(f => f.Number));
foreach (var filePlan in FilePlans)
{
Assert.That(filePlan.Uploaders.Count, Is.EqualTo(uploadersPerFile));
Assert.That(filePlan.Downloaders.Count, Is.EqualTo(downloadersPerFile));
}
foreach (var nodePlan in NodePlans)
{
Assert.That(nodePlan.Uploads.Count, Is.LessThanOrEqualTo(maxUploadsPerNode));
Assert.That(nodePlan.Downloads.Count, Is.LessThanOrEqualTo(maxDownloadsPerNode));
}
}
public void LogPlan(ILog log)
{
log.Log("The plan:");
log.Log("Input:");
log.Log($"numFiles: {numFiles}");
log.Log($"uploadersPerFile: {uploadersPerFile}");
log.Log($"downloadersPerFile: {downloadersPerFile}");
log.Log($"maxUploadsPerNode: {maxUploadsPerNode}");
log.Log($"maxDownloadsPerNode: {maxDownloadsPerNode}");
log.Log("Setup:");
log.Log($"number of nodes: {NodePlans.Count}");
foreach (var filePlan in FilePlans) log.Log(filePlan.ToString());
foreach (var nodePlan in NodePlans) log.Log(nodePlan.ToString());
}
private void AddDownloader(FilePlan filePlan)
{
var nodePlan = GetOrCreateDownloaderNode(filePlan);
filePlan.Downloaders.Add(nodePlan);
nodePlan.Downloads.Add(filePlan);
}
private void AddUploader(FilePlan filePlan)
{
var nodePlan = GetOrCreateUploaderNode(filePlan);
filePlan.Uploaders.Add(nodePlan);
nodePlan.Uploads.Add(filePlan);
}
private NodePlan GetOrCreateDownloaderNode(FilePlan notIn)
{
var available = NodePlans.Where(n =>
n.Downloads.Count < maxDownloadsPerNode && !n.Contains(notIn)
).ToArray();
if (available.Any()) return RandomUtils.GetOneRandom(available);
var newNodePlan = new NodePlan(NodePlans.Count);
NodePlans.Add(newNodePlan);
return newNodePlan;
}
private NodePlan GetOrCreateUploaderNode(FilePlan notIn)
{
var available = NodePlans.Where(n =>
n.Uploads.Count < maxUploadsPerNode && !n.Contains(notIn)
).ToArray();
if (available.Any()) return RandomUtils.GetOneRandom(available);
var newNodePlan = new NodePlan(NodePlans.Count);
NodePlans.Add(newNodePlan);
return newNodePlan;
}
}
}

9
Tests/CodexTests/UtilityTests/DiscordBotTests.cs
View File

@ -23,7 +23,7 @@ namespace CodexTests.UtilityTests
private readonly List<EthAccount> hostAccounts = new List<EthAccount>();
private readonly List<ulong> rewardsSeen = new List<ulong>();
private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1);
private readonly List<string> receivedEvents = new List<string>();
private readonly List<ChainEventMessage> receivedEvents = new List<ChainEventMessage>();
[Test]
[DontDownloadLogs]
@ -73,13 +73,18 @@ namespace CodexTests.UtilityTests
private void AssertEventOccurance(string msg, int expectedCount)
{
Assert.That(receivedEvents.Count(e => e.Contains(msg)), Is.EqualTo(expectedCount),
Assert.That(receivedEvents.Count(e => e.Message.Contains(msg)), Is.EqualTo(expectedCount),
$"Event '{msg}' did not occure correct number of times.");
}
private void OnCommand(string timestamp, GiveRewardsCommand call)
{
Log($"<API call {timestamp}>");
foreach (var e in call.EventsOverview)
{
Assert.That(receivedEvents.All(r => r.BlockNumber < e.BlockNumber), "Received event out of order.");
}
receivedEvents.AddRange(call.EventsOverview);
foreach (var e in call.EventsOverview)
{

54
Tests/FrameworkTests/NethereumWorkflow/BlockTimeFinderTests.cs
View File

@ -19,11 +19,20 @@ namespace FrameworkTests.NethereumWorkflow
{
var start = DateTime.UtcNow.AddDays(-1).AddSeconds(-30);
blocks = new Dictionary<ulong, Block>();
Block? prev = null;
for (ulong i = 0; i < 30; i++)
{
ulong d = 100 + i;
blocks.Add(d, new Block(d, start + TimeSpan.FromSeconds(i * 2)));
var newBlock = new Block(d, start + TimeSpan.FromSeconds(i * 2));
blocks.Add(d, newBlock);
if (prev != null)
{
prev.Next = newBlock;
newBlock.Previous = prev;
}
prev = newBlock;
}
}
@ -99,23 +108,23 @@ namespace FrameworkTests.NethereumWorkflow
}
[Test]
public void FailsToFindBlockBeforeFrontOfChain()
public void FindsGenesisBlockAtFrontOfChain()
{
var first = blocks.First().Value;
var notFound = finder.GetHighestBlockNumberBefore(first.Time);
var firstNumber = finder.GetHighestBlockNumberBefore(first.Time);
Assert.That(notFound, Is.Null);
Assert.That(firstNumber, Is.EqualTo(first.Number));
}
[Test]
public void FailsToFindBlockAfterTailOfChain()
public void FindsCurrentBlockAtTailOfChain()
{
var last = blocks.Last().Value;
var notFound = finder.GetLowestBlockNumberAfter(last.Time);
var lastNumber = finder.GetLowestBlockNumberAfter(last.Time);
Assert.That(notFound, Is.Null);
Assert.That(lastNumber, Is.EqualTo(last.Number));
}
[Test]
@ -143,13 +152,27 @@ namespace FrameworkTests.NethereumWorkflow
{
foreach (var pair in blocks)
{
finder.GetHighestBlockNumberBefore(pair.Value.JustBefore);
finder.GetHighestBlockNumberBefore(pair.Value.Time);
finder.GetHighestBlockNumberBefore(pair.Value.JustAfter);
var block = pair.Value;
finder.GetLowestBlockNumberAfter(pair.Value.JustBefore);
finder.GetLowestBlockNumberAfter(pair.Value.Time);
finder.GetLowestBlockNumberAfter(pair.Value.JustAfter);
AssertLink(block.Previous, finder.GetHighestBlockNumberBefore(block.JustBefore));
AssertLink(block, finder.GetHighestBlockNumberBefore(block.Time));
AssertLink(block, finder.GetHighestBlockNumberBefore(block.JustAfter));
AssertLink(block, finder.GetLowestBlockNumberAfter(block.JustBefore));
AssertLink(block, finder.GetLowestBlockNumberAfter(block.Time));
AssertLink(block.Next, finder.GetLowestBlockNumberAfter(block.JustAfter));
}
}
private void AssertLink(Block? expected, ulong? actual)
{
if (expected == null)
{
Assert.That(actual, Is.Null);
}
else
{
Assert.That(expected.Number, Is.EqualTo(actual!.Value));
}
}
}
@ -167,6 +190,9 @@ namespace FrameworkTests.NethereumWorkflow
public DateTime JustBefore { get { return Time.AddSeconds(-1); } }
public DateTime JustAfter { get { return Time.AddSeconds(1); } }
public Block? Next { get; set; }
public Block? Previous { get; set; }
public override string ToString()
{
return $"[{Number}]";

334
Tests/FrameworkTests/Utils/RunLengthEncoding.cs Normal file
View File

@ -0,0 +1,334 @@
namespace FrameworkTests.Utils
{
public class Run
{
public Run(int start, int length)
{
Start = start;
Length = length;
}
public int Start { get; }
public int Length { get; private set; }
public bool Includes(int index)
{
return index >= Start && index < (Start + Length);
}
public RunUpdate ExpandToInclude(int index)
{
if (Includes(index)) throw new Exception("Run already includes this index. Run: {ToString()} index: {index}");
if (index == (Start + Length))
{
Length++;
return new RunUpdate();
}
if (index == (Start - 1))
{
return new RunUpdate(
newRuns: [new Run(Start - 1, Length + 1)],
removeRuns: [this]
);
}
throw new Exception($"Run cannot expand to include index. Run: {ToString()} index: {index}");
}
public RunUpdate Unset(int index)
{
if (!Includes(index))
{
return new RunUpdate();
}
if (index == Start)
{
// First index: Replace self with new run at next index, unless empty.
if (Length == 1)
{
return new RunUpdate(
newRuns: Array.Empty<Run>(),
removeRuns: [this]
);
}
return new RunUpdate(
newRuns: [new Run(Start + 1, Length - 1)],
removeRuns: [this]
);
}
if (index == (Start + Length - 1))
{
// Last index: Become one smaller.
Length--;
return new RunUpdate();
}
// Split:
var newRunLength = (Start + Length - 1) - index;
Length = index - Start;
return new RunUpdate(
newRuns: [new Run(index + 1, newRunLength)],
removeRuns: Array.Empty<Run>()
);
}
public void Iterate(Action<int> action)
{
for (var i = 0; i < Length; i++)
{
action(Start + i);
}
}
public override string ToString()
{
return $"[{Start},{Length}]";
}
public override bool Equals(object? obj)
{
return obj is Run run &&
Start == run.Start &&
Length == run.Length;
}
public override int GetHashCode()
{
return HashCode.Combine(Start, Length);
}
public static bool operator ==(Run? obj1, Run? obj2)
{
if (ReferenceEquals(obj1, obj2)) return true;
if (ReferenceEquals(obj1, null)) return false;
if (ReferenceEquals(obj2, null)) return false;
return obj1.Equals(obj2);
}
public static bool operator !=(Run? obj1, Run? obj2) => !(obj1 == obj2);
}
public class RunUpdate
{
public RunUpdate()
: this(Array.Empty<Run>(), Array.Empty<Run>())
{
}
public RunUpdate(Run[] newRuns, Run[] removeRuns)
{
NewRuns = newRuns;
RemoveRuns = removeRuns;
}
public Run[] NewRuns { get; }
public Run[] RemoveRuns { get; }
}
public partial class IndexSet
{
private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
public IndexSet()
{
}
public IndexSet(int[] indices)
{
foreach (var i in indices) Set(i);
}
public static IndexSet FromRunLengthEncoded(int[] rle)
{
var set = new IndexSet();
for (var i = 0; i < rle.Length; i += 2)
{
var start = rle[i];
var length = rle[i + 1];
set.runs.Add(start, new Run(start, length));
}
return set;
}
public bool IsSet(int index)
{
if (runs.ContainsKey(index)) return true;
var run = GetRunAt(index);
if (run == null) return false;
return true;
}
public void Set(int index)
{
if (IsSet(index)) return;
var runBefore = GetRunAt(index - 1);
var runAfter = GetRunExact(index + 1);
if (runBefore == null)
{
if (runAfter == null)
{
CreateNewRun(index);
}
else
{
HandleUpdate(runAfter.ExpandToInclude(index));
}
}
else
{
if (runAfter == null)
{
HandleUpdate(runBefore.ExpandToInclude(index));
}
else
{
// new index will connect runBefore with runAfter. We merge!
HandleUpdate(new RunUpdate(
newRuns: [new Run(runBefore.Start, runBefore.Length + 1 + runAfter.Length)],
removeRuns: [runBefore, runAfter]
));
}
}
}
public void Unset(int index)
{
if (runs.ContainsKey(index))
{
HandleUpdate(runs[index].Unset(index));
}
else
{
var run = GetRunAt(index);
if (run == null) return;
HandleUpdate(run.Unset(index));
}
}
public void Iterate(Action<int> onIndex)
{
foreach (var run in runs.Values)
{
run.Iterate(onIndex);
}
}
public int[] RunLengthEncoded()
{
return Encode().ToArray();
}
public override string ToString()
{
return string.Join("&", runs.Select(r => r.ToString()).ToArray());
}
private IEnumerable<int> Encode()
{
foreach (var pair in runs)
{
yield return pair.Value.Start;
yield return pair.Value.Length;
}
}
private Run? GetRunAt(int index)
{
foreach (var run in runs.Values)
{
if (run.Includes(index)) return run;
}
return null;
}
private Run? GetRunExact(int index)
{
if (runs.ContainsKey(index)) return runs[index];
return null;
}
private void HandleUpdate(RunUpdate runUpdate)
{
foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
}
private void CreateNewRun(int index)
{
if (runs.ContainsKey(index + 1))
{
var length = runs[index + 1].Length + 1;
runs.Add(index, new Run(index, length));
runs.Remove(index + 1);
}
else
{
runs.Add(index, new Run(index, 1));
}
}
}
public partial class IndexSet
{
public IndexSet Overlap(IndexSet other)
{
var result = new IndexSet();
Iterate(i =>
{
if (other.IsSet(i)) result.Set(i);
});
return result;
}
public IndexSet Merge(IndexSet other)
{
var result = new IndexSet();
Iterate(result.Set);
other.Iterate(result.Set);
return result;
}
public IndexSet Without(IndexSet other)
{
var result = new IndexSet();
Iterate(i =>
{
if (!other.IsSet(i)) result.Set(i);
});
return result;
}
public override bool Equals(object? obj)
{
if (obj is IndexSet set)
{
if (set.runs.Count != runs.Count) return false;
foreach (var pair in runs)
{
if (!set.runs.ContainsKey(pair.Key)) return false;
if (set.runs[pair.Key] != pair.Value) return false;
}
return true;
}
return false;
}
public override int GetHashCode()
{
return HashCode.Combine(runs);
}
public static bool operator ==(IndexSet? obj1, IndexSet? obj2)
{
if (ReferenceEquals(obj1, obj2)) return true;
if (ReferenceEquals(obj1, null)) return false;
if (ReferenceEquals(obj2, null)) return false;
return obj1.Equals(obj2);
}
public static bool operator !=(IndexSet? obj1, IndexSet? obj2) => !(obj1 == obj2);
}
}

94
Tests/FrameworkTests/Utils/RunLengthEncodingLogicalTests.cs Normal file
View File

@ -0,0 +1,94 @@
using NUnit.Framework;
namespace FrameworkTests.Utils
{
[TestFixture]
public class RunLengthEncodingLogicalTests
{
[Test]
public void EqualityTest()
{
var setA = new IndexSet([1, 2, 3, 4]);
var setB = new IndexSet([1, 2, 3, 4]);
Assert.That(setA, Is.EqualTo(setB));
Assert.That(setA == setB);
}
[Test]
public void InequalityTest1()
{
var setA = new IndexSet([1, 2, 4, 5]);
var setB = new IndexSet([1, 2, 3, 4]);
Assert.That(setA, Is.Not.EqualTo(setB));
Assert.That(setA != setB);
}
[Test]
public void InequalityTest2()
{
var setA = new IndexSet([1, 2, 3]);
var setB = new IndexSet([1, 2, 3, 4]);
Assert.That(setA, Is.Not.EqualTo(setB));
Assert.That(setA != setB);
}
[Test]
public void InequalityTest3()
{
var setA = new IndexSet([2, 3, 4, 5]);
var setB = new IndexSet([1, 2, 3, 4]);
Assert.That(setA, Is.Not.EqualTo(setB));
Assert.That(setA != setB);
}
[Test]
public void InequalityTest()
{
var setA = new IndexSet([2, 3, 4]);
var setB = new IndexSet([1, 2, 3, 4]);
Assert.That(setA, Is.Not.EqualTo(setB));
Assert.That(setA != setB);
}
[Test]
public void Overlap()
{
var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
var expectedSet = new IndexSet([3, 4, 5, 11]);
var set = setA.Overlap(setB);
Assert.That(set, Is.EqualTo(expectedSet));
}
[Test]
public void Merge()
{
var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
var expectedSet = new IndexSet([1, 2, 3, 4, 5, 6, 7, 11, 12, 13, 14]);
var set = setA.Merge(setB);
Assert.That(set, Is.EqualTo(expectedSet));
}
[Test]
public void Without()
{
var setA = new IndexSet([1, 2, 3, 4, 5, 11, 14]);
var setB = new IndexSet([3, 4, 5, 6, 7, 11, 12, 13]);
var expectedSet = new IndexSet([1, 2, 14]);
var set = setA.Without(setB);
Assert.That(set, Is.EqualTo(expectedSet));
}
}
}

166
Tests/FrameworkTests/Utils/RunLengthEncodingRunTests.cs
View File

@ -1,12 +1,40 @@
using NUnit.Framework;
using NUnit.Framework.Interfaces;
using static FrameworkTests.Utils.RunLengthEncodingTests;
namespace FrameworkTests.Utils
{
[TestFixture]
public class RunLengthEncodingRunTests
{
[Test]
public void EqualityTest()
{
var runA = new Run(1, 4);
var runB = new Run(1, 4);
Assert.That(runA, Is.EqualTo(runB));
Assert.That(runA == runB);
}
[Test]
public void InequalityTest1()
{
var runA = new Run(1, 4);
var runB = new Run(1, 5);
Assert.That(runA, Is.Not.EqualTo(runB));
Assert.That(runA != runB);
}
[Test]
public void InequalityTest2()
{
var runA = new Run(1, 4);
var runB = new Run(2, 4);
Assert.That(runA, Is.Not.EqualTo(runB));
Assert.That(runA != runB);
}
[Test]
[Combinatorial]
public void RunIncludes(
@ -33,23 +61,58 @@ namespace FrameworkTests.Utils
}
[Test]
public void RunExpandToInclude()
public void RunExpandThrowsWhenIndexNotAdjacent()
{
var run = new Run(2, 3);
Assert.That(!run.Includes(1));
Assert.That(run.Includes(2));
Assert.That(run.Includes(4));
Assert.That(!run.Includes(5));
Assert.That(run.ExpandToInclude(1), Is.False);
Assert.That(run.ExpandToInclude(2), Is.False);
Assert.That(run.ExpandToInclude(4), Is.False);
Assert.That(run.ExpandToInclude(6), Is.False);
Assert.That(() => run.ExpandToInclude(0), Throws.TypeOf<Exception>());
Assert.That(() => run.ExpandToInclude(6), Throws.TypeOf<Exception>());
}
Assert.That(run.ExpandToInclude(5), Is.True);
[Test]
public void RunExpandThrowsWhenIndexAlreadyIncluded()
{
var run = new Run(2, 3);
Assert.That(!run.Includes(1));
Assert.That(run.Includes(2));
Assert.That(run.Includes(4));
Assert.That(!run.Includes(5));
Assert.That(() => run.ExpandToInclude(2), Throws.TypeOf<Exception>());
Assert.That(() => run.ExpandToInclude(3), Throws.TypeOf<Exception>());
}
[Test]
public void RunExpandToIncludeAfter()
{
var run = new Run(2, 3);
var update = run.ExpandToInclude(5);
Assert.That(update, Is.Not.Null);
Assert.That(update.NewRuns.Length, Is.EqualTo(0));
Assert.That(update.RemoveRuns.Length, Is.EqualTo(0));
Assert.That(run.Includes(5));
Assert.That(!run.Includes(6));
}
[Test]
public void RunExpandToIncludeBefore()
{
var run = new Run(2, 3);
var update = run.ExpandToInclude(1);
Assert.That(update, Is.Not.Null);
Assert.That(update.NewRuns.Length, Is.EqualTo(1));
Assert.That(update.RemoveRuns.Length, Is.EqualTo(1));
Assert.That(update.RemoveRuns[0], Is.SameAs(run));
Assert.That(update.NewRuns[0].Start, Is.EqualTo(1));
Assert.That(update.NewRuns[0].Length, Is.EqualTo(4));
}
[Test]
public void RunCanUnsetLastIndex()
{
@ -100,94 +163,9 @@ namespace FrameworkTests.Utils
{
var run = new Run(2, 4);
var seen = new List<int>();
run.Iterate(i => seen.Add(i));
run.Iterate(seen.Add);
CollectionAssert.AreEqual(new[] { 2, 3, 4, 5 }, seen);
}
}
public class Run
{
public Run(int start, int length)
{
Start = start;
Length = length;
}
public int Start { get; }
public int Length { get; private set; }
public bool Includes(int index)
{
return index >= Start && index < (Start + Length);
}
public bool ExpandToInclude(int index)
{
if (index == (Start + Length))
{
Length++;
return true;
}
return false;
}
public RunUpdate Unset(int index)
{
if (!Includes(index))
{
return new RunUpdate();
}
if (index == Start)
{
// First index: Replace self with new run at next index, unless empty.
if (Length == 1)
{
return new RunUpdate(Array.Empty<Run>(), new[] { this });
}
return new RunUpdate(
newRuns: new[] { new Run(Start + 1, Length - 1) },
removeRuns: new[] { this }
);
}
if (index == (Start + Length - 1))
{
// Last index: Become one smaller.
Length--;
return new RunUpdate();
}
// Split:
var newRunLength = (Start + Length - 1) - index;
Length = index - Start;
return new RunUpdate(new[] { new Run(index + 1, newRunLength) }, Array.Empty<Run>());
}
public void Iterate(Action<int> action)
{
for (var i = 0; i < Length; i++)
{
action(Start + i);
}
}
}
public class RunUpdate
{
public RunUpdate()
: this(Array.Empty<Run>(), Array.Empty<Run>())
{
}
public RunUpdate(Run[] newRuns, Run[] removeRuns)
{
NewRuns = newRuns;
RemoveRuns = removeRuns;
}
public Run[] NewRuns { get; }
public Run[] RemoveRuns { get; }
}
}

135
Tests/FrameworkTests/Utils/RunLengthEncodingTests.cs
View File

@ -1,9 +1,4 @@
using Logging;
using Microsoft.VisualStudio.TestPlatform.Common;
using NuGet.Frameworks;
using NUnit.Framework;
using System.Collections.Concurrent;
using System.Numerics;
using NUnit.Framework;
using Utils;
namespace FrameworkTests.Utils
@ -119,6 +114,19 @@ namespace FrameworkTests.Utils
}, encoded);
}
[Test]
public void SetIndexBetweenRuns()
{
var set = new IndexSet(new[] {8, 9, 10, 12, 13, 14 });
set.Set(11);
var encoded = set.RunLengthEncoded();
CollectionAssert.AreEqual(new[]
{
8, 7
}, encoded);
}
[Test]
public void SetIndexAfterRun()
{
@ -201,120 +209,5 @@ namespace FrameworkTests.Utils
all.Sort();
return all.ToArray();
}
public class IndexSet
{
private readonly SortedList<int, Run> runs = new SortedList<int, Run>();
public IndexSet()
{
}
public IndexSet(int[] indices)
{
foreach (var i in indices) Set(i);
}
public static IndexSet FromRunLengthEncoded(int[] rle)
{
var set = new IndexSet();
for (var i = 0; i < rle.Length; i += 2)
{
var start = rle[i];
var length = rle[i + 1];
set.runs.Add(start, new Run(start, length));
}
return set;
}
public bool IsSet(int index)
{
if (runs.ContainsKey(index)) return true;
var run = GetRunBefore(index);
if (run == null) return false;
return run.Includes(index);
}
public void Set(int index)
{
if (runs.ContainsKey(index)) return;
var run = GetRunBefore(index);
if (run == null || !run.ExpandToInclude(index))
{
CreateNewRun(index);
}
}
public void Unset(int index)
{
if (runs.ContainsKey(index))
{
HandleUpdate(runs[index].Unset(index));
}
else
{
var run = GetRunBefore(index);
if (run == null) return;
HandleUpdate(run.Unset(index));
}
}
public void Iterate(Action<int> onIndex)
{
foreach (var run in runs.Values)
{
run.Iterate(onIndex);
}
}
public int[] RunLengthEncoded()
{
return Encode().ToArray();
}
private IEnumerable<int> Encode()
{
foreach (var pair in runs)
{
yield return pair.Value.Start;
yield return pair.Value.Length;
}
}
private Run? GetRunBefore(int index)
{
Run? result = null;
foreach (var pair in runs)
{
if (pair.Key < index) result = pair.Value;
else return result;
}
return result;
}
private void HandleUpdate(RunUpdate runUpdate)
{
foreach (var newRun in runUpdate.NewRuns) runs.Add(newRun.Start, newRun);
foreach (var removeRun in runUpdate.RemoveRuns) runs.Remove(removeRun.Start);
}
private void CreateNewRun(int index)
{
if (runs.ContainsKey(index + 1))
{
var length = runs[index + 1].Length + 1;
runs.Add(index, new Run(index, length));
runs.Remove(index + 1);
}
else
{
runs.Add(index, new Run(index, 1));
}
}
}
}
}

25
Tools/AutoClient/Purchaser.cs
View File

@ -68,7 +68,7 @@ namespace AutoClient
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
{
using var fileStream = File.OpenWrite(filename);
var fileResponse = await codex.DownloadNetworkAsync(cid);
var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
fileResponse.Stream.CopyTo(fileStream);
}
var time = sw.Elapsed;
@ -84,8 +84,15 @@ namespace AutoClient
private async Task<string> StartNewPurchase()
{
var file = await CreateFile();
var cid = await UploadFile(file);
return await RequestStorage(cid);
try
{
var cid = await UploadFile(file);
return await RequestStorage(cid);
}
finally
{
DeleteFile(file);
}
}
private async Task<string> CreateFile()
@ -93,6 +100,18 @@ namespace AutoClient
return await app.Generator.Generate();
}
private void DeleteFile(string file)
{
try
{
File.Delete(file);
}
catch (Exception exc)
{
app.Log.Error($"Failed to delete file '{file}': {exc}");
}
}
private async Task<ContentId> UploadFile(string filename)
{
using var fileStream = File.OpenRead(filename);

1
Tools/BiblioTech/BaseCommand.cs
View File

@ -1,7 +1,6 @@
using Discord.WebSocket;
using BiblioTech.Options;
using Discord;
using k8s.KubeConfigModels;
namespace BiblioTech
{

1
Tools/BiblioTech/BiblioTech.csproj
View File

@ -12,6 +12,7 @@
<ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
<ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
<ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
</ItemGroup>
</Project>

198
Tools/BiblioTech/CodexCidChecker.cs Normal file
View File

@ -0,0 +1,198 @@
using CodexOpenApi;
using IdentityModel.Client;
using Utils;
namespace BiblioTech
{
public class CodexCidChecker
{
private static readonly string nl = Environment.NewLine;
private readonly Configuration config;
private CodexApi? currentCodexNode;
public CodexCidChecker(Configuration config)
{
this.config = config;
}
public async Task<CheckResponse> PerformCheck(string cid)
{
if (string.IsNullOrEmpty(config.CodexEndpoint))
{
return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
}
try
{
var codex = GetCodex();
var nodeCheck = await CheckCodex(codex);
if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
return await PerformCheck(codex, cid);
}
catch (Exception ex)
{
return new CheckResponse(false, "Internal server error", ex.ToString());
}
}
private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
{
try
{
var manifest = await codex.DownloadNetworkManifestAsync(cid);
return SuccessMessage(manifest);
}
catch (ApiException apiEx)
{
if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
return UnexpectedReturnCode(apiEx.Response);
}
catch (Exception ex)
{
return UnexpectedException(ex);
}
}
#region Response formatting
private CheckResponse SuccessMessage(DataItem content)
{
return FormatResponse(
success: true,
title: $"Success: '{content.Cid}'",
error: "",
$"size: {content.Manifest.OriginalBytes} bytes",
$"blockSize: {content.Manifest.BlockSize} bytes",
$"protected: {content.Manifest.Protected}"
);
}
private CheckResponse UnexpectedException(Exception ex)
{
return FormatResponse(
success: false,
title: "Unexpected error",
error: ex.ToString(),
content: "Details will be sent to the bot-admin channel."
);
}
private CheckResponse UnexpectedReturnCode(string response)
{
var msg = "Unexpected return code. Response: " + response;
return FormatResponse(
success: false,
title: "Unexpected return code",
error: msg,
content: msg
);
}
private CheckResponse FailedToFetch(string response)
{
var msg = "Failed to download content. Response: " + response;
return FormatResponse(
success: false,
title: "Could not download content",
error: msg,
msg,
$"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
);
}
private CheckResponse CidFormatInvalid(string response)
{
return FormatResponse(
success: false,
title: "Invalid format",
error: "",
content: "Provided CID is not formatted correctly."
);
}
private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
{
var msg = string.Join(nl,
new string[]
{
title,
"```"
}
.Concat(content)
.Concat(new string[]
{
"```"
})
) + nl + nl;
return new CheckResponse(success, msg, error);
}
#endregion
#region Codex Node API
private CodexApi GetCodex()
{
if (currentCodexNode == null) currentCodexNode = CreateCodex();
return currentCodexNode;
}
private async Task<bool> CheckCodex(CodexApi codex)
{
try
{
var info = await currentCodexNode!.GetDebugInfoAsync();
if (info == null || string.IsNullOrEmpty(info.Id)) return false;
return true;
}
catch (Exception e)
{
return false;
}
}
private CodexApi CreateCodex()
{
var endpoint = config.CodexEndpoint;
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
var address = new Address(
host: host,
port: port
);
var client = new HttpClient();
if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
{
var tokens = config.CodexEndpointAuth.Split(':');
if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
client.SetBasicAuthentication(tokens[0], tokens[1]);
}
var codex = new CodexApi(client);
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
return codex;
}
#endregion
}
public class CheckResponse
{
public CheckResponse(bool success, string message, string error)
{
Success = success;
Message = message;
Error = error;
}
public bool Success { get; }
public string Message { get; }
public string Error { get; }
}
}

38
Tools/BiblioTech/Commands/CheckCidCommand.cs Normal file
View File

@ -0,0 +1,38 @@
using BiblioTech.Options;
namespace BiblioTech.Commands
{
public class CheckCidCommand : BaseCommand
{
private readonly StringOption cidOption = new StringOption(
name: "cid",
description: "Codex Content-Identifier",
isRequired: true);
private readonly CodexCidChecker checker;
public CheckCidCommand(CodexCidChecker checker)
{
this.checker = checker;
}
public override string Name => "check";
public override string StartingMessage => RandomBusyMessage.Get();
public override string Description => "Checks if content is available in the testnet.";
public override CommandOption[] Options => new[] { cidOption };
protected override async Task Invoke(CommandContext context)
{
var user = context.Command.User;
var cid = await cidOption.Parse(context);
if (string.IsNullOrEmpty(cid))
{
await context.Followup("Option 'cid' was not received.");
return;
}
var response = await checker.PerformCheck(cid);
await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
await context.Followup(response.Message);
}
}
}

6
Tools/BiblioTech/Configuration.cs
View File

@ -38,6 +38,12 @@ namespace BiblioTech
[Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
public int NoDiscord { get; set; } = 0;
[Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")]
public string CodexEndpoint { get; set; } = "http://localhost:8080";
[Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
public string CodexEndpointAuth { get; set; } = "";
public string EndpointsPath => Path.Combine(DataPath, "endpoints");
public string UserDataPath => Path.Combine(DataPath, "users");
public string LogPath => Path.Combine(DataPath, "logs");

3
Tools/BiblioTech/Program.cs
View File

@ -3,7 +3,6 @@ using BiblioTech.Commands;
using BiblioTech.Rewards;
using Discord;
using Discord.WebSocket;
using DiscordRewards;
using Logging;
namespace BiblioTech
@ -81,6 +80,7 @@ namespace BiblioTech
client = new DiscordSocketClient();
client.Log += ClientLog;
var checker = new CodexCidChecker(Config);
var notifyCommand = new NotifyCommand();
var associateCommand = new UserAssociateCommand(notifyCommand);
var sprCommand = new SprCommand();
@ -90,6 +90,7 @@ namespace BiblioTech
sprCommand,
associateCommand,
notifyCommand,
new CheckCidCommand(checker),
new AdminCommand(sprCommand, replacement)
);

47
Tools/BiblioTech/Rewards/ChainEventsSender.cs
View File

@ -1,4 +1,5 @@
using Discord.WebSocket;
using DiscordRewards;
using Logging;
namespace BiblioTech.Rewards
@ -16,24 +17,17 @@ namespace BiblioTech.Rewards
this.eventsChannel = eventsChannel;
}
public async Task ProcessChainEvents(string[] eventsOverview)
public async Task ProcessChainEvents(ChainEventMessage[] eventsOverview, string[] errors)
{
await SendErrorsToAdminChannel(errors);
if (eventsChannel == null || eventsOverview == null || !eventsOverview.Any()) return;
try
{
await Task.Run(async () =>
{
var users = Program.UserRepo.GetAllUserData();
foreach (var e in eventsOverview)
{
if (!string.IsNullOrEmpty(e))
{
var @event = ApplyReplacements(users, e);
await eventsChannel.SendMessageAsync(@event);
await Task.Delay(1000);
}
}
await SendChainEventsInOrder(eventsOverview, eventsChannel, users);
});
}
catch (Exception ex)
@ -42,6 +36,37 @@ namespace BiblioTech.Rewards
}
}
private async Task SendErrorsToAdminChannel(string[] errors)
{
try
{
foreach (var error in errors)
{
await Program.AdminChecker.SendInAdminChannel(error);
}
}
catch (Exception exc)
{
log.Error("Failed to send error messages to admin channel. " + exc);
Environment.Exit(1);
}
}
private async Task SendChainEventsInOrder(ChainEventMessage[] eventsOverview, SocketTextChannel eventsChannel, UserData[] users)
{
eventsOverview = eventsOverview.OrderBy(e => e.BlockNumber).ToArray();
foreach (var e in eventsOverview)
{
var msg = e.Message;
if (!string.IsNullOrEmpty(msg))
{
var @event = ApplyReplacements(users, msg);
await eventsChannel.SendMessageAsync(@event);
await Task.Delay(300);
}
}
}
private string ApplyReplacements(UserData[] users, string msg)
{
var result = ApplyUserAddressReplacements(users, msg);

2
Tools/BiblioTech/Rewards/RoleDriver.cs
View File

@ -31,7 +31,7 @@ namespace BiblioTech.Rewards
await ProcessRewards(rewards);
}
await eventsSender.ProcessChainEvents(rewards.EventsOverview);
await eventsSender.ProcessChainEvents(rewards.EventsOverview, rewards.Errors);
}
private async Task ProcessRewards(GiveRewardsCommand rewards)

3
Tools/MarketInsights/AverageHistory.cs
View File

@ -1,6 +1,5 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using Nethereum.Model;
using TestNetRewarder;
using Utils;
@ -40,7 +39,7 @@ namespace MarketInsights
private MarketTimeSegment BuildContribution(TimeRange timeRange)
{
var builder = new ContributionBuilder(timeRange);
var builder = new ContributionBuilder(appState.Log, timeRange);
mux.Handlers.Add(builder);
chainState.Update(timeRange.To);
mux.Handlers.Remove(builder);

10
Tools/MarketInsights/ContributionBuilder.cs
View File

@ -1,5 +1,6 @@
using CodexContractsPlugin.ChainMonitor;
using GethPlugin;
using Logging;
using System.Numerics;
using Utils;
@ -8,14 +9,16 @@ namespace MarketInsights
public class ContributionBuilder : IChainStateChangeHandler
{
private readonly MarketTimeSegment segment = new MarketTimeSegment();
private readonly ILog log;
public ContributionBuilder(TimeRange timeRange)
public ContributionBuilder(ILog log, TimeRange timeRange)
{
segment = new MarketTimeSegment
{
FromUtc = timeRange.From,
ToUtc = timeRange.To
};
this.log = log;
}
public void OnNewRequest(RequestEvent requestEvent)
@ -55,6 +58,11 @@ namespace MarketInsights
{
}
public void OnError(string msg)
{
log.Error(msg);
}
public MarketTimeSegment GetSegment()
{
return segment;

1
Tools/TestNetRewarder/BotClient.cs
View File

@ -18,7 +18,6 @@ namespace TestNetRewarder
public async Task<bool> IsOnline()
{
var result = await HttpGet();
log.Log("Is DiscordBot online: " + result);
return result == "Pong";
}

35
Tools/TestNetRewarder/EventsFormatter.cs
View File

@ -1,5 +1,6 @@
using CodexContractsPlugin;
using CodexContractsPlugin.ChainMonitor;
using DiscordRewards;
using GethPlugin;
using System.Globalization;
using System.Numerics;
@ -10,19 +11,22 @@ namespace TestNetRewarder
public class EventsFormatter : IChainStateChangeHandler
{
private static readonly string nl = Environment.NewLine;
private readonly List<string> events = new List<string>();
private readonly List<ChainEventMessage> events = new List<ChainEventMessage>();
private readonly List<string> errors = new List<string>();
private readonly EmojiMaps emojiMaps = new EmojiMaps();
public string[] GetEvents()
public ChainEventMessage[] GetEvents()
{
var result = events.ToArray();
events.Clear();
return result;
}
public void AddError(string error)
public string[] GetErrors()
{
AddBlock("📢 **Error**", error);
var result = errors.ToArray();
errors.Clear();
return result;
}
public void OnNewRequest(RequestEvent requestEvent)
@ -82,20 +86,35 @@ namespace TestNetRewarder
$"Slot Index: {slotIndex}"
);
}
public void OnError(string msg)
{
errors.Add(msg);
}
private void AddRequestBlock(RequestEvent requestEvent, string eventName, params string[] content)
{
var blockNumber = $"[{requestEvent.Block.BlockNumber} {FormatDateTime(requestEvent.Block.Utc)}]";
var title = $"{blockNumber} **{eventName}** {FormatRequestId(requestEvent)}";
AddBlock(title, content);
AddBlock(requestEvent.Block.BlockNumber, title, content);
}
private void AddBlock(string title, params string[] content)
private void AddBlock(ulong blockNumber, string title, params string[] content)
{
events.Add(FormatBlock(title, content));
events.Add(FormatBlock(blockNumber, title, content));
}
private string FormatBlock(string title, params string[] content)
private ChainEventMessage FormatBlock(ulong blockNumber, string title, params string[] content)
{
var msg = FormatBlockMessage(title, content);
return new ChainEventMessage
{
BlockNumber = blockNumber,
Message = msg
};
}
private string FormatBlockMessage(string title, string[] content)
{
if (content == null || !content.Any())
{

5
Tools/TestNetRewarder/Processor.cs
View File

@ -48,7 +48,7 @@ namespace TestNetRewarder
{
var msg = "Exception processing time segment: " + ex;
log.Error(msg);
eventsFormatter.AddError(msg);
eventsFormatter.OnError(msg);
throw;
}
}
@ -58,8 +58,9 @@ namespace TestNetRewarder
var numberOfChainEvents = chainState.Update(timeRange.To);
var events = eventsFormatter.GetEvents();
var errors = eventsFormatter.GetErrors();
var request = builder.Build(events);
var request = builder.Build(events, errors);
if (request.HasAny())
{
await client.SendRewards(request);

5
Tools/TestNetRewarder/RequestBuilder.cs
View File

@ -19,7 +19,7 @@ namespace TestNetRewarder
}
}
public GiveRewardsCommand Build(string[] lines)
public GiveRewardsCommand Build(ChainEventMessage[] lines, string[] errors)
{
var result = new GiveRewardsCommand
{
@ -28,7 +28,8 @@ namespace TestNetRewarder
RewardId = p.Key,
UserAddresses = p.Value.Select(v => v.Address).ToArray()
}).ToArray(),
EventsOverview = lines
EventsOverview = lines,
Errors = errors
};
rewards.Clear();

4
Tools/TestNetRewarder/RewardCheck.cs
View File

@ -76,6 +76,10 @@ namespace TestNetRewarder
{
}
public void OnError(string msg)
{
}
private void GiveReward(RewardConfig reward, EthAddress receiver)
{
giver.Give(reward, receiver);