logos-storage/logos-storage-nim-cs-dist-tests
1
0
Fork 0
mirror of https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git synced 2026-05-04 16:43:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

sets up upload and download checking commands

Browse Source
This commit is contained in:
ThatBen 2025-04-10 14:54:49 +02:00
parent 7ccdbd3c26
commit 9ccc4c559c
No known key found for this signature in database
GPG Key ID: E020A7DDCD52E1AB
14 changed files with 583 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Framework/Utils/RandomUtils.cs
View File

@ -34,10 +34,26 @@
var source = items.ToList();
while (source.Any())
{
result.Add(RandomUtils.PickOneRandom(source));
result.Add(PickOneRandom(source));
}
return result.ToArray();
}
}
public static string GenerateRandomString(long requiredLength)
{
lock (@lock)
{
var result = "";
while (result.Length < requiredLength)
{
var bytes = new byte[1024];
random.NextBytes(bytes);
result += string.Join("", bytes.Select(b => b.ToString()));
}
return result;
}
}
}
}

16
Tools/AutoClient/Modes/FolderStore/FolderSaver.cs
View File

@ -1,4 +1,5 @@
using Logging;
using Utils;
namespace AutoClient.Modes.FolderStore
{
@ -126,24 +127,11 @@ namespace AutoClient.Modes.FolderStore
if (info.Length < min)
{
var required = Math.Max(1024, min - info.Length);
status.Padding = paddingMessage + GenerateRandomString(required);
status.Padding = paddingMessage + RandomUtils.GenerateRandomString(required);
statusFile.Save(status);
}
}
private string GenerateRandomString(long required)
{
var result = "";
while (result.Length < required)
{
var bytes = new byte[1024];
random.NextBytes(bytes);
result += string.Join("", bytes.Select(b => b.ToString()));
}
return result;
}
private FileSaver CreateFileSaver(string folderFile, FileStatus entry)
{
var fixedLength = entry.Filename.PadRight(35);

78
Tools/BiblioTech/CodexChecking/CheckRepo.cs Normal file
View File

@ -0,0 +1,78 @@
using Newtonsoft.Json;
namespace BiblioTech.CodexChecking
{
public class CheckRepo
{
private const string modelFilename = "model.json";
private readonly Configuration config;
private readonly object _lock = new object();
private CheckRepoModel? model = null;
public CheckRepo(Configuration config)
{
this.config = config;
}
public CheckReport GetOrCreate(ulong userId)
{
lock (_lock)
{
if (model == null) LoadModel();
var existing = model.Reports.SingleOrDefault(r => r.UserId == userId);
if (existing == null)
{
var newEntry = new CheckReport
{
UserId = userId,
};
model.Reports.Add(newEntry);
SaveChanges();
return newEntry;
}
return existing;
}
}
public void SaveChanges()
{
File.WriteAllText(GetModelFilepath(), JsonConvert.SerializeObject(model, Formatting.Indented));
}
private void LoadModel()
{
if (!File.Exists(GetModelFilepath()))
{
model = new CheckRepoModel();
SaveChanges();
return;
}
model = JsonConvert.DeserializeObject<CheckRepoModel>(File.ReadAllText(GetModelFilepath()));
}
private string GetModelFilepath()
{
return Path.Combine(config.ChecksDataPath, modelFilename);
}
}
public class CheckRepoModel
{
public List<CheckReport> Reports { get; set; } = new List<CheckReport>();
}
public class CheckReport
{
public ulong UserId { get; set; }
public TransferCheck UploadCheck { get; set; } = new TransferCheck();
public TransferCheck DownloadCheck { get; set; } = new TransferCheck();
}
public class TransferCheck
{
public DateTime CompletedUtc { get; set; } = DateTime.MinValue;
public string UniqueData { get; set; } = string.Empty;
}
}

209
Tools/BiblioTech/CodexChecking/CodexTwoWayChecker.cs Normal file
View File

@ -0,0 +1,209 @@
using CodexClient;
using FileUtils;
using Logging;
using Utils;
namespace BiblioTech.CodexChecking
{
public interface ICheckResponseHandler
{
Task CheckNotStarted();
Task NowCompleted();
Task GiveRoleReward();
Task InvalidData();
Task CouldNotDownloadCid();
Task GiveCidToUser(string cid);
Task GiveDataFileToUser(string fileContent);
}
public class CodexTwoWayChecker
{
private readonly ILog log;
private readonly Configuration config;
private readonly CheckRepo repo;
private readonly CodexWrapper codexWrapper;
public CodexTwoWayChecker(ILog log, Configuration config, CheckRepo repo, CodexWrapper codexWrapper)
{
this.log = log;
this.config = config;
this.repo = repo;
this.codexWrapper = codexWrapper;
}
public async Task StartDownloadCheck(ICheckResponseHandler handler, ulong userId)
{
var check = repo.GetOrCreate(userId).DownloadCheck;
if (string.IsNullOrEmpty(check.UniqueData))
{
check.UniqueData = GenerateUniqueData();
repo.SaveChanges();
}
var cid = UploadData(check.UniqueData);
await handler.GiveCidToUser(cid);
}
public async Task VerifyDownloadCheck(ICheckResponseHandler handler, ulong userId, string receivedData)
{
var check = repo.GetOrCreate(userId).DownloadCheck;
if (string.IsNullOrEmpty(check.UniqueData))
{
await handler.CheckNotStarted();
return;
}
if (string.IsNullOrEmpty(receivedData) || receivedData != check.UniqueData)
{
await handler.InvalidData();
return;
}
CheckNowCompleted(handler, check, userId);
}
public async Task StartUploadCheck(ICheckResponseHandler handler, ulong userId)
{
var check = repo.GetOrCreate(userId).UploadCheck;
if (string.IsNullOrEmpty(check.UniqueData))
{
check.UniqueData = GenerateUniqueData();
repo.SaveChanges();
}
await handler.GiveDataFileToUser(check.UniqueData);
}
public async Task VerifyUploadCheck(ICheckResponseHandler handler, ulong userId, string receivedCid)
{
var check = repo.GetOrCreate(userId).UploadCheck;
if (string.IsNullOrEmpty(receivedCid))
{
await handler.InvalidData();
return;
}
var manifest = GetManifest(receivedCid);
if (manifest == null)
{
await handler.CouldNotDownloadCid();
return;
}
if (IsManifestLengthCompatible(check, manifest))
{
if (IsContentCorrect(check, receivedCid))
{
CheckNowCompleted(handler, check, userId);
return;
}
}
await handler.InvalidData();
}
private string GenerateUniqueData()
{
return $"'{RandomBusyMessage.Get()}'{RandomUtils.GenerateRandomString(12)}";
}
private string UploadData(string uniqueData)
{
var filePath = Path.Combine(config.ChecksDataPath, Guid.NewGuid().ToString());
try
{
File.WriteAllText(filePath, uniqueData);
var file = new TrackedFile(log, filePath, "checkData");
return codexWrapper.OnCodex(node =>
{
return node.UploadFile(file).Id;
});
}
catch (Exception ex)
{
log.Error("Exception when uploading data: " + ex);
throw;
}
finally
{
if (File.Exists(filePath)) File.Delete(filePath);
}
}
private Manifest? GetManifest(string receivedCid)
{
try
{
return codexWrapper.OnCodex(node =>
{
return node.DownloadManifestOnly(new ContentId(receivedCid)).Manifest;
});
}
catch
{
return null;
}
}
private bool IsManifestLengthCompatible(TransferCheck check, Manifest manifest)
{
var dataLength = check.UniqueData.Length;
var manifestLength = manifest.OriginalBytes.SizeInBytes;
return
manifestLength > (dataLength - 1) &&
manifestLength < (dataLength + 1);
}
private bool IsContentCorrect(TransferCheck check, string receivedCid)
{
try
{
return codexWrapper.OnCodex(node =>
{
var file = node.DownloadContent(new ContentId(receivedCid));
if (file == null) return false;
try
{
var content = File.ReadAllText(file.Filename).Trim();
return content == check.UniqueData;
}
finally
{
if (File.Exists(file.Filename)) File.Delete(file.Filename);
}
});
}
catch
{
return false;
}
}
private void CheckNowCompleted(ICheckResponseHandler handler, TransferCheck check, ulong userId)
{
if (check.CompletedUtc != DateTime.MinValue) return;
check.CompletedUtc = DateTime.UtcNow;
repo.SaveChanges();
handler.NowCompleted();
CheckUserForRoleRewards(handler, userId);
}
private void CheckUserForRoleRewards(ICheckResponseHandler handler, ulong userId)
{
var check = repo.GetOrCreate(userId);
if (
check.UploadCheck.CompletedUtc != DateTime.MinValue &&
check.DownloadCheck.CompletedUtc != DateTime.MinValue)
{
handler.GiveRoleReward();
}
}
}
}

85
Tools/BiblioTech/CodexChecking/CodexWrapper.cs Normal file
View File

@ -0,0 +1,85 @@
using CodexClient;
using IdentityModel.Client;
using Logging;
using Utils;
using WebUtils;
namespace BiblioTech.CodexChecking
{
public class CodexWrapper
{
private readonly CodexNodeFactory factory;
private readonly ILog log;
private readonly Configuration config;
private readonly object codexLock = new object();
private ICodexNode? currentCodexNode;
public CodexWrapper(ILog log, Configuration config)
{
this.log = log;
this.config = config;
var httpFactory = CreateHttpFactory();
factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath);
}
public void OnCodex(Action<ICodexNode> action)
{
lock (codexLock)
{
action(Get());
}
}
public T OnCodex<T>(Func<ICodexNode, T> func)
{
lock (codexLock)
{
return func(Get());
}
}
private ICodexNode Get()
{
if (currentCodexNode == null)
{
currentCodexNode = CreateCodex();
}
return currentCodexNode;
}
private ICodexNode CreateCodex()
{
var endpoint = config.CodexEndpoint;
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
var address = new Address(
logName: $"cdx@{host}:{port}",
host: host,
port: port
);
var instance = CodexInstance.CreateFromApiEndpoint("ac", address);
return factory.CreateCodexNode(instance);
}
private HttpFactory CreateHttpFactory()
{
if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
{
return new HttpFactory(log);
}
var tokens = config.CodexEndpointAuth.Split(':');
if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
return new HttpFactory(log, onClientCreated: client =>
{
client.SetBasicAuthentication(tokens[0], tokens[1]);
});
}
}
}

59
Tools/BiblioTech/CodexTwoWayChecker.cs
View File

@ -1,59 +0,0 @@
using CodexClient;
using IdentityModel.Client;
using Logging;
using Utils;
using WebUtils;
namespace BiblioTech
{
public class CodexTwoWayChecker
{
private static readonly string nl = Environment.NewLine;
private readonly Configuration config;
private readonly ILog log;
private readonly CodexNodeFactory factory;
private ICodexNode? currentCodexNode;
public CodexTwoWayChecker(Configuration config, ILog log)
{
this.config = config;
this.log = log;
var httpFactory = CreateHttpFactory();
factory = new CodexNodeFactory(log, httpFactory, dataDir: config.DataPath);
}
// down check:
// generate unique data
// upload to cloud node
// give CID to user to download
// user inputs unique data into command to clear this check
// up check:
// generate unique data
// create file and send it to user via discord api
// user uploads and gives CID via command
// download manifest: file is not larger than expected
// download file: contents is unique data -> clear this check
// both checks: altruistic role
private HttpFactory CreateHttpFactory()
{
if (string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
{
return new HttpFactory(log);
}
var tokens = config.CodexEndpointAuth.Split(':');
if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
return new HttpFactory(log, onClientCreated: client =>
{
client.SetBasicAuthentication(tokens[0], tokens[1]);
});
}
}
}

129
Tools/BiblioTech/Commands/CheckCidCommand.cs
View File

@ -1,129 +0,0 @@
using BiblioTech.Options;
using Discord;
using Discord.WebSocket;
namespace BiblioTech.Commands
{
public class CheckCidCommand : BaseCommand
{
private readonly StringOption cidOption = new StringOption(
name: "cid",
description: "Codex Content-Identifier",
isRequired: true);
private readonly CodexTwoWayChecker checker;
private readonly CidStorage cidStorage;
public CheckCidCommand(CodexTwoWayChecker checker)
{
this.checker = checker;
this.cidStorage = new CidStorage(Path.Combine(Program.Config.DataPath, "valid_cids.txt"));
}
public override string Name => "check";
public override string StartingMessage => RandomBusyMessage.Get();
public override string Description => "Checks if content is available in the testnet.";
public override CommandOption[] Options => new[] { cidOption };
protected override async Task Invoke(CommandContext context)
{
var user = context.Command.User;
var cid = await cidOption.Parse(context);
if (string.IsNullOrEmpty(cid))
{
await context.Followup("Option 'cid' was not received.");
return;
}
try
{
await PerformCheck(context, user, cid);
}
catch (Exception ex)
{
await RespondeWithError(context, ex);
}
}
private async Task PerformCheck(CommandContext context, SocketUser user, string cid)
{
var response = checker.PerformCheck(cid);
if (response.Success)
{
await CheckAltruisticRole(context, user, cid, response.Message);
return;
}
await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}'");
await context.Followup(response.Message);
}
private async Task RespondeWithError(CommandContext context, Exception ex)
{
await Program.AdminChecker.SendInAdminChannel("Exception during CheckCidCommand: " + ex);
await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel.");
}
private async Task CheckAltruisticRole(CommandContext context, IUser user, string cid, string responseMessage)
{
if (cidStorage.TryAddCid(cid, user.Id))
{
if (await GiveAltruisticRole(context, user, responseMessage))
{
return;
}
}
else
{
await context.Followup($"{responseMessage}\n\nThis CID has already been used by another user. No role will be granted.");
return;
}
await context.Followup(responseMessage);
}
private async Task<bool> GiveAltruisticRole(CommandContext context, IUser user, string responseMessage)
{
try
{
await Program.RoleDriver.GiveAltruisticRole(user);
await context.Followup($"{responseMessage}\n\nCongratulations! You've been granted the Altruistic Mode role for checking a valid CID!");
return true;
}
catch (Exception ex)
{
await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user {Mention(user)}: {ex.Message}");
return false;
}
}
}
public class CidStorage
{
private readonly string filePath;
private static readonly object _lock = new object();
public CidStorage(string filePath)
{
this.filePath = filePath;
if (!File.Exists(filePath))
{
File.WriteAllText(filePath, string.Empty);
}
}
public bool TryAddCid(string cid, ulong userId)
{
lock (_lock)
{
var existingEntries = File.ReadAllLines(filePath);
if (existingEntries.Any(line => line.Split(',')[0] == cid))
{
return false;
}
File.AppendAllLines(filePath, new[] { $"{cid},{userId}" });
return true;
}
}
}
}

53
Tools/BiblioTech/Commands/CheckDownloadCommand.cs Normal file
View File

@ -0,0 +1,53 @@
using BiblioTech.CodexChecking;
using BiblioTech.Options;
namespace BiblioTech.Commands
{
public class CheckDownloadCommand : BaseCommand
{
private readonly CodexTwoWayChecker checker;
private readonly StringOption contentOption = new StringOption(
name: "content",
description: "Content of the downloaded file",
isRequired: false);
public CheckDownloadCommand(CodexTwoWayChecker checker)
{
this.checker = checker;
}
public override string Name => "checkdownload";
public override string StartingMessage => RandomBusyMessage.Get();
public override string Description => "Checks the download connectivity of your Codex node.";
public override CommandOption[] Options => [contentOption];
protected override async Task Invoke(CommandContext context)
{
var user = context.Command.User;
var content = await contentOption.Parse(context);
try
{
var handler = new CheckResponseHandler(context, user);
if (string.IsNullOrEmpty(content))
{
await checker.StartDownloadCheck(handler, user.Id);
}
else
{
await checker.VerifyDownloadCheck(handler, user.Id, content);
}
}
catch (Exception ex)
{
await RespondWithError(context, ex);
}
}
private async Task RespondWithError(CommandContext context, Exception ex)
{
await Program.AdminChecker.SendInAdminChannel("Exception during CheckDownloadCommand: " + ex);
await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel.");
}
}
}

66
Tools/BiblioTech/Commands/CheckResponseHandler.cs Normal file
View File

@ -0,0 +1,66 @@
using BiblioTech.CodexChecking;
using BiblioTech.Options;
using Discord;
namespace BiblioTech.Commands
{
public class CheckResponseHandler : ICheckResponseHandler
{
private CommandContext context;
private readonly IUser user;
public CheckResponseHandler(CommandContext context, IUser user)
{
this.context = context;
this.user = user;
}
public async Task CheckNotStarted()
{
await context.Followup("Run this command without any arguments first, to begin the check process.");
}
public async Task CouldNotDownloadCid()
{
await context.Followup("Could not download the CID.");
}
public async Task GiveCidToUser(string cid)
{
await context.Followup("Please download this CID using your Codex node. " +
"Then provide the content of the downloaded file as argument to this command. " +
$"`{cid}`");
}
public async Task GiveDataFileToUser(string fileContent)
{
await context.Followup("Please download the attached file. Upload it to your Codex node, " +
"then provide the CID as argument to this command.");
await context.SendFile(fileContent);
}
public async Task GiveRoleReward()
{
try
{
await Program.RoleDriver.GiveAltruisticRole(user);
await context.Followup($"Congratulations! You've been granted the Altruistic Mode role!");
}
catch (Exception ex)
{
await Program.AdminChecker.SendInAdminChannel($"Failed to grant Altruistic Mode role to user <@{user.Id}>: {ex.Message}");
}
}
public async Task InvalidData()
{
await context.Followup("The received data didn't match. Check has failed.");
}
public async Task NowCompleted()
{
await context.Followup("Successfully completed the check!");
}
}
}

53
Tools/BiblioTech/Commands/CheckUploadCommand.cs Normal file
View File

@ -0,0 +1,53 @@
using BiblioTech.CodexChecking;
using BiblioTech.Options;
namespace BiblioTech.Commands
{
public class CheckUploadCommand : BaseCommand
{
private readonly CodexTwoWayChecker checker;
private readonly StringOption cidOption = new StringOption(
name: "cid",
description: "Codex Content-Identifier",
isRequired: false);
public CheckUploadCommand(CodexTwoWayChecker checker)
{
this.checker = checker;
}
public override string Name => "checkupload";
public override string StartingMessage => RandomBusyMessage.Get();
public override string Description => "Checks the upload connectivity of your Codex node.";
public override CommandOption[] Options => [cidOption];
protected override async Task Invoke(CommandContext context)
{
var user = context.Command.User;
var cid = await cidOption.Parse(context);
try
{
var handler = new CheckResponseHandler(context, user);
if (string.IsNullOrEmpty(cid))
{
await checker.StartUploadCheck(handler, user.Id);
}
else
{
await checker.VerifyUploadCheck(handler, user.Id, cid);
}
}
catch (Exception ex)
{
await RespondWithError(context, ex);
}
}
private async Task RespondWithError(CommandContext context, Exception ex)
{
await Program.AdminChecker.SendInAdminChannel("Exception during CheckUploadCommand: " + ex);
await context.Followup("I'm sorry to report something has gone wrong in an unexpected way. Error details are already posted in the admin channel.");
}
}
}

1
Tools/BiblioTech/Configuration.cs
View File

@ -49,6 +49,7 @@ namespace BiblioTech
public string EndpointsPath => Path.Combine(DataPath, "endpoints");
public string UserDataPath => Path.Combine(DataPath, "users");
public string ChecksDataPath => Path.Combine(DataPath, "checks");
public string LogPath => Path.Combine(DataPath, "logs");
public bool DebugNoDiscord => NoDiscord == 1;
}

10
Tools/BiblioTech/Options/CommandContext.cs
View File

@ -49,6 +49,16 @@ namespace BiblioTech.Options
}
}
public async Task SendFile(string fileContent)
{
using var stream = new MemoryStream();
using var writer = new StreamWriter(stream);
writer.Write(fileContent);
stream.Position = 0;
await Command.RespondWithFileAsync(stream, "CheckFile.txt", ephemeral: true);
}
private string FormatChunk(string[] chunk)
{
return string.Join(Environment.NewLine, chunk);

8
Tools/BiblioTech/Program.cs
View File

@ -1,4 +1,5 @@
using ArgsUniform;
using BiblioTech.CodexChecking;
using BiblioTech.Commands;
using BiblioTech.Rewards;
using Discord;
@ -80,7 +81,9 @@ namespace BiblioTech
client = new DiscordSocketClient();
client.Log += ClientLog;
var checker = new CodexTwoWayChecker(Config, Log);
var checkRepo = new CheckRepo(Config);
var codexWrapper = new CodexWrapper(Log, Config);
var checker = new CodexTwoWayChecker(Log, Config, checkRepo, codexWrapper);
var notifyCommand = new NotifyCommand();
var associateCommand = new UserAssociateCommand(notifyCommand);
var sprCommand = new SprCommand();
@ -90,7 +93,8 @@ namespace BiblioTech
sprCommand,
associateCommand,
notifyCommand,
new CheckCidCommand(checker),
new CheckUploadCommand(checker),
new CheckDownloadCommand(checker),
new AdminCommand(sprCommand, replacement)
);

4
Tools/BiblioTech/RandomBusyMessage.cs
View File

@ -14,7 +14,9 @@
"Analyzing the wavelengths...",
"Charging the flux-capacitor...",
"Jumping to hyperspace...",
"Computing the ultimate answer..."
"Computing the ultimate answer...",
"Turning it off and on again...",
"Compiling from sources..."
};
public static string Get()