From 2c88ddfb6b2697694da7dda41834699c447cdbcb Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Oct 2024 12:42:52 +0200 Subject: [PATCH 1/2] brings in api with new calls --- ProjectPlugins/CodexPlugin/openapi.yaml | 60 +++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 4 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index bfe6046..9a20212 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -456,9 +456,35 @@ paths: "/data/{cid}/network": get: - summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able." + summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed." tags: [ Data ] operationId: downloadNetwork + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "File to be downloaded." + responses: + "200": + description: Manifest information for download that has been started. + content: + application/json: + schema: + $ref: "#/components/schemas/DataItem" + "400": + description: Invalid CID is specified + "404": + description: Failed to download dataset manifest + "500": + description: Well it was bad-bad + + "/data/{cid}/network/stream": + get: + summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able." + tags: [ Data ] + operationId: downloadNetworkStream parameters: - in: path name: cid @@ -481,6 +507,32 @@ paths: "500": description: Well it was bad-bad + "/data/{cid}/network/manifest": + get: + summary: "Download only the dataset manifest from the network to the local node if it's not available locally." + tags: [ Data ] + operationId: downloadNetworkManifest + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "File for which the manifest is to be downloaded." + responses: + "200": + description: Manifest information. + content: + application/json: + schema: + $ref: "#/components/schemas/DataItem" + "400": + description: Invalid CID is specified + "404": + description: Failed to download dataset manifest + "500": + description: Well it was bad-bad + "/space": get: summary: "Gets a summary of the storage space allocation of the node." @@ -726,7 +778,7 @@ paths: "503": description: Persistence is not enabled - "/node/spr": + "/spr": get: summary: "Get Node's SPR" operationId: getSPR @@ -744,7 +796,7 @@ paths: "503": description: Node SPR not ready, try again later - "/node/peerid": + "/peerid": get: summary: "Get Node's PeerID" operationId: getPeerId @@ -792,4 +844,4 @@ paths: content: application/json: schema: - $ref: "#/components/schemas/DebugInfo" \ No newline at end of file + $ref: "#/components/schemas/DebugInfo" From fc942b11f854f39c733dae9c6e3dacd815050dc6 Mon Sep 17 00:00:00 2001 From: benbierens Date: Wed, 16 Oct 2024 14:05:02 +0200 Subject: [PATCH 2/2] Implements /check command --- ProjectPlugins/CodexPlugin/ApiChecker.cs | 2 +- ProjectPlugins/CodexPlugin/CodexAccess.cs | 10 +- Tools/AutoClient/Purchaser.cs | 2 +- Tools/BiblioTech/BaseCommand.cs | 1 - Tools/BiblioTech/BiblioTech.csproj | 1 + Tools/BiblioTech/CodexCidChecker.cs | 198 ++++++++++++++++++ Tools/BiblioTech/Commands/CheckCidCommand.cs | 38 ++++ Tools/BiblioTech/Configuration.cs | 6 + Tools/BiblioTech/Program.cs | 3 +- Tools/BiblioTech/Rewards/ChainEventsSender.cs | 1 - 10 files changed, 252 insertions(+), 10 deletions(-) create mode 100644 Tools/BiblioTech/CodexCidChecker.cs create mode 100644 Tools/BiblioTech/Commands/CheckCidCommand.cs diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 5443e11..66d5963 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -10,7 +10,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "AC-19-7F-3A-88-07-CB-43-53-60-4F-21-3D-A6-B1-53-47-65-07-3B-91-C6-88-B9-76-B2-7E-33-6A-1C-69-F4"; + private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 1216ec0..9f3e33f 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -73,7 +73,7 @@ namespace CodexPlugin public Stream DownloadFile(string contentId, Action onFailure) { var fileResponse = OnCodex( - api => api.DownloadNetworkAsync(contentId), + api => api.DownloadNetworkStreamAsync(contentId), CreateRetryConfig(nameof(DownloadFile), onFailure)); if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); @@ -88,25 +88,25 @@ namespace CodexPlugin public StorageAvailability SalesAvailability(StorageAvailability request) { var body = mapper.Map(request); - var read = OnCodex(api => api.OfferStorageAsync(body)); + var read = OnCodex(api => api.OfferStorageAsync(body)); return mapper.Map(read); } public StorageAvailability[] GetAvailabilities() { - var collection = OnCodex>(api => api.GetAvailabilitiesAsync()); + var collection = OnCodex(api => api.GetAvailabilitiesAsync()); return mapper.Map(collection); } public string RequestStorage(StoragePurchaseRequest request) { var body = mapper.Map(request); - return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); + return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); } public CodexSpace Space() { - var space = OnCodex(api => api.SpaceAsync()); + var space = OnCodex(api => api.SpaceAsync()); return mapper.Map(space); } diff --git a/Tools/AutoClient/Purchaser.cs b/Tools/AutoClient/Purchaser.cs index 34e19e4..0baec98 100644 --- a/Tools/AutoClient/Purchaser.cs +++ b/Tools/AutoClient/Purchaser.cs @@ -68,7 +68,7 @@ namespace AutoClient var filename = Guid.NewGuid().ToString().ToLowerInvariant(); { using var fileStream = File.OpenWrite(filename); - var fileResponse = await codex.DownloadNetworkAsync(cid); + var fileResponse = await codex.DownloadNetworkStreamAsync(cid); fileResponse.Stream.CopyTo(fileStream); } var time = sw.Elapsed; diff --git a/Tools/BiblioTech/BaseCommand.cs b/Tools/BiblioTech/BaseCommand.cs index a586471..eb411dd 100644 --- a/Tools/BiblioTech/BaseCommand.cs +++ b/Tools/BiblioTech/BaseCommand.cs @@ -1,7 +1,6 @@ using Discord.WebSocket; using BiblioTech.Options; using Discord; -using k8s.KubeConfigModels; namespace BiblioTech { diff --git a/Tools/BiblioTech/BiblioTech.csproj b/Tools/BiblioTech/BiblioTech.csproj index c896092..fe7e7f2 100644 --- a/Tools/BiblioTech/BiblioTech.csproj +++ b/Tools/BiblioTech/BiblioTech.csproj @@ -12,6 +12,7 @@ + diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs new file mode 100644 index 0000000..21ebc33 --- /dev/null +++ b/Tools/BiblioTech/CodexCidChecker.cs @@ -0,0 +1,198 @@ +using CodexOpenApi; +using IdentityModel.Client; +using Utils; + +namespace BiblioTech +{ + public class CodexCidChecker + { + private static readonly string nl = Environment.NewLine; + private readonly Configuration config; + private CodexApi? currentCodexNode; + + public CodexCidChecker(Configuration config) + { + this.config = config; + } + + public async Task PerformCheck(string cid) + { + if (string.IsNullOrEmpty(config.CodexEndpoint)) + { + return new CheckResponse(false, "Codex CID checker is not (yet) available.", ""); + } + + try + { + var codex = GetCodex(); + var nodeCheck = await CheckCodex(codex); + if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); + + return await PerformCheck(codex, cid); + } + catch (Exception ex) + { + return new CheckResponse(false, "Internal server error", ex.ToString()); + } + } + + private async Task PerformCheck(CodexApi codex, string cid) + { + try + { + var manifest = await codex.DownloadNetworkManifestAsync(cid); + return SuccessMessage(manifest); + } + catch (ApiException apiEx) + { + if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response); + if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response); + return UnexpectedReturnCode(apiEx.Response); + } + catch (Exception ex) + { + return UnexpectedException(ex); + } + } + + #region Response formatting + + private CheckResponse SuccessMessage(DataItem content) + { + return FormatResponse( + success: true, + title: $"Success: '{content.Cid}'", + error: "", + $"size: {content.Manifest.OriginalBytes} bytes", + $"blockSize: {content.Manifest.BlockSize} bytes", + $"protected: {content.Manifest.Protected}" + ); + } + + private CheckResponse UnexpectedException(Exception ex) + { + return FormatResponse( + success: false, + title: "Unexpected error", + error: ex.ToString(), + content: "Details will be sent to the bot-admin channel." + ); + } + + private CheckResponse UnexpectedReturnCode(string response) + { + var msg = "Unexpected return code. Response: " + response; + return FormatResponse( + success: false, + title: "Unexpected return code", + error: msg, + content: msg + ); + } + + private CheckResponse FailedToFetch(string response) + { + var msg = "Failed to download content. Response: " + response; + return FormatResponse( + success: false, + title: "Could not download content", + error: msg, + msg, + $"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'" + ); + } + + private CheckResponse CidFormatInvalid(string response) + { + return FormatResponse( + success: false, + title: "Invalid format", + error: "", + content: "Provided CID is not formatted correctly." + ); + } + + private CheckResponse FormatResponse(bool success, string title, string error, params string[] content) + { + var msg = string.Join(nl, + new string[] + { + title, + "```" + } + .Concat(content) + .Concat(new string[] + { + "```" + }) + ) + nl + nl; + + return new CheckResponse(success, msg, error); + } + + #endregion + + #region Codex Node API + + private CodexApi GetCodex() + { + if (currentCodexNode == null) currentCodexNode = CreateCodex(); + return currentCodexNode; + } + + private async Task CheckCodex(CodexApi codex) + { + try + { + var info = await currentCodexNode!.GetDebugInfoAsync(); + if (info == null || string.IsNullOrEmpty(info.Id)) return false; + return true; + } + catch (Exception e) + { + return false; + } + } + + private CodexApi CreateCodex() + { + var endpoint = config.CodexEndpoint; + var splitIndex = endpoint.LastIndexOf(':'); + var host = endpoint.Substring(0, splitIndex); + var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1)); + + var address = new Address( + host: host, + port: port + ); + + var client = new HttpClient(); + if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + { + var tokens = config.CodexEndpointAuth.Split(':'); + if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + client.SetBasicAuthentication(tokens[0], tokens[1]); + } + + var codex = new CodexApi(client); + codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + return codex; + } + + #endregion + } + + public class CheckResponse + { + public CheckResponse(bool success, string message, string error) + { + Success = success; + Message = message; + Error = error; + } + + public bool Success { get; } + public string Message { get; } + public string Error { get; } + } +} diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs new file mode 100644 index 0000000..01b3b79 --- /dev/null +++ b/Tools/BiblioTech/Commands/CheckCidCommand.cs @@ -0,0 +1,38 @@ +using BiblioTech.Options; + +namespace BiblioTech.Commands +{ + public class CheckCidCommand : BaseCommand + { + private readonly StringOption cidOption = new StringOption( + name: "cid", + description: "Codex Content-Identifier", + isRequired: true); + private readonly CodexCidChecker checker; + + public CheckCidCommand(CodexCidChecker checker) + { + this.checker = checker; + } + + public override string Name => "check"; + public override string StartingMessage => RandomBusyMessage.Get(); + public override string Description => "Checks if content is available in the testnet."; + public override CommandOption[] Options => new[] { cidOption }; + + protected override async Task Invoke(CommandContext context) + { + var user = context.Command.User; + var cid = await cidOption.Parse(context); + if (string.IsNullOrEmpty(cid)) + { + await context.Followup("Option 'cid' was not received."); + return; + } + + var response = await checker.PerformCheck(cid); + await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); + await context.Followup(response.Message); + } + } +} diff --git a/Tools/BiblioTech/Configuration.cs b/Tools/BiblioTech/Configuration.cs index 6f27fd1..7cacf9b 100644 --- a/Tools/BiblioTech/Configuration.cs +++ b/Tools/BiblioTech/Configuration.cs @@ -38,6 +38,12 @@ namespace BiblioTech [Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")] public int NoDiscord { get; set; } = 0; + [Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")] + public string CodexEndpoint { get; set; } = "http://localhost:8080"; + + [Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")] + public string CodexEndpointAuth { get; set; } = ""; + public string EndpointsPath => Path.Combine(DataPath, "endpoints"); public string UserDataPath => Path.Combine(DataPath, "users"); public string LogPath => Path.Combine(DataPath, "logs"); diff --git a/Tools/BiblioTech/Program.cs b/Tools/BiblioTech/Program.cs index 683ef21..ee258e3 100644 --- a/Tools/BiblioTech/Program.cs +++ b/Tools/BiblioTech/Program.cs @@ -3,7 +3,6 @@ using BiblioTech.Commands; using BiblioTech.Rewards; using Discord; using Discord.WebSocket; -using DiscordRewards; using Logging; namespace BiblioTech @@ -81,6 +80,7 @@ namespace BiblioTech client = new DiscordSocketClient(); client.Log += ClientLog; + var checker = new CodexCidChecker(Config); var notifyCommand = new NotifyCommand(); var associateCommand = new UserAssociateCommand(notifyCommand); var sprCommand = new SprCommand(); @@ -90,6 +90,7 @@ namespace BiblioTech sprCommand, associateCommand, notifyCommand, + new CheckCidCommand(checker), new AdminCommand(sprCommand, replacement) ); diff --git a/Tools/BiblioTech/Rewards/ChainEventsSender.cs b/Tools/BiblioTech/Rewards/ChainEventsSender.cs index dadacdd..5cb6aed 100644 --- a/Tools/BiblioTech/Rewards/ChainEventsSender.cs +++ b/Tools/BiblioTech/Rewards/ChainEventsSender.cs @@ -1,7 +1,6 @@ using Discord.WebSocket; using DiscordRewards; using Logging; -using Microsoft.IdentityModel.Tokens; namespace BiblioTech.Rewards {