status-im
/
cs-codex-dist-tests
mirror of
2
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #104 from codex-storage/feature/bot-checks-cids 

Feature/bot checks cids
This commit is contained in:
Ben Bierens 2024-10-18 08:23:58 +02:00 committed by GitHub
parent e352e5c65c fc942b11f8
commit 60b489ced1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 308 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ProjectPlugins/CodexPlugin/ApiChecker.cs
View File

@ -10,7 +10,7 @@ namespace CodexPlugin
public class ApiChecker
{
// <INSERT-OPENAPI-YAML-HASH>
private const string OpenApiYamlHash = "AC-19-7F-3A-88-07-CB-43-53-60-4F-21-3D-A6-B1-53-47-65-07-3B-91-C6-88-B9-76-B2-7E-33-6A-1C-69-F4";
private const string OpenApiYamlHash = "D5-C3-18-71-E8-FF-8F-89-9C-6B-98-3C-F2-C2-D2-37-0A-9F-27-23-35-67-EA-F6-1F-F9-D5-C6-63-34-5A-92";
private const string OpenApiFilePath = "/codex/openapi.yaml";
private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK";

10
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -73,7 +73,7 @@ namespace CodexPlugin
public Stream DownloadFile(string contentId, Action<Failure> onFailure)
{
var fileResponse = OnCodex(
api => api.DownloadNetworkAsync(contentId),
api => api.DownloadNetworkStreamAsync(contentId),
CreateRetryConfig(nameof(DownloadFile), onFailure));
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
@ -88,25 +88,25 @@ namespace CodexPlugin
public StorageAvailability SalesAvailability(StorageAvailability request)
{
var body = mapper.Map(request);
var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
var read = OnCodex(api => api.OfferStorageAsync(body));
return mapper.Map(read);
}
public StorageAvailability[] GetAvailabilities()
{
var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
var collection = OnCodex(api => api.GetAvailabilitiesAsync());
return mapper.Map(collection);
}
public string RequestStorage(StoragePurchaseRequest request)
{
var body = mapper.Map(request);
return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
}
public CodexSpace Space()
{
var space = OnCodex<Space>(api => api.SpaceAsync());
var space = OnCodex(api => api.SpaceAsync());
return mapper.Map(space);
}

60
ProjectPlugins/CodexPlugin/openapi.yaml
View File

@ -456,9 +456,35 @@ paths:
"/data/{cid}/network":
get:
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
summary: "Download a file from the network to the local node if it's not available locally. Note: Download is performed async. Call can return before download is completed."
tags: [ Data ]
operationId: downloadNetwork
parameters:
- in: path
name: cid
required: true
schema:
$ref: "#/components/schemas/Cid"
description: "File to be downloaded."
responses:
"200":
description: Manifest information for download that has been started.
content:
application/json:
schema:
$ref: "#/components/schemas/DataItem"
"400":
description: Invalid CID is specified
"404":
description: Failed to download dataset manifest
"500":
description: Well it was bad-bad
"/data/{cid}/network/stream":
get:
summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able."
tags: [ Data ]
operationId: downloadNetworkStream
parameters:
- in: path
name: cid
@ -481,6 +507,32 @@ paths:
"500":
description: Well it was bad-bad
"/data/{cid}/network/manifest":
get:
summary: "Download only the dataset manifest from the network to the local node if it's not available locally."
tags: [ Data ]
operationId: downloadNetworkManifest
parameters:
- in: path
name: cid
required: true
schema:
$ref: "#/components/schemas/Cid"
description: "File for which the manifest is to be downloaded."
responses:
"200":
description: Manifest information.
content:
application/json:
schema:
$ref: "#/components/schemas/DataItem"
"400":
description: Invalid CID is specified
"404":
description: Failed to download dataset manifest
"500":
description: Well it was bad-bad
"/space":
get:
summary: "Gets a summary of the storage space allocation of the node."
@ -726,7 +778,7 @@ paths:
"503":
description: Persistence is not enabled
"/node/spr":
"/spr":
get:
summary: "Get Node's SPR"
operationId: getSPR
@ -744,7 +796,7 @@ paths:
"503":
description: Node SPR not ready, try again later
"/node/peerid":
"/peerid":
get:
summary: "Get Node's PeerID"
operationId: getPeerId
@ -792,4 +844,4 @@ paths:
content:
application/json:
schema:
$ref: "#/components/schemas/DebugInfo"
$ref: "#/components/schemas/DebugInfo"

2
Tools/AutoClient/Purchaser.cs
View File

@ -68,7 +68,7 @@ namespace AutoClient
var filename = Guid.NewGuid().ToString().ToLowerInvariant();
{
using var fileStream = File.OpenWrite(filename);
var fileResponse = await codex.DownloadNetworkAsync(cid);
var fileResponse = await codex.DownloadNetworkStreamAsync(cid);
fileResponse.Stream.CopyTo(fileStream);
}
var time = sw.Elapsed;

1
Tools/BiblioTech/BaseCommand.cs
View File

@ -1,7 +1,6 @@
using Discord.WebSocket;
using BiblioTech.Options;
using Discord;
using k8s.KubeConfigModels;
namespace BiblioTech
{

1
Tools/BiblioTech/BiblioTech.csproj
View File

@ -12,6 +12,7 @@
<ProjectReference Include="..\..\Framework\ArgsUniform\ArgsUniform.csproj" />
<ProjectReference Include="..\..\Framework\DiscordRewards\DiscordRewards.csproj" />
<ProjectReference Include="..\..\Framework\GethConnector\GethConnector.csproj" />
<ProjectReference Include="..\..\ProjectPlugins\CodexPlugin\CodexPlugin.csproj" />
</ItemGroup>
</Project>

198
Tools/BiblioTech/CodexCidChecker.cs Normal file
View File

@ -0,0 +1,198 @@
using CodexOpenApi;
using IdentityModel.Client;
using Utils;
namespace BiblioTech
{
public class CodexCidChecker
{
private static readonly string nl = Environment.NewLine;
private readonly Configuration config;
private CodexApi? currentCodexNode;
public CodexCidChecker(Configuration config)
{
this.config = config;
}
public async Task<CheckResponse> PerformCheck(string cid)
{
if (string.IsNullOrEmpty(config.CodexEndpoint))
{
return new CheckResponse(false, "Codex CID checker is not (yet) available.", "");
}
try
{
var codex = GetCodex();
var nodeCheck = await CheckCodex(codex);
if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info.");
return await PerformCheck(codex, cid);
}
catch (Exception ex)
{
return new CheckResponse(false, "Internal server error", ex.ToString());
}
}
private async Task<CheckResponse> PerformCheck(CodexApi codex, string cid)
{
try
{
var manifest = await codex.DownloadNetworkManifestAsync(cid);
return SuccessMessage(manifest);
}
catch (ApiException apiEx)
{
if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response);
if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response);
return UnexpectedReturnCode(apiEx.Response);
}
catch (Exception ex)
{
return UnexpectedException(ex);
}
}
#region Response formatting
private CheckResponse SuccessMessage(DataItem content)
{
return FormatResponse(
success: true,
title: $"Success: '{content.Cid}'",
error: "",
$"size: {content.Manifest.OriginalBytes} bytes",
$"blockSize: {content.Manifest.BlockSize} bytes",
$"protected: {content.Manifest.Protected}"
);
}
private CheckResponse UnexpectedException(Exception ex)
{
return FormatResponse(
success: false,
title: "Unexpected error",
error: ex.ToString(),
content: "Details will be sent to the bot-admin channel."
);
}
private CheckResponse UnexpectedReturnCode(string response)
{
var msg = "Unexpected return code. Response: " + response;
return FormatResponse(
success: false,
title: "Unexpected return code",
error: msg,
content: msg
);
}
private CheckResponse FailedToFetch(string response)
{
var msg = "Failed to download content. Response: " + response;
return FormatResponse(
success: false,
title: "Could not download content",
error: msg,
msg,
$"Connection trouble? See 'https://docs.codex.storage/learn/troubleshoot'"
);
}
private CheckResponse CidFormatInvalid(string response)
{
return FormatResponse(
success: false,
title: "Invalid format",
error: "",
content: "Provided CID is not formatted correctly."
);
}
private CheckResponse FormatResponse(bool success, string title, string error, params string[] content)
{
var msg = string.Join(nl,
new string[]
{
title,
"```"
}
.Concat(content)
.Concat(new string[]
{
"```"
})
) + nl + nl;
return new CheckResponse(success, msg, error);
}
#endregion
#region Codex Node API
private CodexApi GetCodex()
{
if (currentCodexNode == null) currentCodexNode = CreateCodex();
return currentCodexNode;
}
private async Task<bool> CheckCodex(CodexApi codex)
{
try
{
var info = await currentCodexNode!.GetDebugInfoAsync();
if (info == null || string.IsNullOrEmpty(info.Id)) return false;
return true;
}
catch (Exception e)
{
return false;
}
}
private CodexApi CreateCodex()
{
var endpoint = config.CodexEndpoint;
var splitIndex = endpoint.LastIndexOf(':');
var host = endpoint.Substring(0, splitIndex);
var port = Convert.ToInt32(endpoint.Substring(splitIndex + 1));
var address = new Address(
host: host,
port: port
);
var client = new HttpClient();
if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":"))
{
var tokens = config.CodexEndpointAuth.Split(':');
if (tokens.Length != 2) throw new Exception("Expected '<username>:<password>' in CodexEndpointAuth parameter.");
client.SetBasicAuthentication(tokens[0], tokens[1]);
}
var codex = new CodexApi(client);
codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
return codex;
}
#endregion
}
public class CheckResponse
{
public CheckResponse(bool success, string message, string error)
{
Success = success;
Message = message;
Error = error;
}
public bool Success { get; }
public string Message { get; }
public string Error { get; }
}
}

38
Tools/BiblioTech/Commands/CheckCidCommand.cs Normal file
View File

@ -0,0 +1,38 @@
using BiblioTech.Options;
namespace BiblioTech.Commands
{
public class CheckCidCommand : BaseCommand
{
private readonly StringOption cidOption = new StringOption(
name: "cid",
description: "Codex Content-Identifier",
isRequired: true);
private readonly CodexCidChecker checker;
public CheckCidCommand(CodexCidChecker checker)
{
this.checker = checker;
}
public override string Name => "check";
public override string StartingMessage => RandomBusyMessage.Get();
public override string Description => "Checks if content is available in the testnet.";
public override CommandOption[] Options => new[] { cidOption };
protected override async Task Invoke(CommandContext context)
{
var user = context.Command.User;
var cid = await cidOption.Parse(context);
if (string.IsNullOrEmpty(cid))
{
await context.Followup("Option 'cid' was not received.");
return;
}
var response = await checker.PerformCheck(cid);
await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'");
await context.Followup(response.Message);
}
}
}

6
Tools/BiblioTech/Configuration.cs
View File

@ -38,6 +38,12 @@ namespace BiblioTech
[Uniform("no-discord", "nd", "NODISCORD", false, "For debugging: Bypasses all Discord API calls.")]
public int NoDiscord { get; set; } = 0;
[Uniform("codex-endpoint", "ce", "CODEXENDPOINT", false, "Codex endpoint. (default 'http://localhost:8080')")]
public string CodexEndpoint { get; set; } = "http://localhost:8080";
[Uniform("codex-endpoint-auth", "cea", "CODEXENDPOINTAUTH", false, "Codex endpoint basic auth. Colon separated username and password. (default: empty, no auth used.)")]
public string CodexEndpointAuth { get; set; } = "";
public string EndpointsPath => Path.Combine(DataPath, "endpoints");
public string UserDataPath => Path.Combine(DataPath, "users");
public string LogPath => Path.Combine(DataPath, "logs");

3
Tools/BiblioTech/Program.cs
View File

@ -3,7 +3,6 @@ using BiblioTech.Commands;
using BiblioTech.Rewards;
using Discord;
using Discord.WebSocket;
using DiscordRewards;
using Logging;
namespace BiblioTech
@ -81,6 +80,7 @@ namespace BiblioTech
client = new DiscordSocketClient();
client.Log += ClientLog;
var checker = new CodexCidChecker(Config);
var notifyCommand = new NotifyCommand();
var associateCommand = new UserAssociateCommand(notifyCommand);
var sprCommand = new SprCommand();
@ -90,6 +90,7 @@ namespace BiblioTech
sprCommand,
associateCommand,
notifyCommand,
new CheckCidCommand(checker),
new AdminCommand(sprCommand, replacement)
);

1
Tools/BiblioTech/Rewards/ChainEventsSender.cs
View File

@ -1,7 +1,6 @@
using Discord.WebSocket;
using DiscordRewards;
using Logging;
using Microsoft.IdentityModel.Tokens;
namespace BiblioTech.Rewards
{