codex-storage
/
cs-codex-dist-tests
mirror of https://github.com/codex-storage/cs-codex-dist-tests.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

sets up http client

This commit is contained in:
Ben 2024-03-25 15:46:45 +01:00
parent 03a3ccb4de
commit c122aa9910
No known key found for this signature in database
GPG Key ID: 541B9D8C9F1426A1
7 changed files with 140 additions and 288 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
Framework/Core/Http.cs
View File

@ -9,14 +9,16 @@ namespace Core
{
public interface IHttp
{
string HttpGetString(string route);
T HttpGetJson<T>(string route);
TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
string HttpPostJson<TRequest>(string route, TRequest body);
TResponse HttpPostString<TResponse>(string route, string body);
string HttpPostStream(string route, Stream stream);
Stream HttpGetStream(string route);
T Deserialize<T>(string json);
//string HttpGetString(string route);
//T HttpGetJson<T>(string route);
//TResponse HttpPostJson<TRequest, TResponse>(string route, TRequest body);
//string HttpPostJson<TRequest>(string route, TRequest body);
//TResponse HttpPostString<TResponse>(string route, string body);
//string HttpPostStream(string route, Stream stream);
//Stream HttpGetStream(string route);
//T Deserialize<T>(string json);
T OnClient<T>(Func<HttpClient, T> action);
}
internal class Http : IHttp
@ -24,26 +26,37 @@ namespace Core
private static readonly object httpLock = new object();
private readonly ILog log;
private readonly ITimeSet timeSet;
private readonly Address address;
private readonly string baseUrl;
private readonly Action<HttpClient> onClientCreated;
private readonly string? logAlias;
internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null)
: this(log, timeSet, address, baseUrl, DoNothing, logAlias)
internal Http(ILog log, ITimeSet timeSet, string? logAlias = null)
: this(log, timeSet, DoNothing, logAlias)
{
}
internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
internal Http(ILog log, ITimeSet timeSet, Action<HttpClient> onClientCreated, string? logAlias = null)
{
this.log = log;
this.timeSet = timeSet;
this.address = address;
this.baseUrl = baseUrl;
this.onClientCreated = onClientCreated;
this.logAlias = logAlias;
if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl;
if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/";
}
public T OnClient<T>(Func<HttpClient, T> action)
{
var client = GetClient();
var description = GetDescription();
return LockRetry(() =>
{
return action(client);
}, description);
}
private string GetDescription()
{
// todo: check this:
return DebugStack.GetCallerName(skipFrames: 2);
}
public string HttpGetString(string route)
@ -190,7 +203,8 @@ namespace Core
private string GetUrl()
{
return $"{address.Host}:{address.Port}{baseUrl}";
//return $"{address.Host}:{address.Port}{baseUrl}";
return "--Obsolete--";
}
private void Log(string url, string message)

18
Framework/Core/PluginTools.cs
View File

@ -22,9 +22,9 @@ namespace Core
public interface IHttpFactoryTool
{
IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null);
IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null);
IHttp CreateHttp(Action<HttpClient> onClientCreated, string? logAlias = null);
IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet timeSet, string? logAlias = null);
IHttp CreateHttp(string? logAlias = null);
}
public interface IFileTool
@ -52,19 +52,19 @@ namespace Core
log = new LogPrefixer(log, prefix);
}
public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, string? logAlias = null)
public IHttp CreateHttp(Action<HttpClient> onClientCreated, string? logAlias = null)
{
return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias);
return CreateHttp(onClientCreated, timeSet, logAlias);
}
public IHttp CreateHttp(Address address, string baseUrl, Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
public IHttp CreateHttp(Action<HttpClient> onClientCreated, ITimeSet ts, string? logAlias = null)
{
return new Http(log, ts, address, baseUrl, onClientCreated, logAlias);
return new Http(log, ts, onClientCreated, logAlias);
}
public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null)
public IHttp CreateHttp(string? logAlias = null)
{
return new Http(log, timeSet, address, baseUrl, logAlias);
return new Http(log, timeSet, logAlias);
}
public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null)

57
ProjectPlugins/CodexPlugin/CodexAccess.cs
View File

@ -1,4 +1,5 @@
using Core;
using CodexOpenApi;
using Core;
using KubernetesWorkflow;
using KubernetesWorkflow.Types;
using Utils;
@ -23,9 +24,9 @@ namespace CodexPlugin
public RunningContainer Container { get; }
public CrashWatcher CrashWatcher { get; }
public CodexDebugResponse GetDebugInfo()
public CodexOpenApi.DebugInfo GetDebugInfo()
{
return Http().HttpGetJson<CodexDebugResponse>("debug/info");
return OnCodex(api => api.GetDebugInfoAsync());
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
@ -107,29 +108,41 @@ namespace CodexPlugin
return workflow.GetPodInfo(Container);
}
private IHttp Http()
private T OnCodex<T>(Func<CodexApi, Task<T>> action)
{
var address = GetAddress();
var api = new CodexOpenApi.CodexApi(new HttpClient());
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
var debugInfo = Time.Wait(api.GetDebugInfoAsync());
using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png");
var cid = Time.Wait(api.UploadAsync(stream));
var file = Time.Wait(api.DownloadNetworkAsync(cid));
while (file.IsPartial) Thread.Sleep(100);
using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png");
file.Stream.CopyTo(outfile);
return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
var result = tools.CreateHttp().OnClient(client =>
{
var api = new CodexApi(client);
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
return Time.Wait(action(api));
});
return result;
}
private IHttp LongHttp()
{
return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
}
//private IHttp Http()
//{
// var address = GetAddress();
// var api = new CodexOpenApi.CodexApi(new HttpClient());
// api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
// var debugInfo = Time.Wait(api.GetDebugInfoAsync());
// using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png");
// var cid = Time.Wait(api.UploadAsync(stream));
// var file = Time.Wait(api.DownloadNetworkAsync(cid));
// while (file.IsPartial) Thread.Sleep(100);
// using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png");
// file.Stream.CopyTo(outfile);
// return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
//}
//private IHttp LongHttp()
//{
// return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
//}
private Address GetAddress()
{

196
ProjectPlugins/CodexPlugin/CodexApiTypes.cs
View File

@ -1,196 +0,0 @@
using Newtonsoft.Json;
namespace CodexPlugin
{
public class CodexDebugResponse
{
public string id { get; set; } = string.Empty;
public string[] addrs { get; set; } = Array.Empty<string>();
public string repo { get; set; } = string.Empty;
public string spr { get; set; } = string.Empty;
public string[] announceAddresses { get; set; } = Array.Empty<string>();
public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty<EnginePeerResponse>();
public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty<SwitchPeerResponse>();
public CodexDebugVersionResponse codex { get; set; } = new();
public CodexDebugTableResponse table { get; set; } = new();
}
public class CodexDebugFutures
{
public int futures { get; set; }
}
public class CodexDebugTableResponse
{
public CodexDebugTableNodeResponse localNode { get; set; } = new();
public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty<CodexDebugTableNodeResponse>();
}
public class CodexDebugTableNodeResponse
{
public string nodeId { get; set; } = string.Empty;
public string peerId { get; set; } = string.Empty;
public string record { get; set; } = string.Empty;
public string address { get; set; } = string.Empty;
public bool seen { get; set; }
}
public class EnginePeerResponse
{
public string peerId { get; set; } = string.Empty;
public EnginePeerContextResponse context { get; set; } = new();
}
public class EnginePeerContextResponse
{
public int blocks { get; set; } = 0;
public int peerWants { get; set; } = 0;
public int exchanged { get; set; } = 0;
public string lastExchange { get; set; } = string.Empty;
}
public class SwitchPeerResponse
{
public string peerId { get; set; } = string.Empty;
public string key { get; set; } = string.Empty;
}
public class CodexDebugVersionResponse
{
public string version { get; set; } = string.Empty;
public string revision { get; set; } = string.Empty;
public bool IsValid()
{
return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision);
}
public override string ToString()
{
return JsonConvert.SerializeObject(this);
}
}
public class CodexDebugPeerResponse
{
public bool IsPeerFound { get; set; }
public string peerId { get; set; } = string.Empty;
public long seqNo { get; set; }
public string[] addresses { get; set; } = Array.Empty<string>();
}
public class CodexDebugThresholdBreaches
{
public string[] breaches { get; set; } = Array.Empty<string>();
}
public class CodexSalesAvailabilityRequest
{
public string totalSize { get; set; } = string.Empty;
public string duration { get; set; } = string.Empty;
public string minPrice { get; set; } = string.Empty;
public string maxCollateral { get; set; } = string.Empty;
}
public class CodexSalesAvailabilityResponse
{
public string id { get; set; } = string.Empty;
public string totalSize { get; set; } = string.Empty;
public string freeSize { get; set; } = string.Empty;
public string duration { get; set; } = string.Empty;
public string minPrice { get; set; } = string.Empty;
public string maxCollateral { get; set; } = string.Empty;
}
public class CodexSalesRequestStorageRequest
{
public string duration { get; set; } = string.Empty;
public string proofProbability { get; set; } = string.Empty;
public string reward { get; set; } = string.Empty;
public string collateral { get; set; } = string.Empty;
public string? expiry { get; set; }
public uint? nodes { get; set; }
public uint? tolerance { get; set; }
}
public class CodexStoragePurchase
{
public string state { get; set; } = string.Empty;
public string error { get; set; } = string.Empty;
}
public class CodexDebugBlockExchangeResponse
{
public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeer>();
public int taskQueue { get; set; }
public int pendingBlocks { get; set; }
public override string ToString()
{
if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty";
return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}";
}
}
public class CodexDebugBlockExchangeResponsePeer
{
public string peerid { get; set; } = string.Empty;
public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerHasBlock>();
public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty<CodexDebugBlockExchangeResponsePeerWant>();
public int exchanged { get; set; }
public override string ToString()
{
return $"(blocks:{hasBlocks.Length} wants:{wants.Length})";
}
}
public class CodexDebugBlockExchangeResponsePeerHasBlock
{
public string cid { get; set; } = string.Empty;
public bool have { get; set; }
public string price { get; set; } = string.Empty;
}
public class CodexDebugBlockExchangeResponsePeerWant
{
public string block { get; set; } = string.Empty;
public int priority { get; set; }
public bool cancel { get; set; }
public string wantType { get; set; } = string.Empty;
public bool sendDontHave { get; set; }
}
public class CodexDebugRepoStoreResponse
{
public string cid { get; set; } = string.Empty;
}
public class CodexLocalData
{
public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest)
{
Cid = cid;
Manifest = manifest;
}
public ContentId Cid { get; }
public CodexLocalDataManifestResponse Manifest { get; }
}
public class CodexLocalDataResponse
{
public string cid { get; set; } = string.Empty;
public CodexLocalDataManifestResponse manifest { get; set; } = new();
}
public class CodexLocalDataManifestResponse
{
public string rootHash { get; set; } = string.Empty;
public int originalBytes { get; set; }
public int blockSize { get; set; }
public bool @protected { get; set; }
}
}

58
ProjectPlugins/CodexPlugin/CodexNode.cs
View File

@ -12,13 +12,13 @@ namespace CodexPlugin
public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress
{
string GetName();
CodexDebugResponse GetDebugInfo();
CodexDebugPeerResponse GetDebugPeer(string peerId);
DebugInfo GetDebugInfo();
DebugPeer GetDebugPeer(string peerId);
ContentId UploadFile(TrackedFile file);
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
CodexLocalData[] LocalFiles();
LocalDataset[] LocalFiles();
void ConnectToPeer(ICodexNode node);
CodexDebugVersionResponse Version { get; }
DebugVersion Version { get; }
IMarketplaceAccess Marketplace { get; }
CrashWatcher CrashWatcher { get; }
PodInfo GetPodInfo();
@ -41,7 +41,7 @@ namespace CodexPlugin
CodexAccess = codexAccess;
Group = group;
Marketplace = marketplaceAccess;
Version = new CodexDebugVersionResponse();
Version = new DebugVersion();
transferSpeeds = new TransferSpeeds();
}
@ -50,7 +50,7 @@ namespace CodexPlugin
public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; }
public CodexNodeGroup Group { get; }
public IMarketplaceAccess Marketplace { get; }
public CodexDebugVersionResponse Version { get; private set; }
public DebugVersion Version { get; private set; }
public ITransferSpeeds TransferSpeeds { get => transferSpeeds; }
public IMetricsScrapeTarget MetricsScrapeTarget
{
@ -73,7 +73,7 @@ namespace CodexPlugin
return CodexAccess.Container.Name;
}
public CodexDebugResponse GetDebugInfo()
public DebugInfo GetDebugInfo()
{
var debugInfo = CodexAccess.GetDebugInfo();
var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId));
@ -81,21 +81,11 @@ namespace CodexPlugin
return debugInfo;
}
public CodexDebugPeerResponse GetDebugPeer(string peerId)
public DebugPeer GetDebugPeer(string peerId)
{
return CodexAccess.GetDebugPeer(peerId);
}
public CodexDebugBlockExchangeResponse GetDebugBlockExchange()
{
return CodexAccess.GetDebugBlockExchange();
}
public CodexDebugRepoStoreResponse[] GetDebugRepoStore()
{
return CodexAccess.GetDebugRepoStore();
}
public ContentId UploadFile(TrackedFile file)
{
using var fileStream = File.OpenRead(file.Filename);
@ -128,9 +118,13 @@ namespace CodexPlugin
return file;
}
public CodexLocalData[] LocalFiles()
public LocalDataset[] LocalFiles()
{
return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray();
return CodexAccess.LocalFiles().Select(l => new LocalDataset()
{
Cid = new ContentId(l.cid)
//, l.manifest
}).ToArray();
}
public void ConnectToPeer(ICodexNode node)
@ -176,9 +170,9 @@ namespace CodexPlugin
Version = debugInfo.codex;
}
private string GetPeerMultiAddress(CodexNode peer, CodexDebugResponse peerInfo)
private string GetPeerMultiAddress(CodexNode peer, DebugInfo peerInfo)
{
var multiAddress = peerInfo.addrs.First();
var multiAddress = peerInfo.Addrs.First();
// Todo: Is there a case where First address in list is not the way?
// The peer we want to connect is in a different pod.
@ -209,24 +203,4 @@ namespace CodexPlugin
tools.GetLog().Log($"{GetName()}: {msg}");
}
}
public class ContentId
{
public ContentId(string id)
{
Id = id;
}
public string Id { get; }
public override bool Equals(object? obj)
{
return obj is ContentId id && Id == id.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
}
}

2
ProjectPlugins/CodexPlugin/CodexPlugin.cs
View File

@ -9,7 +9,7 @@ namespace CodexPlugin
private readonly IPluginTools tools;
private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace;
private const string OpenApiYamlHash = "<CODEX_OPENAPI_HASH_HERE>";
private const string OpenApiYamlHash = "8B-DD-61-54-42-D7-28-8F-5A-A0-AF-C2-A4-53-A7-08-B6-C7-02-FD-59-1A-01-A9-B4-7D-E4-81-FA-84-23-7F";
public CodexPlugin(IPluginTools tools)
{

47
ProjectPlugins/CodexPlugin/CodexTypes.cs Normal file
View File

@ -0,0 +1,47 @@
namespace CodexPlugin
{
public class DebugInfo
{
public string[] Addrs { get; set; } = Array.Empty<string>();
}
public class DebugPeer
{
}
public class LocalDataset
{
public ContentId Cid { get; set; } = new ContentId();
}
public class DebugVersion
{
}
public class ContentId
{
public ContentId()
{
Id = string.Empty;
}
public ContentId(string id)
{
Id = id;
}
public string Id { get; }
public override bool Equals(object? obj)
{
return obj is ContentId id && Id == id.Id;
}
public override int GetHashCode()
{
return HashCode.Combine(Id);
}
}
}