2024-03-25 14:46:45 +00:00
|
|
|
|
using CodexOpenApi;
|
|
|
|
|
using Core;
|
2023-09-11 09:59:33 +00:00
|
|
|
|
using KubernetesWorkflow;
|
2023-11-12 09:07:23 +00:00
|
|
|
|
using KubernetesWorkflow.Types;
|
2024-06-05 07:20:00 +00:00
|
|
|
|
using Logging;
|
2024-04-02 12:53:45 +00:00
|
|
|
|
using Newtonsoft.Json;
|
2023-10-19 09:18:59 +00:00
|
|
|
|
using Utils;
|
2023-04-12 11:53:55 +00:00
|
|
|
|
|
2023-09-11 09:59:33 +00:00
|
|
|
|
namespace CodexPlugin
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2024-06-19 08:39:14 +00:00
|
|
|
|
public class CodexAccess
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
private readonly ILog log;
|
2023-09-12 08:31:55 +00:00
|
|
|
|
private readonly IPluginTools tools;
|
2024-03-26 07:58:16 +00:00
|
|
|
|
private readonly Mapper mapper = new Mapper();
|
2023-04-30 08:08:32 +00:00
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
public CodexAccess(IPluginTools tools, RunningPod container, CrashWatcher crashWatcher)
|
2023-04-12 11:53:55 +00:00
|
|
|
|
{
|
2023-09-12 08:31:55 +00:00
|
|
|
|
this.tools = tools;
|
2024-06-12 08:48:52 +00:00
|
|
|
|
log = tools.GetLog();
|
2023-06-29 14:07:49 +00:00
|
|
|
|
Container = container;
|
2023-09-20 10:55:09 +00:00
|
|
|
|
CrashWatcher = crashWatcher;
|
2023-08-15 09:01:18 +00:00
|
|
|
|
|
2024-06-19 08:39:14 +00:00
|
|
|
|
CrashWatcher.Start();
|
2023-04-18 13:33:12 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-04-13 14:09:17 +00:00
|
|
|
|
public RunningPod Container { get; }
|
2023-09-20 10:55:09 +00:00
|
|
|
|
public CrashWatcher CrashWatcher { get; }
|
2023-05-11 10:44:53 +00:00
|
|
|
|
|
2024-03-26 07:58:16 +00:00
|
|
|
|
public DebugInfo GetDebugInfo()
|
2023-05-04 09:34:43 +00:00
|
|
|
|
{
|
2024-03-26 14:12:28 +00:00
|
|
|
|
return mapper.Map(OnCodex(api => api.GetDebugInfoAsync()));
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
2023-05-04 09:34:43 +00:00
|
|
|
|
|
2024-03-26 09:31:49 +00:00
|
|
|
|
public DebugPeer GetDebugPeer(string peerId)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-03-26 09:31:49 +00:00
|
|
|
|
// Cannot use openAPI: debug/peer endpoint is not specified there.
|
2024-06-14 07:05:56 +00:00
|
|
|
|
return CrashCheck(() =>
|
2023-05-04 09:34:43 +00:00
|
|
|
|
{
|
2024-06-14 07:05:56 +00:00
|
|
|
|
var endpoint = GetEndpoint();
|
|
|
|
|
var str = endpoint.HttpGetString($"debug/peer/{peerId}");
|
|
|
|
|
|
|
|
|
|
if (str.ToLowerInvariant() == "unable to find peer!")
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-06-14 07:05:56 +00:00
|
|
|
|
return new DebugPeer
|
|
|
|
|
{
|
|
|
|
|
IsPeerFound = false
|
|
|
|
|
};
|
|
|
|
|
}
|
2023-06-29 14:07:49 +00:00
|
|
|
|
|
2024-06-14 07:05:56 +00:00
|
|
|
|
var result = endpoint.Deserialize<DebugPeer>(str);
|
|
|
|
|
result.IsPeerFound = true;
|
|
|
|
|
return result;
|
|
|
|
|
});
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 11:14:02 +00:00
|
|
|
|
public void ConnectToPeer(string peerId, string[] peerMultiAddresses)
|
|
|
|
|
{
|
|
|
|
|
OnCodex(api =>
|
|
|
|
|
{
|
|
|
|
|
Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses));
|
|
|
|
|
return Task.FromResult(string.Empty);
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2024-06-06 07:54:50 +00:00
|
|
|
|
public string UploadFile(FileStream fileStream, Action<Failure> onFailure)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-06-12 13:28:08 +00:00
|
|
|
|
return OnCodex(
|
2024-06-05 07:20:00 +00:00
|
|
|
|
api => api.UploadAsync(fileStream),
|
2024-06-06 07:54:50 +00:00
|
|
|
|
CreateRetryConfig(nameof(UploadFile), onFailure));
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-06-06 07:54:50 +00:00
|
|
|
|
public Stream DownloadFile(string contentId, Action<Failure> onFailure)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-06-05 07:20:00 +00:00
|
|
|
|
var fileResponse = OnCodex(
|
|
|
|
|
api => api.DownloadNetworkAsync(contentId),
|
2024-06-06 07:54:50 +00:00
|
|
|
|
CreateRetryConfig(nameof(DownloadFile), onFailure));
|
2024-06-05 07:20:00 +00:00
|
|
|
|
|
2024-03-26 10:39:59 +00:00
|
|
|
|
if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode);
|
|
|
|
|
return fileResponse.Stream;
|
2023-11-10 07:20:08 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 13:07:06 +00:00
|
|
|
|
public LocalDatasetList LocalFiles()
|
2023-11-10 07:20:08 +00:00
|
|
|
|
{
|
2024-03-26 14:12:28 +00:00
|
|
|
|
return mapper.Map(OnCodex(api => api.ListDataAsync()));
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 10:39:59 +00:00
|
|
|
|
public StorageAvailability SalesAvailability(StorageAvailability request)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-03-26 14:12:28 +00:00
|
|
|
|
var body = mapper.Map(request);
|
2024-03-26 10:39:59 +00:00
|
|
|
|
var read = OnCodex<SalesAvailabilityREAD>(api => api.OfferStorageAsync(body));
|
2024-03-26 14:12:28 +00:00
|
|
|
|
return mapper.Map(read);
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-09-23 08:52:12 +00:00
|
|
|
|
public StorageAvailability[] GetAvailabilities()
|
|
|
|
|
{
|
2024-09-24 11:06:00 +00:00
|
|
|
|
var collection = OnCodex<ICollection<SalesAvailabilityREAD>>(api => api.GetAvailabilitiesAsync());
|
2024-09-23 08:52:12 +00:00
|
|
|
|
return mapper.Map(collection);
|
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 10:39:59 +00:00
|
|
|
|
public string RequestStorage(StoragePurchaseRequest request)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-03-26 14:12:28 +00:00
|
|
|
|
var body = mapper.Map(request);
|
2024-03-26 13:07:06 +00:00
|
|
|
|
return OnCodex<string>(api => api.CreateStorageRequestAsync(request.ContentId.Id, body));
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-06-05 07:20:00 +00:00
|
|
|
|
public CodexSpace Space()
|
|
|
|
|
{
|
|
|
|
|
var space = OnCodex<Space>(api => api.SpaceAsync());
|
|
|
|
|
return mapper.Map(space);
|
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 11:23:38 +00:00
|
|
|
|
public StoragePurchase GetPurchaseStatus(string purchaseId)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-06-14 07:05:56 +00:00
|
|
|
|
return CrashCheck(() =>
|
2024-04-02 12:53:45 +00:00
|
|
|
|
{
|
2024-06-14 07:05:56 +00:00
|
|
|
|
var endpoint = GetEndpoint();
|
|
|
|
|
return Time.Retry(() =>
|
|
|
|
|
{
|
|
|
|
|
var str = endpoint.HttpGetString($"storage/purchases/{purchaseId}");
|
|
|
|
|
if (string.IsNullOrEmpty(str)) throw new Exception("Empty response.");
|
|
|
|
|
return JsonConvert.DeserializeObject<StoragePurchase>(str)!;
|
|
|
|
|
}, nameof(GetPurchaseStatus));
|
|
|
|
|
});
|
2024-04-02 12:53:45 +00:00
|
|
|
|
|
|
|
|
|
// TODO: current getpurchase api does not line up with its openapi spec.
|
|
|
|
|
// return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId)));
|
2023-06-29 14:07:49 +00:00
|
|
|
|
}
|
|
|
|
|
|
2023-08-24 09:32:32 +00:00
|
|
|
|
public string GetName()
|
|
|
|
|
{
|
|
|
|
|
return Container.Name;
|
|
|
|
|
}
|
|
|
|
|
|
2023-11-06 13:33:47 +00:00
|
|
|
|
public PodInfo GetPodInfo()
|
|
|
|
|
{
|
|
|
|
|
var workflow = tools.CreateWorkflow();
|
|
|
|
|
return workflow.GetPodInfo(Container);
|
|
|
|
|
}
|
|
|
|
|
|
2024-06-13 06:51:52 +00:00
|
|
|
|
public void DeleteRepoFolder()
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var containerNumber = Container.Containers.First().Recipe.Number;
|
|
|
|
|
var dataDir = $"datadir{containerNumber}";
|
|
|
|
|
var workflow = tools.CreateWorkflow();
|
|
|
|
|
workflow.ExecuteCommand(Container.Containers.First(), "rm", "-Rfv", $"/codex/{dataDir}/repo");
|
|
|
|
|
Log("Deleted repo folder.");
|
|
|
|
|
}
|
|
|
|
|
catch (Exception e)
|
|
|
|
|
{
|
|
|
|
|
Log("Unable to delete repo folder: " + e);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2024-03-25 14:46:45 +00:00
|
|
|
|
private T OnCodex<T>(Func<CodexApi, Task<T>> action)
|
2023-06-29 14:07:49 +00:00
|
|
|
|
{
|
2024-07-25 08:10:11 +00:00
|
|
|
|
var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action));
|
2024-06-05 07:20:00 +00:00
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private T OnCodex<T>(Func<CodexApi, Task<T>> action, Retry retry)
|
|
|
|
|
{
|
2024-07-25 08:10:11 +00:00
|
|
|
|
var result = tools.CreateHttp(GetHttpId(), CheckContainerCrashed).OnClient(client => CallCodex(client, action), retry);
|
2024-03-25 14:46:45 +00:00
|
|
|
|
return result;
|
|
|
|
|
}
|
2024-03-25 10:37:41 +00:00
|
|
|
|
|
2024-06-05 07:20:00 +00:00
|
|
|
|
private T CallCodex<T>(HttpClient client, Func<CodexApi, Task<T>> action)
|
|
|
|
|
{
|
|
|
|
|
var address = GetAddress();
|
|
|
|
|
var api = new CodexApi(client);
|
|
|
|
|
api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1";
|
2024-06-14 07:05:56 +00:00
|
|
|
|
return CrashCheck(() => Time.Wait(action(api)));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private T CrashCheck<T>(Func<T> action)
|
|
|
|
|
{
|
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
return action();
|
|
|
|
|
}
|
|
|
|
|
finally
|
|
|
|
|
{
|
|
|
|
|
CrashWatcher.HasContainerCrashed();
|
|
|
|
|
}
|
2024-06-05 07:20:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-03-26 09:31:49 +00:00
|
|
|
|
private IEndpoint GetEndpoint()
|
|
|
|
|
{
|
|
|
|
|
return tools
|
2024-07-25 08:10:11 +00:00
|
|
|
|
.CreateHttp(GetHttpId(), CheckContainerCrashed)
|
2024-03-26 09:31:49 +00:00
|
|
|
|
.CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name);
|
|
|
|
|
}
|
|
|
|
|
|
2023-10-19 09:18:59 +00:00
|
|
|
|
private Address GetAddress()
|
|
|
|
|
{
|
2024-09-23 08:52:12 +00:00
|
|
|
|
return Container.Containers.Single().GetAddress(CodexContainerRecipe.ApiPortTag);
|
2023-10-10 16:08:21 +00:00
|
|
|
|
}
|
|
|
|
|
|
2024-07-25 08:10:11 +00:00
|
|
|
|
private string GetHttpId()
|
|
|
|
|
{
|
|
|
|
|
return GetAddress().ToString();
|
|
|
|
|
}
|
|
|
|
|
|
2023-08-29 08:09:48 +00:00
|
|
|
|
private void CheckContainerCrashed(HttpClient client)
|
2023-08-15 09:01:18 +00:00
|
|
|
|
{
|
2024-06-19 08:39:14 +00:00
|
|
|
|
if (CrashWatcher.HasContainerCrashed()) throw new Exception($"Container {GetName()} has crashed.");
|
2023-08-15 09:01:18 +00:00
|
|
|
|
}
|
2024-06-05 07:20:00 +00:00
|
|
|
|
|
2024-06-06 07:54:50 +00:00
|
|
|
|
private Retry CreateRetryConfig(string description, Action<Failure> onFailure)
|
2024-06-05 07:20:00 +00:00
|
|
|
|
{
|
|
|
|
|
var timeSet = tools.TimeSet;
|
|
|
|
|
|
|
|
|
|
return new Retry(description, timeSet.HttpRetryTimeout(), timeSet.HttpCallRetryDelay(), failure =>
|
|
|
|
|
{
|
2024-06-06 07:54:50 +00:00
|
|
|
|
onFailure(failure);
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Investigate(failure, timeSet);
|
2024-06-05 07:20:00 +00:00
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
2024-06-12 08:48:52 +00:00
|
|
|
|
private void Investigate(Failure failure, ITimeSet timeSet)
|
2024-06-05 07:20:00 +00:00
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Log($"Retry {failure.TryNumber} took {Time.FormatDuration(failure.Duration)} and failed with '{failure.Exception}'. " +
|
2024-06-07 15:07:35 +00:00
|
|
|
|
$"(HTTP timeout = {Time.FormatDuration(timeSet.HttpCallTimeout())}) " +
|
|
|
|
|
$"Checking if node responds to debug/info...");
|
|
|
|
|
|
2024-06-05 07:20:00 +00:00
|
|
|
|
try
|
|
|
|
|
{
|
|
|
|
|
var debugInfo = GetDebugInfo();
|
|
|
|
|
if (string.IsNullOrEmpty(debugInfo.Spr))
|
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Log("Did not get value debug/info response.");
|
2024-06-05 07:20:00 +00:00
|
|
|
|
Throw(failure);
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Log("Got valid response from debug/info.");
|
2024-06-05 07:20:00 +00:00
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
catch (Exception ex)
|
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Log("Got exception from debug/info call: " + ex);
|
2024-06-05 07:20:00 +00:00
|
|
|
|
Throw(failure);
|
|
|
|
|
}
|
2024-06-07 15:07:35 +00:00
|
|
|
|
|
|
|
|
|
if (failure.Duration < timeSet.HttpCallTimeout())
|
|
|
|
|
{
|
2024-06-12 08:48:52 +00:00
|
|
|
|
Log("Retry failed within HTTP timeout duration.");
|
2024-06-07 15:07:35 +00:00
|
|
|
|
Throw(failure);
|
|
|
|
|
}
|
2024-06-05 07:20:00 +00:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private void Throw(Failure failure)
|
|
|
|
|
{
|
|
|
|
|
throw failure.Exception;
|
|
|
|
|
}
|
|
|
|
|
|
2024-06-12 08:48:52 +00:00
|
|
|
|
private void Log(string msg)
|
|
|
|
|
{
|
2024-06-12 13:28:08 +00:00
|
|
|
|
log.Log($"{GetName()} {msg}");
|
2024-06-12 08:48:52 +00:00
|
|
|
|
}
|
2023-04-18 13:33:12 +00:00
|
|
|
|
}
|
2023-04-12 11:53:55 +00:00
|
|
|
|
}
|