Merge branch 'master' into app/discord-bot
This commit is contained in:
commit
ad70394333
|
@ -196,7 +196,7 @@ namespace Core
|
||||||
|
|
||||||
private T Retry<T>(Func<T> operation, string description)
|
private T Retry<T>(Func<T> operation, string description)
|
||||||
{
|
{
|
||||||
return Time.Retry(operation, timeSet.HttpCallRetryTime(), timeSet.HttpCallRetryDelay(), description);
|
return Time.Retry(operation, timeSet.HttpMaxNumberOfRetries(), timeSet.HttpCallRetryDelay(), description);
|
||||||
}
|
}
|
||||||
|
|
||||||
private HttpClient GetClient()
|
private HttpClient GetClient()
|
||||||
|
|
|
@ -5,11 +5,9 @@ namespace Core
|
||||||
public static class SerializeGate
|
public static class SerializeGate
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// SerializeGate was added to help ensure deployment objects are serializable
|
/// SerializeGate was added to help ensure deployment objects are serializable and remain viable after deserialization.
|
||||||
/// and remain viable after deserialization.
|
|
||||||
/// Tools can be built on top of the core interface that rely on deployment objects being serializable.
|
/// Tools can be built on top of the core interface that rely on deployment objects being serializable.
|
||||||
/// Insert the serialization gate after deployment but before wrapping to ensure any future changes
|
/// Insert the serialization gate after deployment but before wrapping to ensure any future changes don't break this requirement.
|
||||||
/// don't break this requirement.
|
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public static T Gate<T>(T anything)
|
public static T Gate<T>(T anything)
|
||||||
{
|
{
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
public interface ITimeSet
|
public interface ITimeSet
|
||||||
{
|
{
|
||||||
TimeSpan HttpCallTimeout();
|
TimeSpan HttpCallTimeout();
|
||||||
TimeSpan HttpCallRetryTime();
|
int HttpMaxNumberOfRetries();
|
||||||
TimeSpan HttpCallRetryDelay();
|
TimeSpan HttpCallRetryDelay();
|
||||||
TimeSpan WaitForK8sServiceDelay();
|
TimeSpan WaitForK8sServiceDelay();
|
||||||
TimeSpan K8sOperationTimeout();
|
TimeSpan K8sOperationTimeout();
|
||||||
|
@ -13,12 +13,12 @@
|
||||||
{
|
{
|
||||||
public TimeSpan HttpCallTimeout()
|
public TimeSpan HttpCallTimeout()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromMinutes(5);
|
return TimeSpan.FromMinutes(3);
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryTime()
|
public int HttpMaxNumberOfRetries()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromMinutes(1);
|
return 3;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryDelay()
|
public TimeSpan HttpCallRetryDelay()
|
||||||
|
@ -44,9 +44,9 @@
|
||||||
return TimeSpan.FromHours(2);
|
return TimeSpan.FromHours(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryTime()
|
public int HttpMaxNumberOfRetries()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromHours(5);
|
return 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryDelay()
|
public TimeSpan HttpCallRetryDelay()
|
||||||
|
|
|
@ -24,6 +24,8 @@
|
||||||
{
|
{
|
||||||
Name = $"ctnr{Number}";
|
Name = $"ctnr{Number}";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (exposedPorts.Any(p => string.IsNullOrEmpty(p.Tag))) throw new Exception("Port tags are required for all exposed ports.");
|
||||||
}
|
}
|
||||||
|
|
||||||
public string Name { get; }
|
public string Name { get; }
|
||||||
|
@ -65,6 +67,12 @@
|
||||||
|
|
||||||
public int Number { get; }
|
public int Number { get; }
|
||||||
public string Tag { get; }
|
public string Tag { get; }
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
if (string.IsNullOrEmpty(Tag)) return $"untagged-port={Number}";
|
||||||
|
return $"{Tag}={Number}";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public class EnvVar
|
public class EnvVar
|
||||||
|
|
|
@ -50,12 +50,12 @@ namespace KubernetesWorkflow
|
||||||
protected int Index { get; private set; } = 0;
|
protected int Index { get; private set; } = 0;
|
||||||
protected abstract void Initialize(StartupConfig config);
|
protected abstract void Initialize(StartupConfig config);
|
||||||
|
|
||||||
protected Port AddExposedPort(string tag = "")
|
protected Port AddExposedPort(string tag)
|
||||||
{
|
{
|
||||||
return AddExposedPort(factory.CreatePort(tag));
|
return AddExposedPort(factory.CreatePort(tag));
|
||||||
}
|
}
|
||||||
|
|
||||||
protected Port AddExposedPort(int number, string tag = "")
|
protected Port AddExposedPort(int number, string tag)
|
||||||
{
|
{
|
||||||
return AddExposedPort(factory.CreatePort(number, tag));
|
return AddExposedPort(factory.CreatePort(number, tag));
|
||||||
}
|
}
|
||||||
|
@ -67,7 +67,7 @@ namespace KubernetesWorkflow
|
||||||
return p;
|
return p;
|
||||||
}
|
}
|
||||||
|
|
||||||
protected void AddExposedPortAndVar(string name, string tag = "")
|
protected void AddExposedPortAndVar(string name, string tag)
|
||||||
{
|
{
|
||||||
AddEnvVar(name, AddExposedPort(tag));
|
AddEnvVar(name, AddExposedPort(tag));
|
||||||
}
|
}
|
||||||
|
@ -132,11 +132,6 @@ namespace KubernetesWorkflow
|
||||||
|
|
||||||
private Port AddExposedPort(Port port)
|
private Port AddExposedPort(Port port)
|
||||||
{
|
{
|
||||||
if (exposedPorts.Any())
|
|
||||||
{
|
|
||||||
throw new NotImplementedException("Current implementation only support 1 exposed port per container recipe. " +
|
|
||||||
$"Methods for determining container addresses in {nameof(StartupWorkflow)} currently rely on this constraint.");
|
|
||||||
}
|
|
||||||
exposedPorts.Add(port);
|
exposedPorts.Add(port);
|
||||||
return port;
|
return port;
|
||||||
}
|
}
|
||||||
|
|
|
@ -572,16 +572,15 @@ namespace KubernetesWorkflow
|
||||||
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
|
var readback = client.Run(c => c.ReadNamespacedService(serviceSpec.Metadata.Name, K8sNamespace));
|
||||||
foreach (var r in containerRecipes)
|
foreach (var r in containerRecipes)
|
||||||
{
|
{
|
||||||
if (r.ExposedPorts.Any())
|
foreach (var port in r.ExposedPorts)
|
||||||
{
|
{
|
||||||
var firstExposedPort = r.ExposedPorts.First();
|
var portName = GetNameForPort(r, port);
|
||||||
var portName = GetNameForPort(r, firstExposedPort);
|
|
||||||
|
|
||||||
var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
|
var matchingServicePorts = readback.Spec.Ports.Where(p => p.Name == portName);
|
||||||
if (matchingServicePorts.Any())
|
if (matchingServicePorts.Any())
|
||||||
{
|
{
|
||||||
// These service ports belongs to this recipe.
|
// These service ports belongs to this recipe.
|
||||||
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, portName));
|
var optionals = matchingServicePorts.Select(p => MapNodePortIfAble(p, port.Tag));
|
||||||
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
|
var ports = optionals.Where(p => p != null).Select(p => p!).ToArray();
|
||||||
|
|
||||||
result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
|
result.Add(new ContainerRecipePortMapEntry(r.Number, ports));
|
||||||
|
|
|
@ -16,18 +16,26 @@ namespace KubernetesWorkflow
|
||||||
internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
|
internal static RunnerLocation DetermineRunnerLocation(RunningContainer container)
|
||||||
{
|
{
|
||||||
if (knownLocation != null) return knownLocation.Value;
|
if (knownLocation != null) return knownLocation.Value;
|
||||||
|
knownLocation = PingForLocation(container);
|
||||||
|
return knownLocation.Value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static RunnerLocation PingForLocation(RunningContainer container)
|
||||||
|
{
|
||||||
if (PingHost(container.Pod.PodInfo.Ip))
|
if (PingHost(container.Pod.PodInfo.Ip))
|
||||||
{
|
{
|
||||||
knownLocation = RunnerLocation.InternalToCluster;
|
return RunnerLocation.InternalToCluster;
|
||||||
}
|
|
||||||
else if (PingHost(Format(container.ClusterExternalAddress)))
|
|
||||||
{
|
|
||||||
knownLocation = RunnerLocation.ExternalToCluster;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (knownLocation == null) throw new Exception("Unable to determine location relative to kubernetes cluster.");
|
foreach (var port in container.ContainerPorts)
|
||||||
return knownLocation.Value;
|
{
|
||||||
|
if (PingHost(Format(port.ExternalAddress)))
|
||||||
|
{
|
||||||
|
return RunnerLocation.ExternalToCluster;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Exception("Unable to determine location relative to kubernetes cluster.");
|
||||||
}
|
}
|
||||||
|
|
||||||
private static string Format(Address host)
|
private static string Format(Address host)
|
||||||
|
|
|
@ -24,37 +24,46 @@ namespace KubernetesWorkflow
|
||||||
|
|
||||||
public class RunningContainer
|
public class RunningContainer
|
||||||
{
|
{
|
||||||
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, Address clusterExternalAddress, Address clusterInternalAddress)
|
public RunningContainer(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts, string name, ContainerPort[] containerPorts)
|
||||||
{
|
{
|
||||||
Pod = pod;
|
Pod = pod;
|
||||||
Recipe = recipe;
|
Recipe = recipe;
|
||||||
ServicePorts = servicePorts;
|
ServicePorts = servicePorts;
|
||||||
Name = name;
|
Name = name;
|
||||||
ClusterExternalAddress = clusterExternalAddress;
|
ContainerPorts = containerPorts;
|
||||||
ClusterInternalAddress = clusterInternalAddress;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public string Name { get; }
|
public string Name { get; }
|
||||||
public RunningPod Pod { get; }
|
public RunningPod Pod { get; }
|
||||||
public ContainerRecipe Recipe { get; }
|
public ContainerRecipe Recipe { get; }
|
||||||
public Port[] ServicePorts { get; }
|
public Port[] ServicePorts { get; }
|
||||||
public Address ClusterExternalAddress { get; }
|
public ContainerPort[] ContainerPorts { get; }
|
||||||
public Address ClusterInternalAddress { get; }
|
|
||||||
|
|
||||||
[JsonIgnore]
|
public Address GetAddress(string portTag)
|
||||||
public Address Address
|
|
||||||
{
|
{
|
||||||
get
|
var containerPort = ContainerPorts.Single(c => c.Port.Tag == portTag);
|
||||||
|
if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
|
||||||
{
|
{
|
||||||
if (RunnerLocationUtils.DetermineRunnerLocation(this) == RunnerLocation.InternalToCluster)
|
return containerPort.InternalAddress;
|
||||||
{
|
|
||||||
return ClusterInternalAddress;
|
|
||||||
}
|
|
||||||
return ClusterExternalAddress;
|
|
||||||
}
|
}
|
||||||
|
return containerPort.ExternalAddress;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public class ContainerPort
|
||||||
|
{
|
||||||
|
public ContainerPort(Port port, Address externalAddress, Address internalAddress)
|
||||||
|
{
|
||||||
|
Port = port;
|
||||||
|
ExternalAddress = externalAddress;
|
||||||
|
InternalAddress = internalAddress;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Port Port { get; }
|
||||||
|
public Address ExternalAddress { get; }
|
||||||
|
public Address InternalAddress { get; }
|
||||||
|
}
|
||||||
|
|
||||||
public static class RunningContainersExtensions
|
public static class RunningContainersExtensions
|
||||||
{
|
{
|
||||||
public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
|
public static RunningContainer[] Containers(this RunningContainers[] runningContainers)
|
||||||
|
|
|
@ -19,12 +19,10 @@
|
||||||
|
|
||||||
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
|
public Port[] GetServicePortsForContainerRecipe(ContainerRecipe containerRecipe)
|
||||||
{
|
{
|
||||||
if (PortMapEntries.Any(p => p.ContainerNumber == containerRecipe.Number))
|
return PortMapEntries
|
||||||
{
|
.Where(p => p.ContainerNumber == containerRecipe.Number)
|
||||||
return PortMapEntries.Single(p => p.ContainerNumber == containerRecipe.Number).Ports;
|
.SelectMany(p => p.Ports)
|
||||||
}
|
.ToArray();
|
||||||
|
|
||||||
return Array.Empty<Port>();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -118,8 +118,7 @@ namespace KubernetesWorkflow
|
||||||
var name = GetContainerName(r, startupConfig);
|
var name = GetContainerName(r, startupConfig);
|
||||||
|
|
||||||
return new RunningContainer(runningPod, r, servicePorts, name,
|
return new RunningContainer(runningPod, r, servicePorts, name,
|
||||||
GetContainerExternalAddress(runningPod, servicePorts),
|
CreateContainerPorts(runningPod, r, servicePorts));
|
||||||
GetContainerInternalAddress(r));
|
|
||||||
|
|
||||||
}).ToArray();
|
}).ToArray();
|
||||||
}
|
}
|
||||||
|
@ -137,35 +136,39 @@ namespace KubernetesWorkflow
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts)
|
private ContainerPort[] CreateContainerPorts(RunningPod pod, ContainerRecipe recipe, Port[] servicePorts)
|
||||||
{
|
{
|
||||||
return new Address(
|
var result = new List<ContainerPort>();
|
||||||
pod.Cluster.HostAddress,
|
foreach (var exposedPort in recipe.ExposedPorts)
|
||||||
GetServicePort(servicePorts));
|
{
|
||||||
|
result.Add(new ContainerPort(
|
||||||
|
exposedPort,
|
||||||
|
GetContainerExternalAddress(pod, servicePorts, exposedPort),
|
||||||
|
GetContainerInternalAddress(exposedPort)));
|
||||||
|
}
|
||||||
|
|
||||||
|
return result.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
private Address GetContainerInternalAddress(ContainerRecipe recipe)
|
private static Address GetContainerExternalAddress(RunningPod pod, Port[] servicePorts, Port exposedPort)
|
||||||
|
{
|
||||||
|
var servicePort = servicePorts.Single(p => p.Tag == exposedPort.Tag);
|
||||||
|
|
||||||
|
return new Address(
|
||||||
|
pod.Cluster.HostAddress,
|
||||||
|
servicePort.Number);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Address GetContainerInternalAddress(Port exposedPort)
|
||||||
{
|
{
|
||||||
var serviceName = "service-" + numberSource.WorkflowNumber;
|
var serviceName = "service-" + numberSource.WorkflowNumber;
|
||||||
var port = GetInternalPort(recipe);
|
var port = exposedPort.Number;
|
||||||
|
|
||||||
return new Address(
|
return new Address(
|
||||||
$"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
|
$"http://{serviceName}.{k8sNamespace}.svc.cluster.local",
|
||||||
port);
|
port);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static int GetServicePort(Port[] servicePorts)
|
|
||||||
{
|
|
||||||
if (servicePorts.Any()) return servicePorts.First().Number;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private static int GetInternalPort(ContainerRecipe recipe)
|
|
||||||
{
|
|
||||||
if (recipe.ExposedPorts.Any()) return recipe.ExposedPorts.First().Number;
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
private ContainerRecipe[] CreateRecipes(int numberOfContainers, ContainerRecipeFactory recipeFactory, StartupConfig startupConfig)
|
||||||
{
|
{
|
||||||
log.Debug();
|
log.Debug();
|
||||||
|
|
|
@ -10,5 +10,10 @@
|
||||||
|
|
||||||
public string Host { get; }
|
public string Host { get; }
|
||||||
public int Port { get; }
|
public int Port { get; }
|
||||||
|
|
||||||
|
public override string ToString()
|
||||||
|
{
|
||||||
|
return $"{Host}:{Port}";
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -46,33 +46,35 @@
|
||||||
|
|
||||||
public static void Retry(Action action, string description)
|
public static void Retry(Action action, string description)
|
||||||
{
|
{
|
||||||
Retry(action, TimeSpan.FromMinutes(1), description);
|
Retry(action, 1, description);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static T Retry<T>(Func<T> action, string description)
|
public static T Retry<T>(Func<T> action, string description)
|
||||||
{
|
{
|
||||||
return Retry(action, TimeSpan.FromMinutes(1), description);
|
return Retry(action, 1, description);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void Retry(Action action, TimeSpan timeout, string description)
|
public static void Retry(Action action, int maxRetries, string description)
|
||||||
{
|
{
|
||||||
Retry(action, timeout, TimeSpan.FromSeconds(1), description);
|
Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static T Retry<T>(Func<T> action, TimeSpan timeout, string description)
|
public static T Retry<T>(Func<T> action, int maxRetries, string description)
|
||||||
{
|
{
|
||||||
return Retry(action, timeout, TimeSpan.FromSeconds(1), description);
|
return Retry(action, maxRetries, TimeSpan.FromSeconds(1), description);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void Retry(Action action, TimeSpan timeout, TimeSpan retryTime, string description)
|
public static void Retry(Action action, int maxRetries, TimeSpan retryTime, string description)
|
||||||
{
|
{
|
||||||
var start = DateTime.UtcNow;
|
var start = DateTime.UtcNow;
|
||||||
|
var retries = 0;
|
||||||
var exceptions = new List<Exception>();
|
var exceptions = new List<Exception>();
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
if (DateTime.UtcNow - start > timeout)
|
if (retries > maxRetries)
|
||||||
{
|
{
|
||||||
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
|
var duration = DateTime.UtcNow - start;
|
||||||
|
throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
|
@ -83,21 +85,24 @@
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
exceptions.Add(ex);
|
exceptions.Add(ex);
|
||||||
|
retries++;
|
||||||
}
|
}
|
||||||
|
|
||||||
Sleep(retryTime);
|
Sleep(retryTime);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static T Retry<T>(Func<T> action, TimeSpan timeout, TimeSpan retryTime, string description)
|
public static T Retry<T>(Func<T> action, int maxRetries, TimeSpan retryTime, string description)
|
||||||
{
|
{
|
||||||
var start = DateTime.UtcNow;
|
var start = DateTime.UtcNow;
|
||||||
|
var retries = 0;
|
||||||
var exceptions = new List<Exception>();
|
var exceptions = new List<Exception>();
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
if (DateTime.UtcNow - start > timeout)
|
if (retries > maxRetries)
|
||||||
{
|
{
|
||||||
throw new TimeoutException($"Retry '{description}' of {timeout.TotalSeconds} seconds timed out.", new AggregateException(exceptions));
|
var duration = DateTime.UtcNow - start;
|
||||||
|
throw new TimeoutException($"Retry '{description}' timed out after {maxRetries} tries over {Time.FormatDuration(duration)}.", new AggregateException(exceptions));
|
||||||
}
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
|
@ -107,6 +112,7 @@
|
||||||
catch (Exception ex)
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
exceptions.Add(ex);
|
exceptions.Add(ex);
|
||||||
|
retries++;
|
||||||
}
|
}
|
||||||
|
|
||||||
Sleep(retryTime);
|
Sleep(retryTime);
|
||||||
|
|
|
@ -4,7 +4,7 @@ namespace CodexContractsPlugin
|
||||||
{
|
{
|
||||||
public class CodexContractsContainerRecipe : ContainerRecipeFactory
|
public class CodexContractsContainerRecipe : ContainerRecipeFactory
|
||||||
{
|
{
|
||||||
public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:latest-dist-tests";
|
public static string DockerImage { get; } = "codexstorage/codex-contracts-eth:sha-1854dfb-dist-tests";
|
||||||
|
|
||||||
public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
|
public const string MarketplaceAddressFilename = "/hardhat/deployments/codexdisttestnetwork/Marketplace.json";
|
||||||
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
|
public const string MarketplaceArtifactFilename = "/hardhat/artifacts/contracts/Marketplace.sol/Marketplace.json";
|
||||||
|
|
|
@ -1,5 +1,6 @@
|
||||||
using Core;
|
using Core;
|
||||||
using KubernetesWorkflow;
|
using KubernetesWorkflow;
|
||||||
|
using Utils;
|
||||||
|
|
||||||
namespace CodexPlugin
|
namespace CodexPlugin
|
||||||
{
|
{
|
||||||
|
@ -98,12 +99,17 @@ namespace CodexPlugin
|
||||||
|
|
||||||
private IHttp Http()
|
private IHttp Http()
|
||||||
{
|
{
|
||||||
return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
|
return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IHttp LongHttp()
|
private IHttp LongHttp()
|
||||||
{
|
{
|
||||||
return tools.CreateHttp(Container.Address, baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
|
return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name);
|
||||||
|
}
|
||||||
|
|
||||||
|
private Address GetAddress()
|
||||||
|
{
|
||||||
|
return Container.GetAddress(CodexContainerRecipe.ApiPortTag);
|
||||||
}
|
}
|
||||||
|
|
||||||
private void CheckContainerCrashed(HttpClient client)
|
private void CheckContainerCrashed(HttpClient client)
|
||||||
|
|
|
@ -8,8 +8,10 @@ namespace CodexPlugin
|
||||||
private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
|
private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter();
|
||||||
|
|
||||||
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
|
private const string DefaultDockerImage = "codexstorage/nim-codex:latest-dist-tests";
|
||||||
public const string MetricsPortTag = "metrics_port";
|
public const string ApiPortTag = "codex_api_port";
|
||||||
public const string DiscoveryPortTag = "discovery-port";
|
public const string ListenPortTag = "codex_listen_port";
|
||||||
|
public const string MetricsPortTag = "codex_metrics_port";
|
||||||
|
public const string DiscoveryPortTag = "codex_discovery_port";
|
||||||
|
|
||||||
// Used by tests for time-constraint assertions.
|
// Used by tests for time-constraint assertions.
|
||||||
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
|
public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0);
|
||||||
|
@ -27,20 +29,20 @@ namespace CodexPlugin
|
||||||
|
|
||||||
var config = startupConfig.Get<CodexStartupConfig>();
|
var config = startupConfig.Get<CodexStartupConfig>();
|
||||||
|
|
||||||
AddExposedPortAndVar("CODEX_API_PORT");
|
AddExposedPortAndVar("CODEX_API_PORT", ApiPortTag);
|
||||||
AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
|
AddEnvVar("CODEX_API_BINDADDR", "0.0.0.0");
|
||||||
|
|
||||||
var dataDir = $"datadir{ContainerNumber}";
|
var dataDir = $"datadir{ContainerNumber}";
|
||||||
AddEnvVar("CODEX_DATA_DIR", dataDir);
|
AddEnvVar("CODEX_DATA_DIR", dataDir);
|
||||||
AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
|
AddVolume($"codex/{dataDir}", GetVolumeCapacity(config));
|
||||||
|
|
||||||
AddInternalPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
|
AddExposedPortAndVar("CODEX_DISC_PORT", DiscoveryPortTag);
|
||||||
AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
|
AddEnvVar("CODEX_LOG_LEVEL", config.LogLevelWithTopics());
|
||||||
|
|
||||||
// This makes the node announce itself to its local (pod) IP address.
|
// This makes the node announce itself to its local (pod) IP address.
|
||||||
AddEnvVar("NAT_IP_AUTO", "true");
|
AddEnvVar("NAT_IP_AUTO", "true");
|
||||||
|
|
||||||
var listenPort = AddInternalPort();
|
var listenPort = AddExposedPort(ListenPortTag);
|
||||||
AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
|
AddEnvVar("CODEX_LISTEN_ADDRS", $"/ip4/0.0.0.0/tcp/{listenPort.Number}");
|
||||||
|
|
||||||
if (!string.IsNullOrEmpty(config.BootstrapSpr))
|
if (!string.IsNullOrEmpty(config.BootstrapSpr))
|
||||||
|
|
|
@ -13,8 +13,9 @@ namespace CodexPlugin
|
||||||
string GetName();
|
string GetName();
|
||||||
CodexDebugResponse GetDebugInfo();
|
CodexDebugResponse GetDebugInfo();
|
||||||
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
CodexDebugPeerResponse GetDebugPeer(string peerId);
|
||||||
CodexDebugBlockExchangeResponse GetDebugBlockExchange();
|
// These debug methods are not available in master-line Codex. Use only for custom builds.
|
||||||
CodexDebugRepoStoreResponse[] GetDebugRepoStore();
|
//CodexDebugBlockExchangeResponse GetDebugBlockExchange();
|
||||||
|
//CodexDebugRepoStoreResponse[] GetDebugRepoStore();
|
||||||
ContentId UploadFile(TrackedFile file);
|
ContentId UploadFile(TrackedFile file);
|
||||||
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
TrackedFile? DownloadContent(ContentId contentId, string fileLabel = "");
|
||||||
void ConnectToPeer(ICodexNode node);
|
void ConnectToPeer(ICodexNode node);
|
||||||
|
|
|
@ -73,7 +73,7 @@ namespace GethPlugin
|
||||||
|
|
||||||
private NethereumInteraction StartInteraction()
|
private NethereumInteraction StartInteraction()
|
||||||
{
|
{
|
||||||
var address = StartResult.Container.Address;
|
var address = StartResult.Container.GetAddress(GethContainerRecipe.HttpPortTag);
|
||||||
var account = Account;
|
var account = Account;
|
||||||
|
|
||||||
var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
|
var creator = new NethereumInteractionCreator(log, address.Host, address.Port, account.PrivateKey);
|
||||||
|
|
|
@ -13,7 +13,7 @@ namespace MetricsPlugin
|
||||||
public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
|
public MetricsQuery(IPluginTools tools, RunningContainer runningContainer)
|
||||||
{
|
{
|
||||||
RunningContainer = runningContainer;
|
RunningContainer = runningContainer;
|
||||||
http = tools.CreateHttp(RunningContainer.Address, "api/v1");
|
http = tools.CreateHttp(RunningContainer.GetAddress(PrometheusContainerRecipe.PortTag), "api/v1");
|
||||||
log = tools.GetLog();
|
log = tools.GetLog();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -7,11 +7,13 @@ namespace MetricsPlugin
|
||||||
public override string AppName => "prometheus";
|
public override string AppName => "prometheus";
|
||||||
public override string Image => "codexstorage/dist-tests-prometheus:latest";
|
public override string Image => "codexstorage/dist-tests-prometheus:latest";
|
||||||
|
|
||||||
|
public const string PortTag = "prometheus_port_tag";
|
||||||
|
|
||||||
protected override void Initialize(StartupConfig startupConfig)
|
protected override void Initialize(StartupConfig startupConfig)
|
||||||
{
|
{
|
||||||
var config = startupConfig.Get<PrometheusStartupConfig>();
|
var config = startupConfig.Get<PrometheusStartupConfig>();
|
||||||
|
|
||||||
AddExposedPortAndVar("PROM_PORT");
|
AddExposedPortAndVar("PROM_PORT", PortTag);
|
||||||
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
|
AddEnvVar("PROM_CONFIG", config.PrometheusConfigBase64);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -87,7 +87,8 @@ namespace ContinuousTests
|
||||||
{
|
{
|
||||||
cancelToken.ThrowIfCancellationRequested();
|
cancelToken.ThrowIfCancellationRequested();
|
||||||
|
|
||||||
log.Log($"Checking {n.Container.Name} @ '{n.Container.Address.Host}:{n.Container.Address.Port}'...");
|
var address = n.Container.GetAddress(CodexContainerRecipe.ApiPortTag);
|
||||||
|
log.Log($"Checking {n.Container.Name} @ '{address}'...");
|
||||||
|
|
||||||
if (EnsureOnline(log, n))
|
if (EnsureOnline(log, n))
|
||||||
{
|
{
|
||||||
|
@ -95,7 +96,7 @@ namespace ContinuousTests
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
log.Error($"No response from '{n.Container.Address.Host}'.");
|
log.Error($"No response from '{address}'.");
|
||||||
pass = false;
|
pass = false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -57,8 +57,8 @@ namespace ContinuousTests.Tests
|
||||||
|
|
||||||
private void LogRepoStore(ICodexNode codexNode)
|
private void LogRepoStore(ICodexNode codexNode)
|
||||||
{
|
{
|
||||||
var response = codexNode.GetDebugRepoStore();
|
//var response = codexNode.GetDebugRepoStore();
|
||||||
Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
|
//Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}");
|
||||||
}
|
}
|
||||||
|
|
||||||
private void LogStoredBytes(ICodexNode node)
|
private void LogStoredBytes(ICodexNode node)
|
||||||
|
@ -90,8 +90,8 @@ namespace ContinuousTests.Tests
|
||||||
|
|
||||||
private void LogBlockExchangeStatus(ICodexNode codexNode, string msg)
|
private void LogBlockExchangeStatus(ICodexNode codexNode, string msg)
|
||||||
{
|
{
|
||||||
var response = codexNode.GetDebugBlockExchange();
|
//var response = codexNode.GetDebugBlockExchange();
|
||||||
Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
|
//Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -44,4 +44,6 @@ do
|
||||||
--cleanup=1 \
|
--cleanup=1 \
|
||||||
--full-container-logs=1 \
|
--full-container-logs=1 \
|
||||||
--target-duration=172800 # 48 hours
|
--target-duration=172800 # 48 hours
|
||||||
|
|
||||||
|
sleep 30
|
||||||
done
|
done
|
||||||
|
|
|
@ -42,21 +42,22 @@ namespace CodexTests.BasicTests
|
||||||
{
|
{
|
||||||
foreach (var node in nodes)
|
foreach (var node in nodes)
|
||||||
{
|
{
|
||||||
Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
|
// API Call not available in master-line Codex image.
|
||||||
|
//Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private void AssertBlockExchangeIsEmpty(ICodexNode node)
|
//private void AssertBlockExchangeIsEmpty(ICodexNode node)
|
||||||
{
|
//{
|
||||||
var msg = $"BlockExchange for {node.GetName()}: ";
|
// var msg = $"BlockExchange for {node.GetName()}: ";
|
||||||
var response = node.GetDebugBlockExchange();
|
// var response = node.GetDebugBlockExchange();
|
||||||
foreach (var peer in response.peers)
|
// foreach (var peer in response.peers)
|
||||||
{
|
// {
|
||||||
var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
|
// var activeWants = peer.wants.Where(w => !w.cancel).ToArray();
|
||||||
Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
|
// Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants.");
|
||||||
}
|
// }
|
||||||
Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
|
// Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue.");
|
||||||
Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
|
// Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks.");
|
||||||
}
|
//}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -180,9 +180,9 @@ namespace CodexNetDeployer
|
||||||
return TimeSpan.FromSeconds(2);
|
return TimeSpan.FromSeconds(2);
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallRetryTime()
|
public int HttpMaxNumberOfRetries()
|
||||||
{
|
{
|
||||||
return TimeSpan.FromSeconds(2);
|
return 2;
|
||||||
}
|
}
|
||||||
|
|
||||||
public TimeSpan HttpCallTimeout()
|
public TimeSpan HttpCallTimeout()
|
||||||
|
|
Loading…
Reference in New Issue