diff --git a/Framework/FileUtils/TrackedFile.cs b/Framework/FileUtils/TrackedFile.cs index 9ff45d7a..e60bb760 100644 --- a/Framework/FileUtils/TrackedFile.cs +++ b/Framework/FileUtils/TrackedFile.cs @@ -14,6 +14,12 @@ namespace FileUtils Label = label; } + public static TrackedFile FromPath(ILog log, string filepath) + { + // todo: I don't wanne have to do this to call upload. + return new TrackedFile(log, filepath, string.Empty); + } + public string Filename { get; } public string Label { get; } diff --git a/Framework/Utils/Address.cs b/Framework/Utils/Address.cs index 58b8f2ac..010822b9 100644 --- a/Framework/Utils/Address.cs +++ b/Framework/Utils/Address.cs @@ -22,5 +22,21 @@ { return !string.IsNullOrEmpty(Host) && Port > 0; } + + public static Address Empty() + { + return new Address(string.Empty, string.Empty, 0); + } } + + public interface IHasMetricsScrapeTarget + { + Address GetMetricsScrapeTarget(); + } + + public interface IHasManyMetricScrapeTargets + { + Address[] GetMetricsScrapeTargets(); + } + } diff --git a/ProjectPlugins/CodexClient/CodexAccess.cs b/ProjectPlugins/CodexClient/CodexAccess.cs index 434d7946..c7c3dc32 100644 --- a/ProjectPlugins/CodexClient/CodexAccess.cs +++ b/ProjectPlugins/CodexClient/CodexAccess.cs @@ -1,5 +1,4 @@ -using CodexOpenApi; -using Logging; +using Logging; using Newtonsoft.Json; using Utils; using WebUtils; diff --git a/ProjectPlugins/CodexClient/CodexInstance.cs b/ProjectPlugins/CodexClient/CodexInstance.cs index 54c2e9c3..4f77776b 100644 --- a/ProjectPlugins/CodexClient/CodexInstance.cs +++ b/ProjectPlugins/CodexClient/CodexInstance.cs @@ -36,5 +36,19 @@ namespace CodexClient public Address ListenEndpoint { get; } public EthAccount? EthAccount { get; } public Address? MetricsEndpoint { get; } + + public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint) + { + return new CodexInstance( + name, + imageName: "-", + startUtc: DateTime.UtcNow, + discoveryEndpoint: Address.Empty(), + apiEndpoint: apiEndpoint, + listenEndpoint: Address.Empty(), + ethAccount: null, + metricsEndpoint: null + ); + } } } diff --git a/ProjectPlugins/CodexClient/CodexNode.cs b/ProjectPlugins/CodexClient/CodexNode.cs index 45cfcd7b..5cdf7b02 100644 --- a/ProjectPlugins/CodexClient/CodexNode.cs +++ b/ProjectPlugins/CodexClient/CodexNode.cs @@ -5,7 +5,7 @@ using Utils; namespace CodexClient { - public partial interface ICodexNode : IHasEthAddress + public partial interface ICodexNode : IHasEthAddress, IHasMetricsScrapeTarget { string GetName(); string GetImageName(); @@ -30,11 +30,11 @@ namespace CodexClient IMarketplaceAccess Marketplace { get; } ITransferSpeeds TransferSpeeds { get; } EthAccount EthAccount { get; } + StoragePurchase GetPurchaseStatus(string purchaseId); Address GetDiscoveryEndpoint(); Address GetApiEndpoint(); Address GetListenEndpoint(); - Address GetMetricsScrapeTarget(); /// /// Warning! The node is not usable after this. @@ -86,6 +86,11 @@ namespace CodexClient public DebugInfoVersion Version { get; private set; } public ITransferSpeeds TransferSpeeds { get => transferSpeeds; } + public StoragePurchase GetPurchaseStatus(string purchaseId) + { + return codexAccess.GetPurchaseStatus(purchaseId); + } + public EthAddress EthAddress { get diff --git a/ProjectPlugins/CodexClient/CodexNodeFactory.cs b/ProjectPlugins/CodexClient/CodexNodeFactory.cs index d8048a0e..22e7a69e 100644 --- a/ProjectPlugins/CodexClient/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexClient/CodexNodeFactory.cs @@ -9,24 +9,29 @@ namespace CodexClient { private readonly ILog log; private readonly IFileManager fileManager; - private readonly CodexHooksFactory hooksFactor; + private readonly CodexHooksFactory hooksFactory; private readonly IHttpFactory httpFactory; - private readonly IIProcessControlFactory processControlFactory; + private readonly IProcessControlFactory processControlFactory; - public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IIProcessControlFactory processControlFactory) + public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory) { this.log = log; this.fileManager = fileManager; - this.hooksFactor = hooksFactory; + this.hooksFactory = hooksFactory; this.httpFactory = httpFactory; this.processControlFactory = processControlFactory; } + public CodexNodeFactory(ILog log, string dataDir) + : this(log, new FileManager(log, dataDir), new CodexHooksFactory(), new HttpFactory(log), new DoNothingProcessControlFactory()) + { + } + public ICodexNode CreateCodexNode(ICodexInstance instance) { var processControl = processControlFactory.CreateProcessControl(instance); var access = new CodexAccess(log, httpFactory, processControl, instance); - var hooks = hooksFactor.CreateHooks(access.GetName()); + var hooks = hooksFactory.CreateHooks(access.GetName()); var marketplaceAccess = CreateMarketplaceAccess(instance, access, hooks); var node = new CodexNode(log, access, fileManager, marketplaceAccess, hooks); node.Initialize(); diff --git a/ProjectPlugins/CodexClient/ProcessControl.cs b/ProjectPlugins/CodexClient/ProcessControl.cs index f431f58b..2ddfab2e 100644 --- a/ProjectPlugins/CodexClient/ProcessControl.cs +++ b/ProjectPlugins/CodexClient/ProcessControl.cs @@ -2,7 +2,7 @@ namespace CodexClient { - public interface IIProcessControlFactory + public interface IProcessControlFactory { IProcessControl CreateProcessControl(ICodexInstance instance); } @@ -14,4 +14,33 @@ namespace CodexClient void DeleteDataDirFolder(); bool HasCrashed(); } + + public class DoNothingProcessControlFactory : IProcessControlFactory + { + public IProcessControl CreateProcessControl(ICodexInstance instance) + { + return new DoNothingProcessControl(); + } + } + + public class DoNothingProcessControl : IProcessControl + { + public void DeleteDataDirFolder() + { + } + + public IDownloadedLog DownloadLog(LogFile file) + { + throw new NotImplementedException("Not supported by DoNothingProcessControl"); + } + + public bool HasCrashed() + { + return false; + } + + public void Stop(bool waitTillStopped) + { + } + } } diff --git a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs index 7e4f0d00..38084eef 100644 --- a/ProjectPlugins/CodexClient/StoragePurchaseContract.cs +++ b/ProjectPlugins/CodexClient/StoragePurchaseContract.cs @@ -14,6 +14,7 @@ namespace CodexClient void WaitForStorageContractStarted(); void WaitForStorageContractFinished(); void WaitForContractFailed(); + StoragePurchase GetPurchaseStatus(); } public class StoragePurchaseContract : IStoragePurchaseContract @@ -88,9 +89,9 @@ namespace CodexClient WaitForStorageContractState(timeout, "failed"); } - public StoragePurchase GetPurchaseStatus(string purchaseId) + public StoragePurchase GetPurchaseStatus() { - return codexAccess.GetPurchaseStatus(purchaseId); + return codexAccess.GetPurchaseStatus(PurchaseId); } private void WaitForStorageContractState(TimeSpan timeout, string desiredState, int sleep = 1000) diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index 8c82ce22..4a8bd9bb 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -1,6 +1,5 @@ using CodexClient; using Core; -using MetricsPlugin; using System.Collections; using Utils; @@ -42,7 +41,11 @@ namespace CodexPlugin public ICodexNode[] Nodes => nodes; public DebugInfoVersion Version { get; private set; } - public Address[] ScrapeTargets => Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray(); + + public Address[] GetMetricsScrapeTargets() + { + return Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray(); + } public IEnumerator GetEnumerator() { diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index e6521479..1e74a13e 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -8,7 +8,7 @@ using Utils; namespace CodexPlugin { - public class CodexStarter : IIProcessControlFactory + public class CodexStarter : IProcessControlFactory { private readonly IPluginTools pluginTools; private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); diff --git a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs index e632e288..76f6c108 100644 --- a/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/MetricsPlugin/CoreInterfaceExtensions.cs @@ -9,7 +9,7 @@ namespace MetricsPlugin { public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets) { - return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray(), scrapeInterval); + return Plugin(ci).DeployMetricsCollector(scrapeTargets.Select(t => t.GetMetricsScrapeTarget()).ToArray(), scrapeInterval); } public static RunningPod DeployMetricsCollector(this CoreInterface ci, TimeSpan scrapeInterval, params Address[] scrapeTargets) @@ -19,7 +19,7 @@ namespace MetricsPlugin public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, IHasMetricsScrapeTarget scrapeTarget) { - return ci.WrapMetricsCollector(metricsPod, scrapeTarget.MetricsScrapeTarget); + return ci.WrapMetricsCollector(metricsPod, scrapeTarget.GetMetricsScrapeTarget()); } public static IMetricsAccess WrapMetricsCollector(this CoreInterface ci, RunningPod metricsPod, Address scrapeTarget) @@ -29,12 +29,12 @@ namespace MetricsPlugin public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasManyMetricScrapeTargets[] manyScrapeTargets) { - return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.ScrapeTargets).ToArray()); + return ci.GetMetricsFor(scrapeInterval, manyScrapeTargets.SelectMany(t => t.GetMetricsScrapeTargets()).ToArray()); } public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params IHasMetricsScrapeTarget[] scrapeTargets) { - return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.MetricsScrapeTarget).ToArray()); + return ci.GetMetricsFor(scrapeInterval, scrapeTargets.Select(t => t.GetMetricsScrapeTarget()).ToArray()); } public static IMetricsAccess[] GetMetricsFor(this CoreInterface ci, TimeSpan scrapeInterval, params Address[] scrapeTargets) diff --git a/ProjectPlugins/MetricsPlugin/MetricsScrapeTarget.cs b/ProjectPlugins/MetricsPlugin/MetricsScrapeTarget.cs deleted file mode 100644 index 5f497de4..00000000 --- a/ProjectPlugins/MetricsPlugin/MetricsScrapeTarget.cs +++ /dev/null @@ -1,14 +0,0 @@ -using Utils; - -namespace MetricsPlugin -{ - public interface IHasMetricsScrapeTarget - { - Address MetricsScrapeTarget { get; } - } - - public interface IHasManyMetricScrapeTargets - { - Address[] ScrapeTargets { get; } - } -} diff --git a/Tests/CodexContinuousTests/ContinuousTest.cs b/Tests/CodexContinuousTests/ContinuousTest.cs index 00a4c3a7..fa39255c 100644 --- a/Tests/CodexContinuousTests/ContinuousTest.cs +++ b/Tests/CodexContinuousTests/ContinuousTest.cs @@ -1,6 +1,4 @@ -using CodexPlugin; -using Core; -using DistTestCore; +using CodexClient; using FileUtils; using Logging; using MetricsPlugin; @@ -10,7 +8,6 @@ namespace ContinuousTests { public abstract class ContinuousTestLongTimeouts : ContinuousTest { - public override ITimeSet TimeSet => new LongTimeSet(); } public abstract class ContinuousTest @@ -45,13 +42,12 @@ namespace ContinuousTests public ILog Log { get; private set; } = null!; public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; - public virtual ITimeSet TimeSet { get { return new DefaultTimeSet(); } } public CancellationToken CancelToken { get; private set; } = new CancellationToken(); public NodeRunner NodeRunner { get; private set; } = null!; public IMetricsAccess CreateMetricsAccess(IHasMetricsScrapeTarget target) { - return CreateMetricsAccess(target.MetricsScrapeTarget); + return CreateMetricsAccess(target.GetMetricsScrapeTarget()); } public IMetricsAccess CreateMetricsAccess(Address target) diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs index 7d491939..e198650c 100644 --- a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs +++ b/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs @@ -2,6 +2,7 @@ using KubernetesWorkflow.Types; using Logging; using Utils; +using WebUtils; namespace ContinuousTests { diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/CodexContinuousTests/NodeRunner.cs index b51f6222..f2b3c64f 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/CodexContinuousTests/NodeRunner.cs @@ -3,8 +3,7 @@ using Logging; using Utils; using Core; using CodexPlugin; -using KubernetesWorkflow.Types; -using KubernetesWorkflow; +using CodexClient; namespace ContinuousTests { diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/CodexContinuousTests/SingleTestRun.cs index 6617ae5f..643f472e 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/CodexContinuousTests/SingleTestRun.cs @@ -7,6 +7,7 @@ using DistTestCore.Logs; using Core; using KubernetesWorkflow.Types; using TaskFactory = Utils.TaskFactory; +using CodexClient; namespace ContinuousTests { diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/CodexContinuousTests/StartupChecker.cs index ab3e637d..0914b3d1 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/CodexContinuousTests/StartupChecker.cs @@ -1,8 +1,7 @@ -using CodexPlugin; +using CodexClient; using Core; using DistTestCore.Logs; using Logging; -using Newtonsoft.Json; namespace ContinuousTests { diff --git a/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs b/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs index 76457e2e..6eaec611 100644 --- a/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs +++ b/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using FileUtils; using NUnit.Framework; using Utils; diff --git a/Tests/CodexContinuousTests/Tests/PeersTest.cs b/Tests/CodexContinuousTests/Tests/PeersTest.cs index 87bcb825..94a3a090 100644 --- a/Tests/CodexContinuousTests/Tests/PeersTest.cs +++ b/Tests/CodexContinuousTests/Tests/PeersTest.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using CodexTests.Helpers; using ContinuousTests; using NUnit.Framework; diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs index 190cbb61..4a5f7b5a 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/CodexContinuousTests/Tests/TwoClientTest.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using FileUtils; using Logging; using NUnit.Framework; diff --git a/Tests/CodexLongTests/BasicTests/DownloadTests.cs b/Tests/CodexLongTests/BasicTests/DownloadTests.cs index 0a95a643..f77b22e5 100644 --- a/Tests/CodexLongTests/BasicTests/DownloadTests.cs +++ b/Tests/CodexLongTests/BasicTests/DownloadTests.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using DistTestCore; using FileUtils; diff --git a/Tests/CodexLongTests/BasicTests/UploadTests.cs b/Tests/CodexLongTests/BasicTests/UploadTests.cs index 361c3a73..aaa9f1fd 100644 --- a/Tests/CodexLongTests/BasicTests/UploadTests.cs +++ b/Tests/CodexLongTests/BasicTests/UploadTests.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using DistTestCore; using FileUtils; diff --git a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs index fabf51e0..d4597acb 100644 --- a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs +++ b/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using DistTestCore; using NUnit.Framework; using Utils; diff --git a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs index a3f05970..dd92de84 100644 --- a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using DistTestCore; using FileUtils; using NUnit.Framework; @@ -38,11 +38,11 @@ public class ScalabilityTests : CodexDistTest var testFile = GenerateTestFile(fileSizeInMb.MB()); LogNodeStatus(uploader); - var contentId = uploader.UploadFile(testFile, f => LogNodeStatus(uploader)); + var contentId = uploader.UploadFile(testFile); LogNodeStatus(uploader); LogNodeStatus(downloader); - var downloadedFile = downloader.DownloadContent(contentId, f => LogNodeStatus(downloader)); + var downloadedFile = downloader.DownloadContent(contentId); LogNodeStatus(downloader); downloadedFile!.AssertIsEqual(testFile); diff --git a/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs b/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs index 06b2009c..a4fb8192 100644 --- a/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs +++ b/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs @@ -1,4 +1,4 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using FileUtils; using NUnit.Framework; diff --git a/Tests/CodexReleaseTests/DataTests/OneClientTest.cs b/Tests/CodexReleaseTests/DataTests/OneClientTest.cs index 1be82b4d..2abf009a 100644 --- a/Tests/CodexReleaseTests/DataTests/OneClientTest.cs +++ b/Tests/CodexReleaseTests/DataTests/OneClientTest.cs @@ -1,11 +1,6 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Utils; namespace CodexReleaseTests.DataTests diff --git a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs index 3325d35d..427085a5 100644 --- a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/CodexReleaseTests/DataTests/SwarmTest.cs @@ -1,4 +1,5 @@ -using CodexPlugin; +using CodexClient; +using CodexPlugin; using CodexTests; using FileUtils; using NUnit.Framework; diff --git a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs b/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs index 676653e7..8ab9c88d 100644 --- a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs +++ b/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs @@ -1,4 +1,5 @@ -using CodexPlugin; +using CodexClient; +using CodexPlugin; using CodexTests; using NUnit.Framework; using Utils; diff --git a/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs b/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs index d0699a53..2d64e863 100644 --- a/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs +++ b/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs @@ -1,11 +1,6 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace CodexReleaseTests.DataTests { diff --git a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs index dd79c6af..17454e46 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractFailedTest.cs @@ -1,13 +1,6 @@ -using CodexContractsPlugin; +using CodexClient; using CodexContractsPlugin.Marketplace; -using CodexPlugin; -using CodexTests; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Utils; namespace CodexReleaseTests.MarketTests diff --git a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs index d614840f..8788f204 100644 --- a/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/ContractSuccessfulTest.cs @@ -1,7 +1,4 @@ using CodexClient; -using CodexContractsPlugin; -using CodexPlugin; -using GethPlugin; using NUnit.Framework; using Utils; @@ -32,7 +29,7 @@ namespace CodexReleaseTests.MarketTests request.WaitForStorageContractStarted(); AssertContractSlotsAreFilledByHosts(request, hosts); - request.WaitForStorageContractFinished(GetContracts()); + request.WaitForStorageContractFinished(); AssertClientHasPaidForContract(pricePerSlotPerSecond, client, request, hosts); AssertHostsWerePaidForContract(pricePerSlotPerSecond, request, hosts); diff --git a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs index ef181c33..6a4f6dd2 100644 --- a/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MarketplaceAutoBootstrapDistTest.cs @@ -1,4 +1,5 @@ -using CodexContractsPlugin; +using CodexClient; +using CodexContractsPlugin; using CodexContractsPlugin.Marketplace; using CodexPlugin; using CodexTests; @@ -67,7 +68,7 @@ namespace CodexReleaseTests.MarketTests Assert.That(GetTstBalance(host).TstWei, Is.EqualTo(StartingBalanceTST.Tst().TstWei)); Assert.That(GetEthBalance(host).Wei, Is.EqualTo(StartingBalanceEth.Eth().Wei)); - host.Marketplace.MakeStorageAvailable(new CodexPlugin.StorageAvailability( + host.Marketplace.MakeStorageAvailable(new StorageAvailability( totalSpace: HostAvailabilitySize, maxDuration: HostAvailabilityMaxDuration, minPriceForTotalSpace: 1.TstWei(), diff --git a/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs b/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs index 97876f2b..50d1a7c9 100644 --- a/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs +++ b/Tests/CodexReleaseTests/MarketTests/MultipleContractsTest.cs @@ -1,7 +1,4 @@ using CodexClient; -using CodexContractsPlugin; -using CodexPlugin; -using GethPlugin; using NUnit.Framework; using Utils; @@ -36,7 +33,7 @@ namespace CodexReleaseTests.MarketTests All(requests, r => r.WaitForStorageContractStarted()); All(requests, r => AssertContractSlotsAreFilledByHosts(r, hosts)); - All(requests, r => r.WaitForStorageContractFinished(GetContracts())); + All(requests, r => r.WaitForStorageContractFinished()); // todo: removed from codexclient: //contracts.WaitUntilNextPeriod(); diff --git a/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs b/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs index ced42922..74715b66 100644 --- a/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs +++ b/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs @@ -1,13 +1,7 @@ -using CodexPlugin; +using CodexClient; using CodexTests; using CodexTests.Helpers; using NUnit.Framework; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Utils; namespace CodexReleaseTests.NodeTests { diff --git a/Tests/ExperimentalTests/BasicTests/ExampleTests.cs b/Tests/ExperimentalTests/BasicTests/ExampleTests.cs index fc83da3b..716d3fff 100644 --- a/Tests/ExperimentalTests/BasicTests/ExampleTests.cs +++ b/Tests/ExperimentalTests/BasicTests/ExampleTests.cs @@ -1,4 +1,5 @@ -using CodexPlugin; +using CodexClient; +using CodexPlugin; using DistTestCore; using MetricsPlugin; using NUnit.Framework; diff --git a/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs b/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs index 8ce80571..c62db4a0 100644 --- a/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs +++ b/Tests/ExperimentalTests/BasicTests/MarketplaceTests.cs @@ -108,7 +108,7 @@ namespace CodexTests.BasicTests AssertStorageRequest(request, purchase, contracts, client); AssertContractSlot(contracts, request, 0); - purchaseContract.WaitForStorageContractFinished(contracts); + purchaseContract.WaitForStorageContractFinished(); // todo: removed from codexclient: //contracts.WaitUntilNextPeriod(); diff --git a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 88ee0dbb..b0373a32 100644 --- a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,6 +1,7 @@ using CodexContractsPlugin; using CodexClient; using NUnit.Framework; +using Utils; namespace CodexTests.PeerDiscoveryTests { diff --git a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs index 5559c5f6..320c3d86 100644 --- a/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs +++ b/Tests/ExperimentalTests/UtilityTests/DiscordBotTests.cs @@ -20,7 +20,7 @@ namespace CodexTests.UtilityTests private readonly RewardRepo repo = new RewardRepo(); private readonly TestToken hostInitialBalance = 3000000.TstWei(); private readonly TestToken clientInitialBalance = 1000000000.TstWei(); - private readonly EthAccount clientAccount = EthAccount.GenerateNew(); + private readonly EthAccount clientAccount = EthAccountGenerator.GenerateNew(); private readonly List hostAccounts = new List(); private readonly List rewardsSeen = new List(); private readonly TimeSpan rewarderInterval = TimeSpan.FromMinutes(1); @@ -46,7 +46,7 @@ namespace CodexTests.UtilityTests var purchaseContract = ClientPurchasesStorage(client); purchaseContract.WaitForStorageContractStarted(); - purchaseContract.WaitForStorageContractFinished(contracts); + purchaseContract.WaitForStorageContractFinished(); // todo: removed from codexclient: //contracts.WaitUntilNextPeriod(); diff --git a/Tools/AutoClient/App.cs b/Tools/AutoClient/App.cs index cf2573a7..0cdfc673 100644 --- a/Tools/AutoClient/App.cs +++ b/Tools/AutoClient/App.cs @@ -1,4 +1,5 @@ using AutoClient.Modes.FolderStore; +using CodexClient; using Logging; namespace AutoClient @@ -15,7 +16,6 @@ namespace AutoClient ); Generator = CreateGenerator(); - CidRepo = new CidRepo(config); Performance = new Performance(new LogSplitter( new FileLog(Path.Combine(config.LogPath, "performance")), new ConsoleLog() @@ -29,15 +29,17 @@ namespace AutoClient { FolderWorkDispatcher = null!; } + + CodexNodeFactory = new CodexNodeFactory(log: Log, dataDir: Config.DataPath); } public Configuration Config { get; } public ILog Log { get; } public IFileGenerator Generator { get; } public CancellationTokenSource Cts { get; } = new CancellationTokenSource(); - public CidRepo CidRepo { get; } public Performance Performance { get; } public FolderWorkDispatcher FolderWorkDispatcher { get; } + public CodexNodeFactory CodexNodeFactory { get; } private IFileGenerator CreateGenerator() { diff --git a/Tools/AutoClient/AutomaticPurchaser.cs b/Tools/AutoClient/AutomaticPurchaser.cs index 1e217167..1886f321 100644 --- a/Tools/AutoClient/AutomaticPurchaser.cs +++ b/Tools/AutoClient/AutomaticPurchaser.cs @@ -1,24 +1,19 @@ -using CodexOpenApi; -using CodexPlugin; -using Logging; -using Newtonsoft.Json; -using Utils; +using Logging; namespace AutoClient { public class AutomaticPurchaser { + private readonly App app; private readonly ILog log; - private readonly ICodexInstance instance; - private readonly CodexNode codex; + private readonly CodexWrapper node; private Task workerTask = Task.CompletedTask; - private App app => instance.App; - public AutomaticPurchaser(ILog log, ICodexInstance instance, CodexNode codex) + public AutomaticPurchaser(App app, ILog log, CodexWrapper node) { + this.app = app; this.log = log; - this.instance = instance; - this.codex = codex; + this.node = node; } public void Start() @@ -40,7 +35,6 @@ namespace AutoClient { var pid = await StartNewPurchase(); await WaitTillFinished(pid); - await DownloadForeignCid(); } catch (Exception ex) { @@ -50,27 +44,13 @@ namespace AutoClient } } - private async Task DownloadForeignCid() - { - var cid = app.CidRepo.GetForeignCid(instance.NodeId); - if (cid == null) return; - - var size = app.CidRepo.GetSizeForCid(cid); - if (size == null) return; - - var filename = Guid.NewGuid().ToString().ToLowerInvariant(); - await codex.DownloadCid(filename, cid, size); - - DeleteFile(filename); - } - private async Task StartNewPurchase() { var file = await CreateFile(); try { - var cid = await codex.UploadFile(file); - var response = await codex.RequestStorage(cid); + var cid = node.UploadFile(file); + var response = node.RequestStorage(cid); return response.PurchaseId; } finally @@ -92,7 +72,7 @@ namespace AutoClient } catch (Exception exc) { - app.Log.Error($"Failed to delete file '{file}': {exc}"); + log.Error($"Failed to delete file '{file}': {exc}"); } } @@ -103,7 +83,7 @@ namespace AutoClient var emptyResponseTolerance = 10; while (!app.Cts.Token.IsCancellationRequested) { - var purchase = await codex.GetStoragePurchase(pid); + var purchase = node.GetStoragePurchase(pid); if (purchase == null) { await FixedShortDelay(); diff --git a/Tools/AutoClient/CidRepo.cs b/Tools/AutoClient/CidRepo.cs deleted file mode 100644 index 572b5ec9..00000000 --- a/Tools/AutoClient/CidRepo.cs +++ /dev/null @@ -1,86 +0,0 @@ -namespace AutoClient -{ - public class CidRepo - { - private readonly Random random = new Random(); - private readonly object _lock = new object(); - private readonly List entries = new List(); - private readonly Configuration config; - - public CidRepo(Configuration config) - { - this.config = config; - } - - public void Add(string nodeId, string cid, long knownSize) - { - lock (_lock) - { - entries.Add(new CidEntry(nodeId, cid, knownSize)); - if (entries.Count > 1000) entries.Clear(); - } - } - - public void AddEncoded(string originalCid, string encodedCid) - { - lock (_lock) - { - var entry = entries.SingleOrDefault(e => e.Cid == originalCid); - if (entry == null) return; - - entry.Encoded = encodedCid; - } - } - - public string? GetForeignCid(string myNodeId) - { - lock (_lock) - { - while (true) - { - if (!entries.Any()) return null; - var available = entries.Where(e => e.NodeId != myNodeId).ToArray(); - if (!available.Any()) return null; - - var i = random.Next(0, available.Length); - var entry = available[i]; - - if (entry.CreatedUtc < (DateTime.UtcNow + TimeSpan.FromMinutes(config.ContractDurationMinutes))) - { - entries.Remove(entry); - } - else - { - return entry.Cid; - } - } - } - } - - public long? GetSizeForCid(string cid) - { - lock (_lock) - { - var entry = entries.SingleOrDefault(e => e.Cid == cid); - if (entry == null) return null; - return entry.KnownSize; - } - } - } - - public class CidEntry - { - public CidEntry(string nodeId, string cid, long knownSize) - { - NodeId = nodeId; - Cid = cid; - KnownSize = knownSize; - } - - public string NodeId { get; } - public string Cid { get; } - public string Encoded { get; set; } = string.Empty; - public long KnownSize { get; } - public DateTime CreatedUtc { get; } = DateTime.UtcNow; - } -} diff --git a/Tools/AutoClient/CodexContext.cs b/Tools/AutoClient/CodexContext.cs new file mode 100644 index 00000000..98fd1c95 --- /dev/null +++ b/Tools/AutoClient/CodexContext.cs @@ -0,0 +1,32 @@ +using CodexClient; +using Utils; + +namespace AutoClient +{ + public interface ICodexContext + { + string NodeId { get; } + App App { get; } + ICodexNode Codex { get; } + HttpClient Client { get; } + Address Address { get; } + } + + public class CodexContext : ICodexContext + { + public CodexContext(App app, ICodexNode codex, HttpClient client, Address address) + { + App = app; + Codex = codex; + Client = client; + Address = address; + NodeId = Guid.NewGuid().ToString(); + } + + public string NodeId { get; } + public App App { get; } + public ICodexNode Codex { get; } + public HttpClient Client { get; } + public Address Address { get; } + } +} diff --git a/Tools/AutoClient/CodexInstance.cs b/Tools/AutoClient/CodexInstance.cs deleted file mode 100644 index 3be59d45..00000000 --- a/Tools/AutoClient/CodexInstance.cs +++ /dev/null @@ -1,162 +0,0 @@ -using CodexClient; -using CodexOpenApi; -using CodexPlugin; -using Logging; -using Nethereum.Model; -using Newtonsoft.Json; -using Utils; - -namespace AutoClient -{ - public interface ICodexInstance - { - string NodeId { get; } - App App { get; } - CodexApi Codex { get; } - HttpClient Client { get; } - Address Address { get; } - } - - public class CodexInstance : ICodexInstance - { - public CodexInstance(App app, CodexApi codex, HttpClient client, Address address) - { - App = app; - Codex = codex; - Client = client; - Address = address; - NodeId = Guid.NewGuid().ToString(); - } - - public string NodeId { get; } - public App App { get; } - public CodexApi Codex { get; } - public HttpClient Client { get; } - public Address Address { get; } - } - - public class CodexNode - { - private readonly App app; - private readonly ICodexInstance codex; - - public CodexNode(App app, ICodexInstance instance) - { - this.app = app; - codex = instance; - } - - public async Task DownloadCid(string filename, string cid, long? size) - { - try - { - var sw = System.Diagnostics.Stopwatch.StartNew(); - using var fileStream = File.OpenWrite(filename); - var fileResponse = await codex.Codex.DownloadNetworkStreamAsync(cid); - fileResponse.Stream.CopyTo(fileStream); - var time = sw.Elapsed; - app.Performance.DownloadSuccessful(size, time); - } - catch (Exception ex) - { - app.Performance.DownloadFailed(ex); - } - } - - public async Task UploadFile(string filename) - { - using var fileStream = File.OpenRead(filename); - try - { - var info = new FileInfo(filename); - var sw = System.Diagnostics.Stopwatch.StartNew(); - var cid = await UploadStream(fileStream, filename); - var time = sw.Elapsed; - app.Performance.UploadSuccessful(info.Length, time); - app.CidRepo.Add(codex.NodeId, cid.Id, info.Length); - return cid; - } - catch (Exception exc) - { - app.Performance.UploadFailed(exc); - throw; - } - } - - public async Task RequestStorage(ContentId cid) - { - app.Log.Debug("Requesting storage for " + cid.Id); - var result = await codex.Codex.CreateStorageRequestAsync(cid.Id, new StorageRequestCreation() - { - Collateral = app.Config.RequiredCollateral.ToString(), - Duration = (app.Config.ContractDurationMinutes * 60).ToString(), - Expiry = (app.Config.ContractExpiryMinutes * 60).ToString(), - Nodes = app.Config.NumHosts, - Reward = app.Config.Price.ToString(), - ProofProbability = "15", - Tolerance = app.Config.HostTolerance - }, app.Cts.Token); - - app.Log.Debug("Purchase ID: " + result); - - var encoded = await GetEncodedCid(result); - app.CidRepo.AddEncoded(cid.Id, encoded); - - return new RequestStorageResult(result, new ContentId(encoded)); - } - - public class RequestStorageResult - { - public RequestStorageResult(string purchaseId, ContentId encodedCid) - { - PurchaseId = purchaseId; - EncodedCid = encodedCid; - } - - public string PurchaseId { get; } - public ContentId EncodedCid { get; } - - public override string ToString() - { - return $"{PurchaseId} (cid: {EncodedCid})"; - } - } - - public async Task GetStoragePurchase(string pid) - { - // openapi still don't match code. - var str = await codex.Client.GetStringAsync($"{codex.Address.Host}:{codex.Address.Port}/api/codex/v1/storage/purchases/{pid}"); - if (string.IsNullOrEmpty(str)) return null; - return JsonConvert.DeserializeObject(str); - } - - private async Task UploadStream(FileStream fileStream, string filename) - { - app.Log.Debug($"Uploading file..."); - var response = await codex.Codex.UploadAsync( - content_type: "application/octet-stream", - content_disposition: $"attachment; filename=\"{filename}\"", - fileStream, app.Cts.Token); - - if (string.IsNullOrEmpty(response)) FrameworkAssert.Fail("Received empty response."); - if (response.StartsWith("Unable to store block")) FrameworkAssert.Fail("Node failed to store block."); - - app.Log.Debug($"Uploaded file. Received contentId: '{response}'."); - return new ContentId(response); - } - - private async Task GetEncodedCid(string pid) - { - try - { - var sp = (await GetStoragePurchase(pid))!; - return sp.Request.Content.Cid; - } - catch (Exception ex) - { - app.Log.Error(ex.ToString()); - throw; - } - } - } -} diff --git a/Tools/AutoClient/CodexWrapper.cs b/Tools/AutoClient/CodexWrapper.cs new file mode 100644 index 00000000..2369f2f8 --- /dev/null +++ b/Tools/AutoClient/CodexWrapper.cs @@ -0,0 +1,45 @@ +using CodexClient; +using FileUtils; +using Utils; + +namespace AutoClient +{ + public class CodexWrapper + { + private readonly App app; + + public CodexWrapper(App app, ICodexNode node) + { + this.app = app; + Node = node; + } + + public ICodexNode Node { get; } + + public ContentId UploadFile(string filepath) + { + return Node.UploadFile(TrackedFile.FromPath(app.Log, filepath)); + } + + public IStoragePurchaseContract RequestStorage(ContentId cid) + { + app.Log.Debug("Requesting storage for " + cid.Id); + var result = Node.Marketplace.RequestStorage(new StoragePurchaseRequest(cid) + { + RequiredCollateral = app.Config.RequiredCollateral.Tst(), + Duration = TimeSpan.FromMinutes(app.Config.ContractDurationMinutes), + Expiry = TimeSpan.FromMinutes(app.Config.ContractExpiryMinutes), + MinRequiredNumberOfNodes = Convert.ToUInt32(app.Config.NumHosts), + NodeFailureTolerance = Convert.ToUInt32(app.Config.HostTolerance), + PricePerSlotPerSecond = app.Config.Price.Tst(), + ProofProbability = 15 + }); + return result; + } + + public StoragePurchase? GetStoragePurchase(string pid) + { + return Node.GetPurchaseStatus(pid); + } + } +} diff --git a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs b/Tools/AutoClient/Modes/FolderStore/FileWorker.cs index 5953eff3..0459815c 100644 --- a/Tools/AutoClient/Modes/FolderStore/FileWorker.cs +++ b/Tools/AutoClient/Modes/FolderStore/FileWorker.cs @@ -1,30 +1,29 @@ -using Logging; +using CodexClient; +using Logging; namespace AutoClient.Modes.FolderStore { public class FileWorker : FileStatus { private readonly App app; + private readonly CodexWrapper node; private readonly ILog log; - private readonly ICodexInstance instance; private readonly PurchaseInfo purchaseInfo; private readonly string sourceFilename; private readonly Action onFileUploaded; private readonly Action onNewPurchase; - private readonly CodexNode codex; - public FileWorker(App app, ICodexInstance instance, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase) + public FileWorker(App app, CodexWrapper node, PurchaseInfo purchaseInfo, string folder, FileIndex fileIndex, Action onFileUploaded, Action onNewPurchase) : base(app, folder, fileIndex.File + ".json", purchaseInfo) { this.app = app; + this.node = node; log = new LogPrefixer(app.Log, GetFileTag(fileIndex)); - this.instance = instance; this.purchaseInfo = purchaseInfo; sourceFilename = fileIndex.File; if (sourceFilename.ToLowerInvariant().EndsWith(".json")) throw new Exception("Not an era file."); this.onFileUploaded = onFileUploaded; this.onNewPurchase = onNewPurchase; - codex = new CodexNode(app, instance); } public int FailureCounter => State.FailureCounter; @@ -34,14 +33,14 @@ namespace AutoClient.Modes.FolderStore newState.LastUpdate = DateTime.MinValue; } - public async Task Update() + public void Update() { try { if (IsCurrentlyRunning() && UpdatedRecently()) return; Log($"Updating for '{sourceFilename}'..."); - await EnsureRecentPurchase(); + EnsureRecentPurchase(); SaveState(); app.Log.Log(""); } @@ -60,12 +59,12 @@ namespace AutoClient.Modes.FolderStore return State.LastUpdate + TimeSpan.FromMinutes(15) > now; } - private async Task EnsureCid() + private string EnsureCid() { Log($"Checking CID..."); if (!string.IsNullOrEmpty(State.EncodedCid) && - await DoesCidExistInNetwork(State.EncodedCid)) + DoesCidExistInNetwork(State.EncodedCid)) { Log("Encoded-CID successfully found in the network."); // TODO: Using the encoded CID currently would result in double-encoding of the dataset. @@ -75,7 +74,7 @@ namespace AutoClient.Modes.FolderStore } if (!string.IsNullOrEmpty(State.Cid) && - await DoesCidExistInNetwork(State.Cid)) + DoesCidExistInNetwork(State.Cid)) { Log("Basic-CID successfully found in the network."); return State.Cid; @@ -87,7 +86,7 @@ namespace AutoClient.Modes.FolderStore } Log($"Uploading..."); - var cid = await codex.UploadFile(sourceFilename); + var cid = node.UploadFile(sourceFilename); onFileUploaded(); Log("Got Basic-CID: " + cid); State.Cid = cid.Id; @@ -95,7 +94,7 @@ namespace AutoClient.Modes.FolderStore return State.Cid; } - private async Task DoesCidExistInNetwork(string cid) + private bool DoesCidExistInNetwork(string cid) { try { @@ -107,7 +106,7 @@ namespace AutoClient.Modes.FolderStore cts.Cancel(); }); - var manifest = await instance.Codex.DownloadNetworkManifestAsync(cid, cts.Token); + var manifest = node.Node.DownloadManifestOnly(new ContentId(cid)); if (manifest == null) return false; } catch @@ -117,23 +116,23 @@ namespace AutoClient.Modes.FolderStore return true; } - private async Task EnsureRecentPurchase() + private void EnsureRecentPurchase() { Log($"Checking recent purchase..."); var recent = GetMostRecent(); if (recent == null) { Log($"No recent purchase."); - await MakeNewPurchase(); + MakeNewPurchase(); return; } - await UpdatePurchase(recent); + UpdatePurchase(recent); if (recent.Expiry.HasValue) { Log($"Purchase has failed or expired."); - await MakeNewPurchase(); + MakeNewPurchase(); State.FailureCounter++; return; } @@ -141,7 +140,7 @@ namespace AutoClient.Modes.FolderStore if (recent.Finish.HasValue) { Log($"Purchase has finished."); - await MakeNewPurchase(); + MakeNewPurchase(); return; } @@ -149,7 +148,7 @@ namespace AutoClient.Modes.FolderStore if (recent.Started.HasValue && DateTime.UtcNow > safeEnd) { Log($"Purchase is going to expire soon."); - await MakeNewPurchase(); + MakeNewPurchase(); return; } @@ -168,12 +167,12 @@ namespace AutoClient.Modes.FolderStore Log($"Purchase is running."); } - private async Task UpdatePurchase(WorkerPurchase recent) + private void UpdatePurchase(WorkerPurchase recent) { if (string.IsNullOrEmpty(recent.Pid)) throw new Exception("No purchaseID!"); var now = DateTime.UtcNow; - var purchase = await codex.GetStoragePurchase(recent.Pid); + var purchase = node.GetStoragePurchase(recent.Pid); if (purchase == null) { Log($"No purchase information found for PID '{recent.Pid}'. Consider this one expired."); @@ -210,15 +209,15 @@ namespace AutoClient.Modes.FolderStore SaveState(); } - private async Task MakeNewPurchase() + private void MakeNewPurchase() { - var cid = await EnsureCid(); + var cid = EnsureCid(); if (string.IsNullOrEmpty(cid)) throw new Exception("No cid!"); Log($"Creating new purchase..."); - var response = await codex.RequestStorage(new CodexPlugin.ContentId(cid)); + var response = node.RequestStorage(new ContentId(cid)); var purchaseId = response.PurchaseId; - var encodedCid = response.EncodedCid; + var encodedCid = response.ContentId; if (string.IsNullOrEmpty(purchaseId) || purchaseId == "Unable to encode manifest" || purchaseId == "Purchasing not available" || @@ -248,7 +247,7 @@ namespace AutoClient.Modes.FolderStore while (DateTime.UtcNow < timeout) { Thread.Sleep(5000); - await UpdatePurchase(newPurchase); + UpdatePurchase(newPurchase); if (newPurchase.Submitted.HasValue) { Log("New purchase successfully submitted."); diff --git a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs b/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs index 4a5df0eb..aadfe037 100644 --- a/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs +++ b/Tools/AutoClient/Modes/FolderStore/FolderWorkOverview.cs @@ -1,5 +1,4 @@ -using CodexOpenApi; -using System.IO.Compression; +using System.IO.Compression; using static AutoClient.Modes.FolderStore.FolderWorkOverview; namespace AutoClient.Modes.FolderStore @@ -22,7 +21,7 @@ namespace AutoClient.Modes.FolderStore newState.LastOverviewUpdate = DateTime.MinValue; } - public async Task Update(ICodexInstance instance) + public void Update(CodexWrapper instance) { var jsonFiles = Directory.GetFiles(Folder).Where(f => f.ToLowerInvariant().EndsWith(".json") && !f.Contains(OverviewFilename)).ToList(); @@ -54,7 +53,7 @@ namespace AutoClient.Modes.FolderStore State.UncommitedChanges = 0; SaveState(); - await CreateNewOverviewZip(jsonFiles, FilePath, instance); + CreateNewOverviewZip(jsonFiles, FilePath, instance); } } @@ -64,7 +63,7 @@ namespace AutoClient.Modes.FolderStore SaveState(); } - private async Task CreateNewOverviewZip(List jsonFiles, string filePath, ICodexInstance instance) + private void CreateNewOverviewZip(List jsonFiles, string filePath, CodexWrapper node) { Log(""); Log(""); @@ -74,15 +73,14 @@ namespace AutoClient.Modes.FolderStore Log("Uploading to Codex..."); try { - var codex = new CodexNode(app, instance); - var cid = await codex.UploadFile(zipFilename); + var cid = node.UploadFile(zipFilename); Log($"Upload successful: New overview zipfile CID = '{cid.Id}'"); Log("Requesting storage for it..."); - var result = await codex.RequestStorage(cid); - Log("Storage requested. Purchase ID: " + result); + var result = node.RequestStorage(cid); + Log("Storage requested. Purchase ID: " + result.PurchaseId); var outFile = Path.Combine(app.Config.DataPath, "OverviewZip.cid"); - File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.EncodedCid.Id]); + File.AppendAllLines(outFile, [DateTime.UtcNow.ToString("o") + " - " + result.ContentId.Id]); Log($">>> [{outFile}] has been updated. <<<"); } catch (Exception exc) diff --git a/Tools/AutoClient/Modes/FolderStoreMode.cs b/Tools/AutoClient/Modes/FolderStoreMode.cs index 36d73d1d..f0318a64 100644 --- a/Tools/AutoClient/Modes/FolderStoreMode.cs +++ b/Tools/AutoClient/Modes/FolderStoreMode.cs @@ -17,13 +17,13 @@ namespace AutoClient.Modes this.purchaseInfo = purchaseInfo; } - public void Start(ICodexInstance instance, int index) + public void Start(CodexWrapper instance, int index) { - checkTask = Task.Run(async () => + checkTask = Task.Run(() => { try { - await RunChecker(instance); + RunChecker(instance); } catch (Exception ex) { @@ -33,14 +33,14 @@ namespace AutoClient.Modes }); } - private async Task RunChecker(ICodexInstance instance) + private void RunChecker(CodexWrapper instance) { var i = 0; while (!cts.IsCancellationRequested) { Thread.Sleep(2000); - var worker = await ProcessWorkItem(instance); + var worker = ProcessWorkItem(instance); if (worker.FailureCounter > 5) { throw new Exception("Worker has failure count > 5. Stopping AutoClient..."); @@ -51,16 +51,16 @@ namespace AutoClient.Modes { i = 0; var overview = new FolderWorkOverview(app, purchaseInfo, folder); - await overview.Update(instance); + overview.Update(instance); } } } - private async Task ProcessWorkItem(ICodexInstance instance) + private FileWorker ProcessWorkItem(CodexWrapper instance) { var file = app.FolderWorkDispatcher.GetFileToCheck(); var worker = new FileWorker(app, instance, purchaseInfo, folder, file, OnFileUploaded, OnNewPurchase); - await worker.Update(); + worker.Update(); if (worker.IsBusy()) app.FolderWorkDispatcher.WorkerIsBusy(); return worker; } diff --git a/Tools/AutoClient/Modes/Mode.cs b/Tools/AutoClient/Modes/Mode.cs index 5ab33a10..afc76f7a 100644 --- a/Tools/AutoClient/Modes/Mode.cs +++ b/Tools/AutoClient/Modes/Mode.cs @@ -1,14 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace AutoClient.Modes +namespace AutoClient.Modes { public interface IMode { - void Start(ICodexInstance instance, int index); + void Start(CodexWrapper node, int index); void Stop(); } } diff --git a/Tools/AutoClient/Modes/PurchasingMode.cs b/Tools/AutoClient/Modes/PurchasingMode.cs index 52e82215..206ab365 100644 --- a/Tools/AutoClient/Modes/PurchasingMode.cs +++ b/Tools/AutoClient/Modes/PurchasingMode.cs @@ -13,11 +13,11 @@ namespace AutoClient.Modes this.app = app; } - public void Start(ICodexInstance instance, int index) + public void Start(CodexWrapper node, int index) { for (var i = 0; i < app.Config.NumConcurrentPurchases; i++) { - purchasers.Add(new AutomaticPurchaser(new LogPrefixer(app.Log, $"({i}) "), instance, new CodexNode(app, instance))); + purchasers.Add(new AutomaticPurchaser(app, new LogPrefixer(app.Log, $"({i}) "), node)); } var delayPerPurchaser = diff --git a/Tools/AutoClient/Program.cs b/Tools/AutoClient/Program.cs index 8c8aed13..72967d90 100644 --- a/Tools/AutoClient/Program.cs +++ b/Tools/AutoClient/Program.cs @@ -2,7 +2,7 @@ using AutoClient; using AutoClient.Modes; using AutoClient.Modes.FolderStore; -using CodexOpenApi; +using CodexClient; using Utils; public class Program @@ -34,10 +34,11 @@ public class Program public async Task Run() { - var codexInstances = await CreateCodexInstances(); + await Task.CompletedTask; + var codexNodes = CreateCodexWrappers(); var i = 0; - foreach (var cdx in codexInstances) + foreach (var cdx in codexNodes) { var mode = CreateMode(); modes.Add(mode); @@ -74,20 +75,20 @@ public class Program )); } - private async Task CreateCodexInstances() + private CodexWrapper[] CreateCodexWrappers() { var endpointStrs = app.Config.CodexEndpoints.Split(";", StringSplitOptions.RemoveEmptyEntries); - var result = new List(); + var result = new List(); foreach (var e in endpointStrs) { - result.Add(await CreateCodexInstance(e)); + result.Add(CreateCodexWrapper(e)); } return result.ToArray(); } - private async Task CreateCodexInstance(string endpoint) + private CodexWrapper CreateCodexWrapper(string endpoint) { var splitIndex = endpoint.LastIndexOf(':'); var host = endpoint.Substring(0, splitIndex); @@ -99,35 +100,9 @@ public class Program port: port ); - var client = new HttpClient(); - client.Timeout = TimeSpan.FromMinutes(60.0); - var codex = new CodexApi(client); - codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - - app.Log.Log($"Checking Codex at {address}..."); - await CheckCodex(codex); - app.Log.Log("OK"); - - return new CodexInstance( - app, - codex, - client, - address - ); - } - - private async Task CheckCodex(CodexApi codex) - { - try - { - var info = await codex.GetDebugInfoAsync(); - if (string.IsNullOrEmpty(info.Id)) throw new Exception("Failed to fetch Codex node id"); - } - catch (Exception ex) - { - app.Log.Error($"Codex not OK: {ex}"); - throw; - } + var instance = CodexInstance.CreateFromApiEndpoint("ac", address); + var node = app.CodexNodeFactory.CreateCodexNode(instance); + return new CodexWrapper(app, node); } private static void PrintHelp() diff --git a/Tools/BiblioTech/CodexCidChecker.cs b/Tools/BiblioTech/CodexCidChecker.cs index 3b4b8bb9..15728d0d 100644 --- a/Tools/BiblioTech/CodexCidChecker.cs +++ b/Tools/BiblioTech/CodexCidChecker.cs @@ -1,5 +1,4 @@ -using CodexOpenApi; -using IdentityModel.Client; +using CodexClient; using Logging; using Utils; @@ -11,15 +10,26 @@ namespace BiblioTech private readonly Configuration config; private readonly ILog log; private readonly Mutex checkMutex = new Mutex(); - private CodexApi? currentCodexNode; + private readonly CodexNodeFactory factory; + private ICodexNode? currentCodexNode; public CodexCidChecker(Configuration config, ILog log) { this.config = config; this.log = log; + + factory = new CodexNodeFactory(log, dataDir: config.DataPath); + + if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) + { + throw new Exception("Todo: codexnodefactory httpfactory support basicauth!"); + //var tokens = config.CodexEndpointAuth.Split(':'); + //if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); + //client.SetBasicAuthentication(tokens[0], tokens[1]); + } } - public async Task PerformCheck(string cid) + public CheckResponse PerformCheck(string cid) { if (string.IsNullOrEmpty(config.CodexEndpoint)) { @@ -30,10 +40,10 @@ namespace BiblioTech { checkMutex.WaitOne(); var codex = GetCodex(); - var nodeCheck = await CheckCodex(codex); + var nodeCheck = CheckCodex(codex); if (!nodeCheck) return new CheckResponse(false, "Codex node is not available. Cannot perform check.", $"Codex node at '{config.CodexEndpoint}' did not respond correctly to debug/info."); - return await PerformCheck(codex, cid); + return PerformCheck(codex, cid); } catch (Exception ex) { @@ -45,19 +55,13 @@ namespace BiblioTech } } - private async Task PerformCheck(CodexApi codex, string cid) + private CheckResponse PerformCheck(ICodexNode codex, string cid) { try { - var manifest = await codex.DownloadNetworkManifestAsync(cid); + var manifest = codex.DownloadManifestOnly(new ContentId(cid)); return SuccessMessage(manifest); } - catch (ApiException apiEx) - { - if (apiEx.StatusCode == 400) return CidFormatInvalid(apiEx.Response); - if (apiEx.StatusCode == 404) return FailedToFetch(apiEx.Response); - return UnexpectedReturnCode(apiEx.Response); - } catch (Exception ex) { return UnexpectedException(ex); @@ -66,13 +70,13 @@ namespace BiblioTech #region Response formatting - private CheckResponse SuccessMessage(DataItem content) + private CheckResponse SuccessMessage(LocalDataset content) { return FormatResponse( success: true, title: $"Success: '{content.Cid}'", error: "", - $"size: {content.Manifest.DatasetSize} bytes", + $"size: {content.Manifest.OriginalBytes} bytes", $"blockSize: {content.Manifest.BlockSize} bytes", $"protected: {content.Manifest.Protected}" ); @@ -143,17 +147,17 @@ namespace BiblioTech #region Codex Node API - private CodexApi GetCodex() + private ICodexNode GetCodex() { if (currentCodexNode == null) currentCodexNode = CreateCodex(); return currentCodexNode; } - private async Task CheckCodex(CodexApi codex) + private bool CheckCodex(ICodexNode node) { try { - var info = await currentCodexNode!.GetDebugInfoAsync(); + var info = node.GetDebugInfo(); if (info == null || string.IsNullOrEmpty(info.Id)) return false; return true; } @@ -164,7 +168,7 @@ namespace BiblioTech } } - private CodexApi CreateCodex() + private ICodexNode CreateCodex() { var endpoint = config.CodexEndpoint; var splitIndex = endpoint.LastIndexOf(':'); @@ -177,17 +181,8 @@ namespace BiblioTech port: port ); - var client = new HttpClient(); - if (!string.IsNullOrEmpty(config.CodexEndpointAuth) && config.CodexEndpointAuth.Contains(":")) - { - var tokens = config.CodexEndpointAuth.Split(':'); - if (tokens.Length != 2) throw new Exception("Expected ':' in CodexEndpointAuth parameter."); - client.SetBasicAuthentication(tokens[0], tokens[1]); - } - - var codex = new CodexApi(client); - codex.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - return codex; + var instance = CodexInstance.CreateFromApiEndpoint("ac", address); + return factory.CreateCodexNode(instance); } #endregion diff --git a/Tools/BiblioTech/Commands/CheckCidCommand.cs b/Tools/BiblioTech/Commands/CheckCidCommand.cs index a2314280..1e77ce26 100644 --- a/Tools/BiblioTech/Commands/CheckCidCommand.cs +++ b/Tools/BiblioTech/Commands/CheckCidCommand.cs @@ -1,10 +1,5 @@ using BiblioTech.Options; -using System.IO; -using System.Linq; -using System.Threading.Tasks; using Discord; -using BiblioTech.Rewards; -using System.Data; namespace BiblioTech.Commands { @@ -38,7 +33,7 @@ namespace BiblioTech.Commands return; } - var response = await checker.PerformCheck(cid); + var response = checker.PerformCheck(cid); await Program.AdminChecker.SendInAdminChannel($"User {Mention(user)} used '/{Name}' for cid '{cid}'. Lookup-success: {response.Success}. Message: '{response.Message}' Error: '{response.Error}'"); if (response.Success) diff --git a/Tools/BiblioTech/Commands/GetBalanceCommand.cs b/Tools/BiblioTech/Commands/GetBalanceCommand.cs index 8d2460d6..abd19400 100644 --- a/Tools/BiblioTech/Commands/GetBalanceCommand.cs +++ b/Tools/BiblioTech/Commands/GetBalanceCommand.cs @@ -1,6 +1,7 @@ using BiblioTech.Options; using CodexContractsPlugin; using GethPlugin; +using Utils; namespace BiblioTech.Commands { diff --git a/Tools/BiblioTech/Rewards/RoleDriver.cs b/Tools/BiblioTech/Rewards/RoleDriver.cs index fd5cde8e..1964da8e 100644 --- a/Tools/BiblioTech/Rewards/RoleDriver.cs +++ b/Tools/BiblioTech/Rewards/RoleDriver.cs @@ -3,6 +3,7 @@ using Discord.WebSocket; using DiscordRewards; using Logging; using Newtonsoft.Json; +using Utils; namespace BiblioTech.Rewards { @@ -138,7 +139,7 @@ namespace BiblioTech.Rewards { try { - var userData = Program.UserRepo.GetUserDataForAddress(new GethPlugin.EthAddress(address)); + var userData = Program.UserRepo.GetUserDataForAddress(new EthAddress(address)); if (userData != null) log.Log($"User '{userData.Name}' was looked up."); else log.Log($"Lookup for user was unsuccessful. EthAddress: '{address}'"); return userData; diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index d970b016..fc5c9e81 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -1,4 +1,5 @@ -using CodexContractsPlugin; +using CodexClient; +using CodexContractsPlugin; using CodexPlugin; using Core; using GethPlugin; diff --git a/Tools/CodexNetDeployer/Configuration.cs b/Tools/CodexNetDeployer/Configuration.cs index 441e355a..91134837 100644 --- a/Tools/CodexNetDeployer/Configuration.cs +++ b/Tools/CodexNetDeployer/Configuration.cs @@ -1,5 +1,5 @@ using ArgsUniform; -using CodexPlugin; +using CodexClient; using DistTestCore; namespace CodexNetDeployer diff --git a/Tools/CodexNetDeployer/Deployer.cs b/Tools/CodexNetDeployer/Deployer.cs index 1981b81a..1eb97954 100644 --- a/Tools/CodexNetDeployer/Deployer.cs +++ b/Tools/CodexNetDeployer/Deployer.cs @@ -1,4 +1,5 @@ using BlockchainUtils; +using CodexClient; using CodexContractsPlugin; using CodexDiscordBotPlugin; using CodexPlugin; @@ -7,6 +8,7 @@ using GethPlugin; using KubernetesWorkflow.Types; using Logging; using MetricsPlugin; +using WebUtils; namespace CodexNetDeployer { @@ -100,7 +102,7 @@ namespace CodexNetDeployer retryDelay: TimeSpan.FromSeconds(10), kubernetesNamespace: config.KubeNamespace); - var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet()); + var result = new EntryPoint(log, configuration, string.Empty, new FastHttpTimeSet(), new DefaultK8sTimeSet()); configuration.Hooks = new K8sHook(config.TestsTypePodLabel, config.DeployId, result.GetPluginMetadata()); return result; @@ -248,7 +250,7 @@ namespace CodexNetDeployer } } - public class FastHttpTimeSet : ITimeSet + public class FastHttpTimeSet : IWebCallTimeSet { public TimeSpan HttpCallRetryDelay() { @@ -264,15 +266,5 @@ namespace CodexNetDeployer { return TimeSpan.FromSeconds(10); } - - public TimeSpan K8sOperationTimeout() - { - return TimeSpan.FromMinutes(10); - } - - public TimeSpan K8sOperationRetryDelay() - { - return TimeSpan.FromSeconds(30); - } } } \ No newline at end of file