From 32bba67bef03645949fde267c89a16ddcfe02167 Mon Sep 17 00:00:00 2001 From: E M <5089238+emizzle@users.noreply.github.com> Date: Wed, 15 Apr 2026 20:34:49 +1000 Subject: [PATCH] Rename and remove all instances of Codex --- .github/workflows/docker-runner.yml | 2 +- .github/workflows/run-continuous-tests.yaml | 2 +- .github/workflows/run-dist-tests.yaml | 4 +- .github/workflows/run-release-tests.yaml | 2 +- CONTRIBUTINGPLUGINS.MD | 14 +- CONTRIBUTINGTESTS.MD | 32 +- Framework/Logging/ApplicationIds.cs | 10 +- .../CodexClient/CodexNodeFactory.cs | 46 --- .../CodexClient/Hooks/CodexHooksFactory.cs | 61 ---- ProjectPlugins/CodexPlugin/CodexPlugin.cs | 96 ------ ProjectPlugins/CodexPlugin/CodexSetup.cs | 135 --------- .../CodexPlugin/CoreInterfaceExtensions.cs | 51 ---- ProjectPlugins/CodexPlugin/ICodexStarter.cs | 10 - .../DeployAndRunContainerRecipe.cs | 16 +- .../Hooks/LogosStorageHooksFactory.cs | 61 ++++ .../Hooks/StorageNodeHooks.cs} | 14 +- .../LogosStorageAccess.cs} | 44 +-- .../LogosStorageClient.csproj} | 2 +- .../LogosStorageInstance.cs} | 12 +- .../LogosStorageLogLevel.cs} | 4 +- .../LogosStorageLogLine.cs} | 8 +- .../LogosStorageTypes.cs} | 4 +- .../LogosStorageUtils.cs} | 4 +- .../Mapper.cs | 22 +- .../ProcessControl.cs | 6 +- .../StorageNode.cs} | 90 +++--- .../LogosStorageClient/StorageNodeFactory.cs | 46 +++ .../TransferSpeeds.cs | 2 +- .../openapi.yaml | 0 .../CoreInterfaceExtensions.cs | 8 +- .../DiscordBotContainerRecipe.cs | 4 +- .../DiscordBotStartupConfig.cs | 2 +- .../LogosStorageDiscordBotPlugin.cs} | 10 +- .../LogosStorageDiscordBotPlugin.csproj} | 2 +- .../RewarderBotContainerRecipe.cs | 4 +- .../LogosStoragePluginPrebuild.csproj} | 0 .../Program.cs | 22 +- .../PrometheusContainerRecipe.cs | 2 +- ProjectPlugins/MetricsPlugin/dashboard.json | 2 +- .../ApiChecker.cs | 24 +- .../BinaryProcessControl.cs | 8 +- .../BinaryStorageStarter.cs} | 50 ++-- .../ContainerLogosStorageStarter.cs} | 32 +- .../ContainerProcessControl.cs} | 8 +- .../ContainerRecipe.cs} | 32 +- .../StoragePlugin/CoreInterfaceExtensions.cs | 51 ++++ .../Deployment.cs} | 22 +- .../DockerImage.cs} | 8 +- .../FreePortFinder.cs | 2 +- .../StoragePlugin/ILogosStorageStarter.cs | 10 + .../InstanceContainerExtension.cs} | 18 +- .../LocalNodeBuilder.cs} | 34 +-- .../LogosStorageExePath.cs} | 10 +- .../LogosStorageProcessRecipe.cs} | 20 +- .../StoragePlugin/LogosStorageSetup.cs | 135 +++++++++ .../LogosStorageStartupConfig.cs} | 16 +- .../LogosStorageWrapper.cs} | 36 +-- .../OverwatchSupport/IdentityMap.cs | 18 +- .../KademliaPositionFinder.cs | 10 +- .../BlockReceivedLineConverter.cs | 6 +- .../LineConverters/BootstrapLineConverter.cs | 6 +- .../DialSuccessfulLineConverter.cs | 6 +- .../PeerDroppedLineConverter.cs | 6 +- .../LogosStorageLogConverter.cs} | 38 +-- .../LogosStorageTranscriptWriter.cs} | 38 +-- .../LogosStorageTranscriptWriterConfig.cs} | 6 +- .../OverwatchSupport/ModelExtensions.cs | 14 +- .../StorageNodeTranscriptWriter.cs} | 38 +-- .../ProcessControlMap.cs | 12 +- .../StorageNodeGroup.cs} | 32 +- ProjectPlugins/StoragePlugin/StoragePlugin.cs | 96 ++++++ .../StoragePlugin.csproj} | 4 +- README.md | 12 +- Tests/DistTestCore/Configuration.cs | 2 +- Tests/DistTestCore/DistTest.cs | 4 +- .../AutoBootstrapDistTest.cs | 14 +- .../BasicTests/AsyncProfiling.cs | 6 +- .../BasicTests/ExampleTests.cs | 16 +- .../BasicTests/PyramidTests.cs | 20 +- .../DatalayerReliabilityTests.cs | 16 +- .../DetectBlockRetransmitTest.cs | 4 +- .../FullyConnectedDownloadTests.cs | 10 +- .../MultiswarmTests.cs | 8 +- .../ExperimentalTests.csproj | 4 +- .../Helpers/FullConnectivityHelper.cs | 16 +- .../Helpers/PeerConnectionTestHelpers.cs | 8 +- .../Helpers/PeerDownloadTestHelpers.cs | 12 +- ...dexDistTest.cs => LogosStorageDistTest.cs} | 75 +++-- ...r.cs => LogosStorageLogTrackerProvider.cs} | 24 +- .../MetricsAccessExtensions.cs | 2 +- Tests/ExperimentalTests/Parallelism.cs | 2 +- .../LayeredDiscoveryTests.cs | 32 +- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 14 +- Tests/ExperimentalTests/README.md | 14 +- .../UtilityTests/LogHelperTests.cs | 12 +- .../UtilityTests/NetworkIsolationTest.cs | 12 +- Tests/FrameworkTests/FrameworkTests.csproj | 2 +- .../ContentIdTests.cs | 4 +- .../Configuration.cs | 16 +- .../ContinuousTest.cs | 12 +- .../ContinuousTestRunner.cs | 8 +- .../ElasticSearchLogDownloader.cs | 0 .../EntryPointFactory.cs | 2 +- .../LogosStorageContinuousTests.csproj} | 2 +- .../NodeRunner.cs | 16 +- .../Program.cs | 2 +- .../README.md | 13 +- .../SingleTestRun.cs | 20 +- .../StartupChecker.cs | 24 +- .../TestFactory.cs | 0 .../TestHandle.cs | 0 .../TestLoop.cs | 2 +- .../TestMomentAttribute.cs | 0 .../Tests/HoldMyBeerTest.cs | 2 +- .../Tests/MarketplaceTest.cs | 10 +- .../Tests/PeersTest.cs | 8 +- .../Tests/PerformanceTests.cs | 8 +- .../Tests/TransientNodeTest.cs | 16 +- .../Tests/TwoClientTest.cs | 16 +- .../deploy-and-run.sh | 6 +- .../LogosStorageTestNetReport-August2023.md} | 8 +- .../LogosStorageTestNetReport-July2023.md} | 10 +- ...LogosStorageTestNetReport-November2023.md} | 6 +- .../LogosStorageTestNetReport-October2023.md} | 8 +- ...ogosStorageTestNetReport-September2023.md} | 10 +- .../run.sh | 6 +- .../BasicTests/DownloadTests.cs | 10 +- .../BasicTests/LargeFileTests.cs | 14 +- .../BasicTests/TestInfraTests.cs | 10 +- .../BasicTests/UploadTests.cs | 10 +- .../LongFullyConnectedDownloadTests.cs | 6 +- .../LogosStorageTestsLong.csproj} | 0 .../Parallelism.cs | 2 +- .../MultiPeerDownloadTests.cs | 12 +- .../ScalabilityTests/ScalabilityTests.cs | 22 +- .../DataTests/DataExpiryTest.cs | 12 +- .../DataTests/DhtTest.cs | 10 +- .../DataTests/InterruptUploadTest.cs | 14 +- .../DataTests/LocalFilesTest.cs | 8 +- .../DataTests/ManifestOnlyDownloadTest.cs | 10 +- .../DataTests/OneClientTest.cs | 12 +- .../DataTests/StreamlessDownloadTest.cs | 10 +- .../DataTests/SwarmTest.cs | 24 +- .../DataTests/TheseusTest.cs | 16 +- .../DataTests/ThreeClientTest.cs | 8 +- .../DataTests/TwoClientTest.cs | 22 +- .../DataTests/UnknownCidTest.cs | 10 +- .../LogosStorageReleaseTests.csproj} | 0 .../NodeTests/BasicInfoTests.cs | 14 +- .../NodeTests/PeerTableTests.cs | 12 +- .../Parallelism.cs | 2 +- .../RerunAttribute.cs | 2 +- Tools/CsvCombiner/Program.cs | 2 +- Tools/TestClusterStarter/ClusterTestSpec.cs | 6 +- Tools/TestClusterStarter/Program.cs | 2 +- Tools/TranscriptAnalysis/Program.cs | 4 +- Tools/TranscriptAnalysis/ReceiverSet.cs | 8 +- .../Receivers/BaseReceiver.cs | 8 +- .../Receivers/DuplicateBlocksReceived.cs | 8 +- .../Receivers/LogReplaceReceiver.cs | 10 +- .../Receivers/NodesDegree.cs | 12 +- .../TranscriptAnalysis.csproj | 2 +- codexnode-manifest.yml | 26 +- docker/docker-compose.yaml | 4 +- docker/docker-dnr-entrypoint.sh | 2 +- docker/docker-entrypoint.sh | 6 +- docker/job-continuous-tests.yaml | 14 +- docker/job-dist-tests.yaml | 10 +- docker/job-release-tests.yaml | 14 +- docs/Automation.md | 34 +-- docs/CodexHowTo.md | 9 - docs/Continuous-Tests.md | 35 +-- docs/LOCALSETUP.md | 13 - docs/LogosStorageHowTo.md | 9 + docs/RunningReleaseTests.md | 276 ++++++++++++++++++ ...ting.sln => logos-storage-dist-testing.sln | 14 +- 176 files changed, 1661 insertions(+), 1417 deletions(-) delete mode 100644 ProjectPlugins/CodexClient/CodexNodeFactory.cs delete mode 100644 ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs delete mode 100644 ProjectPlugins/CodexPlugin/CodexPlugin.cs delete mode 100644 ProjectPlugins/CodexPlugin/CodexSetup.cs delete mode 100644 ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs delete mode 100644 ProjectPlugins/CodexPlugin/ICodexStarter.cs create mode 100644 ProjectPlugins/LogosStorageClient/Hooks/LogosStorageHooksFactory.cs rename ProjectPlugins/{CodexClient/Hooks/CodexNodeHooks.cs => LogosStorageClient/Hooks/StorageNodeHooks.cs} (77%) rename ProjectPlugins/{CodexClient/CodexAccess.cs => LogosStorageClient/LogosStorageAccess.cs} (80%) rename ProjectPlugins/{CodexClient/CodexClient.csproj => LogosStorageClient/LogosStorageClient.csproj} (94%) rename ProjectPlugins/{CodexClient/CodexInstance.cs => LogosStorageClient/LogosStorageInstance.cs} (72%) rename ProjectPlugins/{CodexClient/CodexLogLevel.cs => LogosStorageClient/LogosStorageLogLevel.cs} (56%) rename ProjectPlugins/{CodexClient/CodexLogLine.cs => LogosStorageClient/LogosStorageLogLine.cs} (94%) rename ProjectPlugins/{CodexClient/CodexTypes.cs => LogosStorageClient/LogosStorageTypes.cs} (98%) rename ProjectPlugins/{CodexClient/CodexUtils.cs => LogosStorageClient/LogosStorageUtils.cs} (86%) rename ProjectPlugins/{CodexClient => LogosStorageClient}/Mapper.cs (84%) rename ProjectPlugins/{CodexClient => LogosStorageClient}/ProcessControl.cs (82%) rename ProjectPlugins/{CodexClient/CodexNode.cs => LogosStorageClient/StorageNode.cs} (80%) create mode 100644 ProjectPlugins/LogosStorageClient/StorageNodeFactory.cs rename ProjectPlugins/{CodexClient => LogosStorageClient}/TransferSpeeds.cs (98%) rename ProjectPlugins/{CodexClient => LogosStorageClient}/openapi.yaml (100%) rename ProjectPlugins/{CodexDiscordBotPlugin => LogosStorageDiscordBotPlugin}/CoreInterfaceExtensions.cs (56%) rename ProjectPlugins/{CodexDiscordBotPlugin => LogosStorageDiscordBotPlugin}/DiscordBotContainerRecipe.cs (92%) rename ProjectPlugins/{CodexDiscordBotPlugin => LogosStorageDiscordBotPlugin}/DiscordBotStartupConfig.cs (98%) rename ProjectPlugins/{CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs => LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.cs} (89%) rename ProjectPlugins/{CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj => LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.csproj} (80%) rename ProjectPlugins/{CodexDiscordBotPlugin => LogosStorageDiscordBotPlugin}/RewarderBotContainerRecipe.cs (91%) rename ProjectPlugins/{CodexPluginPrebuild/CodexPluginPrebuild.csproj => LogosStoragePluginPrebuild/LogosStoragePluginPrebuild.csproj} (100%) rename ProjectPlugins/{CodexPluginPrebuild => LogosStoragePluginPrebuild}/Program.cs (78%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/ApiChecker.cs (75%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/BinaryProcessControl.cs (92%) rename ProjectPlugins/{CodexPlugin/BinaryCodexStarter.cs => StoragePlugin/BinaryStorageStarter.cs} (67%) rename ProjectPlugins/{CodexPlugin/ContainerCodexStarter.cs => StoragePlugin/ContainerLogosStorageStarter.cs} (67%) rename ProjectPlugins/{CodexPlugin/CodexContainerProcessControl.cs => StoragePlugin/ContainerProcessControl.cs} (89%) rename ProjectPlugins/{CodexPlugin/CodexContainerRecipe.cs => StoragePlugin/ContainerRecipe.cs} (78%) create mode 100644 ProjectPlugins/StoragePlugin/CoreInterfaceExtensions.cs rename ProjectPlugins/{CodexPlugin/CodexDeployment.cs => StoragePlugin/Deployment.cs} (73%) rename ProjectPlugins/{CodexPlugin/CodexDockerImage.cs => StoragePlugin/DockerImage.cs} (64%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/FreePortFinder.cs (97%) create mode 100644 ProjectPlugins/StoragePlugin/ILogosStorageStarter.cs rename ProjectPlugins/{CodexPlugin/CodexInstanceContainerExtension.cs => StoragePlugin/InstanceContainerExtension.cs} (62%) rename ProjectPlugins/{CodexPlugin/LocalCodexBuilder.cs => StoragePlugin/LocalNodeBuilder.cs} (77%) rename ProjectPlugins/{CodexPlugin/CodexExePath.cs => StoragePlugin/LogosStorageExePath.cs} (61%) rename ProjectPlugins/{CodexPlugin/CodexProcessRecipe.cs => StoragePlugin/LogosStorageProcessRecipe.cs} (84%) create mode 100644 ProjectPlugins/StoragePlugin/LogosStorageSetup.cs rename ProjectPlugins/{CodexPlugin/CodexStartupConfig.cs => StoragePlugin/LogosStorageStartupConfig.cs} (86%) rename ProjectPlugins/{CodexPlugin/CodexWrapper.cs => StoragePlugin/LogosStorageWrapper.cs} (54%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/IdentityMap.cs (73%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/KademliaPositionFinder.cs (79%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs (86%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/LineConverters/BootstrapLineConverter.cs (94%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs (67%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs (66%) rename ProjectPlugins/{CodexPlugin/OverwatchSupport/CodexLogConverter.cs => StoragePlugin/OverwatchSupport/LogosStorageLogConverter.cs} (67%) rename ProjectPlugins/{CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs => StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriter.cs} (64%) rename ProjectPlugins/{CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs => StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriterConfig.cs} (54%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/OverwatchSupport/ModelExtensions.cs (91%) rename ProjectPlugins/{CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs => StoragePlugin/OverwatchSupport/StorageNodeTranscriptWriter.cs} (74%) rename ProjectPlugins/{CodexPlugin => StoragePlugin}/ProcessControlMap.cs (67%) rename ProjectPlugins/{CodexPlugin/CodexNodeGroup.cs => StoragePlugin/StorageNodeGroup.cs} (66%) create mode 100644 ProjectPlugins/StoragePlugin/StoragePlugin.cs rename ProjectPlugins/{CodexPlugin/CodexPlugin.csproj => StoragePlugin/StoragePlugin.csproj} (87%) rename Tests/ExperimentalTests/{CodexDistTest.cs => LogosStorageDistTest.cs} (71%) rename Tests/ExperimentalTests/{CodexLogTrackerProvider.cs => LogosStorageLogTrackerProvider.cs} (55%) rename Tests/FrameworkTests/{CodexClient => LogosStorageClient}/ContentIdTests.cs (89%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Configuration.cs (79%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/ContinuousTest.cs (81%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/ContinuousTestRunner.cs (94%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/ElasticSearchLogDownloader.cs (100%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/EntryPointFactory.cs (94%) rename Tests/{CodexContinuousTests/CodexContinuousTests.csproj => LogosStorageContinuousTests/LogosStorageContinuousTests.csproj} (86%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/NodeRunner.cs (75%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Program.cs (91%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/README.md (58%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/SingleTestRun.cs (94%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/StartupChecker.cs (88%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/TestFactory.cs (100%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/TestHandle.cs (100%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/TestLoop.cs (97%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/TestMomentAttribute.cs (100%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/HoldMyBeerTest.cs (96%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/MarketplaceTest.cs (85%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/PeersTest.cs (91%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/PerformanceTests.cs (80%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/TransientNodeTest.cs (66%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/Tests/TwoClientTest.cs (81%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/deploy-and-run.sh (87%) rename Tests/{CodexContinuousTests/reports/CodexTestNetReport-August2023.md => LogosStorageContinuousTests/reports/LogosStorageTestNetReport-August2023.md} (93%) rename Tests/{CodexContinuousTests/reports/CodexTestNetReport-July2023.md => LogosStorageContinuousTests/reports/LogosStorageTestNetReport-July2023.md} (74%) rename Tests/{CodexContinuousTests/reports/CodexTestNetReport-November2023.md => LogosStorageContinuousTests/reports/LogosStorageTestNetReport-November2023.md} (86%) rename Tests/{CodexContinuousTests/reports/CodexTestNetReport-October2023.md => LogosStorageContinuousTests/reports/LogosStorageTestNetReport-October2023.md} (91%) rename Tests/{CodexContinuousTests/reports/CodexTestNetReport-September2023.md => LogosStorageContinuousTests/reports/LogosStorageTestNetReport-September2023.md} (87%) rename Tests/{CodexContinuousTests => LogosStorageContinuousTests}/run.sh (61%) rename Tests/{CodexLongTests => LogosStorageLongTests}/BasicTests/DownloadTests.cs (88%) rename Tests/{CodexLongTests => LogosStorageLongTests}/BasicTests/LargeFileTests.cs (81%) rename Tests/{CodexLongTests => LogosStorageLongTests}/BasicTests/TestInfraTests.cs (75%) rename Tests/{CodexLongTests => LogosStorageLongTests}/BasicTests/UploadTests.cs (86%) rename Tests/{CodexLongTests => LogosStorageLongTests}/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs (77%) rename Tests/{CodexLongTests/CodexTestsLong.csproj => LogosStorageLongTests/LogosStorageTestsLong.csproj} (100%) rename Tests/{CodexLongTests => LogosStorageLongTests}/Parallelism.cs (67%) rename Tests/{CodexLongTests => LogosStorageLongTests}/ScalabilityTests/MultiPeerDownloadTests.cs (91%) rename Tests/{CodexLongTests => LogosStorageLongTests}/ScalabilityTests/ScalabilityTests.cs (84%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/DataExpiryTest.cs (84%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/DhtTest.cs (88%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/InterruptUploadTest.cs (76%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/LocalFilesTest.cs (84%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/ManifestOnlyDownloadTest.cs (74%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/OneClientTest.cs (66%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/StreamlessDownloadTest.cs (76%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/SwarmTest.cs (89%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/TheseusTest.cs (78%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/ThreeClientTest.cs (76%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/TwoClientTest.cs (60%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/DataTests/UnknownCidTest.cs (80%) rename Tests/{CodexReleaseTests/CodexReleaseTests.csproj => LogosStorageReleaseTests/LogosStorageReleaseTests.csproj} (100%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/NodeTests/BasicInfoTests.cs (78%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/NodeTests/PeerTableTests.cs (60%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/Parallelism.cs (65%) rename Tests/{CodexReleaseTests => LogosStorageReleaseTests}/RerunAttribute.cs (90%) delete mode 100644 docs/CodexHowTo.md delete mode 100644 docs/LOCALSETUP.md create mode 100644 docs/LogosStorageHowTo.md create mode 100644 docs/RunningReleaseTests.md rename cs-codex-dist-testing.sln => logos-storage-dist-testing.sln (91%) diff --git a/.github/workflows/docker-runner.yml b/.github/workflows/docker-runner.yml index d64ceebe..6b2f825e 100644 --- a/.github/workflows/docker-runner.yml +++ b/.github/workflows/docker-runner.yml @@ -20,6 +20,6 @@ jobs: uses: logos-storage/github-actions/.github/workflows/docker-reusable.yml@master with: docker_file: docker/Dockerfile - dockerhub_repo: logosstorage/cs-codex-dist-tests + dockerhub_repo: logosstorage/logos-storage-dist-tests tag_latest: ${{ github.ref_name == github.event.repository.default_branch }} secrets: inherit diff --git a/.github/workflows/run-continuous-tests.yaml b/.github/workflows/run-continuous-tests.yaml index 0284c27b..04468bed 100644 --- a/.github/workflows/run-continuous-tests.yaml +++ b/.github/workflows/run-continuous-tests.yaml @@ -44,7 +44,7 @@ on: required: false type: string storagedockerimage: - description: Codex Docker image (logosstorage/logos-storage-nim:latest-dist-tests) + description: Logos Storage Docker image (logosstorage/logos-storage-nim:latest-dist-tests) required: false type: string nameprefix: diff --git a/.github/workflows/run-dist-tests.yaml b/.github/workflows/run-dist-tests.yaml index c42a6cfe..a91886eb 100644 --- a/.github/workflows/run-dist-tests.yaml +++ b/.github/workflows/run-dist-tests.yaml @@ -21,7 +21,7 @@ on: required: false type: string command: - description: Command (dotnet test Tests/CodexTests) + description: Command (dotnet test Tests/LogosStorageTests) required: false type: string @@ -31,7 +31,7 @@ env: BRANCH: ${{ github.ref_name }} NAMEPREFIX: d-tests-runner NAMESPACE: default - COMMAND: dotnet test Tests/CodexTests + COMMAND: dotnet test Tests/LogosStorageTests JOB_MANIFEST: docker/job-dist-tests.yaml KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} KUBE_VERSION: v1.28.2 diff --git a/.github/workflows/run-release-tests.yaml b/.github/workflows/run-release-tests.yaml index 3e91cfa1..90228910 100644 --- a/.github/workflows/run-release-tests.yaml +++ b/.github/workflows/run-release-tests.yaml @@ -36,7 +36,7 @@ env: NAMEPREFIX: r-tests NAMESPACE: default JOB_MANIFEST: docker/job-release-tests.yaml - COMMAND: dotnet test Tests/CodexReleaseTests + COMMAND: dotnet test Tests/LogosStorageReleaseTests KUBE_CONFIG: ${{ secrets.KUBE_CONFIG }} KUBE_VERSION: v1.30.5 diff --git a/CONTRIBUTINGPLUGINS.MD b/CONTRIBUTINGPLUGINS.MD index 250a8396..38420918 100644 --- a/CONTRIBUTINGPLUGINS.MD +++ b/CONTRIBUTINGPLUGINS.MD @@ -1,7 +1,7 @@ -# Distributed System Tests for Nim-Codex +# Distributed System Tests for Logos Storage ## Contributing plugins -The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added. +The testing framework was created for testing Logos Storage. However, it's been designed such that other containerized projects can 'easily' be added. In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin. @@ -117,7 +117,7 @@ public static class CoreInterfaceExtensions Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.) -The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing. +The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. ## Container Recipes In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way. @@ -141,18 +141,18 @@ Example: ```C# { var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1"); - var codex = Ci.StartCodexNode(s => s.At(location)); + var node = Ci.StartStorageNode(s => s.At(location)); } ``` -In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container. +In this example, 'Ci' is an instance of the core interface. The StoragePlugin exposes a function 'StartStorageNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Logos Storage plugin starts its container. The available locations array guarantees that each entry corresponds to a different kubernetes host. ```C# { var knownLocations = Ci.GetKnownLocations(); // I don't care where exactly, as long as they are different locations. - var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0))); - var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1))); + var storageAtZero = Ci.StartStorageNode(s => s.At(knownLocations.Get(0))); + var storageAtOne = Ci.StartStorageNode(s => s.At(knownLocations.Get(1))); } ``` diff --git a/CONTRIBUTINGTESTS.MD b/CONTRIBUTINGTESTS.MD index bcad8a6d..115763af 100644 --- a/CONTRIBUTINGTESTS.MD +++ b/CONTRIBUTINGTESTS.MD @@ -1,4 +1,4 @@ -# Distributed System Tests for Nim-Codex +# Distributed System Tests for Logos Storage ## Contributing tests Do you want to write some tests using this distributed test setup? Great! Here's what you do. @@ -9,32 +9,32 @@ Do you want to write some tests using this distributed test setup? Great! Here's 1. Add Project references to `Core`, as well as any project plugin you'll be using. 1. Write tests! Use existing tests for inspiration. -## Tips for writing tests for Codex +## Tips for writing tests for Logos Storage ### Transient tests -1. Add new code files to `Tests/CodexTests` -1. Inherrit from `CodexDistTest` or `AutoBootstrapDistTest`. -1. When using `CodexDistTest`: - 1. You must start your own Codex bootstrap node. You can use `AddCodex(...)` for this. - 1. When you start other Codex nodes with `AddCodex(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option. +1. Add new code files to `Tests/LogosStorageTests` +1. Inherrit from `LogosStorageDistTest` or `AutoBootstrapDistTest`. +1. When using `LogosStorageDistTest`: + 1. You must start your own Codex bootstrap node. You can use `AddLogosStorage(...)` for this. + 1. When you start other Logos Storage nodes with `AddLogosStorage(...)` you can pass the bootstrap node by adding the `.WithBootstrapNode(...)` option. 1. When using `AutoBootstrapDistTest`: - 1. The test-infra creates the bootstrap node for you, and automatically passes it to each Codex node you create in your tests. Handy for keeping your tests clean and to-the-point. - 1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `CodexDistTest` instead. + 1. The test-infra creates the bootstrap node for you, and automatically passes it to each Logos Storage node you create in your tests. Handy for keeping your tests clean and to-the-point. + 1. When using the auto-bootstrap, you have no control over the bootstrap node from your tests. You can't (for example) shut it down during the course of the test. If you need this level of control for your scenario, use the `LogosStorageDistTest` instead. 1. If your test needs a long time to run, add the `[UseLongTimeouts]` function attribute. This will greatly increase maximum time-out values for operations like for example uploading and downloading files. ### Continuous tests -1. Add new code files to `Tests/CodexContinuousTests/Tests` +1. Add new code files to `Tests/LogosStorageContinuousTests/Tests` 1. Inherrit from `ContinuousTest` 1. Define one or more methods and decorate them with the `[TestMoment(...)]` attribute. 1. The TestMoment takes a number of seconds as argument. Each moment will be executed by the continuous test runner applying the given seconds as delay. (Non-cumulative. So two moments at T:10 will be executed one after another without delay, in this case the order of execution should not be depended upon.) -1. Continuous tests automatically receive access to the Codex nodes that the tests are being run against. -1. Additionally, Continuous tests can start their own transient Codex nodes and bootstrap them against the persistent nodes. +1. Continuous tests automatically receive access to the Logos Storage nodes that the tests are being run against. +1. Additionally, Continuous tests can start their own transient Logos Storage nodes and bootstrap them against the persistent nodes. ### Tips for either type of test 1. You can generate files of random test data by calling `GenerateTestFile(...)`. -1. You can enable access to the Codex node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Codex metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.) +1. You can enable access to the Logos Storage node metrics by adding the option `.EnableMetrics()`. Enabling metrics will make the test-infra download and save all Logos Storage metrics in case of a test failure. (The metrics are stored as CSV, in the same location as the test log file.) 1. You can enable access to the blockchain marketplace by adding the option `.EnableMarketplace(...)`. -1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Codex node setup. Please don't enable these features unless your tests need them. -1. Tip: Codex nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer! +1. Enabling metrics and/or enabling the marketplace takes extra resources from the test-infra and increases the time needed during Logos Storage node setup. Please don't enable these features unless your tests need them. +1. Tip: Logos Storage nodes can be named. Use the option `WithName(...)` and make reading your test logs a little nicer! 1. Tip: Commit often. ## Don't forget -1. Once you're happy with your tests, please create a pull-request and ask a Codex core contributor to review your changes. +1. Once you're happy with your tests, please create a pull-request and ask a Logos Storage core contributor to review your changes. diff --git a/Framework/Logging/ApplicationIds.cs b/Framework/Logging/ApplicationIds.cs index 7162a367..06855e30 100644 --- a/Framework/Logging/ApplicationIds.cs +++ b/Framework/Logging/ApplicationIds.cs @@ -2,19 +2,19 @@ { public class ApplicationIds { - public ApplicationIds(string codexId, string gethId, string prometheusId, string codexContractsId, string grafanaId) + public ApplicationIds(string logosStorageId, string gethId, string prometheusId, string logosStorageContractsId, string grafanaId) { - CodexId = codexId; + LogosStorageId = logosStorageId; GethId = gethId; PrometheusId = prometheusId; - CodexContractsId = codexContractsId; + LogosStorageContractsId = logosStorageContractsId; GrafanaId = grafanaId; } - public string CodexId { get; } + public string LogosStorageId { get; } public string GethId { get; } public string PrometheusId { get; } - public string CodexContractsId { get; } + public string LogosStorageContractsId { get; } public string GrafanaId { get; } } } diff --git a/ProjectPlugins/CodexClient/CodexNodeFactory.cs b/ProjectPlugins/CodexClient/CodexNodeFactory.cs deleted file mode 100644 index 0f7423dd..00000000 --- a/ProjectPlugins/CodexClient/CodexNodeFactory.cs +++ /dev/null @@ -1,46 +0,0 @@ -using CodexClient.Hooks; -using FileUtils; -using Logging; -using WebUtils; - -namespace CodexClient -{ - public class CodexNodeFactory - { - private readonly ILog log; - private readonly IFileManager fileManager; - private readonly CodexHooksFactory hooksFactory; - private readonly IHttpFactory httpFactory; - private readonly IProcessControlFactory processControlFactory; - - public CodexNodeFactory(ILog log, IFileManager fileManager, CodexHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory) - { - this.log = log; - this.fileManager = fileManager; - this.hooksFactory = hooksFactory; - this.httpFactory = httpFactory; - this.processControlFactory = processControlFactory; - } - - public CodexNodeFactory(ILog log, HttpFactory httpFactory, string dataDir) - : this(log, new FileManager(log, dataDir), new CodexHooksFactory(), httpFactory, new DoNothingProcessControlFactory()) - { - } - - public CodexNodeFactory(ILog log, string dataDir) - : this(log, new HttpFactory(log), dataDir) - { - } - - public ICodexNode CreateCodexNode(ICodexInstance instance) - { - var processControl = processControlFactory.CreateProcessControl(instance); - var access = new CodexAccess(log, httpFactory, processControl, instance); - var hooks = hooksFactory.CreateHooks(access.GetName()); - var node = new CodexNode(log, access, fileManager, hooks); - node.Initialize(); - return node; - } - - } -} diff --git a/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs b/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs deleted file mode 100644 index d8b782a3..00000000 --- a/ProjectPlugins/CodexClient/Hooks/CodexHooksFactory.cs +++ /dev/null @@ -1,61 +0,0 @@ -using Utils; - -namespace CodexClient.Hooks -{ - public interface ICodexHooksProvider - { - ICodexNodeHooks CreateHooks(string nodeName); - } - - public class CodexHooksFactory - { - public List Providers { get; } = new List(); - - public ICodexNodeHooks CreateHooks(string nodeName) - { - if (Providers.Count == 0) return new DoNothingCodexHooks(); - - var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray(); - return new MuxingCodexNodeHooks(hooks); - } - } - - public class DoNothingHooksProvider : ICodexHooksProvider - { - public ICodexNodeHooks CreateHooks(string nodeName) - { - return new DoNothingCodexHooks(); - } - } - - public class DoNothingCodexHooks : ICodexNodeHooks - { - public void OnFileDownloaded(ByteSize size, ContentId cid) - { - } - - public void OnFileDownloading(ContentId cid) - { - } - - public void OnFileUploaded(string uid, ByteSize size, ContentId cid) - { - } - - public void OnFileUploading(string uid, ByteSize size) - { - } - - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) - { - } - - public void OnNodeStarting(DateTime startUtc, string image) - { - } - - public void OnNodeStopping() - { - } - } -} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs deleted file mode 100644 index f15ea776..00000000 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ /dev/null @@ -1,96 +0,0 @@ -using CodexClient; -using CodexClient.Hooks; -using Core; - -namespace CodexPlugin -{ - public class CodexPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata - { - private const bool UseContainers = true; - - private readonly ICodexStarter codexStarter; - private readonly IPluginTools tools; - private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; - private readonly CodexHooksFactory hooksFactory = new CodexHooksFactory(); - private readonly ProcessControlMap processControlMap = new ProcessControlMap(); - private readonly CodexDockerImage codexDockerImage = new CodexDockerImage(); - private readonly CodexContainerRecipe recipe; - private readonly CodexWrapper codexWrapper; - - public CodexPlugin(IPluginTools tools) - { - this.tools = tools; - - recipe = new CodexContainerRecipe(codexDockerImage); - codexStarter = CreateCodexStarter(); - codexWrapper = new CodexWrapper(tools, processControlMap, hooksFactory); - } - - private ICodexStarter CreateCodexStarter() - { - if (UseContainers) - { - Log("Using Containerized Codex instances"); - return new ContainerCodexStarter(tools, recipe, processControlMap); - } - - Log("Using Binary Codex instances"); - return new BinaryCodexStarter(tools, processControlMap); - } - - public string LogPrefix => "(Codex) "; - - public void Awake(IPluginAccess access) - { - } - - public void Announce() - { - // give codex docker image to contracts plugin. - - Log($"Loaded with Codex ID: '{codexWrapper.GetCodexId()}' - Revision: {codexWrapper.GetCodexRevision()}"); - } - - public void AddMetadata(IAddMetadata metadata) - { - metadata.Add("codexid", codexWrapper.GetCodexId()); - metadata.Add("codexrevision", codexWrapper.GetCodexRevision()); - } - - public void Decommission() - { - codexStarter.Decommission(); - } - - public ICodexInstance[] DeployCodexNodes(int numberOfNodes, Action setup) - { - var codexSetup = GetSetup(numberOfNodes, setup); - return codexStarter.BringOnline(codexSetup); - } - - public ICodexNodeGroup WrapCodexContainers(ICodexInstance[] instances) - { - instances = instances.Select(c => SerializeGate.Gate(c as CodexInstance)).ToArray(); - return codexWrapper.WrapCodexInstances(instances); - } - - public void AddCodexHooksProvider(ICodexHooksProvider hooksProvider) - { - if (hooksFactory.Providers.Contains(hooksProvider)) return; - hooksFactory.Providers.Add(hooksProvider); - } - - private CodexSetup GetSetup(int numberOfNodes, Action setup) - { - var codexSetup = new CodexSetup(numberOfNodes); - codexSetup.LogLevel = defaultLogLevel; - setup(codexSetup); - return codexSetup; - } - - private void Log(string msg) - { - tools.GetLog().Log(msg); - } - } -} diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs deleted file mode 100644 index 65835a50..00000000 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ /dev/null @@ -1,135 +0,0 @@ -using CodexClient; -using KubernetesWorkflow; -using Utils; - -namespace CodexPlugin -{ - public interface ICodexSetup - { - ICodexSetup WithName(string name); - ICodexSetup At(ILocation location); - ICodexSetup WithBootstrapNode(ICodexNode node); - ICodexSetup WithLogLevel(CodexLogLevel level); - ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics); - ICodexSetup WithStorageQuota(ByteSize storageQuota); - ICodexSetup WithBlockTTL(TimeSpan duration); - ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); - ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks); - ICodexSetup EnableMetrics(); - ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig); - } - - public class CodexLogCustomTopics - { - public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p, CodexLogLevel blockExchange) - { - DiscV5 = discV5; - Libp2p = libp2p; - BlockExchange = blockExchange; - } - - public CodexLogCustomTopics(CodexLogLevel discV5, CodexLogLevel libp2p) - { - DiscV5 = discV5; - Libp2p = libp2p; - } - - public CodexLogLevel DiscV5 { get; set; } - public CodexLogLevel Libp2p { get; set; } - public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn; - public CodexLogLevel? BlockExchange { get; } - public CodexLogLevel JsonSerialize { get; set; } = CodexLogLevel.Warn; - public CodexLogLevel MarketplaceInfra { get; set; } = CodexLogLevel.Warn; - } - - public class CodexSetup : CodexStartupConfig, ICodexSetup - { - public int NumberOfNodes { get; } - - public CodexSetup(int numberOfNodes) - { - NumberOfNodes = numberOfNodes; - } - - public ICodexSetup WithName(string name) - { - NameOverride = name; - return this; - } - - public ICodexSetup At(ILocation location) - { - Location = location; - return this; - } - - public ICodexSetup WithBootstrapNode(ICodexNode node) - { - BootstrapSpr = node.GetDebugInfo().Spr; - return this; - } - - public ICodexSetup WithLogLevel(CodexLogLevel level) - { - LogLevel = level; - return this; - } - - public ICodexSetup WithLogLevel(CodexLogLevel level, CodexLogCustomTopics customTopics) - { - LogLevel = level; - CustomTopics = customTopics; - return this; - } - - public ICodexSetup WithStorageQuota(ByteSize storageQuota) - { - StorageQuota = storageQuota; - return this; - } - - public ICodexSetup WithBlockTTL(TimeSpan duration) - { - BlockTTL = Convert.ToInt32(duration.TotalSeconds); - return this; - } - - public ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration) - { - BlockMaintenanceInterval = duration; - return this; - } - - public ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks) - { - BlockMaintenanceNumber = numberOfBlocks; - return this; - } - - public ICodexSetup EnableMetrics() - { - MetricsEnabled = true; - return this; - } - - public ICodexSetup AsPublicTestNet(CodexTestNetConfig testNetConfig) - { - PublicTestNet = testNetConfig; - return this; - } - - public string Describe() - { - var args = string.Join(',', DescribeArgs()); - return $"({NumberOfNodes} CodexNodes with args:[{args}])"; - } - - private IEnumerable DescribeArgs() - { - if (PublicTestNet != null) yield return $"Public TestNet with listenPort: {PublicTestNet.PublicListenPort}"; - yield return $"LogLevel={LogLevelWithTopics()}"; - if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; - if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}"; - } - } -} diff --git a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs deleted file mode 100644 index 3516c815..00000000 --- a/ProjectPlugins/CodexPlugin/CoreInterfaceExtensions.cs +++ /dev/null @@ -1,51 +0,0 @@ -using CodexClient; -using CodexClient.Hooks; -using Core; - -namespace CodexPlugin -{ - public static class CoreInterfaceExtensions - { - public static ICodexInstance[] DeployCodexNodes(this CoreInterface ci, int number, Action setup) - { - return Plugin(ci).DeployCodexNodes(number, setup); - } - - public static ICodexNodeGroup WrapCodexContainers(this CoreInterface ci, ICodexInstance[] instances) - { - return Plugin(ci).WrapCodexContainers(instances); - } - - public static ICodexNode StartCodexNode(this CoreInterface ci) - { - return ci.StartCodexNodes(1)[0]; - } - - public static ICodexNode StartCodexNode(this CoreInterface ci, Action setup) - { - return ci.StartCodexNodes(1, setup)[0]; - } - - public static ICodexNodeGroup StartCodexNodes(this CoreInterface ci, int number, Action setup) - { - var rc = ci.DeployCodexNodes(number, setup); - var result = ci.WrapCodexContainers(rc); - return result; - } - - public static ICodexNodeGroup StartCodexNodes(this CoreInterface ci, int number) - { - return ci.StartCodexNodes(number, s => { }); - } - - public static void AddCodexHooksProvider(this CoreInterface ci, ICodexHooksProvider hooksProvider) - { - Plugin(ci).AddCodexHooksProvider(hooksProvider); - } - - private static CodexPlugin Plugin(CoreInterface ci) - { - return ci.GetPlugin(); - } - } -} diff --git a/ProjectPlugins/CodexPlugin/ICodexStarter.cs b/ProjectPlugins/CodexPlugin/ICodexStarter.cs deleted file mode 100644 index c88cd382..00000000 --- a/ProjectPlugins/CodexPlugin/ICodexStarter.cs +++ /dev/null @@ -1,10 +0,0 @@ -using CodexClient; - -namespace CodexPlugin -{ - public interface ICodexStarter - { - ICodexInstance[] BringOnline(CodexSetup codexSetup); - void Decommission(); - } -} diff --git a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs index 866cfafb..4bece0ba 100644 --- a/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs +++ b/ProjectPlugins/DeployAndRunPlugin/DeployAndRunContainerRecipe.cs @@ -12,9 +12,9 @@ namespace DeployAndRunPlugin { var setup = config.Get(); - if (setup.CodexImageOverride != null) + if (setup.LogosStorageImageOverride != null) { - AddEnvVar("CODEXDOCKERIMAGE", setup.CodexImageOverride); + AddEnvVar("STORAGEDOCKERIMAGE", setup.LogosStorageImageOverride); } AddEnvVar("DNR_REP", setup.Replications.ToString()); @@ -23,28 +23,28 @@ namespace DeployAndRunPlugin AddEnvVar("DNR_DURATION", setup.Duration.TotalSeconds.ToString()); AddEnvVar("KUBECONFIG", "/opt/kubeconfig.yaml"); - AddEnvVar("LOGPATH", "/var/log/codex-continuous-tests"); + AddEnvVar("LOGPATH", "/var/log/storage-continuous-tests"); - AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "codex-dist-tests-app-kubeconfig"); - AddVolume(name: "logs", mountPath: "/var/log/codex-continuous-tests", hostPath: "/var/log/codex-continuous-tests"); + AddVolume(name: "kubeconfig", mountPath: "/opt/kubeconfig.yaml", subPath: "kubeconfig.yaml", secret: "storage-dist-tests-app-kubeconfig"); + AddVolume(name: "logs", mountPath: "/var/log/storage-continuous-tests", hostPath: "/var/log/storage-continuous-tests"); } } public class RunConfig { - public RunConfig(string name, string filter, TimeSpan duration, int replications, string? codexImageOverride = null) + public RunConfig(string name, string filter, TimeSpan duration, int replications, string? logosStorageImageOverride = null) { Name = name; Filter = filter; Duration = duration; Replications = replications; - CodexImageOverride = codexImageOverride; + LogosStorageImageOverride = logosStorageImageOverride; } public string Name { get; } public string Filter { get; } public TimeSpan Duration { get; } public int Replications { get; } - public string? CodexImageOverride { get; } + public string? LogosStorageImageOverride { get; } } } \ No newline at end of file diff --git a/ProjectPlugins/LogosStorageClient/Hooks/LogosStorageHooksFactory.cs b/ProjectPlugins/LogosStorageClient/Hooks/LogosStorageHooksFactory.cs new file mode 100644 index 00000000..33408df7 --- /dev/null +++ b/ProjectPlugins/LogosStorageClient/Hooks/LogosStorageHooksFactory.cs @@ -0,0 +1,61 @@ +using Utils; + +namespace LogosStorageClient.Hooks +{ + public interface ILogosStorageHooksProvider + { + IStorageNodeHooks CreateHooks(string nodeName); + } + + public class LogosStorageHooksFactory + { + public List Providers { get; } = new List(); + + public IStorageNodeHooks CreateHooks(string nodeName) + { + if (Providers.Count == 0) return new DoNothingLogosStorageHooks(); + + var hooks = Providers.Select(p => p.CreateHooks(nodeName)).ToArray(); + return new MuxingStorageNodeHooks(hooks); + } + } + + public class DoNothingHooksProvider : ILogosStorageHooksProvider + { + public IStorageNodeHooks CreateHooks(string nodeName) + { + return new DoNothingLogosStorageHooks(); + } + } + + public class DoNothingLogosStorageHooks : IStorageNodeHooks + { + public void OnFileDownloaded(ByteSize size, ContentId cid) + { + } + + public void OnFileDownloading(ContentId cid) + { + } + + public void OnFileUploaded(string uid, ByteSize size, ContentId cid) + { + } + + public void OnFileUploading(string uid, ByteSize size) + { + } + + public void OnNodeStarted(IStorageNode node, string peerId, string nodeId) + { + } + + public void OnNodeStarting(DateTime startUtc, string image) + { + } + + public void OnNodeStopping() + { + } + } +} diff --git a/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs b/ProjectPlugins/LogosStorageClient/Hooks/StorageNodeHooks.cs similarity index 77% rename from ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs rename to ProjectPlugins/LogosStorageClient/Hooks/StorageNodeHooks.cs index 119cf6d1..e0fddfc7 100644 --- a/ProjectPlugins/CodexClient/Hooks/CodexNodeHooks.cs +++ b/ProjectPlugins/LogosStorageClient/Hooks/StorageNodeHooks.cs @@ -1,11 +1,11 @@ using Utils; -namespace CodexClient.Hooks +namespace LogosStorageClient.Hooks { - public interface ICodexNodeHooks + public interface IStorageNodeHooks { void OnNodeStarting(DateTime startUtc, string image); - void OnNodeStarted(ICodexNode node, string peerId, string nodeId); + void OnNodeStarted(IStorageNode node, string peerId, string nodeId); void OnNodeStopping(); void OnFileUploading(string uid, ByteSize size); void OnFileUploaded(string uid, ByteSize size, ContentId cid); @@ -13,11 +13,11 @@ namespace CodexClient.Hooks void OnFileDownloaded(ByteSize size, ContentId cid); } - public class MuxingCodexNodeHooks : ICodexNodeHooks + public class MuxingStorageNodeHooks : IStorageNodeHooks { - private readonly ICodexNodeHooks[] backingHooks; + private readonly IStorageNodeHooks[] backingHooks; - public MuxingCodexNodeHooks(ICodexNodeHooks[] backingHooks) + public MuxingStorageNodeHooks(IStorageNodeHooks[] backingHooks) { this.backingHooks = backingHooks; } @@ -42,7 +42,7 @@ namespace CodexClient.Hooks foreach (var h in backingHooks) h.OnFileUploading(uid, size); } - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + public void OnNodeStarted(IStorageNode node, string peerId, string nodeId) { foreach (var h in backingHooks) h.OnNodeStarted(node, peerId, nodeId); } diff --git a/ProjectPlugins/CodexClient/CodexAccess.cs b/ProjectPlugins/LogosStorageClient/LogosStorageAccess.cs similarity index 80% rename from ProjectPlugins/CodexClient/CodexAccess.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageAccess.cs index 2b3e4ae7..6a53b1f3 100644 --- a/ProjectPlugins/CodexClient/CodexAccess.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageAccess.cs @@ -1,23 +1,23 @@ using System.Threading; -using CodexOpenApi; +using StorageOpenApi; using Logging; using Newtonsoft.Json; using Utils; using WebUtils; -namespace CodexClient +namespace LogosStorageClient { - public class CodexAccess + public class LogosStorageAccess { public const string API_BASE_URL = "/api/storage/v1"; private readonly ILog log; private readonly IHttpFactory httpFactory; private readonly IProcessControl processControl; - private readonly ICodexInstance instance; + private readonly ILogosStorageInstance instance; private readonly Mapper mapper = new Mapper(); - public CodexAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ICodexInstance instance) + public LogosStorageAccess(ILog log, IHttpFactory httpFactory, IProcessControl processControl, ILogosStorageInstance instance) { this.log = log; this.httpFactory = httpFactory; @@ -49,14 +49,14 @@ namespace CodexClient public DebugInfo GetDebugInfo() { - return mapper.Map(OnCodex(api => api.GetDebugInfoAsync())); + return mapper.Map(OnLogosStorage(api => api.GetDebugInfoAsync())); } public void SetLogLevel(string logLevel) { try { - OnCodex(async api => + OnLogosStorage(async api => { await api.SetDebugLogLevelAsync(logLevel); return string.Empty; @@ -108,7 +108,7 @@ namespace CodexClient public void ConnectToPeer(string peerId, string[] peerMultiAddresses) { - OnCodex(api => + OnLogosStorage(api => { Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses)); return Task.FromResult(string.Empty); @@ -117,36 +117,36 @@ namespace CodexClient public string UploadFile(UploadInput uploadInput) { - return OnCodex(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream)); + return OnLogosStorage(api => api.UploadAsync(uploadInput.ContentType, uploadInput.ContentDisposition, uploadInput.FileStream)); } public Stream DownloadFile(string contentId) { - var fileResponse = OnCodexNoRetry(api => api.DownloadNetworkStreamAsync(contentId)); + var fileResponse = OnLogosStorageNoRetry(api => api.DownloadNetworkStreamAsync(contentId)); if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); return fileResponse.Stream; } public LocalDataset DownloadStreamless(ContentId cid) { - var response = OnCodex(api => api.DownloadNetworkAsync(cid.Id)); + var response = OnLogosStorage(api => api.DownloadNetworkAsync(cid.Id)); return mapper.Map(response); } public LocalDataset DownloadManifestOnly(ContentId cid) { - var response = OnCodex(api => api.DownloadNetworkManifestAsync(cid.Id)); + var response = OnLogosStorage(api => api.DownloadNetworkManifestAsync(cid.Id)); return mapper.Map(response); } public LocalDatasetList LocalFiles() { - return mapper.Map(OnCodex(api => api.ListDataAsync())); + return mapper.Map(OnLogosStorage(api => api.ListDataAsync())); } - public CodexSpace Space() + public LogosStorageSpace Space() { - var space = OnCodex(api => api.SpaceAsync()); + var space = OnLogosStorage(api => api.SpaceAsync()); return mapper.Map(space); } @@ -185,29 +185,29 @@ namespace CodexClient processControl.DeleteDataDirFolder(); } - private T OnCodexNoRetry(Func> action) + private T OnLogosStorageNoRetry(Func> action) { var timeSet = httpFactory.WebCallTimeSet; - var noRetry = new Retry(nameof(OnCodexNoRetry), + var noRetry = new Retry(nameof(OnLogosStorageNoRetry), maxTimeout: TimeSpan.FromSeconds(1.0), sleepAfterFail: TimeSpan.FromSeconds(2.0), onFail: f => { }, failFast: true); - var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action), noRetry); + var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallLogosStorage(client, action), noRetry); return result; } - private T OnCodex(Func> action) + private T OnLogosStorage(Func> action) { - var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallCodex(client, action)); + var result = httpFactory.CreateHttp(GetHttpId(), h => CheckContainerCrashed()).OnClient(client => CallLogosStorage(client, action)); return result; } - private T CallCodex(HttpClient client, Func> action) + private T CallLogosStorage(HttpClient client, Func> action) { var address = GetAddress(); - var api = new CodexApiClient(client); + var api = new StorageApiClient(client); api.BaseUrl = $"{address.Host}:{address.Port}{API_BASE_URL}"; return CrashCheck(() => Time.Wait(action(api))); } diff --git a/ProjectPlugins/CodexClient/CodexClient.csproj b/ProjectPlugins/LogosStorageClient/LogosStorageClient.csproj similarity index 94% rename from ProjectPlugins/CodexClient/CodexClient.csproj rename to ProjectPlugins/LogosStorageClient/LogosStorageClient.csproj index 2336d20a..95e32a7e 100644 --- a/ProjectPlugins/CodexClient/CodexClient.csproj +++ b/ProjectPlugins/LogosStorageClient/LogosStorageClient.csproj @@ -11,7 +11,7 @@ - + diff --git a/ProjectPlugins/CodexClient/CodexInstance.cs b/ProjectPlugins/LogosStorageClient/LogosStorageInstance.cs similarity index 72% rename from ProjectPlugins/CodexClient/CodexInstance.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageInstance.cs index 104c0f03..0f672c2d 100644 --- a/ProjectPlugins/CodexClient/CodexInstance.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageInstance.cs @@ -1,8 +1,8 @@ using Utils; -namespace CodexClient +namespace LogosStorageClient { - public interface ICodexInstance + public interface ILogosStorageInstance { string Name { get; } string ImageName { get; } @@ -14,9 +14,9 @@ namespace CodexClient Address? MetricsEndpoint { get; } } - public class CodexInstance : ICodexInstance + public class LogosStorageInstance : ILogosStorageInstance { - public CodexInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint) + public LogosStorageInstance(string name, string imageName, DateTime startUtc, Address discoveryEndpoint, Address apiEndpoint, Address listenEndpoint, EthAccount? ethAccount, Address? metricsEndpoint) { Name = name; ImageName = imageName; @@ -37,9 +37,9 @@ namespace CodexClient public EthAccount? EthAccount { get; } public Address? MetricsEndpoint { get; } - public static ICodexInstance CreateFromApiEndpoint(string name, Address apiEndpoint, EthAccount? ethAccount = null) + public static ILogosStorageInstance CreateFromApiEndpoint(string name, Address apiEndpoint, EthAccount? ethAccount = null) { - return new CodexInstance( + return new LogosStorageInstance( name, imageName: "-", startUtc: DateTime.UtcNow, diff --git a/ProjectPlugins/CodexClient/CodexLogLevel.cs b/ProjectPlugins/LogosStorageClient/LogosStorageLogLevel.cs similarity index 56% rename from ProjectPlugins/CodexClient/CodexLogLevel.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageLogLevel.cs index 70cd2e3c..6f23145c 100644 --- a/ProjectPlugins/CodexClient/CodexLogLevel.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageLogLevel.cs @@ -1,6 +1,6 @@ -namespace CodexClient +namespace LogosStorageClient { - public enum CodexLogLevel + public enum LogosStorageLogLevel { Trace, Debug, diff --git a/ProjectPlugins/CodexClient/CodexLogLine.cs b/ProjectPlugins/LogosStorageClient/LogosStorageLogLine.cs similarity index 94% rename from ProjectPlugins/CodexClient/CodexLogLine.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageLogLine.cs index 01f90775..a01ff063 100644 --- a/ProjectPlugins/CodexClient/CodexLogLine.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageLogLine.cs @@ -1,10 +1,10 @@ using System.Globalization; -namespace CodexClient +namespace LogosStorageClient { - public class CodexLogLine + public class LogosStorageLogLine { - public static CodexLogLine? Parse(string line) + public static LogosStorageLogLine? Parse(string line) { try { @@ -29,7 +29,7 @@ namespace CodexClient var format = "yyyy-MM-dd HH:mm:ss.fff"; var dt = DateTime.ParseExact(dtLine, format, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal).ToUniversalTime(); - return new CodexLogLine() + return new LogosStorageLogLine() { LogLevel = level, TimestampUtc = dt, diff --git a/ProjectPlugins/CodexClient/CodexTypes.cs b/ProjectPlugins/LogosStorageClient/LogosStorageTypes.cs similarity index 98% rename from ProjectPlugins/CodexClient/CodexTypes.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageTypes.cs index 3a70191a..6298a0ac 100644 --- a/ProjectPlugins/CodexClient/CodexTypes.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageTypes.cs @@ -1,7 +1,7 @@ using Newtonsoft.Json; using Utils; -namespace CodexClient +namespace LogosStorageClient { public class DebugInfo { @@ -110,7 +110,7 @@ namespace CodexClient } } - public class CodexSpace + public class LogosStorageSpace { public long TotalBlocks { get; set; } public long QuotaMaxBytes { get; set; } diff --git a/ProjectPlugins/CodexClient/CodexUtils.cs b/ProjectPlugins/LogosStorageClient/LogosStorageUtils.cs similarity index 86% rename from ProjectPlugins/CodexClient/CodexUtils.cs rename to ProjectPlugins/LogosStorageClient/LogosStorageUtils.cs index b791f049..1da77aae 100644 --- a/ProjectPlugins/CodexClient/CodexUtils.cs +++ b/ProjectPlugins/LogosStorageClient/LogosStorageUtils.cs @@ -1,6 +1,6 @@ -namespace CodexClient +namespace LogosStorageClient { - public static class CodexUtils + public static class LogosStorageUtils { public static string ToShortId(string id) { diff --git a/ProjectPlugins/CodexClient/Mapper.cs b/ProjectPlugins/LogosStorageClient/Mapper.cs similarity index 84% rename from ProjectPlugins/CodexClient/Mapper.cs rename to ProjectPlugins/LogosStorageClient/Mapper.cs index 1257b50d..9d5fc5a0 100644 --- a/ProjectPlugins/CodexClient/Mapper.cs +++ b/ProjectPlugins/LogosStorageClient/Mapper.cs @@ -2,11 +2,11 @@ using System.Numerics; using Utils; -namespace CodexClient +namespace LogosStorageClient { public class Mapper { - public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo) + public DebugInfo Map(StorageOpenApi.DebugInfo debugInfo) { return new DebugInfo { @@ -19,7 +19,7 @@ namespace CodexClient }; } - public LocalDatasetList Map(CodexOpenApi.DataList dataList) + public LocalDatasetList Map(StorageOpenApi.DataList dataList) { return new LocalDatasetList { @@ -27,7 +27,7 @@ namespace CodexClient }; } - public LocalDataset Map(CodexOpenApi.DataItem dataItem) + public LocalDataset Map(StorageOpenApi.DataItem dataItem) { return new LocalDataset { @@ -35,9 +35,9 @@ namespace CodexClient Manifest = MapManifest(dataItem.Manifest) }; } - public CodexSpace Map(CodexOpenApi.Space space) + public LogosStorageSpace Map(StorageOpenApi.Space space) { - return new CodexSpace + return new LogosStorageSpace { QuotaMaxBytes = space.QuotaMaxBytes, QuotaReservedBytes = space.QuotaReservedBytes, @@ -46,7 +46,7 @@ namespace CodexClient }; } - private DebugInfoVersion Map(CodexOpenApi.StorageVersion obj) + private DebugInfoVersion Map(StorageOpenApi.StorageVersion obj) { return new DebugInfoVersion { @@ -55,7 +55,7 @@ namespace CodexClient }; } - private DebugInfoTable Map(CodexOpenApi.PeersTable obj) + private DebugInfoTable Map(StorageOpenApi.PeersTable obj) { return new DebugInfoTable { @@ -64,7 +64,7 @@ namespace CodexClient }; } - private DebugInfoTableNode Map(CodexOpenApi.Node? token) + private DebugInfoTableNode Map(StorageOpenApi.Node? token) { if (token == null) return new DebugInfoTableNode(); return new DebugInfoTableNode @@ -77,7 +77,7 @@ namespace CodexClient }; } - private DebugInfoTableNode[] Map(ICollection nodes) + private DebugInfoTableNode[] Map(ICollection nodes) { if (nodes == null || nodes.Count == 0) { @@ -87,7 +87,7 @@ namespace CodexClient return nodes.Select(Map).ToArray(); } - private Manifest MapManifest(CodexOpenApi.ManifestItem manifest) + private Manifest MapManifest(StorageOpenApi.ManifestItem manifest) { return new Manifest { diff --git a/ProjectPlugins/CodexClient/ProcessControl.cs b/ProjectPlugins/LogosStorageClient/ProcessControl.cs similarity index 82% rename from ProjectPlugins/CodexClient/ProcessControl.cs rename to ProjectPlugins/LogosStorageClient/ProcessControl.cs index 2ddfab2e..c0fba3ce 100644 --- a/ProjectPlugins/CodexClient/ProcessControl.cs +++ b/ProjectPlugins/LogosStorageClient/ProcessControl.cs @@ -1,10 +1,10 @@ using Logging; -namespace CodexClient +namespace LogosStorageClient { public interface IProcessControlFactory { - IProcessControl CreateProcessControl(ICodexInstance instance); + IProcessControl CreateProcessControl(ILogosStorageInstance instance); } public interface IProcessControl @@ -17,7 +17,7 @@ namespace CodexClient public class DoNothingProcessControlFactory : IProcessControlFactory { - public IProcessControl CreateProcessControl(ICodexInstance instance) + public IProcessControl CreateProcessControl(ILogosStorageInstance instance) { return new DoNothingProcessControl(); } diff --git a/ProjectPlugins/CodexClient/CodexNode.cs b/ProjectPlugins/LogosStorageClient/StorageNode.cs similarity index 80% rename from ProjectPlugins/CodexClient/CodexNode.cs rename to ProjectPlugins/LogosStorageClient/StorageNode.cs index d8254595..861b08fd 100644 --- a/ProjectPlugins/CodexClient/CodexNode.cs +++ b/ProjectPlugins/LogosStorageClient/StorageNode.cs @@ -1,11 +1,11 @@ -using CodexClient.Hooks; +using LogosStorageClient.Hooks; using FileUtils; using Logging; using Utils; -namespace CodexClient +namespace LogosStorageClient { - public partial interface ICodexNode : IHasMetricsScrapeTarget + public partial interface IStorageNode : IHasMetricsScrapeTarget { string GetName(); string GetImageName(); @@ -26,8 +26,8 @@ namespace CodexClient LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size); LocalDataset DownloadManifestOnly(ContentId cid); LocalDatasetList LocalFiles(); - CodexSpace Space(); - void ConnectToPeer(ICodexNode node); + LogosStorageSpace Space(); + void ConnectToPeer(IStorageNode node); DebugInfoVersion Version { get; } ITransferSpeeds TransferSpeeds { get; } @@ -45,20 +45,20 @@ namespace CodexClient bool HasCrashed(); } - public class CodexNode : ICodexNode + public class StorageNode : IStorageNode { private const string UploadFailedMessage = "Unable to store block"; private readonly ILog log; - private readonly ICodexNodeHooks hooks; + private readonly IStorageNodeHooks hooks; private readonly TransferSpeeds transferSpeeds; private string peerId = string.Empty; private string nodeId = string.Empty; - private readonly CodexAccess codexAccess; + private readonly LogosStorageAccess logosStorageAccess; private readonly IFileManager fileManager; - public CodexNode(ILog log, CodexAccess codexAccess, IFileManager fileManager, ICodexNodeHooks hooks) + public StorageNode(ILog log, LogosStorageAccess logosStorageAccess, IFileManager fileManager, IStorageNodeHooks hooks) { - this.codexAccess = codexAccess; + this.logosStorageAccess = logosStorageAccess; this.fileManager = fileManager; this.hooks = hooks; Version = new DebugInfoVersion(); @@ -69,14 +69,14 @@ namespace CodexClient public void Awake() { - hooks.OnNodeStarting(codexAccess.GetStartUtc(), codexAccess.GetImageName()); + hooks.OnNodeStarting(logosStorageAccess.GetStartUtc(), logosStorageAccess.GetImageName()); } public void Initialize() { // This is the moment we first connect to a codex node. Sometimes, Kubernetes takes a while to spin up the // container. So we'll adding a custom, generous retry here. - var kubeSpinupRetry = new Retry("CodexNode_Initialize", + var kubeSpinupRetry = new Retry("StorageNode_Initialize", maxTimeout: TimeSpan.FromMinutes(10.0), sleepAfterFail: TimeSpan.FromSeconds(10.0), onFail: f => { }, @@ -94,12 +94,12 @@ namespace CodexClient public string GetName() { - return codexAccess.GetName(); + return logosStorageAccess.GetName(); } public string GetImageName() { - return codexAccess.GetImageName(); + return logosStorageAccess.GetImageName(); } public string GetPeerId() @@ -109,7 +109,7 @@ namespace CodexClient public DebugInfo GetDebugInfo(bool log = false) { - var debugInfo = codexAccess.GetDebugInfo(); + var debugInfo = logosStorageAccess.GetDebugInfo(); if (log) { var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId)); @@ -120,17 +120,17 @@ namespace CodexClient public void SetLogLevel(string logLevel) { - codexAccess.SetLogLevel(logLevel); + logosStorageAccess.SetLogLevel(logLevel); } public string GetSpr() { - return codexAccess.GetSpr(); + return logosStorageAccess.GetSpr(); } public DebugPeer GetDebugPeer(string peerId) { - return codexAccess.GetDebugPeer(peerId); + return logosStorageAccess.GetDebugPeer(peerId); } public ContentId UploadFile(TrackedFile file) @@ -150,7 +150,7 @@ namespace CodexClient var logMessage = $"Uploading file {file.Describe()} with contentType: '{input.ContentType}' and disposition: '{input.ContentDisposition}'..."; var measurement = Stopwatch.Measure(log, logMessage, () => { - return codexAccess.UploadFile(input); + return logosStorageAccess.UploadFile(input); }); var response = measurement.Value; @@ -190,7 +190,7 @@ namespace CodexClient public LocalDataset DownloadStreamless(ContentId cid) { Log($"Downloading streamless '{cid}' (no-wait)"); - return codexAccess.DownloadStreamless(cid); + return logosStorageAccess.DownloadStreamless(cid); } public LocalDataset DownloadStreamlessWait(ContentId cid, ByteSize size) @@ -200,7 +200,7 @@ namespace CodexClient var sw = Stopwatch.Measure(log, nameof(DownloadStreamlessWait), () => { var startSpace = Space(); - var result = codexAccess.DownloadStreamless(cid); + var result = logosStorageAccess.DownloadStreamless(cid); WaitUntilQuotaUsedIncreased(startSpace, size); return result; }); @@ -211,85 +211,85 @@ namespace CodexClient public LocalDataset DownloadManifestOnly(ContentId cid) { Log($"Downloading manifest-only '{cid}'"); - return codexAccess.DownloadManifestOnly(cid); + return logosStorageAccess.DownloadManifestOnly(cid); } public LocalDatasetList LocalFiles() { - return codexAccess.LocalFiles(); + return logosStorageAccess.LocalFiles(); } - public CodexSpace Space() + public LogosStorageSpace Space() { - return codexAccess.Space(); + return logosStorageAccess.Space(); } - public void ConnectToPeer(ICodexNode node) + public void ConnectToPeer(IStorageNode node) { - var peer = (CodexNode)node; + var peer = (StorageNode)node; Log($"Connecting to peer {peer.GetName()}..."); var peerInfo = node.GetDebugInfo(); - codexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo)); + logosStorageAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo)); Log($"Successfully connected to peer {peer.GetName()}."); } public void DeleteDataDirFolder() { - codexAccess.DeleteDataDirFolder(); + logosStorageAccess.DeleteDataDirFolder(); } public void Stop(bool waitTillStopped) { Log("Stopping..."); hooks.OnNodeStopping(); - codexAccess.Stop(waitTillStopped); + logosStorageAccess.Stop(waitTillStopped); } public IDownloadedLog DownloadLog(string additionalName = "") { - return codexAccess.DownloadLog(additionalName); + return logosStorageAccess.DownloadLog(additionalName); } public Address GetDiscoveryEndpoint() { - return codexAccess.GetDiscoveryEndpoint(); + return logosStorageAccess.GetDiscoveryEndpoint(); } public Address GetApiEndpoint() { - return codexAccess.GetApiEndpoint(); + return logosStorageAccess.GetApiEndpoint(); } public Address GetListenEndpoint() { - return codexAccess.GetListenEndpoint(); + return logosStorageAccess.GetListenEndpoint(); } public Address GetMetricsScrapeTarget() { - var address = codexAccess.GetMetricsEndpoint(); + var address = logosStorageAccess.GetMetricsEndpoint(); if (address == null) throw new Exception("Metrics ScrapeTarget accessed, but node was not started with EnableMetrics()"); return address; } public bool HasCrashed() { - return codexAccess.HasCrashed(); + return logosStorageAccess.HasCrashed(); } public override string ToString() { - return $"CodexNode:{GetName()}"; + return $"StorageNode:{GetName()}"; } private void InitializePeerNodeId() { - var debugInfo = codexAccess.GetDebugInfo(); + var debugInfo = logosStorageAccess.GetDebugInfo(); if (!debugInfo.Version.IsValid()) { - throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}"); + throw new Exception($"Invalid version information received from Logos Storage node {GetName()}: {debugInfo.Version}"); } peerId = debugInfo.Id; @@ -302,12 +302,12 @@ namespace CodexClient var nodeName = GetName(); log.AddStringReplace(peerId, nodeName); - log.AddStringReplace(CodexUtils.ToShortId(peerId), nodeName); + log.AddStringReplace(LogosStorageUtils.ToShortId(peerId), nodeName); log.AddStringReplace(nodeId, nodeName); - log.AddStringReplace(CodexUtils.ToShortId(nodeId), nodeName); + log.AddStringReplace(LogosStorageUtils.ToShortId(nodeId), nodeName); } - private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo) + private string[] GetPeerMultiAddresses(StorageNode peer, DebugInfo peerInfo) { var peerId = peer.GetDiscoveryEndpoint().Host .Replace("http://", "") @@ -330,7 +330,7 @@ namespace CodexClient var cts = new CancellationTokenSource(); var downloadTask = Task.Run(() => { - using var downloadStream = codexAccess.DownloadFile(contentId); + using var downloadStream = logosStorageAccess.DownloadFile(contentId); downloadStream.CopyTo(fileStream); }, cts.Token); @@ -351,13 +351,13 @@ namespace CodexClient } } - public void WaitUntilQuotaUsedIncreased(CodexSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed) + public void WaitUntilQuotaUsedIncreased(LogosStorageSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed) { WaitUntilQuotaUsedIncreased(startSpace, expectedIncreaseOfQuotaUsed, TimeSpan.FromMinutes(30)); } public void WaitUntilQuotaUsedIncreased( - CodexSpace startSpace, + LogosStorageSpace startSpace, ByteSize expectedIncreaseOfQuotaUsed, TimeSpan maxTimeout) { diff --git a/ProjectPlugins/LogosStorageClient/StorageNodeFactory.cs b/ProjectPlugins/LogosStorageClient/StorageNodeFactory.cs new file mode 100644 index 00000000..c79939d8 --- /dev/null +++ b/ProjectPlugins/LogosStorageClient/StorageNodeFactory.cs @@ -0,0 +1,46 @@ +using LogosStorageClient.Hooks; +using FileUtils; +using Logging; +using WebUtils; + +namespace LogosStorageClient +{ + public class StorageNodeFactory + { + private readonly ILog log; + private readonly IFileManager fileManager; + private readonly LogosStorageHooksFactory hooksFactory; + private readonly IHttpFactory httpFactory; + private readonly IProcessControlFactory processControlFactory; + + public StorageNodeFactory(ILog log, IFileManager fileManager, LogosStorageHooksFactory hooksFactory, IHttpFactory httpFactory, IProcessControlFactory processControlFactory) + { + this.log = log; + this.fileManager = fileManager; + this.hooksFactory = hooksFactory; + this.httpFactory = httpFactory; + this.processControlFactory = processControlFactory; + } + + public StorageNodeFactory(ILog log, HttpFactory httpFactory, string dataDir) + : this(log, new FileManager(log, dataDir), new LogosStorageHooksFactory(), httpFactory, new DoNothingProcessControlFactory()) + { + } + + public StorageNodeFactory(ILog log, string dataDir) + : this(log, new HttpFactory(log), dataDir) + { + } + + public IStorageNode CreateStorageNode(ILogosStorageInstance instance) + { + var processControl = processControlFactory.CreateProcessControl(instance); + var access = new LogosStorageAccess(log, httpFactory, processControl, instance); + var hooks = hooksFactory.CreateHooks(access.GetName()); + var node = new StorageNode(log, access, fileManager, hooks); + node.Initialize(); + return node; + } + + } +} diff --git a/ProjectPlugins/CodexClient/TransferSpeeds.cs b/ProjectPlugins/LogosStorageClient/TransferSpeeds.cs similarity index 98% rename from ProjectPlugins/CodexClient/TransferSpeeds.cs rename to ProjectPlugins/LogosStorageClient/TransferSpeeds.cs index bb6afeed..545c4999 100644 --- a/ProjectPlugins/CodexClient/TransferSpeeds.cs +++ b/ProjectPlugins/LogosStorageClient/TransferSpeeds.cs @@ -1,6 +1,6 @@ using Utils; -namespace CodexClient +namespace LogosStorageClient { public interface ITransferSpeeds { diff --git a/ProjectPlugins/CodexClient/openapi.yaml b/ProjectPlugins/LogosStorageClient/openapi.yaml similarity index 100% rename from ProjectPlugins/CodexClient/openapi.yaml rename to ProjectPlugins/LogosStorageClient/openapi.yaml diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/LogosStorageDiscordBotPlugin/CoreInterfaceExtensions.cs similarity index 56% rename from ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs rename to ProjectPlugins/LogosStorageDiscordBotPlugin/CoreInterfaceExtensions.cs index c91d7118..794352a1 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CoreInterfaceExtensions.cs +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/CoreInterfaceExtensions.cs @@ -1,11 +1,11 @@ using Core; using KubernetesWorkflow.Types; -namespace CodexDiscordBotPlugin +namespace LogosStorageDiscordBotPlugin { public static class CoreInterfaceExtensions { - public static RunningPod DeployCodexDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) + public static RunningPod DeployLogosStorageDiscordBot(this CoreInterface ci, DiscordBotStartupConfig config) { return Plugin(ci).Deploy(config); } @@ -15,9 +15,9 @@ namespace CodexDiscordBotPlugin return Plugin(ci).DeployRewarder(config); } - private static CodexDiscordBotPlugin Plugin(CoreInterface ci) + private static LogosStorageDiscordBotPlugin Plugin(CoreInterface ci) { - return ci.GetPlugin(); + return ci.GetPlugin(); } } } diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs b/ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotContainerRecipe.cs similarity index 92% rename from ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs rename to ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotContainerRecipe.cs index 0a9d9872..f7a67ba5 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotContainerRecipe.cs +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotContainerRecipe.cs @@ -2,12 +2,12 @@ using KubernetesWorkflow.Recipe; using Utils; -namespace CodexDiscordBotPlugin +namespace LogosStorageDiscordBotPlugin { public class DiscordBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-bibliotech"; - public override string Image => "codexstorage/codex-discordbot:sha-8033da1"; + public override string Image => "logosstorage/logos-storage-discordbot:sha-8033da1"; public static string RewardsPort = "bot_rewards_port"; diff --git a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs b/ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotStartupConfig.cs similarity index 98% rename from ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs rename to ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotStartupConfig.cs index e77552e5..50ee3359 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/DiscordBotStartupConfig.cs +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/DiscordBotStartupConfig.cs @@ -1,4 +1,4 @@ -namespace CodexDiscordBotPlugin +namespace LogosStorageDiscordBotPlugin { public class DiscordBotStartupConfig { diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs b/ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.cs similarity index 89% rename from ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs rename to ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.cs index e1354806..88c8a03e 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.cs +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.cs @@ -3,14 +3,14 @@ using KubernetesWorkflow; using KubernetesWorkflow.Types; using Utils; -namespace CodexDiscordBotPlugin +namespace LogosStorageDiscordBotPlugin { - public class CodexDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata + public class LogosStorageDiscordBotPlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata { private const string ExpectedStartupMessage = "Debug option is set. Discord connection disabled!"; private readonly IPluginTools tools; - public CodexDiscordBotPlugin(IPluginTools tools) + public LogosStorageDiscordBotPlugin(IPluginTools tools) { this.tools = tools; } @@ -23,12 +23,12 @@ namespace CodexDiscordBotPlugin public void Announce() { - tools.GetLog().Log($"Codex DiscordBot (BiblioTech) loaded."); + tools.GetLog().Log($"Logos Storage DiscordBot (BiblioTech) loaded."); } public void AddMetadata(IAddMetadata metadata) { - metadata.Add("codexdiscordbotid", new DiscordBotContainerRecipe().Image); + metadata.Add("logosstoragediscordbotid", new DiscordBotContainerRecipe().Image); } public void Decommission() diff --git a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj b/ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.csproj similarity index 80% rename from ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj rename to ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.csproj index bb5c9beb..c29ab503 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/CodexDiscordBotPlugin.csproj +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/LogosStorageDiscordBotPlugin.csproj @@ -8,7 +8,7 @@ - + diff --git a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs b/ProjectPlugins/LogosStorageDiscordBotPlugin/RewarderBotContainerRecipe.cs similarity index 91% rename from ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs rename to ProjectPlugins/LogosStorageDiscordBotPlugin/RewarderBotContainerRecipe.cs index b35195b3..94bf760f 100644 --- a/ProjectPlugins/CodexDiscordBotPlugin/RewarderBotContainerRecipe.cs +++ b/ProjectPlugins/LogosStorageDiscordBotPlugin/RewarderBotContainerRecipe.cs @@ -2,12 +2,12 @@ using KubernetesWorkflow; using Utils; -namespace CodexDiscordBotPlugin +namespace LogosStorageDiscordBotPlugin { public class RewarderBotContainerRecipe : ContainerRecipeFactory { public override string AppName => "discordbot-rewarder"; - public override string Image => "codexstorage/codex-rewarderbot:sha-8033da1"; + public override string Image => "logosstorage/logos-storage-rewarderbot:sha-8033da1"; protected override void Initialize(StartupConfig startupConfig) { diff --git a/ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj b/ProjectPlugins/LogosStoragePluginPrebuild/LogosStoragePluginPrebuild.csproj similarity index 100% rename from ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj rename to ProjectPlugins/LogosStoragePluginPrebuild/LogosStoragePluginPrebuild.csproj diff --git a/ProjectPlugins/CodexPluginPrebuild/Program.cs b/ProjectPlugins/LogosStoragePluginPrebuild/Program.cs similarity index 78% rename from ProjectPlugins/CodexPluginPrebuild/Program.cs rename to ProjectPlugins/LogosStoragePluginPrebuild/Program.cs index 6e9d3b9b..d8076a69 100644 --- a/ProjectPlugins/CodexPluginPrebuild/Program.cs +++ b/ProjectPlugins/LogosStoragePluginPrebuild/Program.cs @@ -5,16 +5,16 @@ using Utils; public static class Program { private const string Search = ""; - private const string CodexPluginFolderName = "CodexPlugin"; + private const string StoragePluginFolderName = "StoragePlugin"; private const string ProjectPluginsFolderName = "ProjectPlugins"; public static void Main(string[] args) { Console.WriteLine("Injecting hash of 'openapi.yaml'..."); - var pluginRoot = FindCodexPluginFolder(); - var clientRoot = FindCodexClientFolder(); - Console.WriteLine("Located CodexPlugin: " + pluginRoot); + var pluginRoot = FindStoragePluginFolder(); + var clientRoot = FindLogosStorageClientFolder(); + Console.WriteLine("Located StoragePlugin: " + pluginRoot); var openApiFile = Path.Combine(clientRoot, "openapi.yaml"); var clientFile = Path.Combine(clientRoot, "obj", "openapiClient.cs"); var targetFile = Path.Combine(pluginRoot, "ApiChecker.cs"); @@ -29,7 +29,7 @@ public static class Program SearchAndInject(hash, targetFile); - // This program runs as the pre-build trigger for "CodexPlugin". + // This program runs as the pre-build trigger for "StoragePlugin". // You might be wondering why this work isn't done by a shell script. // This is because this project is being run on many different platforms. // (Mac, Unix, Win, but also desktop/cloud containers.) @@ -40,17 +40,17 @@ public static class Program Console.WriteLine("Done!"); } - private static string FindCodexPluginFolder() + private static string FindStoragePluginFolder() { - var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexPlugin"); - if (!Directory.Exists(folder)) throw new Exception("CodexPlugin folder not found. Expected: " + folder); + var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "StoragePlugin"); + if (!Directory.Exists(folder)) throw new Exception("StoragePlugin folder not found. Expected: " + folder); return folder; } - private static string FindCodexClientFolder() + private static string FindLogosStorageClientFolder() { - var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient"); - if (!Directory.Exists(folder)) throw new Exception("CodexClient folder not found. Expected: " + folder); + var folder = Path.Combine(PluginPathUtils.ProjectPluginsDir, "LogosStorageClient"); + if (!Directory.Exists(folder)) throw new Exception("LogosStorageClient folder not found. Expected: " + folder); return folder; } diff --git a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs index 23ebf3a2..9a95f8ed 100644 --- a/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs +++ b/ProjectPlugins/MetricsPlugin/PrometheusContainerRecipe.cs @@ -6,7 +6,7 @@ namespace MetricsPlugin public class PrometheusContainerRecipe : ContainerRecipeFactory { public override string AppName => "prometheus"; - public override string Image => "codexstorage/dist-tests-prometheus:latest"; + public override string Image => "logosstorage/dist-tests-prometheus:latest"; public const string PortTag = "prometheus_port_tag"; diff --git a/ProjectPlugins/MetricsPlugin/dashboard.json b/ProjectPlugins/MetricsPlugin/dashboard.json index 5e1d67ec..b85c0038 100644 --- a/ProjectPlugins/MetricsPlugin/dashboard.json +++ b/ProjectPlugins/MetricsPlugin/dashboard.json @@ -2057,7 +2057,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Codex", + "title": "Logos Storage", "uid": "ca93f344-dd41-48c4-95a7-876c340e3005", "version": 6, "weekStart": "" diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/StoragePlugin/ApiChecker.cs similarity index 75% rename from ProjectPlugins/CodexPlugin/ApiChecker.cs rename to ProjectPlugins/StoragePlugin/ApiChecker.cs index c0356ef6..4e610a3d 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/StoragePlugin/ApiChecker.cs @@ -5,27 +5,27 @@ using System.Security.Cryptography; using System.Text; using Utils; -namespace CodexPlugin +namespace StoragePlugin { public class ApiChecker { // private const string OpenApiYamlHash = "47-B8-22-44-3B-51-EB-82-CC-C1-DD-56-F0-7E-EC-7F-CD-E0-8F-5E-F6-3B-40-E0-02-E6-71-DB-B8-03-65-18"; private const string OpenApiFilePath = "/logosstorage/openapi.yaml"; - private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; + private const string DisableEnvironmentVariable = "StoragePlugin_DISABLE_APICHECK"; private const bool Disable = false; private const string Warning = - "Warning: CodexPlugin was unable to find the openapi.yaml file in the Codex container. Are you running an old version of Codex? " + + "Warning: StoragePlugin was unable to find the openapi.yaml file in the Logos Storage container. Are you running an old version of Logos Storage? " + "Plugin will continue as normal, but API compatibility is not guaranteed!"; private const string Failure = - "Codex API compatibility check failed! " + - "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. The openapi.yaml in " + - "'ProjectPlugins/CodexPlugin' has been overwritten with the container one. " + + "Logos Storage API compatibility check failed! " + + "openapi.yaml used by StoragePlugin does not match openapi.yaml in Logos Storage container. The openapi.yaml in " + + "'ProjectPlugins/StoragePlugin' has been overwritten with the container one. " + "Please and rebuild this project. If you wish to disable API compatibility checking, please set " + - $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'."; + $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/StoragePlugin/ApiChecker.cs'."; private static bool checkPassed = false; @@ -44,7 +44,7 @@ namespace CodexPlugin { if (checkPassed) return; - Log("CodexPlugin is checking API compatibility..."); + Log("StoragePlugin is checking API compatibility..."); if (Disable || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(DisableEnvironmentVariable))) { @@ -81,13 +81,13 @@ namespace CodexPlugin private void OverwriteOpenApiYaml(string containerApi) { - Log("API compatibility check failed. Updating CodexPlugin..."); - var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "CodexClient", "openapi.yaml"); - if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate CodexClient/openapi.yaml. Expected: " + openApiFilePath); + Log("API compatibility check failed. Updating StoragePlugin..."); + var openApiFilePath = Path.Combine(PluginPathUtils.ProjectPluginsDir, "LogosStorageClient", "openapi.yaml"); + if (!File.Exists(openApiFilePath)) throw new Exception("Unable to locate LogosStorageClient/openapi.yaml. Expected: " + openApiFilePath); File.Delete(openApiFilePath); File.WriteAllText(openApiFilePath, containerApi); - Log("CodexClient/openapi.yaml has been updated."); + Log("LogosStorageClient/openapi.yaml has been updated."); } private string Hash(string file) diff --git a/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs b/ProjectPlugins/StoragePlugin/BinaryProcessControl.cs similarity index 92% rename from ProjectPlugins/CodexPlugin/BinaryProcessControl.cs rename to ProjectPlugins/StoragePlugin/BinaryProcessControl.cs index c4ed3ee1..563161c7 100644 --- a/ProjectPlugins/CodexPlugin/BinaryProcessControl.cs +++ b/ProjectPlugins/StoragePlugin/BinaryProcessControl.cs @@ -1,20 +1,20 @@ using System.Diagnostics; -using CodexClient; +using LogosStorageClient; using Logging; -namespace CodexPlugin +namespace StoragePlugin { public class BinaryProcessControl : IProcessControl { private readonly LogFile logFile; private readonly Process process; - private readonly CodexProcessConfig config; + private readonly LogosStorageProcessConfig config; private List logBuffer = new List(); private readonly object bufferLock = new object(); private readonly List streamTasks = new List(); private bool running; - public BinaryProcessControl(ILog log, Process process, CodexProcessConfig config) + public BinaryProcessControl(ILog log, Process process, LogosStorageProcessConfig config) { logFile = log.CreateSubfile(config.Name); diff --git a/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs b/ProjectPlugins/StoragePlugin/BinaryStorageStarter.cs similarity index 67% rename from ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs rename to ProjectPlugins/StoragePlugin/BinaryStorageStarter.cs index 617cb701..49f88281 100644 --- a/ProjectPlugins/CodexPlugin/BinaryCodexStarter.cs +++ b/ProjectPlugins/StoragePlugin/BinaryStorageStarter.cs @@ -1,40 +1,40 @@ -using CodexClient; +using LogosStorageClient; using Core; using Utils; using System.Diagnostics; -namespace CodexPlugin +namespace StoragePlugin { - public class BinaryCodexStarter : ICodexStarter + public class BinaryLogosStorageStarter : ILogosStorageStarter { private readonly IPluginTools pluginTools; private readonly ProcessControlMap processControlMap; private readonly static NumberSource numberSource = new NumberSource(1); private readonly static FreePortFinder freePortFinder = new FreePortFinder(); private readonly static object _lock = new object(); - private readonly static string dataParentDir = "codex_disttest_datadirs"; - private readonly static CodexExePath codexExePath = new CodexExePath(); + private readonly static string dataParentDir = "storage_disttest_datadirs"; + private readonly static LogosStorageExePath logosStorageExePath = new LogosStorageExePath(); - static BinaryCodexStarter() + static BinaryLogosStorageStarter() { - StopAllCodexProcesses(); + StopAllLogosStorageProcesses(); DeleteParentDataDir(); } - public BinaryCodexStarter(IPluginTools pluginTools, ProcessControlMap processControlMap) + public BinaryLogosStorageStarter(IPluginTools pluginTools, ProcessControlMap processControlMap) { this.pluginTools = pluginTools; this.processControlMap = processControlMap; } - public ICodexInstance[] BringOnline(CodexSetup codexSetup) + public ILogosStorageInstance[] BringOnline(LogosStorageSetup logosStorageSetup) { lock (_lock) { LogSeparator(); - Log($"Starting {codexSetup.Describe()}..."); + Log($"Starting {logosStorageSetup.Describe()}..."); - return StartCodexBinaries(codexSetup, codexSetup.NumberOfNodes); + return StartLogosStorageBinaries(logosStorageSetup, logosStorageSetup.NumberOfNodes); } } @@ -46,9 +46,9 @@ namespace CodexPlugin } } - private ICodexInstance[] StartCodexBinaries(CodexStartupConfig startupConfig, int numberOfNodes) + private ILogosStorageInstance[] StartLogosStorageBinaries(LogosStorageStartupConfig startupConfig, int numberOfNodes) { - var result = new List(); + var result = new List(); for (var i = 0; i < numberOfNodes; i++) { result.Add(StartBinary(startupConfig)); @@ -57,14 +57,14 @@ namespace CodexPlugin return result.ToArray(); } - private ICodexInstance StartBinary(CodexStartupConfig config) + private ILogosStorageInstance StartBinary(LogosStorageStartupConfig config) { var name = GetName(config); var dataDir = Path.Combine(dataParentDir, $"datadir_{numberSource.GetNextNumber()}"); - var pconfig = new CodexProcessConfig(name, freePortFinder, dataDir); + var pconfig = new LogosStorageProcessConfig(name, freePortFinder, dataDir); Log(pconfig); - var factory = new CodexProcessRecipe(pconfig, codexExePath); + var factory = new LogosStorageProcessRecipe(pconfig, logosStorageExePath); var recipe = factory.Initialize(config); var startInfo = new ProcessStartInfo( @@ -82,7 +82,7 @@ namespace CodexPlugin } var local = "localhost"; - var instance = new CodexInstance( + var instance = new LogosStorageInstance( name: name, imageName: "binary", startUtc: DateTime.UtcNow, @@ -99,13 +99,13 @@ namespace CodexPlugin return instance; } - private string GetName(CodexStartupConfig config) + private string GetName(LogosStorageStartupConfig config) { if (!string.IsNullOrEmpty(config.NameOverride)) { return config.NameOverride + "_" + numberSource.GetNextNumber(); } - return "codex_" + numberSource.GetNextNumber(); + return "storage_" + numberSource.GetNextNumber(); } private void LogSeparator() @@ -113,7 +113,7 @@ namespace CodexPlugin Log("----------------------------------------------------------------------------"); } - private void Log(CodexProcessConfig pconfig) + private void Log(LogosStorageProcessConfig pconfig) { Log( "NodeConfig:Name=" + pconfig.Name + @@ -137,16 +137,16 @@ namespace CodexPlugin } } - private static void StopAllCodexProcesses() + private static void StopAllLogosStorageProcesses() { var processes = Process.GetProcesses(); - var codexes = processes.Where(p => - p.ProcessName.ToLowerInvariant() == "codex" && + var storageProcesses = processes.Where(p => + p.ProcessName.ToLowerInvariant() == "storage" && p.MainModule != null && - p.MainModule.FileName == codexExePath.Get() + p.MainModule.FileName == logosStorageExePath.Get() ).ToArray(); - foreach (var c in codexes) + foreach (var c in storageProcesses) { c.Kill(); c.WaitForExit(); diff --git a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs b/ProjectPlugins/StoragePlugin/ContainerLogosStorageStarter.cs similarity index 67% rename from ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs rename to ProjectPlugins/StoragePlugin/ContainerLogosStorageStarter.cs index 4103fb8c..b12acd0d 100644 --- a/ProjectPlugins/CodexPlugin/ContainerCodexStarter.cs +++ b/ProjectPlugins/StoragePlugin/ContainerLogosStorageStarter.cs @@ -1,19 +1,19 @@ -using CodexClient; +using LogosStorageClient; using Core; using KubernetesWorkflow; using KubernetesWorkflow.Types; using Utils; -namespace CodexPlugin +namespace StoragePlugin { - public class ContainerCodexStarter : ICodexStarter + public class ContainerLogosStorageStarter : ILogosStorageStarter { private readonly IPluginTools pluginTools; private readonly ProcessControlMap processControlMap; - private readonly CodexContainerRecipe recipe; + private readonly LogosStorageContainerRecipe recipe; private readonly ApiChecker apiChecker; - public ContainerCodexStarter(IPluginTools pluginTools, CodexContainerRecipe recipe, ProcessControlMap processControlMap) + public ContainerLogosStorageStarter(IPluginTools pluginTools, LogosStorageContainerRecipe recipe, ProcessControlMap processControlMap) { this.pluginTools = pluginTools; this.recipe = recipe; @@ -21,14 +21,14 @@ namespace CodexPlugin apiChecker = new ApiChecker(pluginTools); } - public ICodexInstance[] BringOnline(CodexSetup codexSetup) + public ILogosStorageInstance[] BringOnline(LogosStorageSetup logosStorageSetup) { LogSeparator(); - Log($"Starting {codexSetup.Describe()}..."); + Log($"Starting {logosStorageSetup.Describe()}..."); - var startupConfig = CreateStartupConfig(codexSetup); + var startupConfig = CreateStartupConfig(logosStorageSetup); - var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); + var containers = StartLogosStorageContainers(startupConfig, logosStorageSetup.NumberOfNodes, logosStorageSetup.Location); apiChecker.CheckCompatibility(containers); @@ -48,15 +48,15 @@ namespace CodexPlugin { } - private StartupConfig CreateStartupConfig(CodexSetup codexSetup) + private StartupConfig CreateStartupConfig(LogosStorageSetup logosStorageSetup) { var startupConfig = new StartupConfig(); - startupConfig.NameOverride = codexSetup.NameOverride; - startupConfig.Add(codexSetup); + startupConfig.NameOverride = logosStorageSetup.NameOverride; + startupConfig.Add(logosStorageSetup); return startupConfig; } - private RunningPod[] StartCodexContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) + private RunningPod[] StartLogosStorageContainers(StartupConfig startupConfig, int numberOfNodes, ILocation location) { var futureContainers = new List(); for (var i = 0; i < numberOfNodes; i++) @@ -76,10 +76,10 @@ namespace CodexPlugin return workflow.GetPodInfo(rc); } - private ICodexInstance CreateInstance(RunningPod pod) + private ILogosStorageInstance CreateInstance(RunningPod pod) { - var instance = CodexInstanceContainerExtension.CreateFromPod(pod); - var processControl = new CodexContainerProcessControl(pluginTools, pod, onStop: () => + var instance = LogosStorageInstanceContainerExtension.CreateFromPod(pod); + var processControl = new LogosStorageContainerProcessControl(pluginTools, pod, onStop: () => { processControlMap.Remove(instance); }); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs b/ProjectPlugins/StoragePlugin/ContainerProcessControl.cs similarity index 89% rename from ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs rename to ProjectPlugins/StoragePlugin/ContainerProcessControl.cs index 52e3d1dc..dbfd2a5d 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerProcessControl.cs +++ b/ProjectPlugins/StoragePlugin/ContainerProcessControl.cs @@ -1,19 +1,19 @@ -using CodexClient; +using LogosStorageClient; using Core; using KubernetesWorkflow; using KubernetesWorkflow.Types; using Logging; -namespace CodexPlugin +namespace StoragePlugin { - public class CodexContainerProcessControl : IProcessControl + public class LogosStorageContainerProcessControl : IProcessControl { private readonly IPluginTools tools; private readonly RunningPod pod; private readonly Action onStop; private readonly ContainerCrashWatcher crashWatcher; - public CodexContainerProcessControl(IPluginTools tools, RunningPod pod, Action onStop) + public LogosStorageContainerProcessControl(IPluginTools tools, RunningPod pod, Action onStop) { this.tools = tools; this.pod = pod; diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/StoragePlugin/ContainerRecipe.cs similarity index 78% rename from ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs rename to ProjectPlugins/StoragePlugin/ContainerRecipe.cs index e91118b7..7cde8761 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/StoragePlugin/ContainerRecipe.cs @@ -2,26 +2,26 @@ using KubernetesWorkflow; using KubernetesWorkflow.Recipe; using Utils; -namespace CodexPlugin +namespace StoragePlugin { - public class CodexContainerRecipe : ContainerRecipeFactory + public class LogosStorageContainerRecipe : ContainerRecipeFactory { - public const string ApiPortTag = "codex_api_port"; - public const string ListenPortTag = "codex_listen_port"; - public const string MetricsPortTag = "codex_metrics_port"; - public const string DiscoveryPortTag = "codex_discovery_port"; + public const string ApiPortTag = "storage_api_port"; + public const string ListenPortTag = "storage_listen_port"; + public const string MetricsPortTag = "storage_metrics_port"; + public const string DiscoveryPortTag = "storage_discovery_port"; // Used by tests for time-constraint assertions. public static readonly TimeSpan MaxUploadTimePerMegabyte = TimeSpan.FromSeconds(2.0); public static readonly TimeSpan MaxDownloadTimePerMegabyte = TimeSpan.FromSeconds(2.0); - private readonly CodexDockerImage codexDockerImage; + private readonly LogosStorageDockerImage logosStorageDockerImage; - public override string AppName => "codex"; - public override string Image => codexDockerImage.GetCodexDockerImage(); + public override string AppName => "storage"; + public override string Image => logosStorageDockerImage.GetLogosStorageDockerImage(); - public CodexContainerRecipe(CodexDockerImage codexDockerImage) + public LogosStorageContainerRecipe(LogosStorageDockerImage logosStorageDockerImage) { - this.codexDockerImage = codexDockerImage; + this.logosStorageDockerImage = logosStorageDockerImage; } protected override void Initialize(StartupConfig startupConfig) @@ -32,7 +32,7 @@ namespace CodexPlugin SetSchedulingAffinity(notIn: "false"); SetSystemCriticalPriority(); - var config = startupConfig.Get(); + var config = startupConfig.Get(); var apiPort = CreateApiPort(config, ApiPortTag); AddEnvVar("STORAGE_API_PORT", apiPort); @@ -97,27 +97,27 @@ namespace CodexPlugin } } - private Port CreateApiPort(CodexStartupConfig config, string tag) + private Port CreateApiPort(LogosStorageStartupConfig config, string tag) { if (config.PublicTestNet == null) return AddExposedPort(tag); return AddInternalPort(tag); } - private Port CreateListenPort(CodexStartupConfig config) + private Port CreateListenPort(LogosStorageStartupConfig config) { if (config.PublicTestNet == null) return AddInternalPort(ListenPortTag); return AddExposedPort(config.PublicTestNet.PublicListenPort, ListenPortTag); } - private Port CreateDiscoveryPort(CodexStartupConfig config) + private Port CreateDiscoveryPort(LogosStorageStartupConfig config) { if (config.PublicTestNet == null) return AddInternalPort(DiscoveryPortTag, PortProtocol.UDP); return AddExposedPort(config.PublicTestNet.PublicDiscoveryPort, DiscoveryPortTag, PortProtocol.UDP); } - private ByteSize GetVolumeCapacity(CodexStartupConfig config) + private ByteSize GetVolumeCapacity(LogosStorageStartupConfig config) { if (config.StorageQuota != null) return config.StorageQuota.Multiply(1.2); // Default Codex quota: 8 Gb, using +20% to be safe. diff --git a/ProjectPlugins/StoragePlugin/CoreInterfaceExtensions.cs b/ProjectPlugins/StoragePlugin/CoreInterfaceExtensions.cs new file mode 100644 index 00000000..cb46d8f1 --- /dev/null +++ b/ProjectPlugins/StoragePlugin/CoreInterfaceExtensions.cs @@ -0,0 +1,51 @@ +using LogosStorageClient; +using LogosStorageClient.Hooks; +using Core; + +namespace StoragePlugin +{ + public static class CoreInterfaceExtensions + { + public static ILogosStorageInstance[] DeployLogosStorageNodes(this CoreInterface ci, int number, Action setup) + { + return Plugin(ci).DeployLogosStorageNodes(number, setup); + } + + public static IStorageNodeGroup WrapLogosStorageContainers(this CoreInterface ci, ILogosStorageInstance[] instances) + { + return Plugin(ci).WrapLogosStorageContainers(instances); + } + + public static IStorageNode StartStorageNode(this CoreInterface ci) + { + return ci.StartStorageNodes(1)[0]; + } + + public static IStorageNode StartStorageNode(this CoreInterface ci, Action setup) + { + return ci.StartStorageNodes(1, setup)[0]; + } + + public static IStorageNodeGroup StartStorageNodes(this CoreInterface ci, int number, Action setup) + { + var rc = ci.DeployLogosStorageNodes(number, setup); + var result = ci.WrapLogosStorageContainers(rc); + return result; + } + + public static IStorageNodeGroup StartStorageNodes(this CoreInterface ci, int number) + { + return ci.StartStorageNodes(number, s => { }); + } + + public static void AddLogosStorageHooksProvider(this CoreInterface ci, ILogosStorageHooksProvider hooksProvider) + { + Plugin(ci).AddLogosStorageHooksProvider(hooksProvider); + } + + private static StoragePlugin Plugin(CoreInterface ci) + { + return ci.GetPlugin(); + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexDeployment.cs b/ProjectPlugins/StoragePlugin/Deployment.cs similarity index 73% rename from ProjectPlugins/CodexPlugin/CodexDeployment.cs rename to ProjectPlugins/StoragePlugin/Deployment.cs index 7b760d92..7bc4b7a6 100644 --- a/ProjectPlugins/CodexPlugin/CodexDeployment.cs +++ b/ProjectPlugins/StoragePlugin/Deployment.cs @@ -1,24 +1,24 @@ -using CodexClient; +using LogosStorageClient; using KubernetesWorkflow.Types; -namespace CodexPlugin +namespace StoragePlugin { - public class CodexDeployment + public class LogosStorageDeployment { - public CodexDeployment(CodexInstance[] codexInstances, + public LogosStorageDeployment(LogosStorageInstance[] logosStorageInstances, RunningPod? prometheusContainer, RunningPod? discordBotContainer, DeploymentMetadata metadata, string id) { Id = id; - CodexInstances = codexInstances; + LogosStorageInstances = logosStorageInstances; PrometheusContainer = prometheusContainer; DiscordBotContainer = discordBotContainer; Metadata = metadata; } public string Id { get; } - public CodexInstance[] CodexInstances { get; } + public LogosStorageInstance[] LogosStorageInstances { get; } public RunningPod? PrometheusContainer { get; } public RunningPod? DiscordBotContainer { get; } public DeploymentMetadata Metadata { get; } @@ -27,7 +27,7 @@ namespace CodexPlugin public class DeploymentMetadata { public DeploymentMetadata(string name, DateTime startUtc, DateTime finishedUtc, string kubeNamespace, - int numberOfCodexNodes, int numberOfValidators, int storageQuotaMB, CodexLogLevel codexLogLevel, + int numberOfLogosStorageNodes, int numberOfValidators, int storageQuotaMB, LogosStorageLogLevel logosStorageLogLevel, int initialTestTokens, int minPrice, int maxCollateral, int maxDuration, int blockTTL, int blockMI, int blockMN) { @@ -35,10 +35,10 @@ namespace CodexPlugin StartUtc = startUtc; FinishedUtc = finishedUtc; KubeNamespace = kubeNamespace; - NumberOfCodexNodes = numberOfCodexNodes; + NumberOfLogosStorageNodes = numberOfLogosStorageNodes; NumberOfValidators = numberOfValidators; StorageQuotaMB = storageQuotaMB; - CodexLogLevel = codexLogLevel; + LogosStorageLogLevel = logosStorageLogLevel; InitialTestTokens = initialTestTokens; MinPrice = minPrice; MaxCollateral = maxCollateral; @@ -52,10 +52,10 @@ namespace CodexPlugin public DateTime StartUtc { get; } public DateTime FinishedUtc { get; } public string KubeNamespace { get; } - public int NumberOfCodexNodes { get; } + public int NumberOfLogosStorageNodes { get; } public int NumberOfValidators { get; } public int StorageQuotaMB { get; } - public CodexLogLevel CodexLogLevel { get; } + public LogosStorageLogLevel LogosStorageLogLevel { get; } public int InitialTestTokens { get; } public int MinPrice { get; } public int MaxCollateral { get; } diff --git a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs b/ProjectPlugins/StoragePlugin/DockerImage.cs similarity index 64% rename from ProjectPlugins/CodexPlugin/CodexDockerImage.cs rename to ProjectPlugins/StoragePlugin/DockerImage.cs index 092105dd..4bd6650a 100644 --- a/ProjectPlugins/CodexPlugin/CodexDockerImage.cs +++ b/ProjectPlugins/StoragePlugin/DockerImage.cs @@ -1,14 +1,14 @@ -namespace CodexPlugin +namespace StoragePlugin { - public class CodexDockerImage + public class LogosStorageDockerImage { private const string DefaultDockerImage = "logosstorage/logos-storage-nim:latest-dist-tests"; public static string Override { get; set; } = string.Empty; - public string GetCodexDockerImage() + public string GetLogosStorageDockerImage() { - var image = Environment.GetEnvironmentVariable("CODEXDOCKERIMAGE"); + var image = Environment.GetEnvironmentVariable("STORAGEDOCKERIMAGE"); if (!string.IsNullOrEmpty(image)) return image; if (!string.IsNullOrEmpty(Override)) return Override; return DefaultDockerImage; diff --git a/ProjectPlugins/CodexPlugin/FreePortFinder.cs b/ProjectPlugins/StoragePlugin/FreePortFinder.cs similarity index 97% rename from ProjectPlugins/CodexPlugin/FreePortFinder.cs rename to ProjectPlugins/StoragePlugin/FreePortFinder.cs index 9a67899d..6aadee57 100644 --- a/ProjectPlugins/CodexPlugin/FreePortFinder.cs +++ b/ProjectPlugins/StoragePlugin/FreePortFinder.cs @@ -1,6 +1,6 @@ using System.Net.NetworkInformation; -namespace CodexPlugin +namespace StoragePlugin { public class FreePortFinder { diff --git a/ProjectPlugins/StoragePlugin/ILogosStorageStarter.cs b/ProjectPlugins/StoragePlugin/ILogosStorageStarter.cs new file mode 100644 index 00000000..1ed411ab --- /dev/null +++ b/ProjectPlugins/StoragePlugin/ILogosStorageStarter.cs @@ -0,0 +1,10 @@ +using LogosStorageClient; + +namespace StoragePlugin +{ + public interface ILogosStorageStarter + { + ILogosStorageInstance[] BringOnline(LogosStorageSetup logosStorageSetup); + void Decommission(); + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs b/ProjectPlugins/StoragePlugin/InstanceContainerExtension.cs similarity index 62% rename from ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs rename to ProjectPlugins/StoragePlugin/InstanceContainerExtension.cs index 4bed34d5..a3622010 100644 --- a/ProjectPlugins/CodexPlugin/CodexInstanceContainerExtension.cs +++ b/ProjectPlugins/StoragePlugin/InstanceContainerExtension.cs @@ -1,22 +1,22 @@ -using CodexClient; +using LogosStorageClient; using KubernetesWorkflow.Types; using Utils; -namespace CodexPlugin +namespace StoragePlugin { - public static class CodexInstanceContainerExtension + public static class LogosStorageInstanceContainerExtension { - public static ICodexInstance CreateFromPod(RunningPod pod) + public static ILogosStorageInstance CreateFromPod(RunningPod pod) { var container = pod.Containers.Single(); - return new CodexInstance( + return new LogosStorageInstance( name: container.Name, imageName: container.Recipe.Image, startUtc: container.Recipe.RecipeCreatedUtc, - discoveryEndpoint: SetClusterInternalIpAddress(pod, container.GetInternalAddress(CodexContainerRecipe.DiscoveryPortTag)), - apiEndpoint: container.GetAddress(CodexContainerRecipe.ApiPortTag), - listenEndpoint: container.GetInternalAddress(CodexContainerRecipe.ListenPortTag), + discoveryEndpoint: SetClusterInternalIpAddress(pod, container.GetInternalAddress(LogosStorageContainerRecipe.DiscoveryPortTag)), + apiEndpoint: container.GetAddress(LogosStorageContainerRecipe.ApiPortTag), + listenEndpoint: container.GetInternalAddress(LogosStorageContainerRecipe.ListenPortTag), ethAccount: container.Recipe.Additionals.Get(), metricsEndpoint: GetMetricsEndpoint(container) ); @@ -35,7 +35,7 @@ namespace CodexPlugin { try { - return container.GetInternalAddress(CodexContainerRecipe.MetricsPortTag); + return container.GetInternalAddress(LogosStorageContainerRecipe.MetricsPortTag); } catch { diff --git a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs b/ProjectPlugins/StoragePlugin/LocalNodeBuilder.cs similarity index 77% rename from ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs rename to ProjectPlugins/StoragePlugin/LocalNodeBuilder.cs index c37c23cd..060608d6 100644 --- a/ProjectPlugins/CodexPlugin/LocalCodexBuilder.cs +++ b/ProjectPlugins/StoragePlugin/LocalNodeBuilder.cs @@ -1,29 +1,29 @@ -using CodexPlugin; +using StoragePlugin; using Logging; using System.Diagnostics; -namespace CodexNetDeployer +namespace StoragePlugin { - public class LocalCodexBuilder + public class LocalNodeBuilder { private readonly ILog log; private readonly string? repoPath; private readonly string? dockerUsername; - public LocalCodexBuilder(ILog log, string? repoPath, string? dockerUsername) + public LocalNodeBuilder(ILog log, string? repoPath, string? dockerUsername) { - this.log = new LogPrefixer(log, "(LocalCodexBuilder) "); + this.log = new LogPrefixer(log, "(LocalNodeBuilder) "); this.repoPath = repoPath; this.dockerUsername = dockerUsername; } - public LocalCodexBuilder(ILog log, string? repoPath) + public LocalNodeBuilder(ILog log, string? repoPath) : this(log, repoPath, Environment.GetEnvironmentVariable("DOCKERUSERNAME")) { } - public LocalCodexBuilder(ILog log) - : this(log, Environment.GetEnvironmentVariable("CODEXREPOPATH")) + public LocalNodeBuilder(ILog log) + : this(log, Environment.GetEnvironmentVariable("STORAGEREPOPATH")) { } @@ -32,15 +32,15 @@ namespace CodexNetDeployer if (!IsEnabled()) return; if (string.IsNullOrEmpty(dockerUsername)) throw new Exception("Docker username required. (Pass to constructor or set 'DOCKERUSERNAME' environment variable.)"); - if (string.IsNullOrEmpty(repoPath)) throw new Exception("Codex repo path required. (Pass to constructor or set 'CODEXREPOPATH' environment variable.)"); + if (string.IsNullOrEmpty(repoPath)) throw new Exception("Logos Storage repo path required. (Pass to constructor or set 'CODEXREPOPATH' environment variable.)"); if (!Directory.Exists(repoPath)) throw new Exception($"Path '{repoPath}' does not exist."); var files = Directory.GetFiles(repoPath); if (!files.Any(f => f.ToLowerInvariant().EndsWith("codex.nim"))) throw new Exception($"Path '{repoPath}' does not appear to be the Codex repo root."); - Log($"Codex docker image will be built in path '{repoPath}'."); - Log("Please note this can take several minutes. If you're not trying to use a Codex image with local code changes,"); - Log("Consider using the default test image or consider setting the 'CODEXDOCKERIMAGE' environment variable to use an already built image."); - CodexDockerImage.Override = $"Using docker image locally built in path '{repoPath}'."; + Log($"Logos Storage docker image will be built in path '{repoPath}'."); + Log("Please note this can take several minutes. If you're not trying to use a Logos Storage image with local code changes,"); + Log("Consider using the default test image or consider setting the 'STORAGEDOCKERIMAGE' environment variable to use an already built image."); + LogosStorageDockerImage.Override = $"Using docker image locally built in path '{repoPath}'."; } public void Build() @@ -49,7 +49,7 @@ namespace CodexNetDeployer Log("Docker login..."); DockerLogin(); - Log($"Logged in. Building Codex image in path '{repoPath}'..."); + Log($"Logged in. Building Logos Storage image in path '{repoPath}'..."); var customImage = GenerateImageName(); Docker($"build", "-t", customImage, "-f", "./codex.Dockerfile", @@ -62,7 +62,7 @@ namespace CodexNetDeployer Docker("push", customImage); - CodexDockerImage.Override = customImage; + LogosStorageDockerImage.Override = customImage; Log("Image pushed. Good to go!"); } @@ -96,8 +96,8 @@ namespace CodexNetDeployer private string GenerateImageName() { var tag = Environment.GetEnvironmentVariable("DOCKERTAG"); - if (string.IsNullOrEmpty(tag)) return $"{dockerUsername!}/nim-codex-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}"; - return $"{dockerUsername}/nim-codex-autoimage:{tag}"; + if (string.IsNullOrEmpty(tag)) return $"{dockerUsername!}/logos-storage-nim-autoimage:{Guid.NewGuid().ToString().ToLowerInvariant()}"; + return $"{dockerUsername}/logos-storage-nim-autoimage:{tag}"; } private void Docker(params string[] args) diff --git a/ProjectPlugins/CodexPlugin/CodexExePath.cs b/ProjectPlugins/StoragePlugin/LogosStorageExePath.cs similarity index 61% rename from ProjectPlugins/CodexPlugin/CodexExePath.cs rename to ProjectPlugins/StoragePlugin/LogosStorageExePath.cs index 79e9d4d8..3299fa90 100644 --- a/ProjectPlugins/CodexPlugin/CodexExePath.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageExePath.cs @@ -1,15 +1,15 @@ -namespace CodexPlugin +namespace StoragePlugin { - public class CodexExePath + public class LogosStorageExePath { private readonly string[] paths = [ - Path.Combine("d:", "Dev", "nim-codex", "build", "codex.exe"), - Path.Combine("c:", "Projects", "nim-codex", "build", "codex.exe") + Path.Combine("d:", "Dev", "logos-storage-nim", "build", "storage.exe"), + Path.Combine("c:", "Projects", "logos-storage-nim", "build", "storage.exe") ]; private string selectedPath = string.Empty; - public CodexExePath() + public LogosStorageExePath() { foreach (var p in paths) { diff --git a/ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs b/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs similarity index 84% rename from ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs rename to ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs index 7748e50d..912a87cd 100644 --- a/ProjectPlugins/CodexPlugin/CodexProcessRecipe.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageProcessRecipe.cs @@ -1,7 +1,7 @@ using System.Net.Sockets; using System.Net; -namespace CodexPlugin +namespace StoragePlugin { public class ProcessRecipe { @@ -15,9 +15,9 @@ namespace CodexPlugin public string[] Args { get; } } - public class CodexProcessConfig + public class LogosStorageProcessConfig { - public CodexProcessConfig(string name, FreePortFinder freePortFinder, string dataDir) + public LogosStorageProcessConfig(string name, FreePortFinder freePortFinder, string dataDir) { ApiPort = freePortFinder.GetNextFreePort(); DiscPort = freePortFinder.GetNextFreePort(); @@ -38,18 +38,18 @@ namespace CodexPlugin public IPAddress LocalIpAddrs { get; } } - public class CodexProcessRecipe + public class LogosStorageProcessRecipe { - private readonly CodexProcessConfig pc; - private readonly CodexExePath codexExePath; + private readonly LogosStorageProcessConfig pc; + private readonly LogosStorageExePath logosStorageExePath; - public CodexProcessRecipe(CodexProcessConfig pc, CodexExePath codexExePath) + public LogosStorageProcessRecipe(LogosStorageProcessConfig pc, LogosStorageExePath logosStorageExePath) { this.pc = pc; - this.codexExePath = codexExePath; + this.logosStorageExePath = logosStorageExePath; } - public ProcessRecipe Initialize(CodexStartupConfig config) + public ProcessRecipe Initialize(LogosStorageStartupConfig config) { args.Clear(); @@ -104,7 +104,7 @@ namespace CodexPlugin private ProcessRecipe Create() { return new ProcessRecipe( - cmd: codexExePath.Get(), + cmd: logosStorageExePath.Get(), args: args.ToArray()); } diff --git a/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs b/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs new file mode 100644 index 00000000..43cbb364 --- /dev/null +++ b/ProjectPlugins/StoragePlugin/LogosStorageSetup.cs @@ -0,0 +1,135 @@ +using LogosStorageClient; +using KubernetesWorkflow; +using Utils; + +namespace StoragePlugin +{ + public interface ILogosStorageSetup + { + ILogosStorageSetup WithName(string name); + ILogosStorageSetup At(ILocation location); + ILogosStorageSetup WithBootstrapNode(IStorageNode node); + ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level); + ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level, LogosStorageLogCustomTopics customTopics); + ILogosStorageSetup WithStorageQuota(ByteSize storageQuota); + ILogosStorageSetup WithBlockTTL(TimeSpan duration); + ILogosStorageSetup WithBlockMaintenanceInterval(TimeSpan duration); + ILogosStorageSetup WithBlockMaintenanceNumber(int numberOfBlocks); + ILogosStorageSetup EnableMetrics(); + ILogosStorageSetup AsPublicTestNet(LogosStorageTestNetConfig testNetConfig); + } + + public class LogosStorageLogCustomTopics + { + public LogosStorageLogCustomTopics(LogosStorageLogLevel discV5, LogosStorageLogLevel libp2p, LogosStorageLogLevel blockExchange) + { + DiscV5 = discV5; + Libp2p = libp2p; + BlockExchange = blockExchange; + } + + public LogosStorageLogCustomTopics(LogosStorageLogLevel discV5, LogosStorageLogLevel libp2p) + { + DiscV5 = discV5; + Libp2p = libp2p; + } + + public LogosStorageLogLevel DiscV5 { get; set; } + public LogosStorageLogLevel Libp2p { get; set; } + public LogosStorageLogLevel ContractClock { get; set; } = LogosStorageLogLevel.Warn; + public LogosStorageLogLevel? BlockExchange { get; } + public LogosStorageLogLevel JsonSerialize { get; set; } = LogosStorageLogLevel.Warn; + public LogosStorageLogLevel MarketplaceInfra { get; set; } = LogosStorageLogLevel.Warn; + } + + public class LogosStorageSetup : LogosStorageStartupConfig, ILogosStorageSetup + { + public int NumberOfNodes { get; } + + public LogosStorageSetup(int numberOfNodes) + { + NumberOfNodes = numberOfNodes; + } + + public ILogosStorageSetup WithName(string name) + { + NameOverride = name; + return this; + } + + public ILogosStorageSetup At(ILocation location) + { + Location = location; + return this; + } + + public ILogosStorageSetup WithBootstrapNode(IStorageNode node) + { + BootstrapSpr = node.GetDebugInfo().Spr; + return this; + } + + public ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level) + { + LogLevel = level; + return this; + } + + public ILogosStorageSetup WithLogLevel(LogosStorageLogLevel level, LogosStorageLogCustomTopics customTopics) + { + LogLevel = level; + CustomTopics = customTopics; + return this; + } + + public ILogosStorageSetup WithStorageQuota(ByteSize storageQuota) + { + StorageQuota = storageQuota; + return this; + } + + public ILogosStorageSetup WithBlockTTL(TimeSpan duration) + { + BlockTTL = Convert.ToInt32(duration.TotalSeconds); + return this; + } + + public ILogosStorageSetup WithBlockMaintenanceInterval(TimeSpan duration) + { + BlockMaintenanceInterval = duration; + return this; + } + + public ILogosStorageSetup WithBlockMaintenanceNumber(int numberOfBlocks) + { + BlockMaintenanceNumber = numberOfBlocks; + return this; + } + + public ILogosStorageSetup EnableMetrics() + { + MetricsEnabled = true; + return this; + } + + public ILogosStorageSetup AsPublicTestNet(LogosStorageTestNetConfig testNetConfig) + { + PublicTestNet = testNetConfig; + return this; + } + + public string Describe() + { + var args = string.Join(',', DescribeArgs()); + return $"({NumberOfNodes} Logos Storage Nodes with args:[{args}])"; + } + + private IEnumerable DescribeArgs() + { + if (PublicTestNet != null) yield return $"Public TestNet with listenPort: {PublicTestNet.PublicListenPort}"; + yield return $"LogLevel={LogLevelWithTopics()}"; + if (BootstrapSpr != null) yield return $"BootstrapNode={BootstrapSpr}"; + if (StorageQuota != null) yield return $"StorageQuota={StorageQuota}"; + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs similarity index 86% rename from ProjectPlugins/CodexPlugin/CodexStartupConfig.cs rename to ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs index 19c8fec6..0336e802 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageStartupConfig.cs @@ -1,15 +1,15 @@ -using CodexClient; +using LogosStorageClient; using KubernetesWorkflow; using Utils; -namespace CodexPlugin +namespace StoragePlugin { - public class CodexStartupConfig + public class LogosStorageStartupConfig { public string? NameOverride { get; set; } public ILocation Location { get; set; } = KnownLocations.UnspecifiedLocation; - public CodexLogLevel LogLevel { get; set; } - public CodexLogCustomTopics? CustomTopics { get; set; } = new CodexLogCustomTopics(CodexLogLevel.Info, CodexLogLevel.Warn); + public LogosStorageLogLevel LogLevel { get; set; } + public LogosStorageLogCustomTopics? CustomTopics { get; set; } = new LogosStorageLogCustomTopics(LogosStorageLogLevel.Info, LogosStorageLogLevel.Warn); public ByteSize? StorageQuota { get; set; } public bool MetricsEnabled { get; set; } public string? BootstrapSpr { get; set; } @@ -17,7 +17,7 @@ namespace CodexPlugin public bool? EnableValidator { get; set; } public TimeSpan? BlockMaintenanceInterval { get; set; } public int? BlockMaintenanceNumber { get; set; } - public CodexTestNetConfig? PublicTestNet { get; set; } + public LogosStorageTestNetConfig? PublicTestNet { get; set; } public string LogLevelWithTopics() { @@ -84,7 +84,7 @@ namespace CodexPlugin $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" + $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" + $"{CustomTopics.JsonSerialize.ToString().ToLowerInvariant()}:{string.Join(",", jsonSerializeTopics)};" + - $"{CodexLogLevel.Error.ToString()}:{string.Join(",", alwaysIgnoreTopics)}"; + $"{LogosStorageLogLevel.Error.ToString()}:{string.Join(",", alwaysIgnoreTopics)}"; if (CustomTopics.BlockExchange != null) { @@ -95,7 +95,7 @@ namespace CodexPlugin } } - public class CodexTestNetConfig + public class LogosStorageTestNetConfig { public int PublicDiscoveryPort { get; set; } public int PublicListenPort { get; set; } diff --git a/ProjectPlugins/CodexPlugin/CodexWrapper.cs b/ProjectPlugins/StoragePlugin/LogosStorageWrapper.cs similarity index 54% rename from ProjectPlugins/CodexPlugin/CodexWrapper.cs rename to ProjectPlugins/StoragePlugin/LogosStorageWrapper.cs index a4a4eda6..4df7e2d0 100644 --- a/ProjectPlugins/CodexPlugin/CodexWrapper.cs +++ b/ProjectPlugins/StoragePlugin/LogosStorageWrapper.cs @@ -1,57 +1,57 @@ -using CodexClient; -using CodexClient.Hooks; +using LogosStorageClient; +using LogosStorageClient.Hooks; using Core; using Logging; -namespace CodexPlugin +namespace StoragePlugin { - public class CodexWrapper + public class LogosStorageWrapper { private readonly IPluginTools pluginTools; private readonly ProcessControlMap processControlMap; - private readonly CodexHooksFactory hooksFactory; + private readonly LogosStorageHooksFactory hooksFactory; private DebugInfoVersion? versionResponse; - public CodexWrapper(IPluginTools pluginTools, ProcessControlMap processControlMap, CodexHooksFactory hooksFactory) + public LogosStorageWrapper(IPluginTools pluginTools, ProcessControlMap processControlMap, LogosStorageHooksFactory hooksFactory) { this.pluginTools = pluginTools; this.processControlMap = processControlMap; this.hooksFactory = hooksFactory; } - public string GetCodexId() + public string GetLogosStorageId() { if (versionResponse != null) return versionResponse.Version; return "unknown"; } - public string GetCodexRevision() + public string GetLogosStorageRevision() { if (versionResponse != null) return versionResponse.Revision; return "unknown"; } - public ICodexNodeGroup WrapCodexInstances(ICodexInstance[] instances) + public IStorageNodeGroup WrapLogosStorageInstances(ILogosStorageInstance[] instances) { - var codexNodeFactory = new CodexNodeFactory( + var storageNodeFactory = new StorageNodeFactory( log: pluginTools.GetLog(), fileManager: pluginTools.GetFileManager(), hooksFactory: hooksFactory, httpFactory: pluginTools, processControlFactory: processControlMap); - var group = CreateCodexGroup(instances, codexNodeFactory); + var group = CreateStorageNodeGroup(instances, storageNodeFactory); - pluginTools.GetLog().Log($"Codex version: {group.Version}"); + pluginTools.GetLog().Log($"Logos Storage version: {group.Version}"); versionResponse = group.Version; return group; } - private CodexNodeGroup CreateCodexGroup(ICodexInstance[] instances, CodexNodeFactory codexNodeFactory) + private StorageNodeGroup CreateStorageNodeGroup(ILogosStorageInstance[] instances, StorageNodeFactory storageNodeFactory) { - var nodes = instances.Select(codexNodeFactory.CreateCodexNode).ToArray(); - var group = new CodexNodeGroup(pluginTools, nodes); + var nodes = instances.Select(storageNodeFactory.CreateStorageNode).ToArray(); + var group = new StorageNodeGroup(pluginTools, nodes); try { @@ -59,16 +59,16 @@ namespace CodexPlugin } catch { - CodexNodesNotOnline(instances); + LogosStorageNodesNotOnline(instances); throw; } return group; } - private void CodexNodesNotOnline(ICodexInstance[] instances) + private void LogosStorageNodesNotOnline(ILogosStorageInstance[] instances) { - pluginTools.GetLog().Log("Codex nodes failed to start"); + pluginTools.GetLog().Log("Logos Storage nodes failed to start"); var log = pluginTools.GetLog(); foreach (var i in instances) { diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/IdentityMap.cs similarity index 73% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/IdentityMap.cs index a0843fd5..9457ccb2 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/IdentityMap.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/IdentityMap.cs @@ -1,16 +1,16 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { public class IdentityMap { - private readonly List nodes = new List(); + private readonly List nodes = new List(); private readonly Dictionary nameIndexMap = new Dictionary(); private readonly Dictionary shortToLong = new Dictionary(); public void Add(string name, string peerId, string nodeId) { - Add(new CodexNodeIdentity + Add(new StorageNodeIdentity { Name = name, PeerId = peerId, @@ -20,7 +20,7 @@ namespace CodexPlugin.OverwatchSupport nameIndexMap.Add(name, nameIndexMap.Count); } - public void Add(CodexNodeIdentity identity) + public void Add(StorageNodeIdentity identity) { if (string.IsNullOrWhiteSpace(identity.Name)) throw new Exception("Name required"); if (string.IsNullOrWhiteSpace(identity.PeerId) || identity.PeerId.Length < 11) throw new Exception("PeerId invalid"); @@ -28,11 +28,11 @@ namespace CodexPlugin.OverwatchSupport nodes.Add(identity); - shortToLong.Add(CodexUtils.ToShortId(identity.PeerId), identity.PeerId); - shortToLong.Add(CodexUtils.ToNodeIdShortId(identity.NodeId), identity.NodeId); + shortToLong.Add(LogosStorageUtils.ToShortId(identity.PeerId), identity.PeerId); + shortToLong.Add(LogosStorageUtils.ToNodeIdShortId(identity.NodeId), identity.NodeId); } - public CodexNodeIdentity[] Get() + public StorageNodeIdentity[] Get() { return nodes.ToArray(); } @@ -42,7 +42,7 @@ namespace CodexPlugin.OverwatchSupport return nameIndexMap[name]; } - public CodexNodeIdentity GetId(string name) + public StorageNodeIdentity GetId(string name) { return nodes.Single(n => n.Name == name); } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/KademliaPositionFinder.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/KademliaPositionFinder.cs similarity index 79% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/KademliaPositionFinder.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/KademliaPositionFinder.cs index 7e803415..3185ecbd 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/KademliaPositionFinder.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/KademliaPositionFinder.cs @@ -2,11 +2,11 @@ using Utils; using YamlDotNet.Core.Tokens; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { public class KademliaPositionFinder { - public CodexNodeIdentity[] DeterminePositions(CodexNodeIdentity[] identities) + public StorageNodeIdentity[] DeterminePositions(StorageNodeIdentity[] identities) { var zero = identities.First(); var distances = CalculateDistances(zero, identities); @@ -17,9 +17,9 @@ namespace CodexPlugin.OverwatchSupport return identities; } - private Dictionary CalculateDistances(CodexNodeIdentity zero, CodexNodeIdentity[] identities) + private Dictionary CalculateDistances(StorageNodeIdentity zero, StorageNodeIdentity[] identities) { - var result = new Dictionary(); + var result = new Dictionary(); foreach (var id in identities.Skip(1)) { result.Add(id, GetDistance(zero.NodeId, id.NodeId)); @@ -51,7 +51,7 @@ namespace CodexPlugin.OverwatchSupport return result; } - private void CalculateNormalizedPositions(Dictionary distances, BigInteger maxDistance) + private void CalculateNormalizedPositions(Dictionary distances, BigInteger maxDistance) { foreach (var pair in distances) { diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs similarity index 86% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs index 70be0bdd..e9515825 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BlockReceivedLineConverter.cs @@ -1,12 +1,12 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin.OverwatchSupport.LineConverters +namespace StoragePlugin.OverwatchSupport.LineConverters { public class BlockReceivedLineConverter : ILineConverter { public string Interest => "Received blocks from peer"; - public void Process(CodexLogLine line, Action> addEvent) + public void Process(LogosStorageLogLine line, Action> addEvent) { var peer = line.Attributes["peer"]; var blockAddresses = line.Attributes["blocks"]; diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs similarity index 94% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs index 9cec14f0..c95d190b 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/BootstrapLineConverter.cs @@ -1,6 +1,6 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin.OverwatchSupport.LineConverters +namespace StoragePlugin.OverwatchSupport.LineConverters { public class BootstrapLineConverter : ILineConverter { @@ -8,7 +8,7 @@ namespace CodexPlugin.OverwatchSupport.LineConverters public string Interest => "Starting codex node"; - public void Process(CodexLogLine line, Action> addEvent) + public void Process(LogosStorageLogLine line, Action> addEvent) { // "( // configFile: none(InputFile), diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs similarity index 67% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs index 510b8dde..edaf2511 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/DialSuccessfulLineConverter.cs @@ -1,12 +1,12 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin.OverwatchSupport.LineConverters +namespace StoragePlugin.OverwatchSupport.LineConverters { public class DialSuccessfulLineConverter : ILineConverter { public string Interest => "Dial successful"; - public void Process(CodexLogLine line, Action> addEvent) + public void Process(LogosStorageLogLine line, Action> addEvent) { var peerId = line.Attributes["peerId"]; diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs similarity index 66% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs index 858c0f8f..eedb8df8 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LineConverters/PeerDroppedLineConverter.cs @@ -1,12 +1,12 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin.OverwatchSupport.LineConverters +namespace StoragePlugin.OverwatchSupport.LineConverters { public class PeerDroppedLineConverter : ILineConverter { public string Interest => "Dropping peer"; - public void Process(CodexLogLine line, Action> addEvent) + public void Process(LogosStorageLogLine line, Action> addEvent) { var peerId = line.Attributes["peer"]; diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageLogConverter.cs similarity index 67% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageLogConverter.cs index 71be7873..73854665 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexLogConverter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageLogConverter.cs @@ -1,18 +1,18 @@ -using CodexClient; -using CodexPlugin.OverwatchSupport.LineConverters; +using LogosStorageClient; +using StoragePlugin.OverwatchSupport.LineConverters; using Logging; using OverwatchTranscript; using Utils; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { - public class CodexLogConverter + public class LogosStorageLogConverter { private readonly ITranscriptWriter writer; - private readonly CodexTranscriptWriterConfig config; + private readonly LogosStorageTranscriptWriterConfig config; private readonly IdentityMap identityMap; - public CodexLogConverter(ITranscriptWriter writer, CodexTranscriptWriterConfig config, IdentityMap identityMap) + public LogosStorageLogConverter(ITranscriptWriter writer, LogosStorageTranscriptWriterConfig config, IdentityMap identityMap) { this.writer = writer; this.config = config; @@ -43,7 +43,7 @@ namespace CodexPlugin.OverwatchSupport private readonly int nodeIdentityIndex; private readonly ILineConverter[] converters; - public ConversionRunner(ITranscriptWriter writer, CodexTranscriptWriterConfig config, IdentityMap nameIdMap, int nodeIdentityIndex) + public ConversionRunner(ITranscriptWriter writer, LogosStorageTranscriptWriterConfig config, IdentityMap nameIdMap, int nodeIdentityIndex) { this.nodeIdentityIndex = nodeIdentityIndex; this.writer = writer; @@ -52,7 +52,7 @@ namespace CodexPlugin.OverwatchSupport converters = CreateConverters(config).ToArray(); } - private IEnumerable CreateConverters(CodexTranscriptWriterConfig config) + private IEnumerable CreateConverters(LogosStorageTranscriptWriterConfig config) { if (config.IncludeBlockReceivedEvents) { @@ -74,9 +74,9 @@ namespace CodexPlugin.OverwatchSupport }); } - private void AddEvent(DateTime utc, Action action) + private void AddEvent(DateTime utc, Action action) { - var e = new OverwatchCodexEvent + var e = new OverwatchLogosStorageEvent { NodeIdentity = nodeIdentityIndex, }; @@ -89,18 +89,18 @@ namespace CodexPlugin.OverwatchSupport { if (!line.Contains(converter.Interest)) return; - var codexLine = CodexLogLine.Parse(line); + var logosStorageLogLine = LogosStorageLogLine.Parse(line); - if (codexLine == null) throw new Exception("Unable to parse required line"); - EnsureFullIds(codexLine); + if (logosStorageLogLine == null) throw new Exception("Unable to parse required line"); + EnsureFullIds(logosStorageLogLine); - converter.Process(codexLine, (action) => + converter.Process(logosStorageLogLine, (action) => { - AddEvent(codexLine.TimestampUtc, action); + AddEvent(logosStorageLogLine.TimestampUtc, action); }); } - private void EnsureFullIds(CodexLogLine codexLine) + private void EnsureFullIds(LogosStorageLogLine logosStorageLogLine) { // The issue is: node IDs occure both in full and short version. // Downstream tools will assume that a node ID string-equals its own ID. @@ -111,11 +111,11 @@ namespace CodexPlugin.OverwatchSupport // But sometimes, it is part of a larger string: // "thing=abc:123*567890,def" - foreach (var pair in codexLine.Attributes) + foreach (var pair in logosStorageLogLine.Attributes) { if (pair.Value.Contains("*")) { - codexLine.Attributes[pair.Key] = nameIdMap.ReplaceShortIds(pair.Value); + logosStorageLogLine.Attributes[pair.Key] = nameIdMap.ReplaceShortIds(pair.Value); } } } @@ -124,6 +124,6 @@ namespace CodexPlugin.OverwatchSupport public interface ILineConverter { string Interest { get; } - void Process(CodexLogLine line, Action> addEvent); + void Process(LogosStorageLogLine line, Action> addEvent); } } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriter.cs similarity index 64% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriter.cs index 95b8dde6..4ae4c819 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriter.cs @@ -1,33 +1,33 @@ -using CodexClient.Hooks; +using LogosStorageClient.Hooks; using Logging; using OverwatchTranscript; using Utils; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { - public class CodexTranscriptWriter : ICodexHooksProvider + public class LogosStorageTranscriptWriter : ILogosStorageHooksProvider { - private const string CodexHeaderKey = "cdx_h"; + private const string LogosStorageHeaderKey = "cdx_h"; private readonly ILog log; - private readonly CodexTranscriptWriterConfig config; + private readonly LogosStorageTranscriptWriterConfig config; private readonly ITranscriptWriter writer; - private readonly CodexLogConverter converter; + private readonly LogosStorageLogConverter converter; private readonly IdentityMap identityMap = new IdentityMap(); private readonly KademliaPositionFinder positionFinder = new KademliaPositionFinder(); - public CodexTranscriptWriter(ILog log, CodexTranscriptWriterConfig config, ITranscriptWriter transcriptWriter) + public LogosStorageTranscriptWriter(ILog log, LogosStorageTranscriptWriterConfig config, ITranscriptWriter transcriptWriter) { this.log = log; this.config = config; writer = transcriptWriter; - converter = new CodexLogConverter(writer, config, identityMap); + converter = new LogosStorageLogConverter(writer, config, identityMap); } public void FinalizeWriter() { - log.Log("Finalizing Codex transcript..."); + log.Log("Finalizing Logos Storage transcript..."); - writer.AddHeader(CodexHeaderKey, CreateCodexHeader()); + writer.AddHeader(LogosStorageHeaderKey, CreateLogosStorageHeader()); writer.Write(GetOutputFullPath()); log.Log("Done"); @@ -44,10 +44,10 @@ namespace CodexPlugin.OverwatchSupport return outputFile; } - public ICodexNodeHooks CreateHooks(string nodeName) + public IStorageNodeHooks CreateHooks(string nodeName) { nodeName = Str.Between(nodeName, "'", "'"); - return new CodexNodeTranscriptWriter(writer, identityMap, nodeName); + return new StorageNodeTranscriptWriter(writer, identityMap, nodeName); } public void IncludeFile(string filepath) @@ -64,9 +64,9 @@ namespace CodexPlugin.OverwatchSupport // Not all of these logs are necessarily Codex logs. // Check, and process only the Codex ones. - if (IsCodexLog(l)) + if (IsLogosStorageLog(l)) { - log.Log("Processing Codex log: " + l.GetFilepath()); + log.Log("Processing Logos Storage log: " + l.GetFilepath()); converter.ProcessLog(l); } } @@ -74,7 +74,7 @@ namespace CodexPlugin.OverwatchSupport public void AddResult(bool success, string result) { - writer.Add(DateTime.UtcNow, new OverwatchCodexEvent + writer.Add(DateTime.UtcNow, new OverwatchLogosStorageEvent { NodeIdentity = -1, ScenarioFinished = new ScenarioFinishedEvent @@ -85,17 +85,17 @@ namespace CodexPlugin.OverwatchSupport }); } - private OverwatchCodexHeader CreateCodexHeader() + private OverwatchLogosStorageHeader CreateLogosStorageHeader() { - return new OverwatchCodexHeader + return new OverwatchLogosStorageHeader { Nodes = positionFinder.DeterminePositions(identityMap.Get()) }; } - private bool IsCodexLog(IDownloadedLog log) + private bool IsLogosStorageLog(IDownloadedLog log) { - return log.GetLinesContaining("Run Codex node").Any(); + return log.GetLinesContaining("Run Logos Storage node").Any(); } } } diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriterConfig.cs similarity index 54% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriterConfig.cs index 112f5b16..fc9691e5 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexTranscriptWriterConfig.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/LogosStorageTranscriptWriterConfig.cs @@ -1,8 +1,8 @@ -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { - public class CodexTranscriptWriterConfig + public class LogosStorageTranscriptWriterConfig { - public CodexTranscriptWriterConfig(string outputPath, bool includeBlockReceivedEvents) + public LogosStorageTranscriptWriterConfig(string outputPath, bool includeBlockReceivedEvents) { OutputPath = outputPath; IncludeBlockReceivedEvents = includeBlockReceivedEvents; diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/ModelExtensions.cs similarity index 91% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/ModelExtensions.cs index ba9345a1..8cd83ddb 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/ModelExtensions.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/ModelExtensions.cs @@ -1,16 +1,16 @@ -using CodexClient; +using LogosStorageClient; using OverwatchTranscript; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { [Serializable] - public class OverwatchCodexHeader + public class OverwatchLogosStorageHeader { - public CodexNodeIdentity[] Nodes { get; set; } = Array.Empty(); + public StorageNodeIdentity[] Nodes { get; set; } = Array.Empty(); } [Serializable] - public class OverwatchCodexEvent + public class OverwatchLogosStorageEvent { public int NodeIdentity { get; set; } = -1; public ScenarioFinishedEvent? ScenarioFinished { get; set; } @@ -48,7 +48,7 @@ namespace CodexPlugin.OverwatchSupport } [Serializable] - public class CodexNodeIdentity + public class StorageNodeIdentity { public string Name { get; set; } = string.Empty; public string PeerId { get; set; } = string.Empty; @@ -117,7 +117,7 @@ namespace CodexPlugin.OverwatchSupport #endregion - #region Codex Generated Events + #region Logos Storage Generated Events [Serializable] public class BlockReceivedEvent diff --git a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs b/ProjectPlugins/StoragePlugin/OverwatchSupport/StorageNodeTranscriptWriter.cs similarity index 74% rename from ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs rename to ProjectPlugins/StoragePlugin/OverwatchSupport/StorageNodeTranscriptWriter.cs index 3d1b9fb5..ff17ad90 100644 --- a/ProjectPlugins/CodexPlugin/OverwatchSupport/CodexNodeTranscriptWriter.cs +++ b/ProjectPlugins/StoragePlugin/OverwatchSupport/StorageNodeTranscriptWriter.cs @@ -1,19 +1,19 @@ -using CodexClient; -using CodexClient.Hooks; +using LogosStorageClient; +using LogosStorageClient.Hooks; using OverwatchTranscript; using Utils; -namespace CodexPlugin.OverwatchSupport +namespace StoragePlugin.OverwatchSupport { - public class CodexNodeTranscriptWriter : ICodexNodeHooks + public class StorageNodeTranscriptWriter : IStorageNodeHooks { private readonly ITranscriptWriter writer; private readonly IdentityMap identityMap; private readonly string name; private int identityIndex = -1; - private readonly List<(DateTime, OverwatchCodexEvent)> pendingEvents = new List<(DateTime, OverwatchCodexEvent)>(); + private readonly List<(DateTime, OverwatchLogosStorageEvent)> pendingEvents = new List<(DateTime, OverwatchLogosStorageEvent)>(); - public CodexNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name) + public StorageNodeTranscriptWriter(ITranscriptWriter writer, IdentityMap identityMap, string name) { this.writer = writer; this.identityMap = identityMap; @@ -22,7 +22,7 @@ namespace CodexPlugin.OverwatchSupport public void OnNodeStarting(DateTime startUtc, string image) { - WriteCodexEvent(startUtc, e => + WriteLogosStorageEvent(startUtc, e => { e.NodeStarting = new NodeStartingEvent { @@ -31,7 +31,7 @@ namespace CodexPlugin.OverwatchSupport }); } - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + public void OnNodeStarted(IStorageNode node, string peerId, string nodeId) { if (string.IsNullOrEmpty(peerId) || string.IsNullOrEmpty(nodeId)) { @@ -41,7 +41,7 @@ namespace CodexPlugin.OverwatchSupport identityMap.Add(name, peerId, nodeId); identityIndex = identityMap.GetIndex(name); - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.NodeStarted = new NodeStartedEvent { @@ -51,7 +51,7 @@ namespace CodexPlugin.OverwatchSupport public void OnNodeStopping() { - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.NodeStopping = new NodeStoppingEvent { @@ -61,7 +61,7 @@ namespace CodexPlugin.OverwatchSupport public void OnFileDownloading(ContentId cid) { - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.FileDownloading = new FileDownloadingEvent { @@ -72,7 +72,7 @@ namespace CodexPlugin.OverwatchSupport public void OnFileDownloaded(ByteSize size, ContentId cid) { - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.FileDownloaded = new FileDownloadedEvent { @@ -84,7 +84,7 @@ namespace CodexPlugin.OverwatchSupport public void OnFileUploading(string uid, ByteSize size) { - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.FileUploading = new FileUploadingEvent { @@ -96,7 +96,7 @@ namespace CodexPlugin.OverwatchSupport public void OnFileUploaded(string uid, ByteSize size, ContentId cid) { - WriteCodexEvent(e => + WriteLogosStorageEvent(e => { e.FileUploaded = new FileUploadedEvent { @@ -107,14 +107,14 @@ namespace CodexPlugin.OverwatchSupport }); } - private void WriteCodexEvent(Action action) + private void WriteLogosStorageEvent(Action action) { - WriteCodexEvent(DateTime.UtcNow, action); + WriteLogosStorageEvent(DateTime.UtcNow, action); } - private void WriteCodexEvent(DateTime utc, Action action) + private void WriteLogosStorageEvent(DateTime utc, Action action) { - var e = new OverwatchCodexEvent + var e = new OverwatchLogosStorageEvent { NodeIdentity = identityIndex }; @@ -135,7 +135,7 @@ namespace CodexPlugin.OverwatchSupport } } - private void AddToCache(DateTime utc, OverwatchCodexEvent e) + private void AddToCache(DateTime utc, OverwatchLogosStorageEvent e) { pendingEvents.Add((utc, e)); } diff --git a/ProjectPlugins/CodexPlugin/ProcessControlMap.cs b/ProjectPlugins/StoragePlugin/ProcessControlMap.cs similarity index 67% rename from ProjectPlugins/CodexPlugin/ProcessControlMap.cs rename to ProjectPlugins/StoragePlugin/ProcessControlMap.cs index 99498f57..a75b13be 100644 --- a/ProjectPlugins/CodexPlugin/ProcessControlMap.cs +++ b/ProjectPlugins/StoragePlugin/ProcessControlMap.cs @@ -1,27 +1,27 @@ -using CodexClient; +using LogosStorageClient; -namespace CodexPlugin +namespace StoragePlugin { public class ProcessControlMap : IProcessControlFactory { private readonly Dictionary processControlMap = new Dictionary(); - public void Add(ICodexInstance instance, IProcessControl control) + public void Add(ILogosStorageInstance instance, IProcessControl control) { processControlMap.Add(instance.Name, control); } - public void Remove(ICodexInstance instance) + public void Remove(ILogosStorageInstance instance) { processControlMap.Remove(instance.Name); } - public IProcessControl CreateProcessControl(ICodexInstance instance) + public IProcessControl CreateProcessControl(ILogosStorageInstance instance) { return Get(instance); } - public IProcessControl Get(ICodexInstance instance) + public IProcessControl Get(ILogosStorageInstance instance) { return processControlMap[instance.Name]; } diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/StoragePlugin/StorageNodeGroup.cs similarity index 66% rename from ProjectPlugins/CodexPlugin/CodexNodeGroup.cs rename to ProjectPlugins/StoragePlugin/StorageNodeGroup.cs index 70a5316d..783d6fcf 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/StoragePlugin/StorageNodeGroup.cs @@ -1,27 +1,27 @@ -using CodexClient; +using LogosStorageClient; using Core; using System.Collections; using Utils; -namespace CodexPlugin +namespace StoragePlugin { - public interface ICodexNodeGroup : IEnumerable, IHasManyMetricScrapeTargets + public interface IStorageNodeGroup : IEnumerable, IHasManyMetricScrapeTargets { void Stop(bool waitTillStopped); - ICodexNode this[int index] { get; } + IStorageNode this[int index] { get; } } - public class CodexNodeGroup : ICodexNodeGroup + public class StorageNodeGroup : IStorageNodeGroup { - private readonly ICodexNode[] nodes; + private readonly IStorageNode[] nodes; - public CodexNodeGroup(IPluginTools tools, ICodexNode[] nodes) + public StorageNodeGroup(IPluginTools tools, IStorageNode[] nodes) { this.nodes = nodes; Version = new DebugInfoVersion(); } - public ICodexNode this[int index] + public IStorageNode this[int index] { get { @@ -34,12 +34,12 @@ namespace CodexPlugin foreach (var node in Nodes) node.Stop(waitTillStopped); } - public void Stop(CodexNode node, bool waitTillStopped) + public void Stop(StorageNode node, bool waitTillStopped) { node.Stop(waitTillStopped); } - public ICodexNode[] Nodes => nodes; + public IStorageNode[] Nodes => nodes; public DebugInfoVersion Version { get; private set; } public Address[] GetMetricsScrapeTargets() @@ -47,9 +47,9 @@ namespace CodexPlugin return Nodes.Select(n => n.GetMetricsScrapeTarget()).ToArray(); } - public IEnumerator GetEnumerator() + public IEnumerator GetEnumerator() { - return Nodes.Cast().GetEnumerator(); + return Nodes.Cast().GetEnumerator(); } IEnumerator IEnumerable.GetEnumerator() @@ -74,7 +74,7 @@ namespace CodexPlugin var first = versionResponses.First(); if (!versionResponses.All(v => v.Version == first.Version && v.Revision == first.Revision)) { - throw new Exception("Inconsistent version information received from one or more Codex nodes: " + + throw new Exception("Inconsistent version information received from one or more Logos Storage nodes: " + string.Join(",", versionResponses.Select(v => v.ToString()))); } @@ -82,14 +82,14 @@ namespace CodexPlugin } } - public static class CodexNodeGroupExtensions + public static class StorageNodeGroupExtensions { - public static string Names(this ICodexNode[] nodes) + public static string Names(this IStorageNode[] nodes) { return $"[{string.Join(",", nodes.Select(n => n.GetName()))}]"; } - public static string Names(this List nodes) + public static string Names(this List nodes) { return $"[{string.Join(",", nodes.Select(n => n.GetName()))}]"; } diff --git a/ProjectPlugins/StoragePlugin/StoragePlugin.cs b/ProjectPlugins/StoragePlugin/StoragePlugin.cs new file mode 100644 index 00000000..0cbf5a1f --- /dev/null +++ b/ProjectPlugins/StoragePlugin/StoragePlugin.cs @@ -0,0 +1,96 @@ +using LogosStorageClient; +using LogosStorageClient.Hooks; +using Core; + +namespace StoragePlugin +{ + public class StoragePlugin : IProjectPlugin, IHasLogPrefix, IHasMetadata + { + private const bool UseContainers = true; + + private readonly ILogosStorageStarter logosStorageStarter; + private readonly IPluginTools tools; + private readonly LogosStorageLogLevel defaultLogLevel = LogosStorageLogLevel.Trace; + private readonly LogosStorageHooksFactory hooksFactory = new LogosStorageHooksFactory(); + private readonly ProcessControlMap processControlMap = new ProcessControlMap(); + private readonly LogosStorageDockerImage logosStorageDockerImage = new LogosStorageDockerImage(); + private readonly LogosStorageContainerRecipe recipe; + private readonly LogosStorageWrapper logosStorageWrapper; + + public StoragePlugin(IPluginTools tools) + { + this.tools = tools; + + recipe = new LogosStorageContainerRecipe(logosStorageDockerImage); + logosStorageStarter = CreateLogosStorageStarter(); + logosStorageWrapper = new LogosStorageWrapper(tools, processControlMap, hooksFactory); + } + + private ILogosStorageStarter CreateLogosStorageStarter() + { + if (UseContainers) + { + Log("Using Containerized Logos Storage instances"); + return new ContainerLogosStorageStarter(tools, recipe, processControlMap); + } + + Log("Using Binary Logos Storage instances"); + return new BinaryLogosStorageStarter(tools, processControlMap); + } + + public string LogPrefix => "(LogosStorage) "; + + public void Awake(IPluginAccess access) + { + } + + public void Announce() + { + // give codex docker image to contracts plugin. + + Log($"Loaded with Logos Storage ID: '{logosStorageWrapper.GetLogosStorageId()}' - Revision: {logosStorageWrapper.GetLogosStorageRevision()}"); + } + + public void AddMetadata(IAddMetadata metadata) + { + metadata.Add("storageid", logosStorageWrapper.GetLogosStorageId()); + metadata.Add("storagerevision", logosStorageWrapper.GetLogosStorageRevision()); + } + + public void Decommission() + { + logosStorageStarter.Decommission(); + } + + public ILogosStorageInstance[] DeployLogosStorageNodes(int numberOfNodes, Action setup) + { + var logosStorageSetup = GetSetup(numberOfNodes, setup); + return logosStorageStarter.BringOnline(logosStorageSetup); + } + + public IStorageNodeGroup WrapLogosStorageContainers(ILogosStorageInstance[] instances) + { + instances = instances.Select(c => SerializeGate.Gate(c as LogosStorageInstance)).ToArray(); + return logosStorageWrapper.WrapLogosStorageInstances(instances); + } + + public void AddLogosStorageHooksProvider(ILogosStorageHooksProvider hooksProvider) + { + if (hooksFactory.Providers.Contains(hooksProvider)) return; + hooksFactory.Providers.Add(hooksProvider); + } + + private LogosStorageSetup GetSetup(int numberOfNodes, Action setup) + { + var logosStorageSetup = new LogosStorageSetup(numberOfNodes); + logosStorageSetup.LogLevel = defaultLogLevel; + setup(logosStorageSetup); + return logosStorageSetup; + } + + private void Log(string msg) + { + tools.GetLog().Log(msg); + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj b/ProjectPlugins/StoragePlugin/StoragePlugin.csproj similarity index 87% rename from ProjectPlugins/CodexPlugin/CodexPlugin.csproj rename to ProjectPlugins/StoragePlugin/StoragePlugin.csproj index b40af376..d694d9a1 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj +++ b/ProjectPlugins/StoragePlugin/StoragePlugin.csproj @@ -22,12 +22,12 @@ - + - + diff --git a/README.md b/README.md index b3d0c80b..5b05d2a2 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Kubernetes: v1.34.1 Dotnet-kubernetes SDK: v18.0.3 https://github.com/kubernetes-client/csharp Nethereum: v4.14.0 -Currently, this project is mainly used for distributed testing of [Nim-Codex](https://github.com/codex-storage/nim-codex). However, its plugin-structure allows for other projects to be on-boarded (relatively) easily. (See 'contribute a plugin`.) +Currently, this project is mainly used for distributed testing of [Logos Storage](https://github.com/logos-storage/logos-storage-nim). However, its plugin-structure allows for other projects to be on-boarded (relatively) easily. (See 'contribute a plugin`.) ## Tests/DistTestCore Library with generic distributed-testing functionality. Uses NUnit3. Reference this project to build unit-test style scenarios: setup, run test, teardown. The DistTestCore responds to the following env-vars: @@ -15,13 +15,13 @@ Library with generic distributed-testing functionality. Uses NUnit3. Reference t - `DATAFILEPATH` = Path where (temporary) data files will be stored. - `ALWAYS_LOGS` = When set, DistTestCore will always download all container logs at the end of a test run. By default, logs are only downloaded on test failure. -## Tests/CodexTests and Tests/CodexLongTests -These are test assemblies that use DistTestCore to perform tests against transient Codex nodes. -Read more [HERE](/Tests/CodexTests/README.md) +## Tests/LogosStorageTests and Tests/LogosStorageLongTests +These are test assemblies that use DistTestCore to perform tests against transient Logos Storage nodes. +Read more [HERE](/Tests/LogosStorageTests/README.md) ## Tests/ContinuousTests -A console application that runs tests in an endless loop against a persistent deployment of Codex nodes. -Read more [HERE](/Tests/CodexContinuousTests/README.md) +A console application that runs tests in an endless loop against a persistent deployment of Logos Storage nodes. +Read more [HERE](/Tests/LogosStorageContinuousTests/README.md) ## Framework architecture The framework is designed to be extended by project-specific plugins. These plugins contribute functionality and abstractions to the framework. Users of the framework use these to perform tasks such as testing and deploying. diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index c092a27e..e81e3a05 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -12,7 +12,7 @@ namespace DistTestCore public Configuration() { kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); - logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); + logPath = GetEnvVarOrDefault("LOGPATH", "LogosStorageTestLogs"); dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", "")); } diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index 9eeefec1..8cba6af9 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -181,7 +181,7 @@ namespace DistTestCore { if (IsRunningInCluster()) { - Log(" > Detected we're running in the cluster. Using long webCall timeset."); + fixtureLog.Log(" > Detected we're running in the cluster. Using long webCall timeset."); return new LongWebCallTimeSet(); } @@ -193,7 +193,7 @@ namespace DistTestCore { if (IsRunningInCluster()) { - Log(" > Detected we're running in the cluster. Using long kubernetes timeset."); + fixtureLog.Log(" > Detected we're running in the cluster. Using long kubernetes timeset."); return new LongK8sTimeSet(); } diff --git a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs index 93fcf6f7..4d0a7935 100644 --- a/Tests/ExperimentalTests/AutoBootstrapDistTest.cs +++ b/Tests/ExperimentalTests/AutoBootstrapDistTest.cs @@ -1,20 +1,20 @@ -using CodexClient; -using CodexPlugin; +using LogosStorageClient; +using StoragePlugin; using NUnit.Framework; -namespace CodexTests +namespace LogosStorageTests { - public class AutoBootstrapDistTest : CodexDistTest + public class AutoBootstrapDistTest : LogosStorageDistTest { private bool isBooting = false; - public ICodexNode BootstrapNode { get; private set; } = null!; + public IStorageNode BootstrapNode { get; private set; } = null!; [SetUp] public void SetupBootstrapNode() { isBooting = true; - BootstrapNode = StartCodex(s => s.WithName("BOOTSTRAP_" + GetTestNamespace())); + BootstrapNode = StartLogosStorage(s => s.WithName("BOOTSTRAP_" + GetTestNamespace())); isBooting = false; } @@ -24,7 +24,7 @@ namespace CodexTests BootstrapNode.Stop(waitTillStopped: false); } - protected override void OnCodexSetup(ICodexSetup setup) + protected override void OnLogosStorageSetup(ILogosStorageSetup setup) { if (isBooting) return; diff --git a/Tests/ExperimentalTests/BasicTests/AsyncProfiling.cs b/Tests/ExperimentalTests/BasicTests/AsyncProfiling.cs index 2c630975..899297d5 100644 --- a/Tests/ExperimentalTests/BasicTests/AsyncProfiling.cs +++ b/Tests/ExperimentalTests/BasicTests/AsyncProfiling.cs @@ -1,17 +1,17 @@ using NUnit.Framework; using MetricsPlugin; using Utils; -using CodexTests; +using LogosStorageTests; namespace ExperimentalTests.BasicTests { [TestFixture] - public class AsyncProfiling : CodexDistTest + public class AsyncProfiling : LogosStorageDistTest { [Test] public void AsyncProfileMetricsPlz() { - var node = StartCodex(s => s.EnableMetrics()); + var node = StartLogosStorage(s => s.EnableMetrics()); var metrics = Ci.GetMetricsFor(scrapeInterval: TimeSpan.FromSeconds(3.0), node).Single(); var file = GenerateTestFile(100.MB()); diff --git a/Tests/ExperimentalTests/BasicTests/ExampleTests.cs b/Tests/ExperimentalTests/BasicTests/ExampleTests.cs index 724bbd44..02838f67 100644 --- a/Tests/ExperimentalTests/BasicTests/ExampleTests.cs +++ b/Tests/ExperimentalTests/BasicTests/ExampleTests.cs @@ -1,6 +1,6 @@ -using CodexClient; -using CodexPlugin; -using CodexTests; +using LogosStorageClient; +using StoragePlugin; +using LogosStorageTests; using DistTestCore; using MetricsPlugin; using NUnit.Framework; @@ -9,12 +9,12 @@ using Utils; namespace ExperimentalTests.BasicTests { [TestFixture] - public class ExampleTests : CodexDistTest + public class ExampleTests : LogosStorageDistTest { [Test] - public void CodexLogExample() + public void LogosStorageLogExample() { - var primary = StartCodex(s => s.WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Warn, CodexLogLevel.Warn))); + var primary = StartLogosStorage(s => s.WithLogLevel(LogosStorageLogLevel.Trace, new LogosStorageLogCustomTopics(LogosStorageLogLevel.Warn, LogosStorageLogLevel.Warn))); var cid = primary.UploadFile(GenerateTestFile(5.MB())); @@ -29,8 +29,8 @@ namespace ExperimentalTests.BasicTests [Test] public void TwoMetricsExample() { - var group = StartCodex(2, s => s.EnableMetrics()); - var group2 = StartCodex(2, s => s.EnableMetrics()); + var group = StartLogosStorage(2, s => s.EnableMetrics()); + var group2 = StartLogosStorage(2, s => s.EnableMetrics()); var primary = group[0]; var secondary = group[1]; diff --git a/Tests/ExperimentalTests/BasicTests/PyramidTests.cs b/Tests/ExperimentalTests/BasicTests/PyramidTests.cs index 95d43e83..b8b146f5 100644 --- a/Tests/ExperimentalTests/BasicTests/PyramidTests.cs +++ b/Tests/ExperimentalTests/BasicTests/PyramidTests.cs @@ -1,12 +1,12 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; namespace ExperimentalTests.BasicTests { [TestFixture] - public class PyramidTests : CodexDistTest + public class PyramidTests : LogosStorageDistTest { [Test] [CreateTranscript(nameof(PyramidTest))] @@ -22,17 +22,17 @@ namespace ExperimentalTests.BasicTests DownloadAllFilesFromEachNodeInLayer(bottomLayer, cids); } - private List StartLayers(int numberOfLayers) + private List StartLayers(int numberOfLayers) { - var layer = new List(); - layer.Add(StartCodex(s => s.WithName("Top"))); + var layer = new List(); + layer.Add(StartLogosStorage(s => s.WithName("Top"))); for (var i = 0; i < numberOfLayers; i++) { - var newLayer = new List(); + var newLayer = new List(); foreach (var node in layer) { - newLayer.AddRange(StartCodex(2, s => s.WithBootstrapNode(node).WithName("Layer[" + i + "]"))); + newLayer.AddRange(StartLogosStorage(2, s => s.WithBootstrapNode(node).WithName("Layer[" + i + "]"))); } layer.Clear(); @@ -42,7 +42,7 @@ namespace ExperimentalTests.BasicTests return layer; } - private ContentId[] UploadFiles(List layer, ByteSize size) + private ContentId[] UploadFiles(List layer, ByteSize size) { var uploadTasks = new List>(); foreach (var node in layer) @@ -63,7 +63,7 @@ namespace ExperimentalTests.BasicTests return cids; } - private void DownloadAllFilesFromEachNodeInLayer(List layer, ContentId[] cids) + private void DownloadAllFilesFromEachNodeInLayer(List layer, ContentId[] cids) { var downloadTasks = new List(); foreach (var node in layer) diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs index d218e6ec..d753b4fc 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/DatalayerReliabilityTests.cs @@ -1,5 +1,5 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -19,8 +19,8 @@ namespace ExperimentalTests.DownloadConnectivityTests ) { var file = GenerateTestFile(fileSizeMb.MB()); - var uploader = StartCodex(n => n.WithName("uploader")); - var downloaders = StartCodex(numDownloaders, n => n.WithName("downloader")); + var uploader = StartLogosStorage(n => n.WithName("uploader")); + var downloaders = StartLogosStorage(numDownloaders, n => n.WithName("downloader")); var cid = uploader.UploadFile(file); @@ -40,14 +40,14 @@ namespace ExperimentalTests.DownloadConnectivityTests public class TransferPlan { - public ICodexNode Uploader { get; set; } = null!; + public IStorageNode Uploader { get; set; } = null!; public ContentId Cid { get; set; } = null!; public List Downloaders { get; } = new List(); } public class DownloaderPlan { - public ICodexNode Node { get; set; } = null!; + public IStorageNode Node { get; set; } = null!; public List TransferPlans { get; } = new List(); } @@ -102,7 +102,7 @@ namespace ExperimentalTests.DownloadConnectivityTests ) { var plans = new List(); - var uploaders = StartCodex(numDatasets, n => n.WithName("uploader")); + var uploaders = StartLogosStorage(numDatasets, n => n.WithName("uploader")); foreach (var n in uploaders) { plans.Add(new TransferPlan @@ -122,7 +122,7 @@ namespace ExperimentalTests.DownloadConnectivityTests var allDownloaderPlans = available.GetAll(); Assert.That(allDownloaderPlans.Length, Is.LessThan(100)); Log($"Using {allDownloaderPlans.Length} downloaders..."); - var nodes = StartCodex(allDownloaderPlans.Length, n => n.WithName("downloader")); + var nodes = StartLogosStorage(allDownloaderPlans.Length, n => n.WithName("downloader")); for (var i = 0; i < allDownloaderPlans.Length; i++) { allDownloaderPlans[i].Node = nodes[i]; diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs index 601d2cfc..e456ecc5 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/DetectBlockRetransmitTest.cs @@ -1,4 +1,4 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -15,7 +15,7 @@ namespace ExperimentalTests.DownloadConnectivityTests [Values(3, 5, 10, 20)] int numNodes ) { - var nodes = StartCodex(numNodes); + var nodes = StartLogosStorage(numNodes); var file = GenerateTestFile(fileSize.MB()); var cid = nodes[0].UploadFile(file); diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index 1464149c..1e56cff2 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -1,5 +1,5 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -11,7 +11,7 @@ namespace ExperimentalTests.DownloadConnectivityTests [Test] public void MetricsDoesNotInterfereWithPeerDownload() { - var nodes = StartCodex(2, s => s.EnableMetrics()); + var nodes = StartLogosStorage(2, s => s.EnableMetrics()); AssertAllNodesConnected(nodes); } @@ -22,12 +22,12 @@ namespace ExperimentalTests.DownloadConnectivityTests [Values(2, 5)] int numberOfNodes, [Values(1, 10)] int sizeMBs) { - var nodes = StartCodex(numberOfNodes); + var nodes = StartLogosStorage(numberOfNodes); AssertAllNodesConnected(nodes, sizeMBs); } - private void AssertAllNodesConnected(IEnumerable nodes, int sizeMBs = 10) + private void AssertAllNodesConnected(IEnumerable nodes, int sizeMBs = 10) { CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB()); } diff --git a/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs b/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs index a206d475..69c9b24e 100644 --- a/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs +++ b/Tests/ExperimentalTests/DownloadConnectivityTests/MultiswarmTests.cs @@ -1,5 +1,5 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using FileUtils; using Logging; using NUnit.Framework; @@ -30,7 +30,7 @@ namespace ExperimentalTests.DownloadConnectivityTests private void RunThePlan(Plan plan, int fileSizeMb) { foreach (var filePlan in plan.FilePlans) filePlan.File = GenerateTestFile(fileSizeMb.MB()); - var nodes = StartCodex(plan.NodePlans.Count); + var nodes = StartLogosStorage(plan.NodePlans.Count); for (int i = 0; i < plan.NodePlans.Count; i++) plan.NodePlans[i].Node = nodes[i]; // Upload all files to their nodes. @@ -112,7 +112,7 @@ namespace ExperimentalTests.DownloadConnectivityTests } public int Number { get; } - public ICodexNode? Node { get; set; } + public IStorageNode? Node { get; set; } public List Uploads { get; } = new List(); public List Downloads { get; } = new List(); diff --git a/Tests/ExperimentalTests/ExperimentalTests.csproj b/Tests/ExperimentalTests/ExperimentalTests.csproj index 68cab2ff..f9510a78 100644 --- a/Tests/ExperimentalTests/ExperimentalTests.csproj +++ b/Tests/ExperimentalTests/ExperimentalTests.csproj @@ -13,8 +13,8 @@ - - + + diff --git a/Tests/ExperimentalTests/Helpers/FullConnectivityHelper.cs b/Tests/ExperimentalTests/Helpers/FullConnectivityHelper.cs index f8fc6f1d..6b5b55d1 100644 --- a/Tests/ExperimentalTests/Helpers/FullConnectivityHelper.cs +++ b/Tests/ExperimentalTests/Helpers/FullConnectivityHelper.cs @@ -1,9 +1,9 @@ -using CodexClient; -using CodexPlugin; +using LogosStorageClient; +using StoragePlugin; using Logging; using NUnit.Framework; -namespace CodexTests.Helpers +namespace LogosStorageTests.Helpers { public interface IFullConnectivityImplementation { @@ -24,12 +24,12 @@ namespace CodexTests.Helpers this.implementation = implementation; } - public void AssertFullyConnected(IEnumerable nodes) + public void AssertFullyConnected(IEnumerable nodes) { AssertFullyConnected(nodes.ToArray()); } - private void AssertFullyConnected(ICodexNode[] nodes) + private void AssertFullyConnected(IStorageNode[] nodes) { Log($"Asserting '{implementation.Description()}' for nodes: '{nodes.Names()}'..."); Assert.That(nodes.Length, Is.GreaterThan(1)); @@ -70,7 +70,7 @@ namespace CodexTests.Helpers Log($"Connections successful:{Nl}{string.Join(Nl, results)}"); } - private Entry[] CreateEntries(ICodexNode[] nodes) + private Entry[] CreateEntries(IStorageNode[] nodes) { var entries = nodes.Select(n => new Entry(n)).ToArray(); @@ -110,13 +110,13 @@ namespace CodexTests.Helpers public class Entry { - public Entry(ICodexNode node) + public Entry(IStorageNode node) { Node = node; Response = node.GetDebugInfo(); } - public ICodexNode Node { get; } + public IStorageNode Node { get; } public DebugInfo Response { get; } public override string ToString() diff --git a/Tests/ExperimentalTests/Helpers/PeerConnectionTestHelpers.cs b/Tests/ExperimentalTests/Helpers/PeerConnectionTestHelpers.cs index ab4bd6a3..406422da 100644 --- a/Tests/ExperimentalTests/Helpers/PeerConnectionTestHelpers.cs +++ b/Tests/ExperimentalTests/Helpers/PeerConnectionTestHelpers.cs @@ -1,8 +1,8 @@ -using CodexClient; +using LogosStorageClient; using Logging; -using static CodexTests.Helpers.FullConnectivityHelper; +using static LogosStorageTests.Helpers.FullConnectivityHelper; -namespace CodexTests.Helpers +namespace LogosStorageTests.Helpers { public class PeerConnectionTestHelpers : IFullConnectivityImplementation { @@ -13,7 +13,7 @@ namespace CodexTests.Helpers helper = new FullConnectivityHelper(log, this); } - public void AssertFullyConnected(IEnumerable nodes) + public void AssertFullyConnected(IEnumerable nodes) { helper.AssertFullyConnected(nodes); } diff --git a/Tests/ExperimentalTests/Helpers/PeerDownloadTestHelpers.cs b/Tests/ExperimentalTests/Helpers/PeerDownloadTestHelpers.cs index 9f7d309d..a18d3f22 100644 --- a/Tests/ExperimentalTests/Helpers/PeerDownloadTestHelpers.cs +++ b/Tests/ExperimentalTests/Helpers/PeerDownloadTestHelpers.cs @@ -1,10 +1,10 @@ -using CodexClient; +using LogosStorageClient; using FileUtils; using Logging; using Utils; -using static CodexTests.Helpers.FullConnectivityHelper; +using static LogosStorageTests.Helpers.FullConnectivityHelper; -namespace CodexTests.Helpers +namespace LogosStorageTests.Helpers { public class PeerDownloadTestHelpers : IFullConnectivityImplementation { @@ -19,7 +19,7 @@ namespace CodexTests.Helpers this.fileManager = fileManager; } - public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) + public void AssertFullDownloadInterconnectivity(IEnumerable nodes, ByteSize testFileSize) { this.testFileSize = testFileSize; helper.AssertFullyConnected(nodes); @@ -60,12 +60,12 @@ namespace CodexTests.Helpers // Should an exception occur during upload, then this try is inconclusive and we try again next loop. } - private TrackedFile? DownloadFile(ICodexNode node, ContentId contentId, string label) + private TrackedFile? DownloadFile(IStorageNode node, ContentId contentId, string label) { return node.DownloadContent(contentId, label); } - private TrackedFile GenerateTestFile(ICodexNode uploader, ICodexNode downloader) + private TrackedFile GenerateTestFile(IStorageNode uploader, IStorageNode downloader) { var up = uploader.GetName().Replace("<", "").Replace(">", ""); var down = downloader.GetName().Replace("<", "").Replace(">", ""); diff --git a/Tests/ExperimentalTests/CodexDistTest.cs b/Tests/ExperimentalTests/LogosStorageDistTest.cs similarity index 71% rename from Tests/ExperimentalTests/CodexDistTest.cs rename to Tests/ExperimentalTests/LogosStorageDistTest.cs index b060c445..6b6dc57f 100644 --- a/Tests/ExperimentalTests/CodexDistTest.cs +++ b/Tests/ExperimentalTests/LogosStorageDistTest.cs @@ -1,8 +1,7 @@ -using CodexClient; -using CodexNetDeployer; -using CodexPlugin; -using CodexPlugin.OverwatchSupport; -using CodexTests.Helpers; +using LogosStorageClient; +using StoragePlugin; +using StoragePlugin.OverwatchSupport; +using LogosStorageTests.Helpers; using Core; using DistTestCore; using DistTestCore.Logs; @@ -13,61 +12,61 @@ using NUnit.Framework; using OverwatchTranscript; using Utils; -namespace CodexTests +namespace LogosStorageTests { - public class CodexDistTest : DistTest + public class LogosStorageDistTest : DistTest { - private readonly List nodes = new List(); - private CodexTranscriptWriter? writer; + private readonly List nodes = new List(); + private LogosStorageTranscriptWriter? writer; - public CodexDistTest() + public LogosStorageDistTest() { - ProjectPlugin.Load(); + ProjectPlugin.Load(); ProjectPlugin.Load(); } [SetUp] - public void SetupCodexDistTest() + public void SetupLogosStorageDistTest() { writer = SetupTranscript(); } [TearDown] - public void TearDownCodexDistTest() + public void TearDownLogosStorageDistTest() { TeardownTranscript(); } protected override void Initialize(FixtureLog fixtureLog) { - var localBuilder = new LocalCodexBuilder(fixtureLog); + var localBuilder = new LocalNodeBuilder(fixtureLog); localBuilder.Intialize(); localBuilder.Build(); - Ci.AddCodexHooksProvider(new CodexLogTrackerProvider(nodes.Add)); + Ci.AddLogosStorageHooksProvider(new LogosStorageLogTrackerProvider(nodes.Add)); } - public ICodexNode StartCodex() + public IStorageNode StartLogosStorage() { - return StartCodex(s => { }); + return StartLogosStorage(s => { }); } - public ICodexNode StartCodex(Action setup) + public IStorageNode StartLogosStorage(Action setup) { - return StartCodex(1, setup)[0]; + return StartLogosStorage(1, setup)[0]; } - public ICodexNodeGroup StartCodex(int numberOfNodes) + public IStorageNodeGroup StartLogosStorage(int numberOfNodes) { - return StartCodex(numberOfNodes, s => { }); + return StartLogosStorage(numberOfNodes, s => { }); } - public ICodexNodeGroup StartCodex(int numberOfNodes, Action setup) + public IStorageNodeGroup StartLogosStorage(int numberOfNodes, Action setup) { - var group = Ci.StartCodexNodes(numberOfNodes, s => + var group = Ci.StartStorageNodes(numberOfNodes, s => { setup(s); - OnCodexSetup(s); + OnLogosStorageSetup(s); }); return group; @@ -83,12 +82,12 @@ namespace CodexTests return new PeerDownloadTestHelpers(GetTestLog(), GetFileManager()); } - public void CheckLogForErrors(params ICodexNode[] nodes) + public void CheckLogForErrors(params IStorageNode[] nodes) { foreach (var node in nodes) CheckLogForErrors(node); } - public void CheckLogForErrors(ICodexNode node) + public void CheckLogForErrors(IStorageNode node) { Log($"Checking {node.GetName()} log for errors."); var log = node.DownloadLog(); @@ -97,23 +96,23 @@ namespace CodexTests log.AssertLogDoesNotContainLinesStartingWith("ERR "); } - public void LogNodeStatus(ICodexNode node, IMetricsAccess? metrics = null) + public void LogNodeStatus(IStorageNode node, IMetricsAccess? metrics = null) { Log("Status for " + node.GetName() + Environment.NewLine + GetBasicNodeStatus(node)); } - public void WaitAndCheckNodesStaysAlive(TimeSpan duration, ICodexNodeGroup nodes) + public void WaitAndCheckNodesStaysAlive(TimeSpan duration, IStorageNodeGroup nodes) { WaitAndCheckNodesStaysAlive(duration, nodes.ToArray()); } - public void WaitAndCheckNodesStaysAlive(TimeSpan duration, List nodes) + public void WaitAndCheckNodesStaysAlive(TimeSpan duration, List nodes) { WaitAndCheckNodesStaysAlive(duration, nodes.ToArray()); } - public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params ICodexNode[] nodes) + public void WaitAndCheckNodesStaysAlive(TimeSpan duration, params IStorageNode[] nodes) { Log($"{nameof(WaitAndCheckNodesStaysAlive)} {Time.FormatDuration(duration)}..."); @@ -136,12 +135,12 @@ namespace CodexTests Log($"{nameof(WaitAndCheckNodesStaysAlive)} OK"); } - public void AssertNodesContainFile(ContentId cid, ICodexNodeGroup nodes) + public void AssertNodesContainFile(ContentId cid, IStorageNodeGroup nodes) { AssertNodesContainFile(cid, nodes.ToArray()); } - public void AssertNodesContainFile(ContentId cid, params ICodexNode[] nodes) + public void AssertNodesContainFile(ContentId cid, params IStorageNode[] nodes) { Log($"{nameof(AssertNodesContainFile)} {nodes.Names()} {cid}..."); @@ -154,13 +153,13 @@ namespace CodexTests Log($"{nameof(AssertNodesContainFile)} OK"); } - private string GetBasicNodeStatus(ICodexNode node) + private string GetBasicNodeStatus(IStorageNode node) { return JsonConvert.SerializeObject(node.GetDebugInfo(), Formatting.Indented) + Environment.NewLine + node.Space().ToString() + Environment.NewLine; } - protected virtual void OnCodexSetup(ICodexSetup setup) + protected virtual void OnLogosStorageSetup(ILogosStorageSetup setup) { } @@ -171,19 +170,19 @@ namespace CodexTests return null; } - private CodexTranscriptWriter? SetupTranscript() + private LogosStorageTranscriptWriter? SetupTranscript() { var attr = GetTranscriptAttributeOfCurrentTest(); if (attr == null) return null; - var config = new CodexTranscriptWriterConfig( + var config = new LogosStorageTranscriptWriterConfig( attr.OutputFilename, attr.IncludeBlockReceivedEvents ); var log = new LogPrefixer(GetTestLog(), "(Transcript) "); - var writer = new CodexTranscriptWriter(log, config, Transcript.NewWriter(log)); - Ci.AddCodexHooksProvider(writer); + var writer = new LogosStorageTranscriptWriter(log, config, Transcript.NewWriter(log)); + Ci.AddLogosStorageHooksProvider(writer); return writer; } diff --git a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs b/Tests/ExperimentalTests/LogosStorageLogTrackerProvider.cs similarity index 55% rename from Tests/ExperimentalTests/CodexLogTrackerProvider.cs rename to Tests/ExperimentalTests/LogosStorageLogTrackerProvider.cs index dcf2c718..03ffd6f5 100644 --- a/Tests/ExperimentalTests/CodexLogTrackerProvider.cs +++ b/Tests/ExperimentalTests/LogosStorageLogTrackerProvider.cs @@ -1,29 +1,29 @@ -using CodexClient; -using CodexClient.Hooks; +using LogosStorageClient; +using LogosStorageClient.Hooks; using Utils; -namespace CodexTests +namespace LogosStorageTests { - public class CodexLogTrackerProvider : ICodexHooksProvider + public class LogosStorageLogTrackerProvider : ILogosStorageHooksProvider { - private readonly Action addNode; + private readonly Action addNode; - public CodexLogTrackerProvider(Action addNode) + public LogosStorageLogTrackerProvider(Action addNode) { this.addNode = addNode; } // See TestLifecycle.cs DownloadAllLogs() - public ICodexNodeHooks CreateHooks(string nodeName) + public IStorageNodeHooks CreateHooks(string nodeName) { - return new CodexLogTracker(addNode); + return new LogosStorageLogTracker(addNode); } - public class CodexLogTracker : ICodexNodeHooks + public class LogosStorageLogTracker : IStorageNodeHooks { - private readonly Action addNode; + private readonly Action addNode; - public CodexLogTracker(Action addNode) + public LogosStorageLogTracker(Action addNode) { this.addNode = addNode; } @@ -44,7 +44,7 @@ namespace CodexTests { } - public void OnNodeStarted(ICodexNode node, string peerId, string nodeId) + public void OnNodeStarted(IStorageNode node, string peerId, string nodeId) { addNode(node); } diff --git a/Tests/ExperimentalTests/MetricsAccessExtensions.cs b/Tests/ExperimentalTests/MetricsAccessExtensions.cs index 0e26d939..bdbf2735 100644 --- a/Tests/ExperimentalTests/MetricsAccessExtensions.cs +++ b/Tests/ExperimentalTests/MetricsAccessExtensions.cs @@ -3,7 +3,7 @@ using Logging; using MetricsPlugin; using NUnit.Framework.Constraints; -namespace CodexTests +namespace LogosStorageTests { public static class MetricsAccessExtensions { diff --git a/Tests/ExperimentalTests/Parallelism.cs b/Tests/ExperimentalTests/Parallelism.cs index a628e968..4631fcaf 100644 --- a/Tests/ExperimentalTests/Parallelism.cs +++ b/Tests/ExperimentalTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; [assembly: LevelOfParallelism(1)] -namespace CodexTests +namespace LogosStorageTests { } diff --git a/Tests/ExperimentalTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/ExperimentalTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index bf2ed0e1..315b109a 100644 --- a/Tests/ExperimentalTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/ExperimentalTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,19 +1,19 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; namespace ExperimentalTests.PeerDiscoveryTests { [TestFixture] - public class LayeredDiscoveryTests : CodexDistTest + public class LayeredDiscoveryTests : LogosStorageDistTest { [Test] public void TwoLayersTest() { - var root = StartCodex(); - var l1Source = StartCodex(s => s.WithBootstrapNode(root)); - var l1Node = StartCodex(s => s.WithBootstrapNode(root)); - var l2Target = StartCodex(s => s.WithBootstrapNode(l1Node)); + var root = StartLogosStorage(); + var l1Source = StartLogosStorage(s => s.WithBootstrapNode(root)); + var l1Node = StartLogosStorage(s => s.WithBootstrapNode(root)); + var l2Target = StartLogosStorage(s => s.WithBootstrapNode(l1Node)); AssertAllNodesConnected(root, l1Source, l1Node, l2Target); } @@ -21,11 +21,11 @@ namespace ExperimentalTests.PeerDiscoveryTests [Test] public void ThreeLayersTest() { - var root = StartCodex(); - var l1Source = StartCodex(s => s.WithBootstrapNode(root)); - var l1Node = StartCodex(s => s.WithBootstrapNode(root)); - var l2Node = StartCodex(s => s.WithBootstrapNode(l1Node)); - var l3Target = StartCodex(s => s.WithBootstrapNode(l2Node)); + var root = StartLogosStorage(); + var l1Source = StartLogosStorage(s => s.WithBootstrapNode(root)); + var l1Node = StartLogosStorage(s => s.WithBootstrapNode(root)); + var l2Node = StartLogosStorage(s => s.WithBootstrapNode(l1Node)); + var l3Target = StartLogosStorage(s => s.WithBootstrapNode(l2Node)); AssertAllNodesConnected(root, l1Source, l1Node, l2Node, l3Target); } @@ -35,20 +35,20 @@ namespace ExperimentalTests.PeerDiscoveryTests [TestCase(10)] public void NodeChainTest(int chainLength) { - var nodes = new List(); - var node = StartCodex(); + var nodes = new List(); + var node = StartLogosStorage(); nodes.Add(node); for (var i = 1; i < chainLength; i++) { - node = StartCodex(s => s.WithBootstrapNode(node)); + node = StartLogosStorage(s => s.WithBootstrapNode(node)); nodes.Add(node); } AssertAllNodesConnected(nodes.ToArray()); } - private void AssertAllNodesConnected(params ICodexNode[] nodes) + private void AssertAllNodesConnected(params IStorageNode[] nodes) { CreatePeerConnectionTestHelpers().AssertFullyConnected(nodes); } diff --git a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 3748a749..42297f9d 100644 --- a/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/ExperimentalTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -1,7 +1,7 @@ -using CodexClient; +using LogosStorageClient; using NUnit.Framework; using Utils; -using CodexTests; +using LogosStorageTests; namespace ExperimentalTests.PeerDiscoveryTests { @@ -12,7 +12,7 @@ namespace ExperimentalTests.PeerDiscoveryTests public void CanReportUnknownPeerId() { var unknownId = "16Uiu2HAkv2CHWpff3dj5iuVNERAp8AGKGNgpGjPexJZHSqUstfsK"; - var node = StartCodex(); + var node = StartLogosStorage(); var result = node.GetDebugPeer(unknownId); Assert.That(result.IsPeerFound, Is.False); @@ -21,7 +21,7 @@ namespace ExperimentalTests.PeerDiscoveryTests [Test] public void MetricsDoesNotInterfereWithPeerDiscovery() { - var nodes = StartCodex(2, s => s.EnableMetrics()); + var nodes = StartLogosStorage(2, s => s.EnableMetrics()); AssertAllNodesConnected(nodes); } @@ -31,12 +31,12 @@ namespace ExperimentalTests.PeerDiscoveryTests [TestCase(10)] public void VariableNodes(int number) { - var nodes = StartCodex(number); + var nodes = StartLogosStorage(number); AssertAllNodesConnected(nodes); } - private void AssertAllNodesConnected(IEnumerable nodes) + private void AssertAllNodesConnected(IEnumerable nodes) { nodes = nodes.Concat(new[] { BootstrapNode }).ToArray()!; @@ -44,7 +44,7 @@ namespace ExperimentalTests.PeerDiscoveryTests CheckRoutingTable(nodes); } - private void CheckRoutingTable(IEnumerable allNodes) + private void CheckRoutingTable(IEnumerable allNodes) { var allResponses = allNodes.Select(n => n.GetDebugInfo()).ToArray(); diff --git a/Tests/ExperimentalTests/README.md b/Tests/ExperimentalTests/README.md index 568686c8..e9f4ba4d 100644 --- a/Tests/ExperimentalTests/README.md +++ b/Tests/ExperimentalTests/README.md @@ -1,11 +1,11 @@ -# Codex Tests +# Logos Storage Tests This is an NUnit test assembly that can be used with the standard dotnet test runner. For all its CLI options, run `dotnet test --help`. ## Example tests -Running all the tests in the assembly can take a while. In order to check basic viability of your setup as well as the Codex image you're using, consider running only the example tests using the filter option: `dotnet test --filter=Example`. +Running all the tests in the assembly can take a while. In order to check basic viability of your setup as well as the Logos Storage image you're using, consider running only the example tests using the filter option: `dotnet test --filter=Example`. ## Output -The test runner will produce a folder named `CodexTestLogs` with all the test logs. They are sorted by timestamp and reflect the names of the test fixtures and individual tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. The same applies to the fixture log file. The `STATUS` files contain the test results in JSON, for easy machine reading. +The test runner will produce a folder named `LogosStorageTestLogs` with all the test logs. They are sorted by timestamp and reflect the names of the test fixtures and individual tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. The same applies to the fixture log file. The `STATUS` files contain the test results in JSON, for easy machine reading. ## Overrides The following environment variables allow you to override specific aspects of the behaviour of the tests. @@ -14,14 +14,14 @@ The following environment variables allow you to override specific aspects of th |------------------|----------------------------------------------------------------------------------------------------------------| | DEPLOYID | A pod-label 'deployid' is added to each pod created during the tests. Use this to set the value of that label. | | TESTID | Similar to RUNID, except the label is 'testid'. | -| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | +| STORAGEDOCKERIMAGE | If set, this will be used instead of the default Logos Storage docker image. | -## Using a local Codex repository -If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. +## Using a local Logos Storage repository +If you have a clone of the Logos Storage git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Logos Storage's 'vendor' directory will be discarded during the build process. | Variable | Description | |----------------|--------------------------------------------------------------------------------------------------------------------------| -| CODEXREPOPATH | Path to the Codex repository. | +| STORAGEREPOPATH | Path to the Logos Storage repository. | | DOCKERUSERNAME | Username of your Dockerhub account. | | DOCKERPASSWORD | Password OR access-token for your Dockerhub account. You can omit this variable to use your system-default account. | | DOCKERTAG | Optional. Tag used for docker image that will be built and pushed to the Dockerhub account. Random ID used when not set. | diff --git a/Tests/ExperimentalTests/UtilityTests/LogHelperTests.cs b/Tests/ExperimentalTests/UtilityTests/LogHelperTests.cs index 06f220e4..eea74ce7 100644 --- a/Tests/ExperimentalTests/UtilityTests/LogHelperTests.cs +++ b/Tests/ExperimentalTests/UtilityTests/LogHelperTests.cs @@ -1,5 +1,5 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; @@ -12,8 +12,8 @@ namespace ExperimentalTests.UtilityTests [Ignore("Used to find the most common log messages.")] public void FindMostCommonLogMessages() { - var uploader = StartCodex(s => s.WithName("uploader").WithLogLevel(CodexLogLevel.Trace)); - var downloader = StartCodex(s => s.WithName("downloader").WithLogLevel(CodexLogLevel.Trace)); + var uploader = StartLogosStorage(s => s.WithName("uploader").WithLogLevel(LogosStorageLogLevel.Trace)); + var downloader = StartLogosStorage(s => s.WithName("downloader").WithLogLevel(LogosStorageLogLevel.Trace)); var cid = uploader.UploadFile(GenerateTestFile(100.MB())); @@ -34,13 +34,13 @@ namespace ExperimentalTests.UtilityTests } } - private Dictionary GetLogMap(ICodexNode node, DateTime? startUtc = null) + private Dictionary GetLogMap(IStorageNode node, DateTime? startUtc = null) { var log = node.DownloadLog(); var map = new Dictionary(); log.IterateLines(line => { - var log = CodexLogLine.Parse(line); + var log = LogosStorageLogLine.Parse(line); if (log == null) return; if (startUtc.HasValue) diff --git a/Tests/ExperimentalTests/UtilityTests/NetworkIsolationTest.cs b/Tests/ExperimentalTests/UtilityTests/NetworkIsolationTest.cs index ea16dbb4..96e83d54 100644 --- a/Tests/ExperimentalTests/UtilityTests/NetworkIsolationTest.cs +++ b/Tests/ExperimentalTests/UtilityTests/NetworkIsolationTest.cs @@ -1,5 +1,5 @@ -using CodexClient; -using CodexPlugin; +using LogosStorageClient; +using StoragePlugin; using DistTestCore; using NUnit.Framework; using Utils; @@ -13,12 +13,12 @@ namespace ExperimentalTests.UtilityTests [Ignore("Disabled until a solution is implemented.")] public class NetworkIsolationTest : DistTest { - private ICodexNode? node = null; + private IStorageNode? node = null; [Test] public void SetUpANodeAndWait() { - node = Ci.StartCodexNode(); + node = Ci.StartStorageNode(); Time.WaitUntil(() => node == null, TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), nameof(SetUpANodeAndWait)); } @@ -26,7 +26,7 @@ namespace ExperimentalTests.UtilityTests [Test] public void ForeignNodeConnects() { - var myNode = Ci.StartCodexNode(); + var myNode = Ci.StartStorageNode(); Time.WaitUntil(() => node != null, TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(5), nameof(ForeignNodeConnects)); @@ -41,7 +41,7 @@ namespace ExperimentalTests.UtilityTests return; } - Assert.Fail("Connection could be established between two Codex nodes running in different namespaces. " + + Assert.Fail("Connection could be established between two Logos Storage nodes running in different namespaces. " + "This may cause cross-test interference. Network isolation policy should be applied. Test infra failure."); } } diff --git a/Tests/FrameworkTests/FrameworkTests.csproj b/Tests/FrameworkTests/FrameworkTests.csproj index 238c9938..ceb9f248 100644 --- a/Tests/FrameworkTests/FrameworkTests.csproj +++ b/Tests/FrameworkTests/FrameworkTests.csproj @@ -16,7 +16,7 @@ - + diff --git a/Tests/FrameworkTests/CodexClient/ContentIdTests.cs b/Tests/FrameworkTests/LogosStorageClient/ContentIdTests.cs similarity index 89% rename from Tests/FrameworkTests/CodexClient/ContentIdTests.cs rename to Tests/FrameworkTests/LogosStorageClient/ContentIdTests.cs index d960d187..6ebe3b28 100644 --- a/Tests/FrameworkTests/CodexClient/ContentIdTests.cs +++ b/Tests/FrameworkTests/LogosStorageClient/ContentIdTests.cs @@ -1,4 +1,4 @@ -using CodexClient; +using LogosStorageClient; using NUnit.Framework; using System; using System.Collections.Generic; @@ -6,7 +6,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace FrameworkTests.CodexClient +namespace FrameworkTests.LogosStorageClient { [TestFixture] public class ContentIdTests diff --git a/Tests/CodexContinuousTests/Configuration.cs b/Tests/LogosStorageContinuousTests/Configuration.cs similarity index 79% rename from Tests/CodexContinuousTests/Configuration.cs rename to Tests/LogosStorageContinuousTests/Configuration.cs index e6eb556d..a2d601db 100644 --- a/Tests/CodexContinuousTests/Configuration.cs +++ b/Tests/LogosStorageContinuousTests/Configuration.cs @@ -1,5 +1,5 @@ using ArgsUniform; -using CodexPlugin; +using StoragePlugin; using Newtonsoft.Json; namespace ContinuousTests @@ -12,8 +12,8 @@ namespace ContinuousTests [Uniform("data-path", "d", "DATAPATH", true, "Path where temporary data files will be written.")] public string DataPath { get; set; } = "data"; - [Uniform("codex-deployment", "c", "CODEXDEPLOYMENT", true, "Path to codex-deployment JSON file.")] - public string CodexDeploymentJson { get; set; } = string.Empty; + [Uniform("storage-deployment", "c", "CODEXDEPLOYMENT", true, "Path to storage-deployment JSON file.")] + public string LogosStorageDeploymentJson { get; set; } = string.Empty; [Uniform("keep", "k", "KEEP", false, "Set to 1 or 'true' to retain logs of successful tests.")] public bool KeepPassedTestLogs { get; set; } = false; @@ -37,7 +37,7 @@ namespace ContinuousTests " the timestamp of the start of the network deployment. Otherwise, logs will start from the test start timestamp.")] public bool FullContainerLogs { get; set; } = false; - public CodexDeployment CodexDeployment { get; set; } = null!; + public LogosStorageDeployment LogosStorageDeployment { get; set; } = null!; } public class ConfigLoader @@ -47,13 +47,13 @@ namespace ContinuousTests var uniformArgs = new ArgsUniform(PrintHelp, args); var result = uniformArgs.Parse(true); - result.CodexDeployment = ParseCodexDeploymentJson(result.CodexDeploymentJson); + result.LogosStorageDeployment = ParseLogosStorageDeploymentJson(result.LogosStorageDeploymentJson); return result; } - private CodexDeployment ParseCodexDeploymentJson(string filename) + private LogosStorageDeployment ParseLogosStorageDeploymentJson(string filename) { - var d = JsonConvert.DeserializeObject(File.ReadAllText(filename))!; + var d = JsonConvert.DeserializeObject(File.ReadAllText(filename))!; if (d == null) throw new Exception("Unable to parse " + filename); return d; } @@ -61,7 +61,7 @@ namespace ContinuousTests private static void PrintHelp() { var nl = Environment.NewLine; - Console.WriteLine("ContinuousTests will run a set of tests against a codex deployment given a codex-deployment.json file." + nl + + Console.WriteLine("ContinuousTests will run a set of tests against a logos-storage deployment given a storage-deployment.json file." + nl + "The tests will run in an endless loop unless otherwise specified." + nl); } } diff --git a/Tests/CodexContinuousTests/ContinuousTest.cs b/Tests/LogosStorageContinuousTests/ContinuousTest.cs similarity index 81% rename from Tests/CodexContinuousTests/ContinuousTest.cs rename to Tests/LogosStorageContinuousTests/ContinuousTest.cs index fa39255c..f02c146d 100644 --- a/Tests/CodexContinuousTests/ContinuousTest.cs +++ b/Tests/LogosStorageContinuousTests/ContinuousTest.cs @@ -1,4 +1,4 @@ -using CodexClient; +using LogosStorageClient; using FileUtils; using Logging; using MetricsPlugin; @@ -20,7 +20,7 @@ namespace ContinuousTests protected const int DayOne = HourOne * 24; protected const int DayThree = DayOne * 3; - public void Initialize(ICodexNode[] nodes, ILog log, IFileManager fileManager, Configuration configuration, CancellationToken cancelToken) + public void Initialize(IStorageNode[] nodes, ILog log, IFileManager fileManager, Configuration configuration, CancellationToken cancelToken) { Nodes = nodes; Log = log; @@ -38,7 +38,7 @@ namespace ContinuousTests } } - public ICodexNode[] Nodes { get; private set; } = null!; + public IStorageNode[] Nodes { get; private set; } = null!; public ILog Log { get; private set; } = null!; public IFileManager FileManager { get; private set; } = null!; public Configuration Configuration { get; private set; } = null!; @@ -52,11 +52,11 @@ namespace ContinuousTests public IMetricsAccess CreateMetricsAccess(Address target) { - if (Configuration.CodexDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Codex deployment."); + if (Configuration.LogosStorageDeployment.PrometheusContainer == null) throw new Exception("Expected prometheus to be part of Logos Storage deployment."); var entryPointFactory = new EntryPointFactory(); - var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.CodexDeployment.Metadata.KubeNamespace, Log); - return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.CodexDeployment.PrometheusContainer, target); + var entryPoint = entryPointFactory.CreateEntryPoint(Configuration.KubeConfigFile, Configuration.DataPath, Configuration.LogosStorageDeployment.Metadata.KubeNamespace, Log); + return entryPoint.CreateInterface().WrapMetricsCollector(Configuration.LogosStorageDeployment.PrometheusContainer, target); } public abstract int RequiredNumberOfNodes { get; } diff --git a/Tests/CodexContinuousTests/ContinuousTestRunner.cs b/Tests/LogosStorageContinuousTests/ContinuousTestRunner.cs similarity index 94% rename from Tests/CodexContinuousTests/ContinuousTestRunner.cs rename to Tests/LogosStorageContinuousTests/ContinuousTestRunner.cs index edec4e43..ff3ce460 100644 --- a/Tests/CodexContinuousTests/ContinuousTestRunner.cs +++ b/Tests/LogosStorageContinuousTests/ContinuousTestRunner.cs @@ -27,15 +27,15 @@ namespace ContinuousTests var startTime = DateTime.UtcNow; var overviewLog = new LogSplitter( - FixtureLog.Create(logConfig, startTime, config.CodexDeployment.Id, "Overview"), + FixtureLog.Create(logConfig, startTime, config.LogosStorageDeployment.Id, "Overview"), new ConsoleLog() ); - var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.CodexDeployment.Id, + var statusLog = new StatusLog(logConfig, startTime, "continuous-tests", config.LogosStorageDeployment.Id, "ContinuousTestRun"); overviewLog.Log("Initializing..."); - var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, overviewLog); + var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.LogosStorageDeployment.Metadata.KubeNamespace, overviewLog); entryPoint.Announce(); overviewLog.Log("Initialized. Performing startup checks..."); @@ -156,7 +156,7 @@ namespace ContinuousTests if (!config.Cleanup) return; log.Log("Cleaning up test namespace..."); - var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.CodexDeployment.Metadata.KubeNamespace, log); + var entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, config.LogosStorageDeployment.Metadata.KubeNamespace, log); entryPoint.Decommission(deleteKubernetesResources: true, deleteTrackedFiles: true, waitTillDone: true); log.Log("Cleanup finished."); } diff --git a/Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs b/Tests/LogosStorageContinuousTests/ElasticSearchLogDownloader.cs similarity index 100% rename from Tests/CodexContinuousTests/ElasticSearchLogDownloader.cs rename to Tests/LogosStorageContinuousTests/ElasticSearchLogDownloader.cs diff --git a/Tests/CodexContinuousTests/EntryPointFactory.cs b/Tests/LogosStorageContinuousTests/EntryPointFactory.cs similarity index 94% rename from Tests/CodexContinuousTests/EntryPointFactory.cs rename to Tests/LogosStorageContinuousTests/EntryPointFactory.cs index 269b3620..8c17ca83 100644 --- a/Tests/CodexContinuousTests/EntryPointFactory.cs +++ b/Tests/LogosStorageContinuousTests/EntryPointFactory.cs @@ -7,7 +7,7 @@ namespace ContinuousTests { public EntryPointFactory() { - ProjectPlugin.Load(); + ProjectPlugin.Load(); ProjectPlugin.Load(); } diff --git a/Tests/CodexContinuousTests/CodexContinuousTests.csproj b/Tests/LogosStorageContinuousTests/LogosStorageContinuousTests.csproj similarity index 86% rename from Tests/CodexContinuousTests/CodexContinuousTests.csproj rename to Tests/LogosStorageContinuousTests/LogosStorageContinuousTests.csproj index 03403394..3952a601 100644 --- a/Tests/CodexContinuousTests/CodexContinuousTests.csproj +++ b/Tests/LogosStorageContinuousTests/LogosStorageContinuousTests.csproj @@ -13,7 +13,7 @@ - + diff --git a/Tests/CodexContinuousTests/NodeRunner.cs b/Tests/LogosStorageContinuousTests/NodeRunner.cs similarity index 75% rename from Tests/CodexContinuousTests/NodeRunner.cs rename to Tests/LogosStorageContinuousTests/NodeRunner.cs index ae18f7e9..81d80e0c 100644 --- a/Tests/CodexContinuousTests/NodeRunner.cs +++ b/Tests/LogosStorageContinuousTests/NodeRunner.cs @@ -2,20 +2,20 @@ using Logging; using Utils; using Core; -using CodexPlugin; -using CodexClient; +using StoragePlugin; +using LogosStorageClient; namespace ContinuousTests { public class NodeRunner { private readonly EntryPointFactory entryPointFactory = new EntryPointFactory(); - private readonly ICodexNode[] nodes; + private readonly IStorageNode[] nodes; private readonly Configuration config; private readonly ILog log; private readonly string customNamespace; - public NodeRunner(ICodexNode[] nodes, Configuration config, ILog log, string customNamespace) + public NodeRunner(IStorageNode[] nodes, Configuration config, ILog log, string customNamespace) { this.nodes = nodes; this.config = config; @@ -23,24 +23,24 @@ namespace ContinuousTests this.customNamespace = customNamespace; } - public void RunNode(Action setup, Action operation) + public void RunNode(Action setup, Action operation) { RunNode(nodes.ToList().PickOneRandom(), setup, operation); } - public void RunNode(ICodexNode bootstrapNode, Action setup, Action operation) + public void RunNode(IStorageNode bootstrapNode, Action setup, Action operation) { var entryPoint = CreateEntryPoint(); // We have to be sure that the transient node we start is using the same image as whatever's already in the deployed network. // Therefore, we use the image of the bootstrap node. - CodexDockerImage.Override = bootstrapNode.GetImageName(); + LogosStorageDockerImage.Override = bootstrapNode.GetImageName(); try { var debugInfo = bootstrapNode.GetDebugInfo(); Assert.That(!string.IsNullOrEmpty(debugInfo.Spr)); - var node = entryPoint.CreateInterface().StartCodexNode(s => + var node = entryPoint.CreateInterface().StartStorageNode(s => { setup(s); s.WithBootstrapNode(bootstrapNode); diff --git a/Tests/CodexContinuousTests/Program.cs b/Tests/LogosStorageContinuousTests/Program.cs similarity index 91% rename from Tests/CodexContinuousTests/Program.cs rename to Tests/LogosStorageContinuousTests/Program.cs index 7e6fcf28..159b32a6 100644 --- a/Tests/CodexContinuousTests/Program.cs +++ b/Tests/LogosStorageContinuousTests/Program.cs @@ -4,7 +4,7 @@ public class Program { public static void Main(string[] args) { - Console.WriteLine("Codex Continuous-Test-Runner."); + Console.WriteLine("Logos Storage Continuous-Test-Runner."); var runner = new ContinuousTestRunner(args, Cancellation.Cts.Token); diff --git a/Tests/CodexContinuousTests/README.md b/Tests/LogosStorageContinuousTests/README.md similarity index 58% rename from Tests/CodexContinuousTests/README.md rename to Tests/LogosStorageContinuousTests/README.md index ca8eae56..5859c3d7 100644 --- a/Tests/CodexContinuousTests/README.md +++ b/Tests/LogosStorageContinuousTests/README.md @@ -1,11 +1,8 @@ -# Codex Continuous Tests -This CLI tool runs tests in an endless loop, using a network of Codex nodes in a kubernetes cluster. Run `dotnet run -- --help` to view all CLI options. +# Logos Storage Continuous Tests +This CLI tool runs tests in an endless loop, using a network of Logos Storage nodes in a kubernetes cluster. Run `dotnet run -- --help` to view all CLI options. ## Choosing tests -By default, all tests in the `CodexContinuousTests/Tests` folder will be used. If you want to limit your test run to a subset of tests, please delete or disable the other test code files. TODO: We'd like a CLI option for selecting tests. Similar to `dotnet test --filter`, maybe? - -## Where do I get a `codex-deployment.json` -See [THIS](../../Tools/CodexNetDeployer/README.MD) +By default, all tests in the `LogosStorageContinuousTests/Tests` folder will be used. If you want to limit your test run to a subset of tests, please delete or disable the other test code files. TODO: We'd like a CLI option for selecting tests. Similar to `dotnet test --filter`, maybe? ## Output The test runner will produce a folder with all the test logs. They are sorted by timestamp and reflect the names of the tests. When a test fails, the log file for that specific test will be postfixed with `_FAILED`. @@ -16,5 +13,5 @@ While individual tests can pass or fail for a number of times and/or over a leng 1. Passed: The failed condition was not reached within the time specified by the target-duration option. ## Transient nodes -The continuous tests runner is designed to use a network of Codex nodes deployed in a kubernetes cluster. The runner will not influence or manage the lifecycle of the nodes in this deployment. However, some test cases require transient nodes. -A transient node is a node started and managed by the test runner on behalf of a specific test. The runner facilitates the tests to start and stop transient nodes, as well as bootstrap those nodes against (permanent) nodes from the deployed network. The test runner makes sure that the transient nodes use the same docker image as the permanent nodes, to avoid version conflicts. However, the image used for transient nodes can be overwritten by setting the `CODEXDOCKERIMAGE` environment variable. The use of a local Codex repository for building override images is not supported for transient nodes. +The continuous tests runner is designed to use a network of Logos Storage nodes deployed in a kubernetes cluster. The runner will not influence or manage the lifecycle of the nodes in this deployment. However, some test cases require transient nodes. +A transient node is a node started and managed by the test runner on behalf of a specific test. The runner facilitates the tests to start and stop transient nodes, as well as bootstrap those nodes against (permanent) nodes from the deployed network. The test runner makes sure that the transient nodes use the same docker image as the permanent nodes, to avoid version conflicts. However, the image used for transient nodes can be overwritten by setting the `STORAGEDOCKERIMAGE` environment variable. The use of a local Logos Storage repository for building override images is not supported for transient nodes. diff --git a/Tests/CodexContinuousTests/SingleTestRun.cs b/Tests/LogosStorageContinuousTests/SingleTestRun.cs similarity index 94% rename from Tests/CodexContinuousTests/SingleTestRun.cs rename to Tests/LogosStorageContinuousTests/SingleTestRun.cs index 93870406..dcfd7e0f 100644 --- a/Tests/CodexContinuousTests/SingleTestRun.cs +++ b/Tests/LogosStorageContinuousTests/SingleTestRun.cs @@ -2,12 +2,12 @@ using Utils; using NUnit.Framework.Internal; using System.Reflection; -using CodexPlugin; +using StoragePlugin; using DistTestCore.Logs; using Core; using KubernetesWorkflow.Types; using TaskFactory = Utils.TaskFactory; -using CodexClient; +using LogosStorageClient; namespace ContinuousTests { @@ -21,7 +21,7 @@ namespace ContinuousTests private readonly StatusLog statusLog; private readonly TestHandle handle; private readonly CancellationToken cancelToken; - private readonly ICodexNode[] nodes; + private readonly IStorageNode[] nodes; private readonly FixtureLog fixtureLog; private readonly string testName; private static int failureCount = 0; @@ -39,7 +39,7 @@ namespace ContinuousTests testName = handle.Test.GetType().Name; fixtureLog = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, deployId, testName); entryPoint = entryPointFactory.CreateEntryPoint(config.KubeConfigFile, config.DataPath, - config.CodexDeployment.Metadata.KubeNamespace, fixtureLog); + config.LogosStorageDeployment.Metadata.KubeNamespace, fixtureLog); ApplyLogReplacements(fixtureLog, startupChecker); nodes = CreateRandomNodes(); @@ -112,7 +112,7 @@ namespace ContinuousTests var effectiveStart = testStart.Subtract(TimeSpan.FromSeconds(30)); if (config.FullContainerLogs) { - effectiveStart = config.CodexDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30)); + effectiveStart = config.LogosStorageDeployment.Metadata.StartUtc.Subtract(TimeSpan.FromSeconds(30)); } var effectiveEnd = DateTime.UtcNow; @@ -284,20 +284,20 @@ namespace ContinuousTests return $"({string.Join(",", nodes.Select(n => n.GetName()))})"; } - private ICodexNode[] CreateRandomNodes() + private IStorageNode[] CreateRandomNodes() { var instances = SelectRandomInstance(); fixtureLog.Log("Selected nodes: " + string.Join(",", instances.Select(c => c.Name))); - return entryPoint.CreateInterface().WrapCodexContainers(instances).ToArray(); + return entryPoint.CreateInterface().WrapLogosStorageContainers(instances).ToArray(); } - private ICodexInstance[] SelectRandomInstance() + private ILogosStorageInstance[] SelectRandomInstance() { var number = handle.Test.RequiredNumberOfNodes; - var instances = config.CodexDeployment.CodexInstances.ToList(); + var instances = config.LogosStorageDeployment.LogosStorageInstances.ToList(); if (number == -1) return instances.ToArray(); - var result = new ICodexInstance[number]; + var result = new ILogosStorageInstance[number]; for (var i = 0; i < number; i++) { result[i] = instances.PickOneRandom(); diff --git a/Tests/CodexContinuousTests/StartupChecker.cs b/Tests/LogosStorageContinuousTests/StartupChecker.cs similarity index 88% rename from Tests/CodexContinuousTests/StartupChecker.cs rename to Tests/LogosStorageContinuousTests/StartupChecker.cs index 2febf14e..cb00af4e 100644 --- a/Tests/CodexContinuousTests/StartupChecker.cs +++ b/Tests/LogosStorageContinuousTests/StartupChecker.cs @@ -1,4 +1,4 @@ -using CodexClient; +using LogosStorageClient; using Core; using DistTestCore.Logs; using Logging; @@ -22,14 +22,14 @@ namespace ContinuousTests public void Check() { - var log = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, config.CodexDeployment.Id, + var log = FixtureLog.Create(new LogConfig(config.LogPath), DateTime.UtcNow, config.LogosStorageDeployment.Id, "StartupChecks"); log.Log("Starting continuous test run..."); IncludeDeploymentConfiguration(log); log.Log("Checking configuration..."); PreflightCheck(config); - log.Log("Contacting Codex nodes..."); - CheckCodexNodes(log, config); + log.Log("Contacting Logos Storage nodes..."); + CheckLogosStorageNodes(log, config); log.Log("All OK."); } @@ -40,9 +40,9 @@ namespace ContinuousTests log.Log(""); throw new NotImplementedException(); - //var deployment = config.CodexDeployment; + //var deployment = config.LogosStorageDeployment; //var workflow = entryPoint.Tools.CreateWorkflow(); - //foreach (var instance in deployment.CodexInstances) + //foreach (var instance in deployment.LogosStorageInstances) //{ // foreach (var container in instance.Pod.Containers) // { @@ -90,12 +90,12 @@ namespace ContinuousTests } } - private void CheckCodexNodes(ILog log, Configuration config) + private void CheckLogosStorageNodes(ILog log, Configuration config) { throw new NotImplementedException(); //var nodes = entryPoint.CreateInterface() - // .WrapCodexContainers(config.CodexDeployment.CodexInstances.Select(i => i.Pod).ToArray()); + // .WrapLogosStorageContainers(config.LogosStorageDeployment.LogosStorageInstances.Select(i => i.Pod).ToArray()); //var pass = true; //foreach (var n in nodes) //{ @@ -121,14 +121,14 @@ namespace ContinuousTests //} } - private bool EnsureOnline(BaseLog log, ICodexNode n) + private bool EnsureOnline(BaseLog log, IStorageNode n) { try { var info = n.GetDebugInfo(); if (info == null || string.IsNullOrEmpty(info.Id)) return false; - log.Log($"Codex version: '{info.Version.Version}' revision: '{info.Version.Revision}'"); + log.Log($"Logos Storage version: '{info.Version.Version}' revision: '{info.Version.Revision}'"); LogReplacements.Add(new BaseLogStringReplacement(info.Id, n.GetName())); } catch @@ -185,10 +185,10 @@ namespace ContinuousTests errors.Add( $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Test must require > 0 nodes, or -1 to select all nodes."); } - else if (test.RequiredNumberOfNodes > config.CodexDeployment.CodexInstances.Length) + else if (test.RequiredNumberOfNodes > config.LogosStorageDeployment.LogosStorageInstances.Length) { errors.Add( - $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.CodexDeployment.CodexInstances.Length}"); + $"Test '{test.Name}' requires {test.RequiredNumberOfNodes} nodes. Deployment only has {config.LogosStorageDeployment.LogosStorageInstances.Length}"); } } } diff --git a/Tests/CodexContinuousTests/TestFactory.cs b/Tests/LogosStorageContinuousTests/TestFactory.cs similarity index 100% rename from Tests/CodexContinuousTests/TestFactory.cs rename to Tests/LogosStorageContinuousTests/TestFactory.cs diff --git a/Tests/CodexContinuousTests/TestHandle.cs b/Tests/LogosStorageContinuousTests/TestHandle.cs similarity index 100% rename from Tests/CodexContinuousTests/TestHandle.cs rename to Tests/LogosStorageContinuousTests/TestHandle.cs diff --git a/Tests/CodexContinuousTests/TestLoop.cs b/Tests/LogosStorageContinuousTests/TestLoop.cs similarity index 97% rename from Tests/CodexContinuousTests/TestLoop.cs rename to Tests/LogosStorageContinuousTests/TestLoop.cs index aa0d6364..1ab0c913 100644 --- a/Tests/CodexContinuousTests/TestLoop.cs +++ b/Tests/LogosStorageContinuousTests/TestLoop.cs @@ -78,7 +78,7 @@ namespace ContinuousTests var test = (ContinuousTest)Activator.CreateInstance(testType)!; var handle = new TestHandle(test); var run = new SingleTestRun(entryPointFactory, taskFactory, config, overviewLog, statusLog, handle, - startupChecker, cancelToken, config.CodexDeployment.Id); + startupChecker, cancelToken, config.LogosStorageDeployment.Id); runFinishedHandle.Reset(); run.Run(runFinishedHandle, result => diff --git a/Tests/CodexContinuousTests/TestMomentAttribute.cs b/Tests/LogosStorageContinuousTests/TestMomentAttribute.cs similarity index 100% rename from Tests/CodexContinuousTests/TestMomentAttribute.cs rename to Tests/LogosStorageContinuousTests/TestMomentAttribute.cs diff --git a/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs b/Tests/LogosStorageContinuousTests/Tests/HoldMyBeerTest.cs similarity index 96% rename from Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs rename to Tests/LogosStorageContinuousTests/Tests/HoldMyBeerTest.cs index 6eaec611..fa289a6e 100644 --- a/Tests/CodexContinuousTests/Tests/HoldMyBeerTest.cs +++ b/Tests/LogosStorageContinuousTests/Tests/HoldMyBeerTest.cs @@ -1,4 +1,4 @@ -using CodexClient; +using LogosStorageClient; using FileUtils; using NUnit.Framework; using Utils; diff --git a/Tests/CodexContinuousTests/Tests/MarketplaceTest.cs b/Tests/LogosStorageContinuousTests/Tests/MarketplaceTest.cs similarity index 85% rename from Tests/CodexContinuousTests/Tests/MarketplaceTest.cs rename to Tests/LogosStorageContinuousTests/Tests/MarketplaceTest.cs index 09170f7a..08644d8f 100644 --- a/Tests/CodexContinuousTests/Tests/MarketplaceTest.cs +++ b/Tests/LogosStorageContinuousTests/Tests/MarketplaceTest.cs @@ -31,9 +31,9 @@ // file = FileManager.GenerateTestFile(fileSize); -// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// NodeRunner.RunNode((logosStorageAccess, marketplaceAccess) => // { -// cid = UploadFile(codexAccess.Node, file); +// cid = UploadFile(logosStorageAccess.Node, file); // Assert.That(cid, Is.Not.Null); // purchaseId = marketplaceAccess.RequestStorage( @@ -46,16 +46,16 @@ // Assert.That(!string.IsNullOrEmpty(purchaseId)); -// WaitForContractToStart(codexAccess, purchaseId); +// WaitForContractToStart(logosStorageAccess, purchaseId); // }); // } // [TestMoment(t: MinuteFive + MinuteOne)] // public void StoredDataIsAvailableAfterThreeDays() // { -// NodeRunner.RunNode((codexAccess, marketplaceAccess) => +// NodeRunner.RunNode((logosStorageAccess, marketplaceAccess) => // { -// var result = DownloadFile(codexAccess.Node, cid!); +// var result = DownloadFile(logosStorageAccess.Node, cid!); // file.AssertIsEqual(result); // }); diff --git a/Tests/CodexContinuousTests/Tests/PeersTest.cs b/Tests/LogosStorageContinuousTests/Tests/PeersTest.cs similarity index 91% rename from Tests/CodexContinuousTests/Tests/PeersTest.cs rename to Tests/LogosStorageContinuousTests/Tests/PeersTest.cs index 94a3a090..ebef6316 100644 --- a/Tests/CodexContinuousTests/Tests/PeersTest.cs +++ b/Tests/LogosStorageContinuousTests/Tests/PeersTest.cs @@ -1,9 +1,9 @@ -using CodexClient; -using CodexTests.Helpers; +using LogosStorageClient; +using LogosStorageTests.Helpers; using ContinuousTests; using NUnit.Framework; -namespace CodexContinuousTests.Tests +namespace LogosStorageContinuousTests.Tests { public class PeersTest : ContinuousTest { @@ -38,7 +38,7 @@ namespace CodexContinuousTests.Tests } } - private string AreAllPresent(ICodexNode n, string[] allIds) + private string AreAllPresent(IStorageNode n, string[] allIds) { var info = n.GetDebugInfo(); var known = info.Table.Nodes.Select(n => n.NodeId).ToArray(); diff --git a/Tests/CodexContinuousTests/Tests/PerformanceTests.cs b/Tests/LogosStorageContinuousTests/Tests/PerformanceTests.cs similarity index 80% rename from Tests/CodexContinuousTests/Tests/PerformanceTests.cs rename to Tests/LogosStorageContinuousTests/Tests/PerformanceTests.cs index b6094a0b..7bdb143a 100644 --- a/Tests/CodexContinuousTests/Tests/PerformanceTests.cs +++ b/Tests/LogosStorageContinuousTests/Tests/PerformanceTests.cs @@ -42,7 +42,7 @@ // public override TimeSpan RunTestEvery => TimeSpan.FromMinutes(10); // public override TestFailMode TestFailMode => TestFailMode.AlwaysRunAllMoments; -// public void UploadTest(int megabytes, CodexAccess uploadNode) +// public void UploadTest(int megabytes, LogosStorageAccess uploadNode) // { // var file = FileManager.GenerateTestFile(megabytes.MB()); @@ -53,10 +53,10 @@ // var timePerMB = time / megabytes; -// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); +// Assert.That(timePerMB, Is.LessThan(LogosStorageContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); // } -// public void DownloadTest(int megabytes, CodexAccess uploadNode, CodexAccess downloadNode) +// public void DownloadTest(int megabytes, LogosStorageAccess uploadNode, LogosStorageAccess downloadNode) // { // var file = FileManager.GenerateTestFile(megabytes.MB()); @@ -73,7 +73,7 @@ // var timePerMB = time / megabytes; -// Assert.That(timePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); +// Assert.That(timePerMB, Is.LessThan(LogosStorageContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); // } // private static TimeSpan Measure(Action action) diff --git a/Tests/CodexContinuousTests/Tests/TransientNodeTest.cs b/Tests/LogosStorageContinuousTests/Tests/TransientNodeTest.cs similarity index 66% rename from Tests/CodexContinuousTests/Tests/TransientNodeTest.cs rename to Tests/LogosStorageContinuousTests/Tests/TransientNodeTest.cs index fe2ae344..95307999 100644 --- a/Tests/CodexContinuousTests/Tests/TransientNodeTest.cs +++ b/Tests/LogosStorageContinuousTests/Tests/TransientNodeTest.cs @@ -15,24 +15,24 @@ // private TestFile file = null!; // private ContentId cid = null!; -// private CodexAccess UploadBootstapNode { get { return Nodes[0]; } } -// private CodexAccess DownloadBootstapNode { get { return Nodes[1]; } } -// private CodexAccess IntermediateNode { get { return Nodes[2]; } } +// private LogosStorageAccess UploadBootstapNode { get { return Nodes[0]; } } +// private LogosStorageAccess DownloadBootstapNode { get { return Nodes[1]; } } +// private LogosStorageAccess IntermediateNode { get { return Nodes[2]; } } // [TestMoment(t: 0)] // public void UploadWithTransientNode() // { // file = FileManager.GenerateTestFile(10.MB()); -// NodeRunner.RunNode(UploadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// NodeRunner.RunNode(UploadBootstapNode, (logosStorageAccess, marketplaceAccess, lifecycle) => // { -// cid = UploadFile(codexAccess, file)!; +// cid = UploadFile(logosStorageAccess, file)!; // Assert.That(cid, Is.Not.Null); // var dlt = Task.Run(() => // { // Thread.Sleep(10000); -// lifecycle.DownloadLog(codexAccess.Container); +// lifecycle.DownloadLog(logosStorageAccess.Container); // }); // var resultFile = DownloadFile(IntermediateNode, cid); @@ -44,9 +44,9 @@ // [TestMoment(t: 30)] // public void DownloadWithTransientNode() // { -// NodeRunner.RunNode(DownloadBootstapNode, (codexAccess, marketplaceAccess, lifecycle) => +// NodeRunner.RunNode(DownloadBootstapNode, (logosStorageAccess, marketplaceAccess, lifecycle) => // { -// var resultFile = DownloadFile(codexAccess, cid); +// var resultFile = DownloadFile(logosStorageAccess, cid); // file.AssertIsEqual(resultFile); // }); // } diff --git a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs b/Tests/LogosStorageContinuousTests/Tests/TwoClientTest.cs similarity index 81% rename from Tests/CodexContinuousTests/Tests/TwoClientTest.cs rename to Tests/LogosStorageContinuousTests/Tests/TwoClientTest.cs index 4a5f7b5a..e2b88dca 100644 --- a/Tests/CodexContinuousTests/Tests/TwoClientTest.cs +++ b/Tests/LogosStorageContinuousTests/Tests/TwoClientTest.cs @@ -1,4 +1,4 @@ -using CodexClient; +using LogosStorageClient; using FileUtils; using Logging; using NUnit.Framework; @@ -54,13 +54,13 @@ namespace ContinuousTests.Tests LogBlockExchangeStatus(Nodes[1], "After download"); } - private void LogRepoStore(ICodexNode codexNode) + private void LogRepoStore(IStorageNode storageNode) { - //var response = codexNode.GetDebugRepoStore(); - //Log.Log($"{codexNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}"); + //var response = storageNode.GetDebugRepoStore(); + //Log.Log($"{storageNode.GetName()} has {string.Join(",", response.Select(r => r.cid))}"); } - private void LogStoredBytes(ICodexNode node) + private void LogStoredBytes(IStorageNode node) { var metrics = CreateMetricsAccess(node); var metric = metrics.GetMetric(BytesStoredMetric); @@ -87,10 +87,10 @@ namespace ContinuousTests.Tests Log.Log($"Bytes per millisecond: {bytesPerMs}"); } - private void LogBlockExchangeStatus(ICodexNode codexNode, string msg) + private void LogBlockExchangeStatus(IStorageNode storageNode, string msg) { - //var response = codexNode.GetDebugBlockExchange(); - //Log.Log($"{codexNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}"); + //var response = storageNode.GetDebugBlockExchange(); + //Log.Log($"{storageNode.GetName()} {msg}: {JsonConvert.SerializeObject(response)}"); } } } diff --git a/Tests/CodexContinuousTests/deploy-and-run.sh b/Tests/LogosStorageContinuousTests/deploy-and-run.sh similarity index 87% rename from Tests/CodexContinuousTests/deploy-and-run.sh rename to Tests/LogosStorageContinuousTests/deploy-and-run.sh index 3d778adf..dce9cb86 100644 --- a/Tests/CodexContinuousTests/deploy-and-run.sh +++ b/Tests/LogosStorageContinuousTests/deploy-and-run.sh @@ -33,13 +33,13 @@ do cp codex-deployment-$name-$i.json ../../Tests/CodexContinuousTests done echo "Starting tests..." -cd ../../Tests/CodexContinuousTests +cd ../../Tests/LogosStorageContinuousTests for i in $( seq 0 $replication) do screen -d -m dotnet run \ --kube-config=/opt/kubeconfig.yaml \ - --codex-deployment=codex-deployment-$name-$i.json \ - --log-path=/var/log/codex-continuous-tests/logs-$name-$i \ + --storage-deployment=storage-deployment-$name-$i.json \ + --log-path=/var/log/storage-continuous-tests/logs-$name-$i \ --data-path=data-$name-$i \ --keep=1 \ --stop=1 \ diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-August2023.md b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-August2023.md similarity index 93% rename from Tests/CodexContinuousTests/reports/CodexTestNetReport-August2023.md rename to Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-August2023.md index 82ea08ad..3af42e0a 100644 --- a/Tests/CodexContinuousTests/reports/CodexTestNetReport-August2023.md +++ b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-August2023.md @@ -1,4 +1,4 @@ -# Codex Continuous Test-net Report +# Logos Storage Continuous Test-net Report Date: 05-09-2023 Report for: 08-2023 @@ -15,7 +15,7 @@ Report for: 08-2023 ## Deployment Configuration Continuous Test-net is deployed to the kubernetes cluster with the following configuration: -5x Codex Nodes: +5x Logos Storage Nodes: - Log-level: Trace - Storage quota: 2048 MB - Storage sell: 1024 MB @@ -29,7 +29,7 @@ Continuous Test-net is deployed to the kubernetes cluster with the following con 3 of these 5 nodes have: - Validator: true -Kubernetes namespace: 'codex-continuous-tests' +Kubernetes namespace: 'storage-continuous-tests' ## Test Overview | Changes | Test | Description | Status | Results | @@ -42,7 +42,7 @@ Kubernetes namespace: 'codex-continuous-tests' ## Resulting changes As a result of the testing efforts in 08-2023, these three changes were made: 1. Block announcement performance - Block iteration (as part of the advertising loop) was found to be performing too much disc IO. Reducing this greatly increased client stability. -1. Component initialization order - A timing issue in the initialization order of the several Codex components caused nodes to improperly announce their addresses over the network in case the marketplace support was enabled. +1. Component initialization order - A timing issue in the initialization order of the several Logos Storage components caused nodes to improperly announce their addresses over the network in case the marketplace support was enabled. 1. Block maintenance performance - Block iteration (again but this time in maintenance) was causing the node to not respond to Ping messages quickly enough, causing other nodes to kick it from the network. By slowing down block maintenance this is solved. A future update of the DHT logic will also impact this behavior. ## Action Points diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-July2023.md b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-July2023.md similarity index 74% rename from Tests/CodexContinuousTests/reports/CodexTestNetReport-July2023.md rename to Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-July2023.md index 9b48b96d..2d7ba4a1 100644 --- a/Tests/CodexContinuousTests/reports/CodexTestNetReport-July2023.md +++ b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-July2023.md @@ -1,4 +1,4 @@ -# Codex Continuous Test-net Report +# Logos Storage Continuous Test-net Report Date: 02-08-2023 Report for: 07-2023 @@ -13,7 +13,7 @@ Report for: 07-2023 ## Deployment Configuration Continuous Test-net is deployed to the kubernetes cluster with the following configuration: -5x Codex Nodes: +5x Logos Storage Nodes: - Log-level: Trace - Storage quota: 2048 MB - Storage sell: 1024 MB @@ -25,14 +25,14 @@ Continuous Test-net is deployed to the kubernetes cluster with the following con 3 of these 5 nodes have: - Validator: true -Kubernetes namespace: 'codex-continuous-tests' +Kubernetes namespace: 'storage-continuous-tests' ## Test Overview | Changes | Test | Description | Status | Results | |----------------|---------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------|--------------------------------------------------------------------------------------| | New in 07-2023 | Two-client test | Every 30 seconds, two nodes are chosen at random. A 10 MB file is generated and uploaded to one. 10 seconds later, it is downloaded from the other. File contents are asserted to be equal. | Faulted | Test reliably fails after 30 to 45 minutes. Both upload and download failures occur. | -| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Codex node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Codex node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted. | +| New in 07-2023 | Transient-node test | Every 1 minute, a new, transient Logos Storage node is started and bootstrapped against a random node of the test net. A 10 MB file is generated and uploaded to the transient node. The file is then downloaded from a second random test net node and file equality is asserted. After that, the transient node is shut down. 30 seconds later, a new transient Logos Storage node is started and bootstrapped against a third (guaranteed different from first and second) node from the test net. The same file is then downloaded from the new transient node. File equality is asserted. | Not running | Test was not run because the previous more rudamentory test has faulted. | ## Action Points -- Codex logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours. +- Logos Storage logs (from these long-running containers) are often incomplete when downloaded after a test failure. A reliable way of maintaining these logs is needed. The logs can quickly explode in size, even for test-nets that run only for a few hours. - The distributed testing setup can and has been used to reproduce the failure of the Two-client test in a local environment. Investigation is on-going. \ No newline at end of file diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-November2023.md similarity index 86% rename from Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md rename to Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-November2023.md index ebe4b827..264832c5 100644 --- a/Tests/CodexContinuousTests/reports/CodexTestNetReport-November2023.md +++ b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-November2023.md @@ -1,16 +1,16 @@ -# Codex Continuous Test-net Report +# Logos Storage Continuous Test-net Report Date: 05-12-2023 Report for: 11-2023 ## Continuous test-net Status -Continuous test runs (which can take many hours or days) can easily be started by team member from the github actions UI. Results are collected and displayed in Grafana. For the time being, we're suspending the effort to have a network of Codex nodes "always online" and continuously being tested, until overal reliability improves. +Continuous test runs (which can take many hours or days) can easily be started by team member from the github actions UI. Results are collected and displayed in Grafana. For the time being, we're suspending the effort to have a network of Logos Storage nodes "always online" and continuously being tested, until overal reliability improves. ## Deployment Configuration Continous Test-net is deployed to the kubernetes cluster with the following configuration: -5x Codex Nodes: +5x Logos Storage Nodes: - Log-level: Trace - Storage quota: 20480 MB - Storage sell: 1024 MB diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-October2023.md b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-October2023.md similarity index 91% rename from Tests/CodexContinuousTests/reports/CodexTestNetReport-October2023.md rename to Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-October2023.md index ad796c4a..9dad0f9a 100644 --- a/Tests/CodexContinuousTests/reports/CodexTestNetReport-October2023.md +++ b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-October2023.md @@ -1,4 +1,4 @@ -# Codex Continuous Test-net Report +# Logos Storage Continuous Test-net Report Date: 13-11-2023 Report for: 10-2023 @@ -13,7 +13,7 @@ Report for: 10-2023 ## Deployment Configuration Continous Test-net is deployed to the kubernetes cluster with the following configuration: -5x Codex Nodes: +5x Logos Storage Nodes: - Log-level: Trace - Storage quota: 2048 MB - Storage sell: 1024 MB @@ -27,7 +27,7 @@ Continous Test-net is deployed to the kubernetes cluster with the following conf 3 of these 5 nodes have: - Validator: true -Kubernetes namespace: 'codex-continuous-tests' +Kubernetes namespace: 'storage-continuous-tests' * Some tests have been performed with alternative (disabled) maintenance parameters: - Block-TTL: 99999999 seconds - Block-MI: 99999999 seconds @@ -44,7 +44,7 @@ Kubernetes namespace: 'codex-continuous-tests' ## Resulting changes As a result of the testing efforts in 10-2023, these changes were made: 1. Consolidation of test logs and metrics using grafana and elastic-search. -1. Investment made in profiling instrumentation in Codex codebase. +1. Investment made in profiling instrumentation in Logos Storage codebase. 1. Some testing effort has been diverted to preparing the necessary infrastructure for the creation of a public testnet by 1-December-2023. ## Action Points diff --git a/Tests/CodexContinuousTests/reports/CodexTestNetReport-September2023.md b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-September2023.md similarity index 87% rename from Tests/CodexContinuousTests/reports/CodexTestNetReport-September2023.md rename to Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-September2023.md index 15a6639e..339be070 100644 --- a/Tests/CodexContinuousTests/reports/CodexTestNetReport-September2023.md +++ b/Tests/LogosStorageContinuousTests/reports/LogosStorageTestNetReport-September2023.md @@ -1,4 +1,4 @@ -# Codex Continuous Test-net Report +# Logos Storage Continuous Test-net Report Date: 05-10-2023 Report for: 09-2023 @@ -13,7 +13,7 @@ Report for: 09-2023 ## Deployment Configuration Continuous Test-net is deployed to the kubernetes cluster with the following configuration: -5x Codex Nodes: +5x Logos Storage Nodes: - Log-level: Trace - Storage quota: 2048 MB - Storage sell: 1024 MB @@ -27,7 +27,7 @@ Continuous Test-net is deployed to the kubernetes cluster with the following con 3 of these 5 nodes have: - Validator: true -Kubernetes namespace: 'codex-continuous-tests' +Kubernetes namespace: 'storage-continuous-tests' * Some tests have been performed with alternative (disabled) maintenance parameters: - Block-TTL: 99999999 seconds - Block-MI: 99999999 seconds @@ -43,9 +43,9 @@ Kubernetes namespace: 'codex-continuous-tests' ## Resulting changes As a result of the testing efforts in 09-2023, these changes were made: -1. Cloud logging infrastructure was upgraded to reliably and verifiably retrieve Codex container logs. +1. Cloud logging infrastructure was upgraded to reliably and verifiably retrieve Logos Storage container logs. 1. Test runners and tools were upgraded to support automated deploy and run from CI environment. -1. Test runners and tools were upgraded to support to automatic building and use of custom locally-built docker images of Codex. (This allows to use of local modifications for quick debugging without the need to go through commit/review/CI workflow.) +1. Test runners and tools were upgraded to support to automatic building and use of custom locally-built docker images of Logos Storage. (This allows to use of local modifications for quick debugging without the need to go through commit/review/CI workflow.) ## Action Points - We'll be using the upgrades to debug the Two-client test. diff --git a/Tests/CodexContinuousTests/run.sh b/Tests/LogosStorageContinuousTests/run.sh similarity index 61% rename from Tests/CodexContinuousTests/run.sh rename to Tests/LogosStorageContinuousTests/run.sh index f9e34b51..34c1d7a5 100644 --- a/Tests/CodexContinuousTests/run.sh +++ b/Tests/LogosStorageContinuousTests/run.sh @@ -1,10 +1,10 @@ dotnet run \ - --project "${TESTS_PATH:-Tests/CodexContinuousTests}" \ + --project "${TESTS_PATH:-Tests/LogosStorageContinuousTests}" \ --kube-config="${KUBECONFIG:-/opt/kubeconfig.yaml}" \ - --codex-deployment="${TESTS_CODEX_DEPLOYMENT:-codex-deployment.json}" \ + --storage-deployment="${TESTS_CODEX_DEPLOYMENT:-storage-deployment.json}" \ --filter="${TESTS_FILTER:-}" \ --data-path="${TESTS_DATA_PATH:-data}" \ - --log-path="${LOGPATH:-/var/log/codex-continuous-tests}" \ + --log-path="${LOGPATH:-/var/log/storage-continuous-tests}" \ --full-container-logs=1 \ --keep=${TESTS_KEEP:-1} \ --stop=${TESTS_STOP:-10} \ diff --git a/Tests/CodexLongTests/BasicTests/DownloadTests.cs b/Tests/LogosStorageLongTests/BasicTests/DownloadTests.cs similarity index 88% rename from Tests/CodexLongTests/BasicTests/DownloadTests.cs rename to Tests/LogosStorageLongTests/BasicTests/DownloadTests.cs index f77b22e5..17ccdcf5 100644 --- a/Tests/CodexLongTests/BasicTests/DownloadTests.cs +++ b/Tests/LogosStorageLongTests/BasicTests/DownloadTests.cs @@ -1,11 +1,11 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using DistTestCore; using FileUtils; using NUnit.Framework; using Utils; -namespace CodexLongTests.BasicTests +namespace LogosStorageLongTests.BasicTests { [TestFixture] public class DownloadTests : AutoBootstrapDistTest @@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests [Values(1, 3, 5)] int numberOfFiles, [Values(10, 50, 100)] int filesizeMb) { - var host = StartCodex(); - var client = StartCodex(); + var host = StartLogosStorage(); + var client = StartLogosStorage(); var testfiles = new List(); var contentIds = new List(); diff --git a/Tests/CodexLongTests/BasicTests/LargeFileTests.cs b/Tests/LogosStorageLongTests/BasicTests/LargeFileTests.cs similarity index 81% rename from Tests/CodexLongTests/BasicTests/LargeFileTests.cs rename to Tests/LogosStorageLongTests/BasicTests/LargeFileTests.cs index 0470e35a..9e2f8205 100644 --- a/Tests/CodexLongTests/BasicTests/LargeFileTests.cs +++ b/Tests/LogosStorageLongTests/BasicTests/LargeFileTests.cs @@ -1,14 +1,14 @@ -using CodexPlugin; -using CodexTests; +using StoragePlugin; +using LogosStorageTests; using DistTestCore; using NUnit.Framework; using NUnit.Framework.Interfaces; using Utils; -namespace CodexLongTests.BasicTests +namespace LogosStorageLongTests.BasicTests { [TestFixture] - public class LargeFileTests : CodexDistTest + public class LargeFileTests : LogosStorageDistTest { #region Abort test run after first failure @@ -48,7 +48,7 @@ namespace CodexLongTests.BasicTests var expectedFile = GenerateTestFile(sizeMB); - var node = StartCodex(s => s.WithStorageQuota((size + 10).MB())); + var node = StartLogosStorage(s => s.WithStorageQuota((size + 10).MB())); var uploadStart = DateTime.UtcNow; var cid = node.UploadFile(expectedFile); @@ -66,10 +66,10 @@ namespace CodexLongTests.BasicTests var uploadTimePerMB = (uploadStart - downloadStart) / sizeInMB; var downloadTimePerMB = (downloadStart - downloadFinished) / sizeInMB; - Assert.That(uploadTimePerMB, Is.LessThan(CodexContainerRecipe.MaxUploadTimePerMegabyte), + Assert.That(uploadTimePerMB, Is.LessThan(LogosStorageContainerRecipe.MaxUploadTimePerMegabyte), "MaxUploadTimePerMegabyte performance threshold breached."); - Assert.That(downloadTimePerMB, Is.LessThan(CodexContainerRecipe.MaxDownloadTimePerMegabyte), + Assert.That(downloadTimePerMB, Is.LessThan(LogosStorageContainerRecipe.MaxDownloadTimePerMegabyte), "MaxDownloadTimePerMegabyte performance threshold breached."); } } diff --git a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs b/Tests/LogosStorageLongTests/BasicTests/TestInfraTests.cs similarity index 75% rename from Tests/CodexLongTests/BasicTests/TestInfraTests.cs rename to Tests/LogosStorageLongTests/BasicTests/TestInfraTests.cs index 7dcf02b0..41157f22 100644 --- a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs +++ b/Tests/LogosStorageLongTests/BasicTests/TestInfraTests.cs @@ -1,17 +1,17 @@ -using CodexTests; +using LogosStorageTests; using DistTestCore; using NUnit.Framework; -namespace CodexLongTests.BasicTests +namespace LogosStorageLongTests.BasicTests { - public class TestInfraTests : CodexDistTest + public class TestInfraTests : LogosStorageDistTest { [Test] [UseLongTimeouts] [Ignore("Not supported atm")] public void TestInfraShouldHave1000AddressSpacesPerPod() { - var group = StartCodex(1000, s => s.EnableMetrics()); + var group = StartLogosStorage(1000, s => s.EnableMetrics()); var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray(); @@ -26,7 +26,7 @@ namespace CodexLongTests.BasicTests { for (var i = 0; i < 20; i++) { - var n = StartCodex(); + var n = StartLogosStorage(); Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id)); } diff --git a/Tests/CodexLongTests/BasicTests/UploadTests.cs b/Tests/LogosStorageLongTests/BasicTests/UploadTests.cs similarity index 86% rename from Tests/CodexLongTests/BasicTests/UploadTests.cs rename to Tests/LogosStorageLongTests/BasicTests/UploadTests.cs index aaa9f1fd..64420d3d 100644 --- a/Tests/CodexLongTests/BasicTests/UploadTests.cs +++ b/Tests/LogosStorageLongTests/BasicTests/UploadTests.cs @@ -1,11 +1,11 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using DistTestCore; using FileUtils; using NUnit.Framework; using Utils; -namespace CodexLongTests.BasicTests +namespace LogosStorageLongTests.BasicTests { [TestFixture] public class UploadTests : AutoBootstrapDistTest @@ -17,8 +17,8 @@ namespace CodexLongTests.BasicTests [Values(1, 3, 5)] int numberOfFiles, [Values(10, 50, 100)] int filesizeMb) { - var host = StartCodex(); - var client = StartCodex(); + var host = StartLogosStorage(); + var client = StartLogosStorage(); var testfiles = new List(); var contentIds = new List(); diff --git a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs b/Tests/LogosStorageLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs similarity index 77% rename from Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs rename to Tests/LogosStorageLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs index eaa999eb..e75f9f6a 100644 --- a/Tests/CodexLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs +++ b/Tests/LogosStorageLongTests/DownloadConnectivityTests/LongFullyConnectedDownloadTests.cs @@ -1,9 +1,9 @@ -using CodexTests; +using LogosStorageTests; using DistTestCore; using NUnit.Framework; using Utils; -namespace CodexLongTests.DownloadConnectivityTests +namespace LogosStorageLongTests.DownloadConnectivityTests { [TestFixture] public class LongFullyConnectedDownloadTests : AutoBootstrapDistTest @@ -15,7 +15,7 @@ namespace CodexLongTests.DownloadConnectivityTests [Values(10, 15, 20)] int numberOfNodes, [Values(10, 100)] int sizeMBs) { - var nodes = StartCodex(numberOfNodes); + var nodes = StartLogosStorage(numberOfNodes); CreatePeerDownloadTestHelpers().AssertFullDownloadInterconnectivity(nodes, sizeMBs.MB()); } diff --git a/Tests/CodexLongTests/CodexTestsLong.csproj b/Tests/LogosStorageLongTests/LogosStorageTestsLong.csproj similarity index 100% rename from Tests/CodexLongTests/CodexTestsLong.csproj rename to Tests/LogosStorageLongTests/LogosStorageTestsLong.csproj diff --git a/Tests/CodexLongTests/Parallelism.cs b/Tests/LogosStorageLongTests/Parallelism.cs similarity index 67% rename from Tests/CodexLongTests/Parallelism.cs rename to Tests/LogosStorageLongTests/Parallelism.cs index 51709b8d..a3acc93e 100644 --- a/Tests/CodexLongTests/Parallelism.cs +++ b/Tests/LogosStorageLongTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; [assembly: LevelOfParallelism(1)] -namespace CodexLongTests +namespace LogosStorageLongTests { } diff --git a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs b/Tests/LogosStorageLongTests/ScalabilityTests/MultiPeerDownloadTests.cs similarity index 91% rename from Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs rename to Tests/LogosStorageLongTests/ScalabilityTests/MultiPeerDownloadTests.cs index d4597acb..37f82fee 100644 --- a/Tests/CodexLongTests/ScalabilityTests/MultiPeerDownloadTests.cs +++ b/Tests/LogosStorageLongTests/ScalabilityTests/MultiPeerDownloadTests.cs @@ -1,9 +1,9 @@ -using CodexClient; +using LogosStorageClient; using DistTestCore; using NUnit.Framework; using Utils; -namespace CodexTests.ScalabilityTests +namespace LogosStorageTests.ScalabilityTests { [TestFixture] public class MultiPeerDownloadTests : AutoBootstrapDistTest @@ -17,7 +17,7 @@ namespace CodexTests.ScalabilityTests [Values(100, 1000)] int fileSize ) { - var hosts = StartCodex(numberOfHosts, s => s.WithName("host").WithLogLevel(CodexLogLevel.Trace)); + var hosts = StartLogosStorage(numberOfHosts, s => s.WithName("host").WithLogLevel(LogosStorageLogLevel.Trace)); var file = GenerateTestFile(fileSize.MB()); var uploadTasks = hosts.Select(h => @@ -37,7 +37,7 @@ namespace CodexTests.ScalabilityTests .FindLinesThatContain("Block Stored") .Select(s => { - var line = CodexLogLine.Parse(s)!; + var line = LogosStorageLogLine.Parse(s)!; return line.Attributes["cid"]; }) .ToArray(); @@ -46,7 +46,7 @@ namespace CodexTests.ScalabilityTests - var client = StartCodex(s => s.WithName("client").WithLogLevel(CodexLogLevel.Trace)); + var client = StartLogosStorage(s => s.WithName("client").WithLogLevel(LogosStorageLogLevel.Trace)); var resultFile = client.DownloadContent(cid); resultFile!.AssertIsEqual(file); @@ -56,7 +56,7 @@ namespace CodexTests.ScalabilityTests downloadLog .IterateLines(s => { - var line = CodexLogLine.Parse(s)!; + var line = LogosStorageLogLine.Parse(s)!; var peer = line.Attributes["peer"]; var blockAddresses = line.Attributes["blocks"]; diff --git a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs b/Tests/LogosStorageLongTests/ScalabilityTests/ScalabilityTests.cs similarity index 84% rename from Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs rename to Tests/LogosStorageLongTests/ScalabilityTests/ScalabilityTests.cs index dd92de84..90b6c2db 100644 --- a/Tests/CodexLongTests/ScalabilityTests/ScalabilityTests.cs +++ b/Tests/LogosStorageLongTests/ScalabilityTests/ScalabilityTests.cs @@ -1,13 +1,13 @@ -using CodexClient; +using LogosStorageClient; using DistTestCore; using FileUtils; using NUnit.Framework; using Utils; -namespace CodexTests.ScalabilityTests; +namespace LogosStorageTests.ScalabilityTests; [TestFixture] -public class ScalabilityTests : CodexDistTest +public class ScalabilityTests : LogosStorageDistTest { /// /// We upload a file to node A, then download it with B. @@ -23,10 +23,10 @@ public class ScalabilityTests : CodexDistTest [Values(4000, 5000, 6000, 7000, 8000, 9000, 10000)] int fileSizeInMb ) { - var logLevel = CodexLogLevel.Trace; + var logLevel = LogosStorageLogLevel.Trace; - var bootstrap = StartCodex(s => s.WithLogLevel(logLevel)); - var nodes = StartCodex(numberOfNodes - 1, s => s + var bootstrap = StartLogosStorage(s => s.WithLogLevel(logLevel)); + var nodes = StartLogosStorage(numberOfNodes - 1, s => s .WithBootstrapNode(bootstrap) .WithLogLevel(logLevel) .WithStorageQuota((fileSizeInMb + 50).MB()) @@ -74,10 +74,10 @@ public class ScalabilityTests : CodexDistTest [Values(100, 1000, 5000, 10000)] int fileSizeInMb ) { - var logLevel = CodexLogLevel.Info; + var logLevel = LogosStorageLogLevel.Info; - var bootstrap = StartCodex(s => s.WithLogLevel(logLevel)); - var nodes = StartCodex(numberOfNodes - 1, s => s + var bootstrap = StartLogosStorage(s => s.WithLogLevel(logLevel)); + var nodes = StartLogosStorage(numberOfNodes - 1, s => s .WithBootstrapNode(bootstrap) .WithLogLevel(logLevel) .WithStorageQuota((fileSizeInMb + 50).MB()) @@ -118,14 +118,14 @@ public class ScalabilityTests : CodexDistTest public class NodeFilePair { - public NodeFilePair(ICodexNode node, TrackedFile file, ContentId cid) + public NodeFilePair(IStorageNode node, TrackedFile file, ContentId cid) { Node = node; File = file; Cid = cid; } - public ICodexNode Node { get; } + public IStorageNode Node { get; } public TrackedFile File { get; } public ContentId Cid { get; } } diff --git a/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs b/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs similarity index 84% rename from Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs index 8e989a73..0c4f8424 100644 --- a/Tests/CodexReleaseTests/DataTests/DataExpiryTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/DataExpiryTest.cs @@ -1,18 +1,18 @@ -using CodexPlugin; -using CodexTests; +using StoragePlugin; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class DataExpiryTest : CodexDistTest + public class DataExpiryTest : LogosStorageDistTest { private readonly TimeSpan blockTtl = TimeSpan.FromMinutes(1.0); private readonly TimeSpan blockInterval = TimeSpan.FromSeconds(10.0); private readonly int blockCount = 100000; - private ICodexSetup WithFastBlockExpiry(ICodexSetup setup) + private ILogosStorageSetup WithFastBlockExpiry(ILogosStorageSetup setup) { return setup .WithBlockTTL(blockTtl) @@ -24,7 +24,7 @@ namespace CodexReleaseTests.DataTests public void DeletesExpiredData() { var fileSize = 3.MB(); - var node = StartCodex(s => WithFastBlockExpiry(s)); + var node = StartLogosStorage(s => WithFastBlockExpiry(s)); var startSpace = node.Space(); Assert.That(startSpace.QuotaUsedBytes, Is.EqualTo(0)); diff --git a/Tests/CodexReleaseTests/DataTests/DhtTest.cs b/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs similarity index 88% rename from Tests/CodexReleaseTests/DataTests/DhtTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs index e4a99b8e..5246204a 100644 --- a/Tests/CodexReleaseTests/DataTests/DhtTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/DhtTest.cs @@ -1,10 +1,10 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using Logging; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests.DHT +namespace LogosStorageReleaseTests.DataTests.DHT { [Ignore("work in progress")] [TestFixture(10, 1000)] @@ -31,7 +31,7 @@ namespace CodexReleaseTests.DataTests.DHT public void PressureTest() { var cids = new List(); - var nodes = StartCodex(numNodes); + var nodes = StartLogosStorage(numNodes); for (var i = 0; i < numFilesPerNode; i++) { @@ -45,7 +45,7 @@ namespace CodexReleaseTests.DataTests.DHT var estimate = numNodes * numFilesPerNode * 2; Log($"Estimate of DHT records: {estimate}"); - var node = StartCodex(); + var node = StartLogosStorage(); for (var i = 0; i < numToFetch; i++) { var timing = Stopwatch.Measure(GetTestLog(), nameof(PressureTest) + i, () => diff --git a/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs similarity index 76% rename from Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs index be385f48..fea8b06c 100644 --- a/Tests/CodexReleaseTests/DataTests/InterruptUploadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/InterruptUploadTest.cs @@ -1,18 +1,18 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using FileUtils; using NUnit.Framework; using System.Diagnostics; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { - public class InterruptUploadTest : CodexDistTest + public class InterruptUploadTest : LogosStorageDistTest { [Test] public void UploadInterruptTest() { - var nodes = StartCodex(10); + var nodes = StartLogosStorage(10); var tasks = nodes.Select(n => Task.Run(() => RunInterruptUploadTest(n))); Task.WaitAll(tasks.ToArray()); @@ -22,7 +22,7 @@ namespace CodexReleaseTests.DataTests WaitAndCheckNodesStaysAlive(TimeSpan.FromMinutes(2), nodes); } - private bool RunInterruptUploadTest(ICodexNode node) + private bool RunInterruptUploadTest(IStorageNode node) { var file = GenerateTestFile(300.MB()); @@ -36,7 +36,7 @@ namespace CodexReleaseTests.DataTests return !log.GetLinesContaining("Unhandled exception in async proc, aborting").Any(); } - private Process StartCurlUploadProcess(ICodexNode node, TrackedFile file) + private Process StartCurlUploadProcess(IStorageNode node, TrackedFile file) { var apiAddress = node.GetApiEndpoint(); var codexUrl = $"{apiAddress}/api/storage/v1/data"; diff --git a/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs b/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs similarity index 84% rename from Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs index ef1b8526..05e47d45 100644 --- a/Tests/CodexReleaseTests/DataTests/LocalFilesTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/LocalFilesTest.cs @@ -1,4 +1,4 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using System; using System.Collections.Generic; @@ -7,15 +7,15 @@ using System.Text; using System.Threading.Tasks; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class LocalFilesTest : CodexDistTest + public class LocalFilesTest : LogosStorageDistTest { [Test] public void ShouldShowLocalFiles() { - var node = StartCodex(); + var node = StartLogosStorage(); var size1 = 123.KB(); var size2 = 23.MB(); diff --git a/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs similarity index 74% rename from Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs index f1fcc84c..af4a21ea 100644 --- a/Tests/CodexReleaseTests/DataTests/ManifestOnlyDownloadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/ManifestOnlyDownloadTest.cs @@ -1,17 +1,17 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class ManifestOnlyDownloadTest : CodexDistTest + public class ManifestOnlyDownloadTest : LogosStorageDistTest { [Test] public void ManifestOnlyTest() { - var uploader = StartCodex(); - var downloader = StartCodex(s => s.WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(); + var downloader = StartLogosStorage(s => s.WithBootstrapNode(uploader)); var file = GenerateTestFile(2.GB()); var size = file.GetFilesize().SizeInBytes; diff --git a/Tests/CodexReleaseTests/DataTests/OneClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs similarity index 66% rename from Tests/CodexReleaseTests/DataTests/OneClientTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs index 2abf009a..2bd057c7 100644 --- a/Tests/CodexReleaseTests/DataTests/OneClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/OneClientTest.cs @@ -1,24 +1,24 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class OneClientTest : CodexDistTest + public class OneClientTest : LogosStorageDistTest { [Test] public void OneClient() { - var node = StartCodex(); + var node = StartLogosStorage(); PerformOneClientTest(node); LogNodeStatus(node); } - private void PerformOneClientTest(ICodexNode primary) + private void PerformOneClientTest(IStorageNode primary) { var testFile = GenerateTestFile(1.MB()); diff --git a/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs b/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs similarity index 76% rename from Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs index 93b89f19..ad377258 100644 --- a/Tests/CodexReleaseTests/DataTests/StreamlessDownloadTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/StreamlessDownloadTest.cs @@ -1,17 +1,17 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class StreamlessDownloadTest : CodexDistTest + public class StreamlessDownloadTest : LogosStorageDistTest { [Test] public void StreamlessTest() { - var uploader = StartCodex(); - var downloader = StartCodex(s => s.WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(); + var downloader = StartLogosStorage(s => s.WithBootstrapNode(uploader)); var size = 10.MB(); var file = GenerateTestFile(size); diff --git a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs b/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs similarity index 89% rename from Tests/CodexReleaseTests/DataTests/SwarmTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs index 2379fb08..9e084994 100644 --- a/Tests/CodexReleaseTests/DataTests/SwarmTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/SwarmTest.cs @@ -1,11 +1,11 @@ -using CodexClient; -using CodexPlugin; -using CodexTests; +using LogosStorageClient; +using StoragePlugin; +using LogosStorageTests; using FileUtils; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { namespace SwarmTests { @@ -16,7 +16,7 @@ namespace CodexReleaseTests.DataTests { private readonly int numberOfNodes; private readonly int filesizeMb; - private ICodexNodeGroup nodes = null!; + private IStorageNodeGroup nodes = null!; public SwarmTests(int numberOfNodes, int filesizeMb) { @@ -40,7 +40,7 @@ namespace CodexReleaseTests.DataTests public void Stream() { var filesize = filesizeMb.MB(); - nodes = StartCodex(numberOfNodes); + nodes = StartLogosStorage(numberOfNodes); var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); var tasks = ParallelDownloadEachFile(files); @@ -53,7 +53,7 @@ namespace CodexReleaseTests.DataTests public void Streamless() { var filesize = filesizeMb.MB(); - nodes = StartCodex(numberOfNodes); + nodes = StartLogosStorage(numberOfNodes); var files = nodes.Select(n => UploadUniqueFilePerNode(n, filesize)).ToArray(); var tasks = ParallelStreamlessDownloadEachFile(files); @@ -62,7 +62,7 @@ namespace CodexReleaseTests.DataTests AssertAllFilesStreamlesslyDownloadedCorrectly(files); } - private SwarmTestNetworkFile UploadUniqueFilePerNode(ICodexNode node, ByteSize fileSize) + private SwarmTestNetworkFile UploadUniqueFilePerNode(IStorageNode node, ByteSize fileSize) { var file = GenerateTestFile(fileSize); var cid = node.UploadFile(file); @@ -93,7 +93,7 @@ namespace CodexReleaseTests.DataTests return tasks.ToArray(); } - private Task StartDownload(ICodexNode node, SwarmTestNetworkFile[] files) + private Task StartDownload(IStorageNode node, SwarmTestNetworkFile[] files) { return Task.Run(() => { @@ -118,7 +118,7 @@ namespace CodexReleaseTests.DataTests }); } - private Task StartStreamlessDownload(ICodexNode node, SwarmTestNetworkFile[] files) + private Task StartStreamlessDownload(IStorageNode node, SwarmTestNetworkFile[] files) { return Task.Run(() => { @@ -176,7 +176,7 @@ namespace CodexReleaseTests.DataTests private class SwarmTestNetworkFile { - public SwarmTestNetworkFile(ICodexNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) + public SwarmTestNetworkFile(IStorageNode uploader, ByteSize originalSize, TrackedFile original, ContentId cid) { Uploader = uploader; OriginalSize = originalSize; @@ -184,7 +184,7 @@ namespace CodexReleaseTests.DataTests Cid = cid; } - public ICodexNode Uploader { get; } + public IStorageNode Uploader { get; } public ByteSize OriginalSize { get; } public TrackedFile Original { get; } public ContentId Cid { get; } diff --git a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs b/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs similarity index 78% rename from Tests/CodexReleaseTests/DataTests/TheseusTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs index 6f07a34b..c444dedf 100644 --- a/Tests/CodexReleaseTests/DataTests/TheseusTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/TheseusTest.cs @@ -1,16 +1,16 @@ -using CodexClient; -using CodexPlugin; -using CodexTests; +using LogosStorageClient; +using StoragePlugin; +using LogosStorageTests; using FileUtils; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] public class TheseusTest : AutoBootstrapDistTest { - private readonly List nodes = new List(); + private readonly List nodes = new List(); private TrackedFile file = null!; private ContentId cid = new ContentId(); @@ -29,7 +29,7 @@ namespace CodexReleaseTests.DataTests Assert.That(remainingNodes, Is.GreaterThan(0)); Assert.That(steps, Is.GreaterThan(remainingNodes + 1)); - nodes.AddRange(StartCodex(remainingNodes + 1)); + nodes.AddRange(StartLogosStorage(remainingNodes + 1)); cid = nodes.First().UploadFile(file); AllNodesHaveFile(); @@ -40,7 +40,7 @@ namespace CodexReleaseTests.DataTests nodes[0].Stop(waitTillStopped: true); nodes.RemoveAt(0); - nodes.Add(StartCodex()); + nodes.Add(StartLogosStorage()); AllNodesHaveFile(); } @@ -52,7 +52,7 @@ namespace CodexReleaseTests.DataTests foreach (var n in nodes) HasFile(n); } - private void HasFile(ICodexNode n) + private void HasFile(IStorageNode n) { var downloaded = n.DownloadContent(cid); file.AssertIsEqual(downloaded); diff --git a/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs similarity index 76% rename from Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs index 97326853..69de0fbf 100644 --- a/Tests/CodexReleaseTests/DataTests/ThreeClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/ThreeClientTest.cs @@ -1,16 +1,16 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { public class ThreeClientTest : AutoBootstrapDistTest { [Test] public void ThreeClient() { - var primary = StartCodex(); - var secondary = StartCodex(); + var primary = StartLogosStorage(); + var secondary = StartLogosStorage(); var testFile = GenerateTestFile(10.MB()); diff --git a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs b/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs similarity index 60% rename from Tests/CodexReleaseTests/DataTests/TwoClientTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs index 8ab9c88d..71e6d4ad 100644 --- a/Tests/CodexReleaseTests/DataTests/TwoClientTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/TwoClientTest.cs @@ -1,19 +1,19 @@ -using CodexClient; -using CodexPlugin; -using CodexTests; +using LogosStorageClient; +using StoragePlugin; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class TwoClientTests : CodexDistTest + public class TwoClientTests : LogosStorageDistTest { [Test] public void TwoClientTest() { - var uploader = StartCodex(s => s.WithName("Uploader")); - var downloader = StartCodex(s => s.WithName("Downloader").WithBootstrapNode(uploader)); + var uploader = StartLogosStorage(s => s.WithName("Uploader")); + var downloader = StartLogosStorage(s => s.WithName("Downloader").WithBootstrapNode(uploader)); PerformTwoClientTest(uploader, downloader); } @@ -29,18 +29,18 @@ namespace CodexReleaseTests.DataTests return; } - var uploader = Ci.StartCodexNode(s => s.WithName("Uploader").At(locations.Get(0))); - var downloader = Ci.StartCodexNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1))); + var uploader = Ci.StartStorageNode(s => s.WithName("Uploader").At(locations.Get(0))); + var downloader = Ci.StartStorageNode(s => s.WithName("Downloader").WithBootstrapNode(uploader).At(locations.Get(1))); PerformTwoClientTest(uploader, downloader); } - private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader) + private void PerformTwoClientTest(IStorageNode uploader, IStorageNode downloader) { PerformTwoClientTest(uploader, downloader, 10.MB()); } - private void PerformTwoClientTest(ICodexNode uploader, ICodexNode downloader, ByteSize size) + private void PerformTwoClientTest(IStorageNode uploader, IStorageNode downloader, ByteSize size) { var testFile = GenerateTestFile(size); diff --git a/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs b/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs similarity index 80% rename from Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs rename to Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs index 40b7f9f1..180a0f92 100644 --- a/Tests/CodexReleaseTests/DataTests/UnknownCidTest.cs +++ b/Tests/LogosStorageReleaseTests/DataTests/UnknownCidTest.cs @@ -1,16 +1,16 @@ -using CodexClient; -using CodexTests; +using LogosStorageClient; +using LogosStorageTests; using NUnit.Framework; -namespace CodexReleaseTests.DataTests +namespace LogosStorageReleaseTests.DataTests { [TestFixture] - public class UnknownCidTest : CodexDistTest + public class UnknownCidTest : LogosStorageDistTest { [Test] public void DownloadingUnknownCidDoesNotCauseCrash() { - var node = StartCodex(); + var node = StartLogosStorage(); var unknownCid = new ContentId("zDvZRwzkzHsok3Z8yMoiXE9EDBFwgr8WygB8s4ddcLzzSwwXAxLZ"); diff --git a/Tests/CodexReleaseTests/CodexReleaseTests.csproj b/Tests/LogosStorageReleaseTests/LogosStorageReleaseTests.csproj similarity index 100% rename from Tests/CodexReleaseTests/CodexReleaseTests.csproj rename to Tests/LogosStorageReleaseTests/LogosStorageReleaseTests.csproj diff --git a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs b/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs similarity index 78% rename from Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs rename to Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs index f6cba228..139e709d 100644 --- a/Tests/CodexReleaseTests/NodeTests/BasicInfoTests.cs +++ b/Tests/LogosStorageReleaseTests/NodeTests/BasicInfoTests.cs @@ -1,17 +1,17 @@ -using CodexTests; +using LogosStorageTests; using NUnit.Framework; using Utils; -namespace CodexReleaseTests.NodeTests +namespace LogosStorageReleaseTests.NodeTests { [TestFixture] - public class BasicInfoTests : CodexDistTest + public class BasicInfoTests : LogosStorageDistTest { [Test] public void QuotaTest() { var size = 3.GB(); - var node = StartCodex(s => s.WithStorageQuota(size)); + var node = StartLogosStorage(s => s.WithStorageQuota(size)); var space = node.Space(); Assert.That(space.QuotaMaxBytes, Is.EqualTo(size.SizeInBytes)); @@ -20,7 +20,7 @@ namespace CodexReleaseTests.NodeTests [Test] public void Spr() { - var node = StartCodex(); + var node = StartLogosStorage(); var info = node.GetDebugInfo(); Assert.That(!string.IsNullOrEmpty(info.Spr)); @@ -34,7 +34,7 @@ namespace CodexReleaseTests.NodeTests [Test] public void VersionInfo() { - var node = StartCodex(); + var node = StartLogosStorage(); var info = node.GetDebugInfo(); Assert.That(!string.IsNullOrEmpty(info.Version.Version)); @@ -44,7 +44,7 @@ namespace CodexReleaseTests.NodeTests [Test] public void AnnounceAddress() { - var node = StartCodex(); + var node = StartLogosStorage(); var addr = node.GetListenEndpoint(); var info = node.GetDebugInfo(); diff --git a/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs b/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs similarity index 60% rename from Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs rename to Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs index 74715b66..81acfa7e 100644 --- a/Tests/CodexReleaseTests/NodeTests/PeerTableTests.cs +++ b/Tests/LogosStorageReleaseTests/NodeTests/PeerTableTests.cs @@ -1,9 +1,9 @@ -using CodexClient; -using CodexTests; -using CodexTests.Helpers; +using LogosStorageClient; +using LogosStorageTests; +using LogosStorageTests.Helpers; using NUnit.Framework; -namespace CodexReleaseTests.NodeTests +namespace LogosStorageReleaseTests.NodeTests { [TestFixture] public class PeerTableTests : AutoBootstrapDistTest @@ -11,12 +11,12 @@ namespace CodexReleaseTests.NodeTests [Test] public void PeerTableCompleteness() { - var nodes = StartCodex(10); + var nodes = StartLogosStorage(10); AssertAllNodesSeeEachOther(nodes.Concat([BootstrapNode!])); } - private void AssertAllNodesSeeEachOther(IEnumerable nodes) + private void AssertAllNodesSeeEachOther(IEnumerable nodes) { var helper = new PeerConnectionTestHelpers(GetTestLog()); helper.AssertFullyConnected(nodes); diff --git a/Tests/CodexReleaseTests/Parallelism.cs b/Tests/LogosStorageReleaseTests/Parallelism.cs similarity index 65% rename from Tests/CodexReleaseTests/Parallelism.cs rename to Tests/LogosStorageReleaseTests/Parallelism.cs index 44731c5e..09111c87 100644 --- a/Tests/CodexReleaseTests/Parallelism.cs +++ b/Tests/LogosStorageReleaseTests/Parallelism.cs @@ -1,6 +1,6 @@ using NUnit.Framework; [assembly: LevelOfParallelism(2)] -namespace CodexReleaseTests +namespace LogosStorageReleaseTests { } diff --git a/Tests/CodexReleaseTests/RerunAttribute.cs b/Tests/LogosStorageReleaseTests/RerunAttribute.cs similarity index 90% rename from Tests/CodexReleaseTests/RerunAttribute.cs rename to Tests/LogosStorageReleaseTests/RerunAttribute.cs index d30b010d..38be38c7 100644 --- a/Tests/CodexReleaseTests/RerunAttribute.cs +++ b/Tests/LogosStorageReleaseTests/RerunAttribute.cs @@ -1,6 +1,6 @@ using NUnit.Framework; -namespace CodexReleaseTests +namespace LogosStorageReleaseTests { public class RerunAttribute : ValuesAttribute { diff --git a/Tools/CsvCombiner/Program.cs b/Tools/CsvCombiner/Program.cs index 6e71cd07..ae292671 100644 --- a/Tools/CsvCombiner/Program.cs +++ b/Tools/CsvCombiner/Program.cs @@ -4,7 +4,7 @@ public class Program { public static void Main(string[] args) { - args = ["d:\\CodexTestLogs\\BlockExchange\\experiment2-fetchbatched"]; + args = ["d:\\LogosStorageTestLogs\\BlockExchange\\experiment2-fetchbatched"]; var p = new Program(args[0]); p.Run(); } diff --git a/Tools/TestClusterStarter/ClusterTestSpec.cs b/Tools/TestClusterStarter/ClusterTestSpec.cs index f44be214..593a67e1 100644 --- a/Tools/TestClusterStarter/ClusterTestSpec.cs +++ b/Tools/TestClusterStarter/ClusterTestSpec.cs @@ -14,20 +14,20 @@ namespace TestClusterStarter public class ClusterTestSpec { - public ClusterTestSpec(string name, string filter, int replication, int durationSeconds, string? codexImageOverride) + public ClusterTestSpec(string name, string filter, int replication, int durationSeconds, string? logosStorageImageOverride) { Name = name; Filter = filter; Replication = replication; DurationSeconds = durationSeconds; - CodexImageOverride = codexImageOverride; + LogosStorageImageOverride = logosStorageImageOverride; } public string Name { get; } public string Filter { get; } public int Replication { get; } public int DurationSeconds { get; } - public string? CodexImageOverride { get; } + public string? LogosStorageImageOverride { get; } } public class ClusterTestDeployment diff --git a/Tools/TestClusterStarter/Program.cs b/Tools/TestClusterStarter/Program.cs index 9fe44cec..7a4edf9b 100644 --- a/Tools/TestClusterStarter/Program.cs +++ b/Tools/TestClusterStarter/Program.cs @@ -39,7 +39,7 @@ public class Program filter: spec.Filter, duration: TimeSpan.FromSeconds(spec.DurationSeconds), replications: spec.Replication, - codexImageOverride: spec.CodexImageOverride)); + logosStorageImageOverride: spec.LogosStorageImageOverride)); rcs.Add(rc); } diff --git a/Tools/TranscriptAnalysis/Program.cs b/Tools/TranscriptAnalysis/Program.cs index 47868c41..320e98b2 100644 --- a/Tools/TranscriptAnalysis/Program.cs +++ b/Tools/TranscriptAnalysis/Program.cs @@ -1,4 +1,4 @@ -using CodexPlugin.OverwatchSupport; +using StoragePlugin.OverwatchSupport; using Logging; using OverwatchTranscript; using TranscriptAnalysis; @@ -30,7 +30,7 @@ public static class Program CloseReader(reader); }; - var header = reader.GetHeader("cdx_h"); + var header = reader.GetHeader("cdx_h"); var receivers = new ReceiverSet(args[0], log, reader, header); receivers.InitAll(); diff --git a/Tools/TranscriptAnalysis/ReceiverSet.cs b/Tools/TranscriptAnalysis/ReceiverSet.cs index 35c2a700..869b4e4d 100644 --- a/Tools/TranscriptAnalysis/ReceiverSet.cs +++ b/Tools/TranscriptAnalysis/ReceiverSet.cs @@ -1,4 +1,4 @@ -using CodexPlugin.OverwatchSupport; +using StoragePlugin.OverwatchSupport; using Logging; using OverwatchTranscript; using TranscriptAnalysis.Receivers; @@ -7,7 +7,7 @@ namespace TranscriptAnalysis { public interface IEventReceiver { - void Init(string sourceFilename, ILog log, OverwatchCodexHeader header); + void Init(string sourceFilename, ILog log, OverwatchLogosStorageHeader header); void Finish(); } @@ -21,10 +21,10 @@ namespace TranscriptAnalysis private readonly string sourceFilename; private readonly ILog log; private readonly ITranscriptReader reader; - private readonly OverwatchCodexHeader header; + private readonly OverwatchLogosStorageHeader header; private readonly List receivers = new List(); - public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchCodexHeader header) + public ReceiverSet(string sourceFilename, ILog log, ITranscriptReader reader, OverwatchLogosStorageHeader header) { this.sourceFilename = sourceFilename; this.log = log; diff --git a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs index c8ed20db..9c8e203d 100644 --- a/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs +++ b/Tools/TranscriptAnalysis/Receivers/BaseReceiver.cs @@ -1,4 +1,4 @@ -using CodexPlugin.OverwatchSupport; +using StoragePlugin.OverwatchSupport; using Logging; using OverwatchTranscript; @@ -7,7 +7,7 @@ namespace TranscriptAnalysis.Receivers public abstract class BaseReceiver : IEventReceiver { protected ILog log { get; private set; } = new NullLog(); - protected OverwatchCodexHeader Header { get; private set; } = null!; + protected OverwatchLogosStorageHeader Header { get; private set; } = null!; protected CsvWriter CsvWriter { get; private set; } protected string SourceFilename { get; private set; } = string.Empty; @@ -20,7 +20,7 @@ namespace TranscriptAnalysis.Receivers CsvWriter = new CsvWriter(log); } - public void Init(string sourceFilename, ILog log, OverwatchCodexHeader header) + public void Init(string sourceFilename, ILog log, OverwatchLogosStorageHeader header) { this.log = new LogPrefixer(log, $"({Name}) "); Header = header; @@ -37,7 +37,7 @@ namespace TranscriptAnalysis.Receivers return GetIdentity(nodeIndex)?.Name; } - protected CodexNodeIdentity? GetIdentity(int nodeIndex) + protected StorageNodeIdentity? GetIdentity(int nodeIndex) { if (nodeIndex < 0) return null; return Header.Nodes[nodeIndex]; diff --git a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs index cbdd71cc..d03f9bcf 100644 --- a/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs +++ b/Tools/TranscriptAnalysis/Receivers/DuplicateBlocksReceived.cs @@ -1,16 +1,16 @@ -using CodexPlugin.OverwatchSupport; +using StoragePlugin.OverwatchSupport; using OverwatchTranscript; namespace TranscriptAnalysis.Receivers { - public class DuplicateBlocksReceived : BaseReceiver + public class DuplicateBlocksReceived : BaseReceiver { public static List Counts = new List(); private long uploadSize; public override string Name => "BlocksReceived"; - public override void Receive(ActivateEvent @event) + public override void Receive(ActivateEvent @event) { if (@event.Payload.BlockReceived != null) { @@ -65,7 +65,7 @@ namespace TranscriptAnalysis.Receivers private int seen = 0; private readonly Dictionary> peerIdBlockAddrCount = new Dictionary>(); - private void Handle(OverwatchCodexEvent payload, BlockReceivedEvent blockReceived) + private void Handle(OverwatchLogosStorageEvent payload, BlockReceivedEvent blockReceived) { var receiverPeerId = GetPeerId(payload.NodeIdentity); if (receiverPeerId == null) return; diff --git a/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs b/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs index a20f656c..69cacf82 100644 --- a/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs +++ b/Tools/TranscriptAnalysis/Receivers/LogReplaceReceiver.cs @@ -1,16 +1,16 @@ -using CodexClient; -using CodexPlugin.OverwatchSupport; +using LogosStorageClient; +using StoragePlugin.OverwatchSupport; using OverwatchTranscript; namespace TranscriptAnalysis.Receivers { - public class LogReplaceReceiver : BaseReceiver + public class LogReplaceReceiver : BaseReceiver { public override string Name => "LogReplacer"; private readonly List seen = new List(); - public override void Receive(ActivateEvent @event) + public override void Receive(ActivateEvent @event) { var peerId = GetPeerId(@event.Payload.NodeIdentity); var name = GetName(@event.Payload.NodeIdentity); @@ -22,7 +22,7 @@ namespace TranscriptAnalysis.Receivers seen.Add(peerId); log.AddStringReplace(peerId, name); - log.AddStringReplace(CodexUtils.ToShortId(peerId), name); + log.AddStringReplace(LogosStorageUtils.ToShortId(peerId), name); } } diff --git a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs index d37b3d38..a076a601 100644 --- a/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs +++ b/Tools/TranscriptAnalysis/Receivers/NodesDegree.cs @@ -1,10 +1,10 @@ -using CodexClient; -using CodexPlugin.OverwatchSupport; +using LogosStorageClient; +using StoragePlugin.OverwatchSupport; using OverwatchTranscript; namespace TranscriptAnalysis.Receivers { - public class NodesDegree : BaseReceiver + public class NodesDegree : BaseReceiver { public class Dial { @@ -50,7 +50,7 @@ namespace TranscriptAnalysis.Receivers public override string Name => "NodesDegree"; - public override void Receive(ActivateEvent @event) + public override void Receive(ActivateEvent @event) { if (@event.Payload.DialSuccessful != null) { @@ -109,8 +109,8 @@ namespace TranscriptAnalysis.Receivers private void AddDial(string peerId, string targetPeerId) { - peerId = CodexUtils.ToShortId(peerId); - targetPeerId = CodexUtils.ToShortId(targetPeerId); + peerId = LogosStorageUtils.ToShortId(peerId); + targetPeerId = LogosStorageUtils.ToShortId(targetPeerId); var peer = GetNode(peerId); var target = GetNode(targetPeerId); ; diff --git a/Tools/TranscriptAnalysis/TranscriptAnalysis.csproj b/Tools/TranscriptAnalysis/TranscriptAnalysis.csproj index ab2d295e..bbe8e923 100644 --- a/Tools/TranscriptAnalysis/TranscriptAnalysis.csproj +++ b/Tools/TranscriptAnalysis/TranscriptAnalysis.csproj @@ -9,7 +9,7 @@ - + diff --git a/codexnode-manifest.yml b/codexnode-manifest.yml index fdd42620..67e5059c 100644 --- a/codexnode-manifest.yml +++ b/codexnode-manifest.yml @@ -1,30 +1,30 @@ apiVersion: v1 kind: Namespace metadata: - name: codex-test-namespace + name: storage-test-namespace labels: - name: codex-test-namespace + name: storage-test-namespace --- apiVersion: apps/v1 kind: Deployment metadata: - name: codex-demo - namespace: codex-test-namespace + name: storage-demo + namespace: storage-test-namespace spec: replicas: 1 selector: matchLabels: - codex-node: dist-test + storage-node: dist-test template: metadata: labels: - codex-node: dist-test + storage-node: dist-test spec: containers: - - name: codex-node1 - image: codexstorage/nim-codex:sha-7b88ea0 + - name: storage-node1 + image: logosstorage/logos-storage-nim:sha-7b88ea0 ports: - containerPort: 8080 name: api-1 @@ -37,8 +37,8 @@ spec: value: "8081" - name: LISTEN_ADDRS value: "/ip4/0.0.0.0/tcp/8082" - - name: codex-node2 - image: codexstorage/nim-codex:sha-7b88ea0 + - name: storage-node2 + image: logosstorage/logos-storage-nim:sha-7b88ea0 ports: - containerPort: 8083 name: api-2 @@ -57,12 +57,12 @@ spec: apiVersion: v1 kind: Service metadata: - name: codex-entrypoint - namespace: codex-test-namespace + name: storage-entrypoint + namespace: storage-test-namespace spec: type: NodePort selector: - codex-node: dist-test + storage-node: dist-test ports: - name: "node1" protocol: TCP diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 9d32992f..16a6bd29 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -4,7 +4,7 @@ services: context: .. dockerfile: docker/Dockerfile environment: - - CODEXDOCKERIMAGE=logosstorage/logos-storage-nim:sha-14c5270 + - STORAGEDOCKERIMAGE=logosstorage/logos-storage-nim:sha-14c5270 - BRANCH="feature/docker-image-testruns" - KUBECONFIG=/opt/kubeconfig - LOGPATH=/opt/logs @@ -13,7 +13,7 @@ services: continuous-test-run: image: thatbenbierens/dist-tests-deployandrun:initial environment: - # - CODEXDOCKERIMAGE=imageoverride + # - STORAGEDOCKERIMAGE=imageoverride - DNR_REP=3 - DNR_NAME=Tryout - DNR_FILTER=PeernBeer diff --git a/docker/docker-dnr-entrypoint.sh b/docker/docker-dnr-entrypoint.sh index 5475acb3..ed7754a4 100644 --- a/docker/docker-dnr-entrypoint.sh +++ b/docker/docker-dnr-entrypoint.sh @@ -1,5 +1,5 @@ #!/bin/bash echo "Running continuous tests..." -cd /app/Tests/CodexContinuousTests +cd /app/Tests/LogosStorageContinuousTests exec "$@" diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh index 6246d70b..88176561 100644 --- a/docker/docker-entrypoint.sh +++ b/docker/docker-entrypoint.sh @@ -2,14 +2,14 @@ # Variables ## Common -SOURCE="${SOURCE:-https://github.com/codex-storage/cs-codex-dist-tests.git}" +SOURCE="${SOURCE:-https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git}" BRANCH="${BRANCH:-master}" -FOLDER="${FOLDER:-/opt/cs-codex-dist-tests}" +FOLDER="${FOLDER:-/opt/logos-storage-dist-tests}" ## Tests specific DEPLOYMENT_CODEXNETDEPLOYER_PATH="${DEPLOYMENT_CODEXNETDEPLOYER_PATH:-Tools/CodexNetDeployer}" DEPLOYMENT_CODEXNETDEPLOYER_RUNNER="${DEPLOYMENT_CODEXNETDEPLOYER_RUNNER:-deploy-continuous-testnet.sh}" -CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/CodexContinuousTests}" +CONTINUOUS_TESTS_FOLDER="${CONTINUOUS_TESTS_FOLDER:-Tests/LogosStorageContinuousTests}" CONTINUOUS_TESTS_RUNNER="${CONTINUOUS_TESTS_RUNNER:-run.sh}" # Get code diff --git a/docker/job-continuous-tests.yaml b/docker/job-continuous-tests.yaml index ecf8ea8b..4850a9ad 100644 --- a/docker/job-continuous-tests.yaml +++ b/docker/job-continuous-tests.yaml @@ -22,7 +22,7 @@ spec: workload-type: "tests-runners-ci" containers: - name: ${NAMEPREFIX}-runner - image: codexstorage/cs-codex-dist-tests:latest + image: logosstorage/logos-storage-dist-tests:latest imagePullPolicy: Always resources: requests: @@ -31,7 +31,7 @@ spec: - name: KUBECONFIG value: "/opt/kubeconfig.yaml" - name: LOGPATH - value: "/var/log/codex-continuous-tests/${DEPLOYMENT_NAMESPACE}" + value: "/var/log/storage-continuous-tests/${DEPLOYMENT_NAMESPACE}" - name: NAMESPACE value: "${NAMESPACE}" - name: BRANCH @@ -40,8 +40,8 @@ spec: value: "${SOURCE}" - name: RUNID value: "${RUNID}" - - name: CODEXDOCKERIMAGE - value: "${CODEXDOCKERIMAGE}" + - name: STORAGEDOCKERIMAGE + value: "${STORAGEDOCKERIMAGE}" - name: TESTID value: "${TESTID}" - name: DEPLOYMENT_NAMESPACE @@ -59,12 +59,12 @@ spec: mountPath: /opt/kubeconfig.yaml subPath: kubeconfig.yaml - name: logs - mountPath: /var/log/codex-continuous-tests + mountPath: /var/log/storage-continuous-tests restartPolicy: Never volumes: - name: kubeconfig secret: - secretName: codex-dist-tests-app-kubeconfig + secretName: storage-dist-tests-app-kubeconfig - name: logs hostPath: - path: /var/log/codex-continuous-tests + path: /var/log/storage-continuous-tests diff --git a/docker/job-dist-tests.yaml b/docker/job-dist-tests.yaml index b6fdb728..9c44b78d 100644 --- a/docker/job-dist-tests.yaml +++ b/docker/job-dist-tests.yaml @@ -21,7 +21,7 @@ spec: workload-type: "tests-runners-ci" containers: - name: ${NAMEPREFIX}-runner - image: codexstorage/cs-codex-dist-tests:latest + image: logosstorage/logos-storage-dist-tests:latest imagePullPolicy: Always resources: requests: @@ -30,7 +30,7 @@ spec: - name: KUBECONFIG value: "/opt/kubeconfig.yaml" - name: LOGPATH - value: "/var/log/codex-dist-tests" + value: "/var/log/storage-dist-tests" - name: NAMESPACE value: "${NAMESPACE}" - name: BRANCH @@ -46,13 +46,13 @@ spec: mountPath: /opt/kubeconfig.yaml subPath: kubeconfig.yaml - name: logs - mountPath: /var/log/codex-dist-tests + mountPath: /var/log/storage-dist-tests args: ${COMMAND} restartPolicy: Never volumes: - name: kubeconfig secret: - secretName: codex-dist-tests-app-kubeconfig + secretName: storage-dist-tests-app-kubeconfig - name: logs hostPath: - path: /var/log/codex-dist-tests + path: /var/log/storage-dist-tests diff --git a/docker/job-release-tests.yaml b/docker/job-release-tests.yaml index 4c1b636e..0c753f29 100644 --- a/docker/job-release-tests.yaml +++ b/docker/job-release-tests.yaml @@ -22,7 +22,7 @@ spec: workload-type: "tests-runners-ci" containers: - name: runner - image: logosstorage/cs-codex-dist-tests:latest + image: logosstorage/logos-storage-dist-tests:latest imagePullPolicy: Always resources: requests: @@ -31,7 +31,7 @@ spec: - name: KUBECONFIG value: "/opt/kubeconfig.yaml" - name: LOGPATH - value: "/var/log/codex-${TEST_TYPE}" + value: "/var/log/storage-${TEST_TYPE}" - name: NAMESPACE value: "${NAMESPACE}" - name: BRANCH @@ -40,8 +40,8 @@ spec: value: "${SOURCE}" - name: RUNID value: "${RUNID}" - - name: CODEXDOCKERIMAGE - value: "${CODEXDOCKERIMAGE}" + - name: STORAGEDOCKERIMAGE + value: "${STORAGEDOCKERIMAGE}" - name: TESTID value: "${TESTID}" - name: TESTS_TYPE @@ -51,13 +51,13 @@ spec: mountPath: /opt/kubeconfig.yaml subPath: kubeconfig.yaml - name: logs - mountPath: /var/log/codex-${TEST_TYPE} + mountPath: /var/log/storage-${TEST_TYPE} args: ${COMMAND} restartPolicy: Never volumes: - name: kubeconfig secret: - secretName: codex-dist-tests-app-kubeconfig + secretName: storage-dist-tests-app-kubeconfig - name: logs hostPath: - path: /var/log/codex-${TEST_TYPE} + path: /var/log/storage-${TEST_TYPE} diff --git a/docs/Automation.md b/docs/Automation.md index 5443294d..33f2a1a3 100644 --- a/docs/Automation.md +++ b/docs/Automation.md @@ -11,11 +11,11 @@ ## Description - The goal of [Distributed System Tests for Nim-Codex](../../) is to test how [Codex](https://codex.storage) works in different topologies in the distributed network and to be able to detect regressions during development. + The goal of [Distributed System Tests for Logos Storage](../../) is to test how Logos Storage works in different topologies in the distributed network and to be able to detect regressions during development. We can [run Tests locally](LOCALSETUP.md) and it works well, but in order to scale that we may need to run Tests in an automatic way using remote Kubernetes cluster. - Initially, we are considering to run dist-tests on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](Continuous-Tests.md) which are called to detect issues on continuous Codex runs. + Initially, we are considering to run dist-tests on [logos-storage-nim](https://github.com/logos-storage/logos-storage-nim) master branch merge, to be able to determine regressions. And we also working on [Continuous Tests](Continuous-Tests.md) which are called to detect issues on continuous Logos Storage runs. ## Architecture @@ -35,7 +35,7 @@ GitHub --> CI --> Kubernetes --> Job | 3. Job run Dist-Tests runner Pod with specified parameters 4. Dists-Tests runner run the tests from inside the Kubernetes and generate the logs 5. Vector ship the logs of the Dists-Tests Pods - 6. Prometheus collect the metrics of the Dists-Tests Codex Pods + 6. Prometheus collect the metrics of the Dists-Tests Logos Storage Pods 7. We can see the status of the Dist-Test @@ -43,7 +43,7 @@ GitHub --> CI --> Kubernetes --> Job | | Component | Description | | --------------------------------------- | --------------------------------------- | -| [cs-codex-dist-tests](/) | Distributed System Tests | +| [logos-storage-dist-tests](/) | Distributed System Tests | | [Kubernetes](https://kubernetes.io) | Environment where we run Tests | | [Vector](https://vector.dev) | Ship logs to the Elasticsearch | | [Elasticsearch](https://www.elastic.co) | Store and index logs | @@ -116,15 +116,15 @@ GitHub --> CI --> Kubernetes --> Job | mountPath: /opt/kubeconfig.yaml subPath: kubeconfig.yaml - name: logs - mountPath: /var/log/cs-codex-dist-tests + mountPath: /var/log/logos-storage-dist-tests restartPolicy: Never volumes: - name: kubeconfig secret: - secretName: cs-codex-dist-tests-app-kubeconfig + secretName: logos-storage-dist-tests-app-kubeconfig - name: logs hostPath: - path: /var/log/cs-codex-dist-tests + path: /var/log/logos-storage-dist-tests ``` @@ -145,7 +145,7 @@ GitHub --> CI --> Kubernetes --> Job | ```shell folder="/opt/dist-tests" - git clone https://github.com/codex-storage/cs-codex-dist-tests.git $folder + git clone https://github.com/logos-storage/logos-storage-dist-tests.git $folder cd $folder ``` @@ -153,7 +153,7 @@ GitHub --> CI --> Kubernetes --> Job | ```shell # RUNNERLOCATION # defined at Pod creation # KUBECONFIG # defined at Pod creation - export LOGPATH="/var/log/cs-codex-dist-tests" # Logs from that location will be send in Elasticsearch + export LOGPATH="/var/log/logos-storage-dist-tests" # Logs from that location will be send in Elasticsearch export RUNID=$(date +%Y%m%d-%H%M%S) # Run ID to show in Kibana/Grafana export TESTID=$(git rev-parse --short HEAD) # Test ID to show in Kibana/Grafana ``` @@ -170,7 +170,7 @@ GitHub --> CI --> Kubernetes --> Job | dotnet test LongTests # Specific test - dotnet test --filter=CodexLogExample + dotnet test --filter=LogosStorageLogExample ``` 6. We can see in OpenLens Pods started by dist-tests app @@ -180,7 +180,7 @@ GitHub --> CI --> Kubernetes --> Job | ### Run tests automatically - Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [nim-codex](https://github.com/codex-storage/nim-codex) master branch merge. + Now we use GitHub Actions to trigger dist-tests run manually and considering to run them on [logos-storage-nim](https://github.com/logos-storage/logos-storage-nim) master branch merge. **It works in the following way** 1. Github Actions secrets contains [kubeconfig to interact with the Kubernetes cluster](../../../issues/19). @@ -190,7 +190,7 @@ GitHub --> CI --> Kubernetes --> Job | - `namespace` - Namespace where Dist-test runner will de created - `nameprefix` - Dist-test runner name prefix - 3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [codexstorage/cs-codex-dist-tests](https://hub.docker.com/r/codexstorage/cs-codex-dist-tests/tags). + 3. Kubernetes Job will run the Pod with a custom [Docker image](/docker/Dockerfile) - [logosstorage/logos-storage-dist-tests](https://hub.docker.com/r/logosstorage/logos-storage-dist-tests/tags). Image [entrypoint](/docker/docker-entrypoint.sh) is customizable and we can pass the following variables - `SOURCE` - Dist-tests source repository, useful when we work with forks - `default="current repository"` - `BRANCH` - Repository branch, useful when we would like to run tests from the custom branch - `default="master"` @@ -202,7 +202,7 @@ GitHub --> CI --> Kubernetes --> Job | ```shell RUNNERLOCATION=InternalToCluster KUBECONFIG=/opt/kubeconfig.yaml - LOGPATH=/var/log/cs-codex-dist-tests + LOGPATH=/var/log/logos-storage-dist-tests NAMESPACE=default SOURCE=current repository BRANCH=master @@ -212,7 +212,7 @@ GitHub --> CI --> Kubernetes --> Job | 5. Dist-tests runner will use [kubeconfig to interact with the Kubernetes and run the tests](../../../issues/21), which is set by `KUBECONFIG` variable. - 6. Runner will execute all tests and will write the logs to the `/var/log/cs-codex-dist-tests` folder. + 6. Runner will execute all tests and will write the logs to the `/var/log/logos-storage-dist-tests` folder. 7. Kubernetes Job status will be changed from `Running` to the `Completed` or `Failed`. @@ -225,7 +225,7 @@ GitHub --> CI --> Kubernetes --> Job | > **Note:** This part is not finished yet and it is under development - We use Elasticsearch to store and discover the logs of the tests execution and Codex Pods which are run during the tests. + We use Elasticsearch to store and discover the logs of the tests execution and Logos Storage Pods which are run during the tests. We can use [Kibana](#kibana) to discover all the logs and [Grafana](#grafana) to see tests execution status. @@ -234,8 +234,8 @@ GitHub --> CI --> Kubernetes --> Job | | App | URL | Authentication | | ------- | ---------------------------------------------------------------------------- | -------------- | - | Kibana | [kibana.dist-tests.codex.storage](https://kibana.dist-tests.codex.storage) | GitHub account | - | Grafana | [grafana.dist-tests.codex.storage](https://grafana.dist-tests.codex.storage) | GitHub account | + | Kibana | [kibana.dist-tests.logos.storage](https://kibana.dist-tests.logos.storage) | GitHub account | + | Grafana | [grafana.dist-tests.logos.storage](https://grafana.dist-tests.logos.storage) | GitHub account | #### Kibana diff --git a/docs/CodexHowTo.md b/docs/CodexHowTo.md deleted file mode 100644 index 6ef8384f..00000000 --- a/docs/CodexHowTo.md +++ /dev/null @@ -1,9 +0,0 @@ -# Codex Tests - How To -Follow these steps to run the distributed tests for Codex. These tests should pass for any (future) release of Codex. - -## Build a docker image. -If you already have a dist-tests compatible docker image, skip this step. -![BuildImage](./CodexHowTo_BuildImage.png) - -## Run tests. -![RunTests](./CodexHowTo_RunReleaseTests.png) diff --git a/docs/Continuous-Tests.md b/docs/Continuous-Tests.md index 8ed59bc1..7b9ebef9 100644 --- a/docs/Continuous-Tests.md +++ b/docs/Continuous-Tests.md @@ -12,8 +12,7 @@ Usually, we are running Continuous Tests manually and for automated runs, please refer to the [Tests automation](Automation.md). - We have two projects in the repository - - [CodexNetDeployer](../CodexNetDeployer) - Prepare environment to run the tests + There is one project in the repository: - [ContinuousTests](../ContinuousTests) - Continuous Tests And they are used to prepare environment and run Continuous Tests. @@ -53,15 +52,15 @@ # mountPath: /opt/kubeconfig.yaml # subPath: kubeconfig.yaml # - name: logs - # mountPath: /var/log/codex-dist-tests + # mountPath: /var/log/storage-dist-tests command: ["sleep", "infinity"] # volumes: # - name: kubeconfig # secret: - # secretName: codex-dist-tests-app-kubeconfig + # secretName: storage-dist-tests-app-kubeconfig # - name: logs # hostPath: - # path: /var/log/codex-dist-tests + # path: /var/log/storage-dist-tests ``` ```shell @@ -70,7 +69,7 @@ 2. Copy kubeconfig to the runner Pod using the name you set in the previous step ```shell - kubectl cp ~/.kube/codex-dist-tests.yaml tests-runner:/opt/kubeconfig.yaml + kubectl cp ~/.kube/storage-dist-tests.yaml tests-runner:/opt/kubeconfig.yaml ``` 3. Exec into the runner Pod using the name you set in the previous step @@ -93,26 +92,12 @@ tmux cd /opt - git clone https://github.com/codex-storage/cs-codex-dist-tests.git + git clone https://github.com/logos-storage/logos-storage-dist-tests.git ``` - 6. Run `CodexNetDeployer` + 6. Run `ContinuousTests` ```shell - # Usually take ~ 10 minutes - cd cs-codex-dist-tests/Tools/CodexNetDeployer - - # Adjust values - vi deploy-continuous-testnet.sh - - # Deploy Codex Netwotk - export RUNID=$(date +%Y%m%d-%H%M%S) - bash deploy-continuous-testnet.sh - ``` - - 7. Run `ContinuousTests` - ```shell - cd ../../Tests/CodexContinuousTests - cp ../../Tools/CodexNetDeployer/codex-deployment.json . + cd ../../Tests/LogosStorageContinuousTests # Adjust values vi run.sh @@ -121,7 +106,7 @@ bash run.sh ``` - 8. [Tmux sessions](https://tmuxcheatsheet.com) + 7. [Tmux sessions](https://tmuxcheatsheet.com) ```shell # Detach Ctrl + b --> d @@ -136,4 +121,4 @@ ## Analyze logs - We should check the logs in the `/opt/cs-codex-dist-tests/Tests/CodexContinuousTests/logs` folder + We should check the logs in the `/opt/logos-storage-dist-tests/Tests/LogosStorageContinuousTests/logs` folder diff --git a/docs/LOCALSETUP.md b/docs/LOCALSETUP.md deleted file mode 100644 index 326c67ec..00000000 --- a/docs/LOCALSETUP.md +++ /dev/null @@ -1,13 +0,0 @@ -# Distributed System Tests for Nim-Codex - -## Local setup -These steps will help you set up everything you need to run and debug the tests on your local system. - -### Installing the requirements. -1. Install dotnet v10.0 or newer. (If you install a newer version, consider updating the .csproj files by replacing all mention of `net10.0` with your version.) -1. Set up a nice C# IDE or plugin for your current IDE. -1. Install docker desktop. -1. In the docker-desktop settings, enable kubernetes using kubeadm. (This might take a few minutes.) Note that the version of the `KubernetesClient` package in the `KubernetesWorkflow` project [must be compatible with](https://github.com/kubernetes-client/csharp#version-compatibility) the version of Kubernetes that `kubeadm` exposes. For example, the current version of Kubernetes that kubeadm exposes is `1.34.1`, therefore `KubernetesClient` must use version `18.x`. See https://github.com/kubernetes-client/csharp#version-compatibility for more information. - -### Running the tests -Most IDEs will let you run individual tests or test fixtures straight from the code file. If you want to run all the tests, you can use `dotnet test`. You can control which tests to run by specifying which folder of tests to run. `dotnet test Tests/CodexTests` will run only the tests in `/Tests/CodexTests` and exclude the long tests. diff --git a/docs/LogosStorageHowTo.md b/docs/LogosStorageHowTo.md new file mode 100644 index 00000000..eec4fc7b --- /dev/null +++ b/docs/LogosStorageHowTo.md @@ -0,0 +1,9 @@ +# Logos Storage Tests - How To +Follow these steps to run the distributed tests for Logos Storage. These tests should pass for any (future) release of Logos Storage. + +## Build a docker image. +If you already have a dist-tests compatible docker image, skip this step. +![BuildImage](./LogosStorageHowTo_BuildImage.png) + +## Run tests. +![RunTests](./LogosStorageHowTo_RunReleaseTests.png) diff --git a/docs/RunningReleaseTests.md b/docs/RunningReleaseTests.md new file mode 100644 index 00000000..a029918b --- /dev/null +++ b/docs/RunningReleaseTests.md @@ -0,0 +1,276 @@ +# Running Release Tests + +This guide covers running the release tests both **locally** (against a local Kubernetes cluster) and **on Digital Ocean** (or any remote Kubernetes cluster). + +--- + +## 1. Running Locally (Docker Desktop Kubernetes) + +### Prerequisites + +1. Install [.NET 10.0+](https://dotnet.microsoft.com/download). (If you install a newer version, update all `net10.0` references in the `.csproj` files to match.) +2. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/). +3. Enable Kubernetes in Docker Desktop: **Settings → Kubernetes → Enable Kubernetes (kubeadm) → Apply & Restart**. This may take a few minutes. + +> **Note on Kubernetes client compatibility:** The `KubernetesClient` package version in the `KubernetesWorkflow` project [must be compatible](https://github.com/kubernetes-client/csharp#version-compatibility) with the Kubernetes version that kubeadm exposes. For example, if kubeadm exposes Kubernetes `1.34.1`, then `KubernetesClient` must be version `18.x`. + +### How it works + +When you run `dotnet test` from your machine, the framework detects it is running **outside** the cluster (by checking whether the `KUBERNETES_PORT` and `KUBERNETES_SERVICE_HOST` environment variables are set). In this mode it connects to the cluster via your local `~/.kube/config`, which Docker Desktop automatically configures. + +Each test creates its own isolated namespace in the cluster, starts the required Storage nodes as pods, runs the test, then tears everything down. + +### Verify the cluster is working + +Before running tests, confirm Docker Desktop's Kubernetes context is active: + +```bash +kubectl config current-context # should show "docker-desktop" +kubectl get nodes # should show a Ready node +``` + +If the current context is not `docker-desktop` (e.g. it points to a remote cluster), switch it: +```bash +kubectl config use-context docker-desktop +``` + +### Run the tests + +Most IDEs let you run individual tests or test fixtures directly from the code file. To run from the command line: + +```bash +cd /path/to/logos-storage-nim-cs-dist-tests + +# Run all release tests +dotnet test Tests/LogosStorageReleaseTests + +# Run a specific test by name +dotnet test Tests/LogosStorageReleaseTests --filter=OneClientTest + +# Run with verbose output +dotnet test Tests/LogosStorageReleaseTests --logger="console;verbosity=detailed" +``` + +### Useful environment variables + +| Variable | Default | Description | +|---|---|---| +| `STORAGEDOCKERIMAGE` | `logosstorage/logos-storage-nim:latest-dist-tests` | Storage node image to test | +| `KUBECONFIG` | `~/.kube/config` | Path to kubeconfig file (optional when using Docker Desktop) | +| `LOGPATH` | `LogosStorageTestLogs` (relative) | Directory for test logs | +| `DATAFILEPATH` | `TestDataFiles` (relative) | Directory for test data files | +| `ALWAYS_LOGS` | _(unset)_ | Set to any non-empty value to always download container logs (not just on failure) | +| `TEST_TYPE` | _(unset)_ | Set to `release-tests` only when running inside the cluster (see §2c). Do **not** set this for local runs — it activates long in-cluster timeouts. | + +Example — run against a specific Storage image: + +```bash +STORAGEDOCKERIMAGE=logosstorage/logos-storage-nim:v0.1.8 dotnet test Tests/LogosStorageReleaseTests +``` + +### Troubleshooting + +**`NullReferenceException` at `DistTest..ctor()`** + +If every test fails immediately with a stack trace ending at `DistTest.GetWebCallTimeSet()` or `DistTest.GetK8sTimeSet()`, check whether `TEST_TYPE` is set to `release-tests` in your shell: + +```bash +echo $TEST_TYPE +``` + +If it is, unset it before running locally: + +```bash +unset TEST_TYPE +dotnet test Tests/LogosStorageReleaseTests +``` + +Setting `TEST_TYPE=release-tests` triggers in-cluster detection which tries to log a message through an object that hasn't been constructed yet, causing the crash. + +**Tests fail with `kubectl` errors / wrong cluster** + +Ensure the active context is `docker-desktop`, not a remote cluster: + +```bash +kubectl config current-context # confirm "docker-desktop" +kubectl config use-context docker-desktop # switch if needed +kubectl get nodes # should show a Ready node +``` + +**Image not found** + +The `STORAGEDOCKERIMAGE` must be pullable from Docker Desktop. Either use a published image or build and push locally: + +```bash +docker pull logosstorage/logos-storage-nim:latest-dist-tests +``` + +--- + +## 2. Running on Digital Ocean (Remote Kubernetes) + +### Overview + +On a remote cluster the test runner itself must run **inside** the cluster, because the framework needs direct pod-to-pod networking. The CI workflow does this automatically by creating a Kubernetes Job that runs the test runner image. You can also do it manually. + +### Prerequisites + +- A running Digital Ocean Kubernetes (DOKS) cluster +- `kubectl` configured to talk to that cluster +- The cluster must be pre-configured (see below) + +### 2a. Cluster pre-configuration + +Do these steps once per cluster. + +**1. Label the worker nodes that will run the test runner pod** + +```bash +kubectl label node workload-type=tests-runners-ci +``` + +The job manifest uses a `nodeSelector` for this label, so at least one node must have it. + +**2. Create the kubeconfig secret for the test runner** + +The test runner pod itself needs a kubeconfig to manage pods inside the cluster. Use a static service-account-based kubeconfig — avoid copying your local `~/.kube/config` directly, as it likely uses an exec credential plugin (e.g. `doctl`) that won't be available inside the pod. + +```bash +# Create a service account and grant it cluster-admin access +kubectl create serviceaccount dist-tests-app +kubectl create clusterrolebinding dist-tests-app \ + --clusterrole=cluster-admin \ + --serviceaccount=default:dist-tests-app + +# Create a long-lived static token +kubectl create token dist-tests-app --duration=8760h > /tmp/sa-token.txt + +# Build a static kubeconfig using the token +CLUSTER=$(kubectl config view --minify -o jsonpath='{.clusters[0].name}') +SERVER=$(kubectl config view --minify -o jsonpath='{.clusters[0].cluster.server}') +kubectl config view --minify --raw -o jsonpath='{.clusters[0].cluster.certificate-authority-data}' | base64 --decode > /tmp/ca.crt + +kubectl --kubeconfig=/tmp/static-kubeconfig.yaml config set-cluster "$CLUSTER" \ + --server="$SERVER" \ + --certificate-authority=/tmp/ca.crt \ + --embed-certs=true +kubectl --kubeconfig=/tmp/static-kubeconfig.yaml config set-credentials dist-tests-app \ + --token=$(cat /tmp/sa-token.txt) +kubectl --kubeconfig=/tmp/static-kubeconfig.yaml config set-context default \ + --cluster="$CLUSTER" --user=dist-tests-app +kubectl --kubeconfig=/tmp/static-kubeconfig.yaml config use-context default + +# Store it as a secret +kubectl create secret generic storage-dist-tests-app-kubeconfig \ + --from-file=kubeconfig.yaml=/tmp/static-kubeconfig.yaml \ + -n default +``` + +The pod mounts this at `/opt/kubeconfig.yaml` and passes it via `KUBECONFIG`. + +**3. (If not already present) Create the `system-node-critical` priority class** + +The job manifest requests `priorityClassName: system-node-critical`. On most clusters this exists already; check with: + +```bash +kubectl get priorityclass system-node-critical +``` + +If missing, create it or change the priority class name in [docker/job-release-tests.yaml](../docker/job-release-tests.yaml). + +### 2b. Running via GitHub Actions (recommended) + +This is the standard automated path. + +**Required GitHub secret:** + +| Secret | Value | +|---|---| +| `KUBE_CONFIG` | Base64-encoded kubeconfig with permissions to create Jobs/Pods in the cluster | + +Encode your kubeconfig: +```bash +base64 -i ~/.kube/do-cluster-config | pbcopy # macOS +base64 -w0 ~/.kube/do-cluster-config # Linux +``` + +**Trigger the workflow:** + +Go to **Actions → Run Release Tests → Run workflow** and provide: + +| Input | Description | +|---|---| +| `storagedockerimage` | Image to test, e.g. `logosstorage/logos-storage-nim:v0.1.8-dist-tests` | + +Optional inputs (for non-default branches/repos): + +| Input | Description | +|---|---| +| `source` | Repository URL (defaults to current repo) | +| `branch` | Branch to clone for running tests (defaults to current branch) | + +**What happens:** + +1. GitHub Actions decodes `KUBE_CONFIG` and runs `kubectl` +2. Creates a Kubernetes Job from [docker/job-release-tests.yaml](../docker/job-release-tests.yaml) with substituted variables +3. The Job runs the `logosstorage/logos-storage-dist-tests:latest` image +4. The entrypoint clones this repo, runs `dotnet test Tests/LogosStorageReleaseTests` +5. The workflow streams pod logs and fails if the Job does not complete successfully + +### 2c. Running manually with kubectl + +Useful for debugging or one-off runs without CI. + +**Set the required variables:** + +```bash +export NAMESPACE=default +export NAMEPREFIX=r-tests-manual +export RUNID=$(date +%Y%m%d-%H%M%S) +export TESTID=$(git rev-parse --short HEAD) +export TEST_TYPE=release-tests +export SOURCE=https://github.com/logos-storage/logos-storage-nim-cs-dist-tests.git +export BRANCH=master +export STORAGEDOCKERIMAGE=logosstorage/logos-storage-nim:latest-dist-tests +export COMMAND='["dotnet","test","Tests/LogosStorageReleaseTests"]' +``` + +**Apply the job:** + +```bash +envsubst < docker/job-release-tests.yaml | kubectl apply -f - +``` + +**Follow the logs:** + +```bash +# Wait for pod to start +kubectl get pod --selector job-name=$NAMEPREFIX -w + +# Stream logs +POD=$(kubectl get pod --selector job-name=$NAMEPREFIX -o jsonpath='{.items[0].metadata.name}') +kubectl logs $POD -f + +# Check final job status +kubectl get job $NAMEPREFIX -o jsonpath='{.status.conditions[0].type}' +# Should print "Complete" +``` + +**Cleanup:** + +Jobs are auto-deleted after 24 hours (TTL configured in the manifest). To delete immediately: + +```bash +kubectl delete job $NAMEPREFIX +``` + +### Key differences: local vs. remote + +| | Local (Docker Desktop) | Remote (Digital Ocean) | +|---|---|---| +| Runner location | Your machine (external to cluster) | Inside a pod in the cluster | +| Kubeconfig | `~/.kube/config` (auto) | Mounted secret `storage-dist-tests-app-kubeconfig` | +| Network access to pods | Via `kubectl port-forward` / node IP | Direct pod-to-pod | +| `RUNNERLOCATION` detection | `ExternalToCluster` (automatic) | `InternalToCluster` (automatic inside pod) | +| How to run | `dotnet test` on your machine | Kubernetes Job | +| Image required | No (builds from source) | `logosstorage/logos-storage-dist-tests:latest` | diff --git a/cs-codex-dist-testing.sln b/logos-storage-dist-testing.sln similarity index 91% rename from cs-codex-dist-testing.sln rename to logos-storage-dist-testing.sln index 14a291ce..18d182ae 100644 --- a/cs-codex-dist-testing.sln +++ b/logos-storage-dist-testing.sln @@ -23,17 +23,17 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Logging", "Framework\Loggin EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Utils", "Framework\Utils\Utils.csproj", "{8D264872-5361-4AC5-8A99-908137E13A22}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexPlugin", "ProjectPlugins\CodexPlugin\CodexPlugin.csproj", "{F36DFCB1-C33F-426B-851B-FB1DEE7F028E}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StoragePlugin", "ProjectPlugins\StoragePlugin\StoragePlugin.csproj", "{F36DFCB1-C33F-426B-851B-FB1DEE7F028E}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MetricsPlugin", "ProjectPlugins\MetricsPlugin\MetricsPlugin.csproj", "{8DE8FF65-23CB-4FB3-8BE5-6C0BEC4BAA97}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexContinuousTests", "Tests\CodexContinuousTests\CodexContinuousTests.csproj", "{ADEC06CF-6F3A-44C5-AA57-EAB94124AC82}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogosStorageContinuousTests", "Tests\LogosStorageContinuousTests\LogosStorageContinuousTests.csproj", "{ADEC06CF-6F3A-44C5-AA57-EAB94124AC82}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexTestsLong", "Tests\CodexLongTests\CodexTestsLong.csproj", "{0C2D067F-053C-45A8-AE0D-4EB388E77C89}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogosStorageTestsLong", "Tests\LogosStorageLongTests\LogosStorageTestsLong.csproj", "{0C2D067F-053C-45A8-AE0D-4EB388E77C89}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DistTestCore", "Tests\DistTestCore\DistTestCore.csproj", "{E849B7BA-FDCC-4CFF-998F-845ED2F1BF40}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexDiscordBotPlugin", "ProjectPlugins\CodexDiscordBotPlugin\CodexDiscordBotPlugin.csproj", "{FB96A58B-F7F0-490A-9A85-72A96A018042}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogosStorageDiscordBotPlugin", "ProjectPlugins\LogosStorageDiscordBotPlugin\LogosStorageDiscordBotPlugin.csproj", "{FB96A58B-F7F0-490A-9A85-72A96A018042}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestClusterStarter", "Tools\TestClusterStarter\TestClusterStarter.csproj", "{3E38A906-C2FC-43DC-8CA2-FC07C79CF3CA}" EndProject @@ -41,7 +41,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeployAndRunPlugin", "Proje EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FrameworkTests", "Tests\FrameworkTests\FrameworkTests.csproj", "{25E72004-4D71-4D1E-A193-FC125D12FF96}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexPluginPrebuild", "ProjectPlugins\CodexPluginPrebuild\CodexPluginPrebuild.csproj", "{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogosStoragePluginPrebuild", "ProjectPlugins\LogosStoragePluginPrebuild\LogosStoragePluginPrebuild.csproj", "{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FD6E81BA-93E8-4BB7-94F8-98C5427E02B0}" ProjectSection(SolutionItems) = preProject @@ -54,11 +54,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TranscriptAnalysis", "Tools EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CsvCombiner", "Tools\CsvCombiner\CsvCombiner.csproj", "{6230347F-5045-4E25-8E7A-13D7221B7444}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexReleaseTests", "Tests\CodexReleaseTests\CodexReleaseTests.csproj", "{639A0603-4E80-465B-BB59-AB02F1DEEF5A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "LogosStorageReleaseTests", "Tests\LogosStorageReleaseTests\LogosStorageReleaseTests.csproj", "{639A0603-4E80-465B-BB59-AB02F1DEEF5A}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ExperimentalTests", "Tests\ExperimentalTests\ExperimentalTests.csproj", "{BA7369CD-7C2F-4075-8E35-98BCC19EE203}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CodexClient", "ProjectPlugins\CodexClient\CodexClient.csproj", "{9AF12703-29AF-416D-9781-204223D6D0E5}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LogosStorageClient", "ProjectPlugins\LogosStorageClient\LogosStorageClient.csproj", "{9AF12703-29AF-416D-9781-204223D6D0E5}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "WebUtils", "Framework\WebUtils\WebUtils.csproj", "{372C9E5D-5453-4D45-9948-E9324E21AD65}" EndProject