From a27b29403167dfe2429cd02b73e8c3017f4d0709 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 22 Mar 2024 11:15:22 +0100 Subject: [PATCH 01/25] Updates codex marketplace API types --- ProjectPlugins/CodexPlugin/CodexApiTypes.cs | 5 +++-- ProjectPlugins/CodexPlugin/MarketplaceTypes.cs | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs index b1965cd..96d19ef 100644 --- a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs @@ -87,7 +87,7 @@ namespace CodexPlugin public class CodexSalesAvailabilityRequest { - public string size { get; set; } = string.Empty; + public string totalSize { get; set; } = string.Empty; public string duration { get; set; } = string.Empty; public string minPrice { get; set; } = string.Empty; public string maxCollateral { get; set; } = string.Empty; @@ -96,7 +96,8 @@ namespace CodexPlugin public class CodexSalesAvailabilityResponse { public string id { get; set; } = string.Empty; - public string size { get; set; } = string.Empty; + public string totalSize { get; set; } = string.Empty; + public string freeSize { get; set; } = string.Empty; public string duration { get; set; } = string.Empty; public string minPrice { get; set; } = string.Empty; public string maxCollateral { get; set; } = string.Empty; diff --git a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs index bfdf1ff..dc7f7e5 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs @@ -67,7 +67,7 @@ namespace CodexPlugin { return new CodexSalesAvailabilityRequest { - size = ToDecInt(TotalSpace.SizeInBytes), + totalSize = ToDecInt(TotalSpace.SizeInBytes), duration = ToDecInt(MaxDuration.TotalSeconds), maxCollateral = ToDecInt(MaxCollateral), minPrice = ToDecInt(MinPriceForTotalSpace) @@ -77,7 +77,7 @@ namespace CodexPlugin public void Log(ILog log) { log.Log($"Making storage available... (" + - $"size: {TotalSpace}, " + + $"totalSize: {TotalSpace}, " + $"maxDuration: {Time.FormatDuration(MaxDuration)}, " + $"minPriceForTotalSpace: {MinPriceForTotalSpace}, " + $"maxCollateral: {MaxCollateral})"); From 7d28b62206d96d6f4ff2a967340c11acb4f9dd6e Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 22 Mar 2024 13:36:33 +0100 Subject: [PATCH 02/25] new image --- Framework/FileUtils/FileManager.cs | 2 +- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Framework/FileUtils/FileManager.cs b/Framework/FileUtils/FileManager.cs index 9b25c56..10ecf24 100644 --- a/Framework/FileUtils/FileManager.cs +++ b/Framework/FileUtils/FileManager.cs @@ -45,7 +45,7 @@ namespace FileUtils { var sw = Stopwatch.Begin(log); var result = GenerateRandomFile(size, label); - sw.End($"Generated file '{result.Describe()}'."); + sw.End($"Generated file {result.Describe()}."); return result; } diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index ca619b2..08a21be 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -9,7 +9,7 @@ namespace CodexPlugin { private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-e4ddb94-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-2a34c48-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; From 042285e664364318eef1f55074045c663187ae85 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 22 Mar 2024 14:16:55 +0100 Subject: [PATCH 03/25] increases block cache size --- Framework/NethereumWorkflow/BlockUtils/BlockCache.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs b/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs index 3fdf803..1954963 100644 --- a/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs +++ b/Framework/NethereumWorkflow/BlockUtils/BlockCache.cs @@ -4,7 +4,7 @@ { public delegate void CacheClearedEvent(); - private const int MaxEntries = 1024; + private const int MaxEntries = 1024 * 1024 * 5; private readonly Dictionary entries = new Dictionary(); public event CacheClearedEvent? OnCacheCleared; From 617b656cf79a6f06c2ef09f6a8c3afb8f33783d6 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:13:25 +0200 Subject: [PATCH 04/25] Use just folder as a path for digests upload (#99) --- .github/workflows/docker-reusable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-reusable.yml b/.github/workflows/docker-reusable.yml index cc01f1e..4cda961 100644 --- a/.github/workflows/docker-reusable.yml +++ b/.github/workflows/docker-reusable.yml @@ -104,7 +104,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: digests-${{ matrix.target.arch }} - path: /tmp/digests/* + path: /tmp/digests if-no-files-found: error retention-days: 1 From e8213b95151d5c3703ae62207142468b71cb1676 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Mar 2024 10:17:32 +0100 Subject: [PATCH 05/25] Updates codex image --- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 08a21be..9d041c6 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -9,7 +9,7 @@ namespace CodexPlugin { private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-2a34c48-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-c219a5f-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; From bd5fe0cae124116447615bf7cd80f8f85962518c Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Mar 2024 10:20:44 +0100 Subject: [PATCH 06/25] Sets up client generation with NSwag --- ProjectPlugins/CodexPlugin/CodexPlugin.csproj | 16 + ProjectPlugins/CodexPlugin/openapi.yaml | 717 ++++++++++++++++++ 2 files changed, 733 insertions(+) create mode 100644 ProjectPlugins/CodexPlugin/openapi.yaml diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj index 19c3b60..47899a0 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj @@ -7,7 +7,23 @@ + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml new file mode 100644 index 0000000..adfc887 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -0,0 +1,717 @@ +openapi: 3.0.3 + +info: + version: 0.0.1 + title: Codex API + description: "List of endpoints and interfaces available to Codex API users" + +security: + - { } + +components: + schemas: + MultiAddress: + type: string + description: Address of node as specified by the multi-address specification https://multiformats.io/multiaddr/ + example: /ip4/127.0.0.1/tcp/8080 + + PeerId: + type: string + description: Peer Identity reference as specified at https://docs.libp2p.io/concepts/fundamentals/peers/ + example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N + + Id: + type: string + description: 32bits identifier encoded in hex-decimal string. + example: 0x... + + BigInt: + type: string + description: Integer represented as decimal string + + Cid: + type: string + description: Content Identifier as specified at https://github.com/multiformats/cid + example: QmYyQSo1c1Ym7orWxLYvCrM2EmxFTANf8wXmmE7DWjhx5N + + SlotId: + type: string + description: Keccak hash of the abi encoded tuple (RequestId, slot index) + example: 268a781e0db3f7cf36b18e5f4fdb7f586ec9edd08e5500b17c0e518a769f114a + + LogLevel: + type: string + description: "One of the log levels: TRACE, DEBUG, INFO, NOTICE, WARN, ERROR or FATAL" + example: DEBUG + + EthereumAddress: + type: string + description: Address of Ethereum address + + Reward: + type: string + description: The maximum amount of tokens paid per second per slot to hosts the client is willing to pay + + Duration: + type: string + description: The duration of the request in seconds as decimal string + + ProofProbability: + type: string + description: How often storage proofs are required as decimal string + + Expiry: + type: string + description: A timestamp as seconds since unix epoch at which this request expires if the Request does not find requested amount of nodes to host the data. + default: 10 minutes + + ErasureParameters: + type: object + properties: + totalChunks: + type: number + + PoRParameters: + description: Parameters for Proof of Retrievability + type: object + properties: + u: + type: string + publicKey: + type: string + name: + type: string + + Content: + type: object + description: Parameters specifying the content + properties: + cid: + $ref: "#/components/schemas/Cid" + erasure: + $ref: "#/components/schemas/ErasureParameters" + por: + $ref: "#/components/schemas/PoRParameters" + + DebugInfo: + type: object + properties: + id: + $ref: "#/components/schemas/PeerId" + addrs: + type: array + items: + $ref: "#/components/schemas/MultiAddress" + repo: + type: string + description: Path of the data repository where all nodes data are stored + spr: + type: string + description: Signed Peer Record to advertise DHT connection information + + SalesAvailability: + type: object + properties: + id: + $ref: "#/components/schemas/Id" + totalSize: + type: string + description: Total size of availability's storage in bytes as decimal string + duration: + $ref: "#/components/schemas/Duration" + minPrice: + type: string + description: Minimum price to be paid (in amount of tokens) as decimal string + maxCollateral: + type: string + description: Maximum collateral user is willing to pay per filled Slot (in amount of tokens) as decimal string + + SalesAvailabilityREAD: + allOf: + - $ref: "#/components/schemas/SalesAvailability" + - type: object + properties: + freeSize: + type: string + description: Unused size of availability's storage in bytes as decimal string + + SalesAvailabilityCREATE: + allOf: + - $ref: "#/components/schemas/SalesAvailability" + - required: + - totalSize + - minPrice + - maxCollateral + - duration + + Slot: + type: object + properties: + id: + $ref: "#/components/schemas/SlotId" + request: + $ref: "#/components/schemas/StorageRequest" + slotIndex: + type: string + description: Slot Index as hexadecimal string + + Reservation: + type: object + properties: + id: + $ref: "#/components/schemas/Id" + availabilityId: + $ref: "#/components/schemas/Id" + size: + $ref: "#/components/schemas/BigInt" + requestId: + $ref: "#/components/schemas/Id" + slotIndex: + type: string + description: Slot Index as hexadecimal string + + StorageRequestCreation: + type: object + required: + - reward + - duration + - proofProbability + - collateral + - expiry + properties: + duration: + $ref: "#/components/schemas/Duration" + reward: + $ref: "#/components/schemas/Reward" + proofProbability: + $ref: "#/components/schemas/ProofProbability" + nodes: + type: number + description: Minimal number of nodes the content should be stored on + default: 1 + tolerance: + type: number + description: Additional number of nodes on top of the `nodes` property that can be lost before pronouncing the content lost + default: 0 + collateral: + type: string + description: Number as decimal string that represents how much collateral is asked from hosts that wants to fill a slots + expiry: + type: string + description: Number as decimal string that represents expiry time of the request (in unix timestamp) + + StorageAsk: + type: object + required: + - reward + properties: + slots: + type: number + description: Number of slots (eq. hosts) that the Request want to have the content spread over + slotSize: + type: string + description: Amount of storage per slot (in bytes) as decimal string + duration: + $ref: "#/components/schemas/Duration" + proofProbability: + $ref: "#/components/schemas/ProofProbability" + reward: + $ref: "#/components/schemas/Reward" + maxSlotLoss: + type: number + description: Max slots that can be lost without data considered to be lost + + StorageRequest: + type: object + properties: + id: + type: string + description: Request ID + client: + $ref: "#/components/schemas/EthereumAddress" + ask: + $ref: "#/components/schemas/StorageAsk" + content: + $ref: "#/components/schemas/Content" + expiry: + $ref: "#/components/schemas/Expiry" + nonce: + type: string + description: Random data + + Purchase: + type: object + properties: + state: + type: string + description: Description of the Request's state + error: + type: string + description: If Request failed, then here is presented the error message + request: + $ref: "#/components/schemas/StorageRequest" + + DataList: + type: object + properties: + content: + type: array + items: + $ref: "#/components/schemas/DataItem" + + DataItem: + type: object + properties: + cid: + $ref: "#/components/schemas/Cid" + manifest: + $ref: "#/components/schemas/ManifestItem" + + ManifestItem: + type: object + properties: + rootHash: + $ref: "#/components/schemas/Cid" + description: "Root hash of the content" + originalBytes: + type: number + description: "Length of original content in bytes" + blockSize: + type: number + description: "Size of blocks" + protected: + type: boolean + description: "Indicates if content is protected by erasure-coding" + + Space: + type: object + properties: + totalBlocks: + type: number + description: "Number of blocks stored by the node" + quotaMaxBytes: + type: number + description: "Maximum storage space used by the node" + quotaUsedBytes: + type: number + description: "Amount of storage space currently in use" + quotaReservedBytes: + type: number + description: "Amount of storage space reserved" + +servers: + - url: "http://localhost:8080/api/codex/v1" + +tags: + - name: Marketplace + description: Marketplace information and operations + - name: Data + description: Data operations + - name: Node + description: Node management + - name: Debug + description: Debugging configuration + +paths: + "/connect/{peerId}": + get: + summary: "Connect to a peer" + description: | + If `addrs` param is supplied, it will be used to dial the peer, otherwise the `peerId` is used + to invoke peer discovery, if it succeeds the returned addresses will be used to dial. + tags: [ Node ] + operationId: connectPeer + parameters: + - in: path + name: peerId + required: true + schema: + $ref: "#/components/schemas/PeerId" + description: Peer that should be dialed. + - in: query + name: addrs + schema: + type: array + nullable: true + items: + $ref: "#/components/schemas/MultiAddress" + description: | + If supplied, it will be used to dial the peer. + The address has to target the listening address of the peer, + which is specified with the `--listen-addrs` CLI flag. + + responses: + "200": + description: Successfully connected to peer + "400": + description: Peer either not found or was not possible to dial + + "/data": + get: + summary: "Lists manifest CIDs stored locally in node." + tags: [ Data ] + operationId: listData + responses: + "200": + description: Retrieved list of content CIDs + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DataList" + "400": + description: Invalid CID is specified + "404": + description: Content specified by the CID is not found + "500": + description: Well it was bad-bad + post: + summary: "Upload a file in a streaming manner. Once finished, the file is stored in the node and can be retrieved by any node in the network using the returned CID." + tags: [ Data ] + operationId: upload + requestBody: + content: + application/octet-stream: + schema: + type: string + format: binary + responses: + "200": + description: CID of uploaded file + content: + text/plain: + schema: + type: string + "500": + description: Well it was bad-bad and the upload did not work out + + "/data/{cid}": + get: + summary: "Download a file from the local node in a streaming manner. If the file is not available locally, a 404 is returned." + tags: [ Data ] + operationId: downloadLocal + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: File to be downloaded. + + responses: + "200": + description: Retrieved content specified by CID + content: + application/octet-stream: + schema: + type: string + format: binary + "400": + description: Invalid CID is specified + "404": + description: Content specified by the CID is unavailable locally + "500": + description: Well it was bad-bad + + "/data/{cid}/network": + get: + summary: "Download a file from the network in a streaming manner. If the file is not available locally, it will be retrieved from other nodes in the network if able." + tags: [ Data ] + operationId: downloadNetwork + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: "File to be downloaded." + responses: + "200": + description: Retrieved content specified by CID + content: + application/octet-stream: + schema: + type: string + format: binary + "400": + description: Invalid CID is specified + "404": + description: Content specified by the CID is not found + "500": + description: Well it was bad-bad + + "/space": + get: + summary: "Gets a summary of the storage space allocation of the node." + tags: [ Data ] + operationId: space + responses: + "200": + description: "Summary of storage allocation" + content: + application/json: + schema: + $ref: "#/components/schemas/Space" + + "500": + description: "It's not working as planned" + + "/sales/slots": + get: + summary: "Returns active slots" + tags: [ Marketplace ] + operationId: getActiveSlots + responses: + "200": + description: Retrieved active slots + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Slot" + + "503": + description: Sales are unavailable + + "/sales/slots/{slotId}": + get: + summary: "Returns active slot with id {slotId} for the host" + tags: [ Marketplace ] + operationId: getActiveSlotById + parameters: + - in: path + name: slotId + required: true + schema: + $ref: "#/components/schemas/Cid" + description: File to be downloaded. + responses: + "200": + description: Retrieved active slot + content: + application/json: + schema: + $ref: "#/components/schemas/Slot" + + "400": + description: Invalid or missing SlotId + + "404": + description: Host is not in an active sale for the slot + + "503": + description: Sales are unavailable + + "/sales/availability": + get: + summary: "Returns storage that is for sale" + tags: [ Marketplace ] + operationId: getOfferedStorage + responses: + "200": + description: Retrieved storage availabilities of the node + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/SalesAvailability" + "500": + description: Error getting unused availabilities + "503": + description: Sales are unavailable + + post: + summary: "Offers storage for sale" + operationId: offerStorage + tags: [ Marketplace ] + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SalesAvailabilityCREATE" + responses: + "201": + description: Created storage availability + content: + application/json: + schema: + $ref: "#/components/schemas/SalesAvailabilityREAD" + "400": + description: Invalid data input + "422": + description: Not enough node's storage quota available + "500": + description: Error reserving availability + "503": + description: Sales are unavailable + "/sales/availability/{id}": + patch: + summary: "Updates availability" + description: | + The new parameters will be only considered for new requests. + Existing Requests linked to this Availability will continue as is. + operationId: updateOfferedStorage + tags: [ Marketplace ] + parameters: + - in: path + name: id + required: true + schema: + type: string + description: ID of Availability + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/SalesAvailability" + responses: + "204": + description: Availability successfully updated + "400": + description: Invalid data input + "404": + description: Availability not found + "422": + description: Not enough node's storage quota available + "500": + description: Error reserving availability + "503": + description: Sales are unavailable + + "/sales/availability/{id}/reservations": + patch: + summary: "Get availability's reservations" + description: Return's list of Reservations for ongoing Storage Requests that the node hosts. + operationId: getReservations + tags: [ Marketplace ] + parameters: + - in: path + name: id + required: true + schema: + type: string + description: ID of Availability + responses: + "200": + description: Retrieved storage availabilities of the node + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Reservation" + "400": + description: Invalid Availability ID + "404": + description: Availability not found + "500": + description: Error getting reservations + "503": + description: Sales are unavailable + + "/storage/request/{cid}": + post: + summary: "Creates a new Request for storage" + tags: [ Marketplace ] + operationId: createStorageRequest + parameters: + - in: path + name: cid + required: true + schema: + $ref: "#/components/schemas/Cid" + description: CID of the uploaded data that should be stored + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/StorageRequestCreation" + responses: + "200": + description: Returns the Request ID as decimal string + "400": + description: Invalid or missing Request ID + "404": + description: Request ID not found + "503": + description: Purchasing is unavailable + + "/storage/purchases": + get: + summary: "Returns list of purchase IDs" + tags: [ Marketplace ] + operationId: getPurchases + responses: + "200": + description: Gets all purchase IDs stored in node + content: + application/json: + schema: + type: array + items: + type: string + "503": + description: Purchasing is unavailable + + "/storage/purchases/{id}": + get: + summary: "Returns purchase details" + tags: [ Marketplace ] + operationId: getPurchase + parameters: + - in: path + name: id + required: true + schema: + type: string + description: Hexadecimal ID of a Purchase + responses: + "200": + description: Purchase details + content: + application/json: + schema: + $ref: "#/components/schemas/Purchase" + "400": + description: Invalid or missing Purchase ID + "404": + description: Purchase not found + "503": + description: Purchasing is unavailable + + "/debug/chronicles/loglevel": + post: + summary: "Set log level at run time" + tags: [ Debug ] + operationId: setDebugLogLevel + + parameters: + - in: query + name: level + required: true + schema: + $ref: "#/components/schemas/LogLevel" + + responses: + "200": + description: Successfully log level set + "400": + description: Invalid or missing log level + "500": + description: Well it was bad-bad + + "/debug/info": + get: + summary: "Gets node information" + operationId: getDebugInfo + tags: [ Debug ] + responses: + "200": + description: Node's information + content: + application/json: + schema: + $ref: "#/components/schemas/DebugInfo" From 87ec67778bc7437f6b2bd4e5f2d4bdc1161d9520 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Mar 2024 11:37:41 +0100 Subject: [PATCH 07/25] poc: upload and download streams working --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 14 ++++++++++++++ ProjectPlugins/CodexPlugin/CodexNode.cs | 3 --- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index a305d76..f2410f7 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -109,6 +109,20 @@ namespace CodexPlugin private IHttp Http() { + var address = GetAddress(); + var api = new CodexOpenApi.CodexApi(new HttpClient()); + api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + + var debugInfo = Time.Wait(api.GetDebugInfoAsync()); + + using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png"); + var cid = Time.Wait(api.UploadAsync(stream)); + + var file = Time.Wait(api.DownloadNetworkAsync(cid)); + while (file.IsPartial) Thread.Sleep(100); + using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png"); + file.Stream.CopyTo(outfile); + return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); } diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index de7f10f..40136dc 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -14,9 +14,6 @@ namespace CodexPlugin string GetName(); CodexDebugResponse GetDebugInfo(); CodexDebugPeerResponse GetDebugPeer(string peerId); - // These debug methods are not available in master-line Codex. Use only for custom builds. - //CodexDebugBlockExchangeResponse GetDebugBlockExchange(); - //CodexDebugRepoStoreResponse[] GetDebugRepoStore(); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); CodexLocalData[] LocalFiles(); From 03a3ccb4de7a7480889ba39ea8e08e2cc73d85cd Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Mar 2024 13:48:20 +0100 Subject: [PATCH 08/25] prototype of pre-build yaml hash injection --- ProjectPlugins/CodexPlugin/CodexPlugin.cs | 4 ++- ProjectPlugins/CodexPlugin/CodexPlugin.csproj | 4 +++ .../CodexPluginPrebuild.csproj | 10 ++++++ ProjectPlugins/CodexPluginPrebuild/Program.cs | 36 +++++++++++++++++++ cs-codex-dist-testing.sln | 7 ++++ 5 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj create mode 100644 ProjectPlugins/CodexPluginPrebuild/Program.cs diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 3e40a99..624862e 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -1,4 +1,4 @@ -using Core; +using Core; using KubernetesWorkflow.Types; namespace CodexPlugin @@ -9,6 +9,8 @@ namespace CodexPlugin private readonly IPluginTools tools; private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; + private const string OpenApiYamlHash = ""; + public CodexPlugin(IPluginTools tools) { codexStarter = new CodexStarter(tools); diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj index 47899a0..2bb1256 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj @@ -34,4 +34,8 @@ + + + + diff --git a/ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj b/ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj new file mode 100644 index 0000000..f02677b --- /dev/null +++ b/ProjectPlugins/CodexPluginPrebuild/CodexPluginPrebuild.csproj @@ -0,0 +1,10 @@ + + + + Exe + net7.0 + enable + enable + + + diff --git a/ProjectPlugins/CodexPluginPrebuild/Program.cs b/ProjectPlugins/CodexPluginPrebuild/Program.cs new file mode 100644 index 0000000..f33246d --- /dev/null +++ b/ProjectPlugins/CodexPluginPrebuild/Program.cs @@ -0,0 +1,36 @@ +using System.Security.Cryptography; + +public static class Program +{ + private const string OpenApiFile = "../CodexPlugin/openapi.yaml"; + private const string Search = ""; + private const string TargetFile = "CodexPlugin.cs"; + + private static string CreateHash() + { + var fileBytes = File.ReadAllBytes(OpenApiFile); + var sha = SHA256.Create(); + var hash = sha.ComputeHash(fileBytes); + return BitConverter.ToString(hash); + } + + private static void SearchAndReplace(string hash) + { + var lines = File.ReadAllLines(TargetFile); + lines = lines.Select(l => l.Replace(Search, hash)).ToArray(); + File.WriteAllLines(TargetFile, lines); + } + + public static void Main(string[] args) + { + Console.WriteLine("Injecting hash of 'openapi.yaml'..."); + // This hash is used to verify that the Codex docker image being used is compatible + // with the openapi.yaml being used by the Codex plugin. + // If the openapi.yaml files don't match, an exception is thrown. + + var hash = CreateHash(); + SearchAndReplace(hash); + + Console.WriteLine("Done!"); + } +} \ No newline at end of file diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index 3a403fe..b9b5693 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -59,6 +59,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GethConnector", "Framework\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DiscordRewards", "Framework\DiscordRewards\DiscordRewards.csproj", "{B07820C4-309F-4454-BCC1-1D4902C9C67B}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexPluginPrebuild", "ProjectPlugins\CodexPluginPrebuild\CodexPluginPrebuild.csproj", "{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -161,6 +163,10 @@ Global {B07820C4-309F-4454-BCC1-1D4902C9C67B}.Debug|Any CPU.Build.0 = Debug|Any CPU {B07820C4-309F-4454-BCC1-1D4902C9C67B}.Release|Any CPU.ActiveCfg = Release|Any CPU {B07820C4-309F-4454-BCC1-1D4902C9C67B}.Release|Any CPU.Build.0 = Release|Any CPU + {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {88C212E9-308A-46A4-BAAD-468E8EBD8EDF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -190,6 +196,7 @@ Global {570C0DBE-0EF1-47B5-9A3B-E1F7895722A5} = {7591C5B3-D86E-4AE4-8ED2-B272D17FE7E3} {F730DA73-1C92-4107-BCFB-D33759DAB0C3} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} {B07820C4-309F-4454-BCC1-1D4902C9C67B} = {81AE04BC-CBFA-4E6F-B039-8208E9AFAAE7} + {88C212E9-308A-46A4-BAAD-468E8EBD8EDF} = {8F1F1C2A-E313-4E0C-BE40-58FB0BA91124} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {237BF0AA-9EC4-4659-AD9A-65DEB974250C} From c122aa99103a7913a4200f3fe4a95e48c18d9e0b Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 25 Mar 2024 15:46:45 +0100 Subject: [PATCH 09/25] sets up http client --- Framework/Core/Http.cs | 50 +++-- Framework/Core/PluginTools.cs | 18 +- ProjectPlugins/CodexPlugin/CodexAccess.cs | 57 +++--- ProjectPlugins/CodexPlugin/CodexApiTypes.cs | 196 -------------------- ProjectPlugins/CodexPlugin/CodexNode.cs | 58 ++---- ProjectPlugins/CodexPlugin/CodexPlugin.cs | 2 +- ProjectPlugins/CodexPlugin/CodexTypes.cs | 47 +++++ 7 files changed, 140 insertions(+), 288 deletions(-) delete mode 100644 ProjectPlugins/CodexPlugin/CodexApiTypes.cs create mode 100644 ProjectPlugins/CodexPlugin/CodexTypes.cs diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 192152f..b3feca3 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -9,14 +9,16 @@ namespace Core { public interface IHttp { - string HttpGetString(string route); - T HttpGetJson(string route); - TResponse HttpPostJson(string route, TRequest body); - string HttpPostJson(string route, TRequest body); - TResponse HttpPostString(string route, string body); - string HttpPostStream(string route, Stream stream); - Stream HttpGetStream(string route); - T Deserialize(string json); + //string HttpGetString(string route); + //T HttpGetJson(string route); + //TResponse HttpPostJson(string route, TRequest body); + //string HttpPostJson(string route, TRequest body); + //TResponse HttpPostString(string route, string body); + //string HttpPostStream(string route, Stream stream); + //Stream HttpGetStream(string route); + //T Deserialize(string json); + + T OnClient(Func action); } internal class Http : IHttp @@ -24,26 +26,37 @@ namespace Core private static readonly object httpLock = new object(); private readonly ILog log; private readonly ITimeSet timeSet; - private readonly Address address; - private readonly string baseUrl; private readonly Action onClientCreated; private readonly string? logAlias; - internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, string? logAlias = null) - : this(log, timeSet, address, baseUrl, DoNothing, logAlias) + internal Http(ILog log, ITimeSet timeSet, string? logAlias = null) + : this(log, timeSet, DoNothing, logAlias) { } - internal Http(ILog log, ITimeSet timeSet, Address address, string baseUrl, Action onClientCreated, string? logAlias = null) + internal Http(ILog log, ITimeSet timeSet, Action onClientCreated, string? logAlias = null) { this.log = log; this.timeSet = timeSet; - this.address = address; - this.baseUrl = baseUrl; this.onClientCreated = onClientCreated; this.logAlias = logAlias; - if (!this.baseUrl.StartsWith("/")) this.baseUrl = "/" + this.baseUrl; - if (!this.baseUrl.EndsWith("/")) this.baseUrl += "/"; + } + + public T OnClient(Func action) + { + var client = GetClient(); + var description = GetDescription(); + + return LockRetry(() => + { + return action(client); + }, description); + } + + private string GetDescription() + { + // todo: check this: + return DebugStack.GetCallerName(skipFrames: 2); } public string HttpGetString(string route) @@ -190,7 +203,8 @@ namespace Core private string GetUrl() { - return $"{address.Host}:{address.Port}{baseUrl}"; + //return $"{address.Host}:{address.Port}{baseUrl}"; + return "--Obsolete--"; } private void Log(string url, string message) diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs index ebd9d38..a493563 100644 --- a/Framework/Core/PluginTools.cs +++ b/Framework/Core/PluginTools.cs @@ -22,9 +22,9 @@ namespace Core public interface IHttpFactoryTool { - IHttp CreateHttp(Address address, string baseUrl, Action onClientCreated, string? logAlias = null); - IHttp CreateHttp(Address address, string baseUrl, Action onClientCreated, ITimeSet timeSet, string? logAlias = null); - IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null); + IHttp CreateHttp(Action onClientCreated, string? logAlias = null); + IHttp CreateHttp(Action onClientCreated, ITimeSet timeSet, string? logAlias = null); + IHttp CreateHttp(string? logAlias = null); } public interface IFileTool @@ -52,19 +52,19 @@ namespace Core log = new LogPrefixer(log, prefix); } - public IHttp CreateHttp(Address address, string baseUrl, Action onClientCreated, string? logAlias = null) + public IHttp CreateHttp(Action onClientCreated, string? logAlias = null) { - return CreateHttp(address, baseUrl, onClientCreated, timeSet, logAlias); + return CreateHttp(onClientCreated, timeSet, logAlias); } - public IHttp CreateHttp(Address address, string baseUrl, Action onClientCreated, ITimeSet ts, string? logAlias = null) + public IHttp CreateHttp(Action onClientCreated, ITimeSet ts, string? logAlias = null) { - return new Http(log, ts, address, baseUrl, onClientCreated, logAlias); + return new Http(log, ts, onClientCreated, logAlias); } - public IHttp CreateHttp(Address address, string baseUrl, string? logAlias = null) + public IHttp CreateHttp(string? logAlias = null) { - return new Http(log, timeSet, address, baseUrl, logAlias); + return new Http(log, timeSet, logAlias); } public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index f2410f7..9434f21 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -1,4 +1,5 @@ -using Core; +using CodexOpenApi; +using Core; using KubernetesWorkflow; using KubernetesWorkflow.Types; using Utils; @@ -23,9 +24,9 @@ namespace CodexPlugin public RunningContainer Container { get; } public CrashWatcher CrashWatcher { get; } - public CodexDebugResponse GetDebugInfo() + public CodexOpenApi.DebugInfo GetDebugInfo() { - return Http().HttpGetJson("debug/info"); + return OnCodex(api => api.GetDebugInfoAsync()); } public CodexDebugPeerResponse GetDebugPeer(string peerId) @@ -107,29 +108,41 @@ namespace CodexPlugin return workflow.GetPodInfo(Container); } - private IHttp Http() + private T OnCodex(Func> action) { var address = GetAddress(); - var api = new CodexOpenApi.CodexApi(new HttpClient()); - api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - - var debugInfo = Time.Wait(api.GetDebugInfoAsync()); - - using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png"); - var cid = Time.Wait(api.UploadAsync(stream)); - - var file = Time.Wait(api.DownloadNetworkAsync(cid)); - while (file.IsPartial) Thread.Sleep(100); - using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png"); - file.Stream.CopyTo(outfile); - - return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); + var result = tools.CreateHttp().OnClient(client => + { + var api = new CodexApi(client); + api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + return Time.Wait(action(api)); + }); + return result; } - private IHttp LongHttp() - { - return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); - } + //private IHttp Http() + //{ + // var address = GetAddress(); + // var api = new CodexOpenApi.CodexApi(new HttpClient()); + // api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; + + // var debugInfo = Time.Wait(api.GetDebugInfoAsync()); + + // using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png"); + // var cid = Time.Wait(api.UploadAsync(stream)); + + // var file = Time.Wait(api.DownloadNetworkAsync(cid)); + // while (file.IsPartial) Thread.Sleep(100); + // using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png"); + // file.Stream.CopyTo(outfile); + + // return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); + //} + + //private IHttp LongHttp() + //{ + // return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); + //} private Address GetAddress() { diff --git a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs b/ProjectPlugins/CodexPlugin/CodexApiTypes.cs deleted file mode 100644 index 96d19ef..0000000 --- a/ProjectPlugins/CodexPlugin/CodexApiTypes.cs +++ /dev/null @@ -1,196 +0,0 @@ -using Newtonsoft.Json; - -namespace CodexPlugin -{ - public class CodexDebugResponse - { - public string id { get; set; } = string.Empty; - public string[] addrs { get; set; } = Array.Empty(); - public string repo { get; set; } = string.Empty; - public string spr { get; set; } = string.Empty; - public string[] announceAddresses { get; set; } = Array.Empty(); - public EnginePeerResponse[] enginePeers { get; set; } = Array.Empty(); - public SwitchPeerResponse[] switchPeers { get; set; } = Array.Empty(); - public CodexDebugVersionResponse codex { get; set; } = new(); - public CodexDebugTableResponse table { get; set; } = new(); - } - - public class CodexDebugFutures - { - public int futures { get; set; } - } - - public class CodexDebugTableResponse - { - public CodexDebugTableNodeResponse localNode { get; set; } = new(); - public CodexDebugTableNodeResponse[] nodes { get; set; } = Array.Empty(); - } - - public class CodexDebugTableNodeResponse - { - public string nodeId { get; set; } = string.Empty; - public string peerId { get; set; } = string.Empty; - public string record { get; set; } = string.Empty; - public string address { get; set; } = string.Empty; - public bool seen { get; set; } - } - - public class EnginePeerResponse - { - public string peerId { get; set; } = string.Empty; - public EnginePeerContextResponse context { get; set; } = new(); - } - - public class EnginePeerContextResponse - { - public int blocks { get; set; } = 0; - public int peerWants { get; set; } = 0; - public int exchanged { get; set; } = 0; - public string lastExchange { get; set; } = string.Empty; - } - - public class SwitchPeerResponse - { - public string peerId { get; set; } = string.Empty; - public string key { get; set; } = string.Empty; - } - - public class CodexDebugVersionResponse - { - public string version { get; set; } = string.Empty; - public string revision { get; set; } = string.Empty; - - public bool IsValid() - { - return !string.IsNullOrEmpty(version) && !string.IsNullOrEmpty(revision); - } - - public override string ToString() - { - return JsonConvert.SerializeObject(this); - } - } - - public class CodexDebugPeerResponse - { - public bool IsPeerFound { get; set; } - - public string peerId { get; set; } = string.Empty; - public long seqNo { get; set; } - public string[] addresses { get; set; } = Array.Empty(); - } - - public class CodexDebugThresholdBreaches - { - public string[] breaches { get; set; } = Array.Empty(); - } - - public class CodexSalesAvailabilityRequest - { - public string totalSize { get; set; } = string.Empty; - public string duration { get; set; } = string.Empty; - public string minPrice { get; set; } = string.Empty; - public string maxCollateral { get; set; } = string.Empty; - } - - public class CodexSalesAvailabilityResponse - { - public string id { get; set; } = string.Empty; - public string totalSize { get; set; } = string.Empty; - public string freeSize { get; set; } = string.Empty; - public string duration { get; set; } = string.Empty; - public string minPrice { get; set; } = string.Empty; - public string maxCollateral { get; set; } = string.Empty; - } - - public class CodexSalesRequestStorageRequest - { - public string duration { get; set; } = string.Empty; - public string proofProbability { get; set; } = string.Empty; - public string reward { get; set; } = string.Empty; - public string collateral { get; set; } = string.Empty; - public string? expiry { get; set; } - public uint? nodes { get; set; } - public uint? tolerance { get; set; } - } - - public class CodexStoragePurchase - { - public string state { get; set; } = string.Empty; - public string error { get; set; } = string.Empty; - } - - public class CodexDebugBlockExchangeResponse - { - public CodexDebugBlockExchangeResponsePeer[] peers { get; set; } = Array.Empty(); - public int taskQueue { get; set; } - public int pendingBlocks { get; set; } - - public override string ToString() - { - if (peers.Length == 0 && taskQueue == 0 && pendingBlocks == 0) return "all-empty"; - - return $"taskqueue: {taskQueue} pendingblocks: {pendingBlocks} peers: {string.Join(",", peers.Select(p => p.ToString()))}"; - } - } - - public class CodexDebugBlockExchangeResponsePeer - { - public string peerid { get; set; } = string.Empty; - public CodexDebugBlockExchangeResponsePeerHasBlock[] hasBlocks { get; set; } = Array.Empty(); - public CodexDebugBlockExchangeResponsePeerWant[] wants { get; set; } = Array.Empty(); - public int exchanged { get; set; } - - public override string ToString() - { - return $"(blocks:{hasBlocks.Length} wants:{wants.Length})"; - } - } - - public class CodexDebugBlockExchangeResponsePeerHasBlock - { - public string cid { get; set; } = string.Empty; - public bool have { get; set; } - public string price { get; set; } = string.Empty; - } - - public class CodexDebugBlockExchangeResponsePeerWant - { - public string block { get; set; } = string.Empty; - public int priority { get; set; } - public bool cancel { get; set; } - public string wantType { get; set; } = string.Empty; - public bool sendDontHave { get; set; } - } - - public class CodexDebugRepoStoreResponse - { - public string cid { get; set; } = string.Empty; - } - - public class CodexLocalData - { - public CodexLocalData(ContentId cid, CodexLocalDataManifestResponse manifest) - { - Cid = cid; - Manifest = manifest; - } - - public ContentId Cid { get; } - public CodexLocalDataManifestResponse Manifest { get; } - } - - public class CodexLocalDataResponse - { - public string cid { get; set; } = string.Empty; - public CodexLocalDataManifestResponse manifest { get; set; } = new(); - } - - public class CodexLocalDataManifestResponse - { - public string rootHash { get; set; } = string.Empty; - public int originalBytes { get; set; } - public int blockSize { get; set; } - public bool @protected { get; set; } - } -} diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 40136dc..5c78b5d 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -12,13 +12,13 @@ namespace CodexPlugin public interface ICodexNode : IHasContainer, IHasMetricsScrapeTarget, IHasEthAddress { string GetName(); - CodexDebugResponse GetDebugInfo(); - CodexDebugPeerResponse GetDebugPeer(string peerId); + DebugInfo GetDebugInfo(); + DebugPeer GetDebugPeer(string peerId); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); - CodexLocalData[] LocalFiles(); + LocalDataset[] LocalFiles(); void ConnectToPeer(ICodexNode node); - CodexDebugVersionResponse Version { get; } + DebugVersion Version { get; } IMarketplaceAccess Marketplace { get; } CrashWatcher CrashWatcher { get; } PodInfo GetPodInfo(); @@ -41,7 +41,7 @@ namespace CodexPlugin CodexAccess = codexAccess; Group = group; Marketplace = marketplaceAccess; - Version = new CodexDebugVersionResponse(); + Version = new DebugVersion(); transferSpeeds = new TransferSpeeds(); } @@ -50,7 +50,7 @@ namespace CodexPlugin public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; } public CodexNodeGroup Group { get; } public IMarketplaceAccess Marketplace { get; } - public CodexDebugVersionResponse Version { get; private set; } + public DebugVersion Version { get; private set; } public ITransferSpeeds TransferSpeeds { get => transferSpeeds; } public IMetricsScrapeTarget MetricsScrapeTarget { @@ -73,7 +73,7 @@ namespace CodexPlugin return CodexAccess.Container.Name; } - public CodexDebugResponse GetDebugInfo() + public DebugInfo GetDebugInfo() { var debugInfo = CodexAccess.GetDebugInfo(); var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); @@ -81,21 +81,11 @@ namespace CodexPlugin return debugInfo; } - public CodexDebugPeerResponse GetDebugPeer(string peerId) + public DebugPeer GetDebugPeer(string peerId) { return CodexAccess.GetDebugPeer(peerId); } - public CodexDebugBlockExchangeResponse GetDebugBlockExchange() - { - return CodexAccess.GetDebugBlockExchange(); - } - - public CodexDebugRepoStoreResponse[] GetDebugRepoStore() - { - return CodexAccess.GetDebugRepoStore(); - } - public ContentId UploadFile(TrackedFile file) { using var fileStream = File.OpenRead(file.Filename); @@ -128,9 +118,13 @@ namespace CodexPlugin return file; } - public CodexLocalData[] LocalFiles() + public LocalDataset[] LocalFiles() { - return CodexAccess.LocalFiles().Select(l => new CodexLocalData(new ContentId(l.cid), l.manifest)).ToArray(); + return CodexAccess.LocalFiles().Select(l => new LocalDataset() + { + Cid = new ContentId(l.cid) + //, l.manifest + }).ToArray(); } public void ConnectToPeer(ICodexNode node) @@ -176,9 +170,9 @@ namespace CodexPlugin Version = debugInfo.codex; } - private string GetPeerMultiAddress(CodexNode peer, CodexDebugResponse peerInfo) + private string GetPeerMultiAddress(CodexNode peer, DebugInfo peerInfo) { - var multiAddress = peerInfo.addrs.First(); + var multiAddress = peerInfo.Addrs.First(); // Todo: Is there a case where First address in list is not the way? // The peer we want to connect is in a different pod. @@ -209,24 +203,4 @@ namespace CodexPlugin tools.GetLog().Log($"{GetName()}: {msg}"); } } - - public class ContentId - { - public ContentId(string id) - { - Id = id; - } - - public string Id { get; } - - public override bool Equals(object? obj) - { - return obj is ContentId id && Id == id.Id; - } - - public override int GetHashCode() - { - return HashCode.Combine(Id); - } - } } diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 624862e..a7e0f57 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -9,7 +9,7 @@ namespace CodexPlugin private readonly IPluginTools tools; private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; - private const string OpenApiYamlHash = ""; + private const string OpenApiYamlHash = "8B-DD-61-54-42-D7-28-8F-5A-A0-AF-C2-A4-53-A7-08-B6-C7-02-FD-59-1A-01-A9-B4-7D-E4-81-FA-84-23-7F"; public CodexPlugin(IPluginTools tools) { diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs new file mode 100644 index 0000000..f9848ea --- /dev/null +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -0,0 +1,47 @@ +namespace CodexPlugin +{ + public class DebugInfo + { + public string[] Addrs { get; set; } = Array.Empty(); + } + + public class DebugPeer + { + + } + + public class LocalDataset + { + public ContentId Cid { get; set; } = new ContentId(); + } + + public class DebugVersion + { + + } + + public class ContentId + { + public ContentId() + { + Id = string.Empty; + } + + public ContentId(string id) + { + Id = id; + } + + public string Id { get; } + + public override bool Equals(object? obj) + { + return obj is ContentId id && Id == id.Id; + } + + public override int GetHashCode() + { + return HashCode.Combine(Id); + } + } +} From b69059fd370955351db8a98c1f11efacda6bd333 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 08:58:16 +0100 Subject: [PATCH 10/25] restoring debug-info --- Framework/Core/Http.cs | 16 ++++++++-------- ProjectPlugins/CodexPlugin/CodexAccess.cs | 10 ++++++++-- ProjectPlugins/CodexPlugin/CodexDeployment.cs | 4 ++-- ProjectPlugins/CodexPlugin/CodexNodeGroup.cs | 6 +++--- ProjectPlugins/CodexPlugin/CodexPlugin.csproj | 2 +- ProjectPlugins/CodexPlugin/CodexSetup.cs | 2 +- ProjectPlugins/CodexPlugin/CodexStarter.cs | 6 +++--- ProjectPlugins/CodexPlugin/CodexTypes.cs | 7 ++++++- ProjectPlugins/CodexPlugin/Mapper.cs | 18 ++++++++++++++++++ .../BasicTests/TestInfraTests.cs | 4 ++-- .../Helpers/FullConnectivityHelper.cs | 6 +++--- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 6 +++--- Tools/CodexNetDeployer/CodexNodeStarter.cs | 2 +- 13 files changed, 59 insertions(+), 30 deletions(-) create mode 100644 ProjectPlugins/CodexPlugin/Mapper.cs diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index b3feca3..252a300 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -9,14 +9,14 @@ namespace Core { public interface IHttp { - //string HttpGetString(string route); - //T HttpGetJson(string route); - //TResponse HttpPostJson(string route, TRequest body); - //string HttpPostJson(string route, TRequest body); - //TResponse HttpPostString(string route, string body); - //string HttpPostStream(string route, Stream stream); - //Stream HttpGetStream(string route); - //T Deserialize(string json); + string HttpGetString(string route); + T HttpGetJson(string route); + TResponse HttpPostJson(string route, TRequest body); + string HttpPostJson(string route, TRequest body); + TResponse HttpPostString(string route, string body); + string HttpPostStream(string route, Stream stream); + Stream HttpGetStream(string route); + T Deserialize(string json); T OnClient(Func action); } diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 9434f21..f576cfd 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -9,6 +9,7 @@ namespace CodexPlugin public class CodexAccess : ILogHandler { private readonly IPluginTools tools; + private readonly Mapper mapper = new Mapper(); private bool hasContainerCrashed; public CodexAccess(IPluginTools tools, RunningContainer container, CrashWatcher crashWatcher) @@ -24,9 +25,9 @@ namespace CodexPlugin public RunningContainer Container { get; } public CrashWatcher CrashWatcher { get; } - public CodexOpenApi.DebugInfo GetDebugInfo() + public DebugInfo GetDebugInfo() { - return OnCodex(api => api.GetDebugInfoAsync()); + return Map(OnCodex(api => api.GetDebugInfoAsync())); } public CodexDebugPeerResponse GetDebugPeer(string peerId) @@ -108,6 +109,11 @@ namespace CodexPlugin return workflow.GetPodInfo(Container); } + private dynamic Map(dynamic input) + { + return mapper.Map(input); + } + private T OnCodex(Func> action) { var address = GetAddress(); diff --git a/ProjectPlugins/CodexPlugin/CodexDeployment.cs b/ProjectPlugins/CodexPlugin/CodexDeployment.cs index ccf5ef2..9f74dd3 100644 --- a/ProjectPlugins/CodexPlugin/CodexDeployment.cs +++ b/ProjectPlugins/CodexPlugin/CodexDeployment.cs @@ -31,14 +31,14 @@ namespace CodexPlugin public class CodexInstance { - public CodexInstance(RunningContainers containers, CodexDebugResponse info) + public CodexInstance(RunningContainers containers, DebugInfo info) { Containers = containers; Info = info; } public RunningContainers Containers { get; } - public CodexDebugResponse Info { get; } + public DebugInfo Info { get; } } public class DeploymentMetadata diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index e9eceb7..8b6264c 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -20,7 +20,7 @@ namespace CodexPlugin this.starter = starter; Containers = containers; Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); - Version = new CodexDebugVersionResponse(); + Version = new DebugVersion(); } public ICodexNode this[int index] @@ -41,7 +41,7 @@ namespace CodexPlugin public RunningContainers[] Containers { get; private set; } public CodexNode[] Nodes { get; private set; } - public CodexDebugVersionResponse Version { get; private set; } + public DebugVersion Version { get; private set; } public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray(); public IEnumerator GetEnumerator() @@ -65,7 +65,7 @@ namespace CodexPlugin var versionResponses = Nodes.Select(n => n.Version); var first = versionResponses.First(); - if (!versionResponses.All(v => v.version == first.version && v.revision == first.revision)) + if (!versionResponses.All(v => v.Version == first.Version && v.Revision == first.Revision)) { throw new Exception("Inconsistent version information received from one or more Codex nodes: " + string.Join(",", versionResponses.Select(v => v.ToString()))); diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj index 2bb1256..ffdff42 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.csproj +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.csproj @@ -35,7 +35,7 @@ - + diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index 54be6a9..a7afc9f 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -75,7 +75,7 @@ namespace CodexPlugin public ICodexSetup WithBootstrapNode(ICodexNode node) { - BootstrapSpr = node.GetDebugInfo().spr; + BootstrapSpr = node.GetDebugInfo().Spr; return this; } diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 57045cc..7a202f8 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -9,7 +9,7 @@ namespace CodexPlugin { private readonly IPluginTools pluginTools; private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); - private CodexDebugVersionResponse? versionResponse; + private DebugVersion? versionResponse; public CodexStarter(IPluginTools pluginTools) { @@ -62,13 +62,13 @@ namespace CodexPlugin public string GetCodexId() { - if (versionResponse != null) return versionResponse.version; + if (versionResponse != null) return versionResponse.Version; return recipe.Image; } public string GetCodexRevision() { - if (versionResponse != null) return versionResponse.revision; + if (versionResponse != null) return versionResponse.Revision; return "unknown"; } diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index f9848ea..5a54f9e 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -3,10 +3,14 @@ public class DebugInfo { public string[] Addrs { get; set; } = Array.Empty(); + public string Spr { get; set; } = string.Empty; + public string Id { get; set; } = string.Empty; + public string[] AnnounceAddresses { get; set; } = Array.Empty(); } public class DebugPeer { + public bool IsPeerFound { get; set; } } @@ -17,7 +21,8 @@ public class DebugVersion { - + public string Version { get; internal set; } = string.Empty; + public string Revision { get; internal set; } = string.Empty; } public class ContentId diff --git a/ProjectPlugins/CodexPlugin/Mapper.cs b/ProjectPlugins/CodexPlugin/Mapper.cs new file mode 100644 index 0000000..d7905ab --- /dev/null +++ b/ProjectPlugins/CodexPlugin/Mapper.cs @@ -0,0 +1,18 @@ +using Newtonsoft.Json.Linq; + +namespace CodexPlugin +{ + public class Mapper + { + public DebugInfo Map(CodexOpenApi.DebugInfo debugInfo) + { + return new DebugInfo + { + Id = debugInfo.Id, + Spr = debugInfo.Spr, + Addrs = debugInfo.Addrs.ToArray(), + AnnounceAddresses = ((JArray) debugInfo.AdditionalProperties["announceAddresses"]).Select(x => x.ToString()).ToArray(), + }; + } + } +} diff --git a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs b/Tests/CodexLongTests/BasicTests/TestInfraTests.cs index 832f13a..5720457 100644 --- a/Tests/CodexLongTests/BasicTests/TestInfraTests.cs +++ b/Tests/CodexLongTests/BasicTests/TestInfraTests.cs @@ -11,7 +11,7 @@ namespace CodexLongTests.BasicTests { var group = AddCodex(1000, s => s.EnableMetrics()); - var nodeIds = group.Select(n => n.GetDebugInfo().id).ToArray(); + var nodeIds = group.Select(n => n.GetDebugInfo().Id).ToArray(); Assert.That(nodeIds.Length, Is.EqualTo(nodeIds.Distinct().Count()), "Not all created nodes provided a unique id."); @@ -24,7 +24,7 @@ namespace CodexLongTests.BasicTests { var n = AddCodex(); - Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().id)); + Assert.That(!string.IsNullOrEmpty(n.GetDebugInfo().Id)); } } } diff --git a/Tests/CodexTests/Helpers/FullConnectivityHelper.cs b/Tests/CodexTests/Helpers/FullConnectivityHelper.cs index 1504c78..8db17ed 100644 --- a/Tests/CodexTests/Helpers/FullConnectivityHelper.cs +++ b/Tests/CodexTests/Helpers/FullConnectivityHelper.cs @@ -114,12 +114,12 @@ namespace CodexTests.Helpers } public ICodexNode Node { get; } - public CodexDebugResponse Response { get; } + public DebugInfo Response { get; } public override string ToString() { - if (Response == null || string.IsNullOrEmpty(Response.id)) return "UNKNOWN"; - return Response.id; + if (Response == null || string.IsNullOrEmpty(Response.Id)) return "UNKNOWN"; + return Response.Id; } } diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 10c1cde..53d78d2 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -70,7 +70,7 @@ namespace CodexTests.PeerDiscoveryTests } } - private string AreAllPresent(CodexDebugResponse info, CodexDebugResponse[] allResponses) + private string AreAllPresent(DebugInfo info, DebugInfo[] allResponses) { var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray(); var allOthers = GetAllOtherResponses(info, allResponses); @@ -84,9 +84,9 @@ namespace CodexTests.PeerDiscoveryTests return string.Empty; } - private CodexDebugResponse[] GetAllOtherResponses(CodexDebugResponse exclude, CodexDebugResponse[] allResponses) + private DebugInfo[] GetAllOtherResponses(DebugInfo exclude, DebugInfo[] allResponses) { - return allResponses.Where(r => r.id != exclude.id).ToArray(); + return allResponses.Where(r => r.Id != exclude.Id).ToArray(); } } } diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index 663659b..73e3c68 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -61,7 +61,7 @@ namespace CodexNetDeployer }); var debugInfo = codexNode.GetDebugInfo(); - if (!string.IsNullOrWhiteSpace(debugInfo.spr)) + if (!string.IsNullOrWhiteSpace(debugInfo.Spr)) { Console.Write("Online\t"); From a3253b42d8453b04b603bab11074d89681b4875d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 09:10:06 +0100 Subject: [PATCH 11/25] Introduces endpoint component from old http functionality --- Framework/Core/Endpoint.cs | 193 ++++++++++++++++++ Framework/Core/Http.cs | 194 ++----------------- Framework/Core/PluginTools.cs | 1 - ProjectPlugins/MetricsPlugin/MetricsQuery.cs | 12 +- 4 files changed, 215 insertions(+), 185 deletions(-) create mode 100644 Framework/Core/Endpoint.cs diff --git a/Framework/Core/Endpoint.cs b/Framework/Core/Endpoint.cs new file mode 100644 index 0000000..28b0c61 --- /dev/null +++ b/Framework/Core/Endpoint.cs @@ -0,0 +1,193 @@ +using Logging; +using Newtonsoft.Json; +using Serialization = Newtonsoft.Json.Serialization; +using System.Net.Http.Headers; +using System.Net.Http.Json; +using Utils; + +namespace Core +{ + public interface IEndpoint + { + string HttpGetString(string route); + T HttpGetJson(string route); + TResponse HttpPostJson(string route, TRequest body); + string HttpPostJson(string route, TRequest body); + TResponse HttpPostString(string route, string body); + string HttpPostStream(string route, Stream stream); + Stream HttpGetStream(string route); + T Deserialize(string json); + } + + internal class Endpoint : IEndpoint + { + private readonly ILog log; + private readonly IHttp http; + private readonly Address address; + private readonly string baseUrl; + private readonly string? logAlias; + + public Endpoint(ILog log, IHttp http, Address address, string baseUrl, string? logAlias) + { + this.log = log; + this.http = http; + this.address = address; + this.baseUrl = baseUrl; + this.logAlias = logAlias; + } + + public string HttpGetString(string route) + { + return http.OnClient(client => + { + return GetString(client, route); + }, $"HTTP-GET:{route}"); + } + + public T HttpGetJson(string route) + { + return http.OnClient(client => + { + var json = GetString(client, route); + return Deserialize(json); + }, $"HTTP-GET:{route}"); + } + + public TResponse HttpPostJson(string route, TRequest body) + { + return http.OnClient(client => + { + var response = PostJson(client, route, body); + var json = Time.Wait(response.Content.ReadAsStringAsync()); + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException(json); + } + Log(GetUrl() + route, json); + return Deserialize(json); + }, $"HTTP-POST-JSON: {route}"); + } + + public string HttpPostJson(string route, TRequest body) + { + return http.OnClient(client => + { + var response = PostJson(client, route, body); + return Time.Wait(response.Content.ReadAsStringAsync()); + }, $"HTTP-POST-JSON: {route}"); + } + + public TResponse HttpPostString(string route, string body) + { + return http.OnClient(client => + { + var response = PostJsonString(client, route, body); + if (response == null) throw new Exception("Received no response."); + var result = Deserialize(response); + if (result == null) throw new Exception("Failed to deserialize response"); + return result; + }, $"HTTO-POST-JSON: {route}"); + } + + public string HttpPostStream(string route, Stream stream) + { + return http.OnClient(client => + { + var url = GetUrl() + route; + Log(url, "~ STREAM ~"); + var content = new StreamContent(stream); + content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); + var response = Time.Wait(client.PostAsync(url, content)); + var str = Time.Wait(response.Content.ReadAsStringAsync()); + Log(url, str); + return str; + }, $"HTTP-POST-STREAM: {route}"); + } + + public Stream HttpGetStream(string route) + { + return http.OnClient(client => + { + var url = GetUrl() + route; + Log(url, "~ STREAM ~"); + return Time.Wait(client.GetStreamAsync(url)); + }, $"HTTP-GET-STREAM: {route}"); + } + + public T Deserialize(string json) + { + var errors = new List(); + var deserialized = JsonConvert.DeserializeObject(json, new JsonSerializerSettings() + { + Error = delegate (object? sender, Serialization.ErrorEventArgs args) + { + if (args.CurrentObject == args.ErrorContext.OriginalObject) + { + errors.Add($""" + Member: '{args.ErrorContext.Member?.ToString() ?? ""}' + Path: {args.ErrorContext.Path} + Error: {args.ErrorContext.Error.Message} + """); + args.ErrorContext.Handled = true; + } + } + }); + if (errors.Count > 0) + { + throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}"); + } + else if (deserialized == null) + { + throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null"); + } + return deserialized; + } + + private string GetString(HttpClient client, string route) + { + var url = GetUrl() + route; + Log(url, ""); + var result = Time.Wait(client.GetAsync(url)); + var str = Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; + } + + private HttpResponseMessage PostJson(HttpClient client, string route, TRequest body) + { + var url = GetUrl() + route; + using var content = JsonContent.Create(body); + Log(url, JsonConvert.SerializeObject(body)); + return Time.Wait(client.PostAsync(url, content)); + } + + private string PostJsonString(HttpClient client, string route, string body) + { + var url = GetUrl() + route; + Log(url, body); + var content = new StringContent(body); + content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); + var result = Time.Wait(client.PostAsync(url, content)); + var str = Time.Wait(result.Content.ReadAsStringAsync()); + Log(url, str); + return str; + } + + private string GetUrl() + { + return $"{address.Host}:{address.Port}{baseUrl}"; + } + + private void Log(string url, string message) + { + if (logAlias != null) + { + log.Debug($"({logAlias})({url}) = '{message}'", 3); + } + else + { + log.Debug($"({url}) = '{message}'", 3); + } + } + } +} diff --git a/Framework/Core/Http.cs b/Framework/Core/Http.cs index 252a300..61d13f8 100644 --- a/Framework/Core/Http.cs +++ b/Framework/Core/Http.cs @@ -1,24 +1,13 @@ using Logging; -using Newtonsoft.Json; -using Serialization = Newtonsoft.Json.Serialization; -using System.Net.Http.Headers; -using System.Net.Http.Json; using Utils; namespace Core { public interface IHttp { - string HttpGetString(string route); - T HttpGetJson(string route); - TResponse HttpPostJson(string route, TRequest body); - string HttpPostJson(string route, TRequest body); - TResponse HttpPostString(string route, string body); - string HttpPostStream(string route, Stream stream); - Stream HttpGetStream(string route); - T Deserialize(string json); - T OnClient(Func action); + T OnClient(Func action, string description); + IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null); } internal class Http : IHttp @@ -27,25 +16,27 @@ namespace Core private readonly ILog log; private readonly ITimeSet timeSet; private readonly Action onClientCreated; - private readonly string? logAlias; - internal Http(ILog log, ITimeSet timeSet, string? logAlias = null) - : this(log, timeSet, DoNothing, logAlias) + internal Http(ILog log, ITimeSet timeSet) + : this(log, timeSet, DoNothing) { } - internal Http(ILog log, ITimeSet timeSet, Action onClientCreated, string? logAlias = null) + internal Http(ILog log, ITimeSet timeSet, Action onClientCreated) { this.log = log; this.timeSet = timeSet; this.onClientCreated = onClientCreated; - this.logAlias = logAlias; } public T OnClient(Func action) + { + return OnClient(action, GetDescription()); + } + + public T OnClient(Func action, string description) { var client = GetClient(); - var description = GetDescription(); return LockRetry(() => { @@ -53,172 +44,17 @@ namespace Core }, description); } + public IEndpoint CreateEndpoint(Address address, string baseUrl, string? logAlias = null) + { + return new Endpoint(log, this, address, baseUrl, logAlias); + } + private string GetDescription() { // todo: check this: return DebugStack.GetCallerName(skipFrames: 2); } - public string HttpGetString(string route) - { - return LockRetry(() => - { - return GetString(route); - }, $"HTTP-GET:{route}"); - } - - public T HttpGetJson(string route) - { - return LockRetry(() => - { - var json = GetString(route); - return Deserialize(json); - }, $"HTTP-GET:{route}"); - } - - public TResponse HttpPostJson(string route, TRequest body) - { - return LockRetry(() => - { - var response = PostJson(route, body); - var json = Time.Wait(response.Content.ReadAsStringAsync()); - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException(json); - } - Log(GetUrl() + route, json); - return Deserialize(json); - }, $"HTTP-POST-JSON: {route}"); - } - - public string HttpPostJson(string route, TRequest body) - { - return LockRetry(() => - { - var response = PostJson(route, body); - return Time.Wait(response.Content.ReadAsStringAsync()); - }, $"HTTP-POST-JSON: {route}"); - } - - public TResponse HttpPostString(string route, string body) - { - return LockRetry(() => - { - var response = PostJsonString(route, body); - if (response == null) throw new Exception("Received no response."); - var result = Deserialize(response); - if (result == null) throw new Exception("Failed to deserialize response"); - return result; - }, $"HTTO-POST-JSON: {route}"); - } - - public string HttpPostStream(string route, Stream stream) - { - return LockRetry(() => - { - using var client = GetClient(); - var url = GetUrl() + route; - Log(url, "~ STREAM ~"); - var content = new StreamContent(stream); - content.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream"); - var response = Time.Wait(client.PostAsync(url, content)); - var str = Time.Wait(response.Content.ReadAsStringAsync()); - Log(url, str); - return str; - }, $"HTTP-POST-STREAM: {route}"); - } - - public Stream HttpGetStream(string route) - { - return LockRetry(() => - { - var client = GetClient(); - var url = GetUrl() + route; - Log(url, "~ STREAM ~"); - return Time.Wait(client.GetStreamAsync(url)); - }, $"HTTP-GET-STREAM: {route}"); - } - - public T Deserialize(string json) - { - var errors = new List(); - var deserialized = JsonConvert.DeserializeObject(json, new JsonSerializerSettings() - { - Error = delegate(object? sender, Serialization.ErrorEventArgs args) - { - if (args.CurrentObject == args.ErrorContext.OriginalObject) - { - errors.Add($""" - Member: '{args.ErrorContext.Member?.ToString() ?? ""}' - Path: {args.ErrorContext.Path} - Error: {args.ErrorContext.Error.Message} - """); - args.ErrorContext.Handled = true; - } - } - }); - if (errors.Count > 0) - { - throw new JsonSerializationException($"Failed to deserialize JSON '{json}' with exception(s): \n{string.Join("\n", errors)}"); - } - else if (deserialized == null) - { - throw new JsonSerializationException($"Failed to deserialize JSON '{json}': resulting deserialized object is null"); - } - return deserialized; - } - - private string GetString(string route) - { - using var client = GetClient(); - var url = GetUrl() + route; - Log(url, ""); - var result = Time.Wait(client.GetAsync(url)); - var str = Time.Wait(result.Content.ReadAsStringAsync()); - Log(url, str); - return str; - } - - private HttpResponseMessage PostJson(string route, TRequest body) - { - using var client = GetClient(); - var url = GetUrl() + route; - using var content = JsonContent.Create(body); - Log(url, JsonConvert.SerializeObject(body)); - return Time.Wait(client.PostAsync(url, content)); - } - - private string PostJsonString(string route, string body) - { - using var client = GetClient(); - var url = GetUrl() + route; - Log(url, body); - var content = new StringContent(body); - content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json"); - var result = Time.Wait(client.PostAsync(url, content)); - var str = Time.Wait(result.Content.ReadAsStringAsync()); - Log(url, str); - return str; - } - - private string GetUrl() - { - //return $"{address.Host}:{address.Port}{baseUrl}"; - return "--Obsolete--"; - } - - private void Log(string url, string message) - { - if (logAlias != null) - { - log.Debug($"({logAlias})({url}) = '{message}'", 3); - } - else - { - log.Debug($"({url}) = '{message}'", 3); - } - } - private T LockRetry(Func operation, string description) { lock (httpLock) diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs index a493563..0b9b87d 100644 --- a/Framework/Core/PluginTools.cs +++ b/Framework/Core/PluginTools.cs @@ -1,7 +1,6 @@ using FileUtils; using KubernetesWorkflow; using Logging; -using Utils; namespace Core { diff --git a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs index 427d35d..96d81aa 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs @@ -7,14 +7,16 @@ namespace MetricsPlugin { public class MetricsQuery { - private readonly IHttp http; + private readonly IEndpoint endpoint; private readonly ILog log; public MetricsQuery(IPluginTools tools, RunningContainer runningContainer) { RunningContainer = runningContainer; log = tools.GetLog(); - http = tools.CreateHttp(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1"); + endpoint = tools + .CreateHttp() + .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1"); } public RunningContainer RunningContainer { get; } @@ -51,7 +53,7 @@ namespace MetricsPlugin public Metrics? GetAllMetricsForNode(IMetricsScrapeTarget target) { - var response = http.HttpGetJson($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}"); + var response = endpoint.HttpGetJson($"query?query={GetInstanceStringForNode(target)}{GetQueryTimeRange()}"); if (response.status != "success") return null; var result = MapResponseToMetrics(response); Log(target, result); @@ -60,14 +62,14 @@ namespace MetricsPlugin private PrometheusQueryResponse? GetLastOverTime(string metricName, string instanceString) { - var response = http.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); + var response = endpoint.HttpGetJson($"query?query=last_over_time({metricName}{instanceString}{GetQueryTimeRange()})"); if (response.status != "success") return null; return response; } private PrometheusQueryResponse? GetAll(string metricName) { - var response = http.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); + var response = endpoint.HttpGetJson($"query?query={metricName}{GetQueryTimeRange()}"); if (response.status != "success") return null; return response; } From 05d28d1d0a3955303a801aed3796154cc6e13bb1 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 10:03:52 +0100 Subject: [PATCH 12/25] Restores debug info --- Framework/Core/PluginTools.cs | 18 ++--- ProjectPlugins/CodexPlugin/CodexNode.cs | 23 +++---- ProjectPlugins/CodexPlugin/CodexNodeGroup.cs | 4 +- ProjectPlugins/CodexPlugin/CodexStarter.cs | 2 +- ProjectPlugins/CodexPlugin/CodexTypes.cs | 43 ++++++++++-- ProjectPlugins/CodexPlugin/Mapper.cs | 66 ++++++++++++++++++- .../BasicTests/ContinuousSubstitute.cs | 6 +- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 6 +- 8 files changed, 131 insertions(+), 37 deletions(-) diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs index 0b9b87d..ff77bae 100644 --- a/Framework/Core/PluginTools.cs +++ b/Framework/Core/PluginTools.cs @@ -21,9 +21,9 @@ namespace Core public interface IHttpFactoryTool { - IHttp CreateHttp(Action onClientCreated, string? logAlias = null); - IHttp CreateHttp(Action onClientCreated, ITimeSet timeSet, string? logAlias = null); - IHttp CreateHttp(string? logAlias = null); + IHttp CreateHttp(Action onClientCreated); + IHttp CreateHttp(Action onClientCreated, ITimeSet timeSet); + IHttp CreateHttp(); } public interface IFileTool @@ -51,19 +51,19 @@ namespace Core log = new LogPrefixer(log, prefix); } - public IHttp CreateHttp(Action onClientCreated, string? logAlias = null) + public IHttp CreateHttp(Action onClientCreated) { - return CreateHttp(onClientCreated, timeSet, logAlias); + return CreateHttp(onClientCreated, timeSet); } - public IHttp CreateHttp(Action onClientCreated, ITimeSet ts, string? logAlias = null) + public IHttp CreateHttp(Action onClientCreated, ITimeSet ts) { - return new Http(log, ts, onClientCreated, logAlias); + return new Http(log, ts, onClientCreated); } - public IHttp CreateHttp(string? logAlias = null) + public IHttp CreateHttp() { - return new Http(log, timeSet, logAlias); + return new Http(log, timeSet); } public IStartupWorkflow CreateWorkflow(string? namespaceOverride = null) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 5c78b5d..6698d38 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -3,7 +3,6 @@ using FileUtils; using GethPlugin; using KubernetesWorkflow; using KubernetesWorkflow.Types; -using Logging; using MetricsPlugin; using Utils; @@ -18,7 +17,7 @@ namespace CodexPlugin TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); LocalDataset[] LocalFiles(); void ConnectToPeer(ICodexNode node); - DebugVersion Version { get; } + DebugInfoVersion Version { get; } IMarketplaceAccess Marketplace { get; } CrashWatcher CrashWatcher { get; } PodInfo GetPodInfo(); @@ -41,7 +40,7 @@ namespace CodexPlugin CodexAccess = codexAccess; Group = group; Marketplace = marketplaceAccess; - Version = new DebugVersion(); + Version = new DebugInfoVersion(); transferSpeeds = new TransferSpeeds(); } @@ -50,8 +49,9 @@ namespace CodexPlugin public CrashWatcher CrashWatcher { get => CodexAccess.CrashWatcher; } public CodexNodeGroup Group { get; } public IMarketplaceAccess Marketplace { get; } - public DebugVersion Version { get; private set; } + public DebugInfoVersion Version { get; private set; } public ITransferSpeeds TransferSpeeds { get => transferSpeeds; } + public IMetricsScrapeTarget MetricsScrapeTarget { get @@ -59,6 +59,7 @@ namespace CodexPlugin return new MetricsScrapeTarget(CodexAccess.Container, CodexContainerRecipe.MetricsPortTag); } } + public EthAddress EthAddress { get @@ -76,8 +77,8 @@ namespace CodexPlugin public DebugInfo GetDebugInfo() { var debugInfo = CodexAccess.GetDebugInfo(); - var known = string.Join(",", debugInfo.table.nodes.Select(n => n.peerId)); - Log($"Got DebugInfo with id: '{debugInfo.id}'. This node knows: {known}"); + var known = string.Join(",", debugInfo.Table.Nodes.Select(n => n.PeerId)); + Log($"Got DebugInfo with id: '{debugInfo.Id}'. This node knows: {known}"); return debugInfo; } @@ -156,18 +157,18 @@ namespace CodexPlugin public void EnsureOnlineGetVersionResponse() { var debugInfo = Time.Retry(CodexAccess.GetDebugInfo, "ensure online"); - var nodePeerId = debugInfo.id; + var nodePeerId = debugInfo.Id; var nodeName = CodexAccess.Container.Name; - if (!debugInfo.codex.IsValid()) + if (!debugInfo.Version.IsValid()) { - throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.codex}"); + throw new Exception($"Invalid version information received from Codex node {GetName()}: {debugInfo.Version}"); } var log = tools.GetLog(); log.AddStringReplace(nodePeerId, nodeName); - log.AddStringReplace(debugInfo.table.localNode.nodeId, nodeName); - Version = debugInfo.codex; + log.AddStringReplace(debugInfo.Table.LocalNode.NodeId, nodeName); + Version = debugInfo.Version; } private string GetPeerMultiAddress(CodexNode peer, DebugInfo peerInfo) diff --git a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs index 8b6264c..1f7ed4e 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeGroup.cs @@ -20,7 +20,7 @@ namespace CodexPlugin this.starter = starter; Containers = containers; Nodes = containers.Containers().Select(c => CreateOnlineCodexNode(c, tools, codexNodeFactory)).ToArray(); - Version = new DebugVersion(); + Version = new DebugInfoVersion(); } public ICodexNode this[int index] @@ -41,7 +41,7 @@ namespace CodexPlugin public RunningContainers[] Containers { get; private set; } public CodexNode[] Nodes { get; private set; } - public DebugVersion Version { get; private set; } + public DebugInfoVersion Version { get; private set; } public IMetricsScrapeTarget[] ScrapeTargets => Nodes.Select(n => n.MetricsScrapeTarget).ToArray(); public IEnumerator GetEnumerator() diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index 7a202f8..d10f502 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -9,7 +9,7 @@ namespace CodexPlugin { private readonly IPluginTools pluginTools; private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); - private DebugVersion? versionResponse; + private DebugInfoVersion? versionResponse; public CodexStarter(IPluginTools pluginTools) { diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index 5a54f9e..1226a2f 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -1,4 +1,6 @@ -namespace CodexPlugin +using Newtonsoft.Json; + +namespace CodexPlugin { public class DebugInfo { @@ -6,6 +8,39 @@ public string Spr { get; set; } = string.Empty; public string Id { get; set; } = string.Empty; public string[] AnnounceAddresses { get; set; } = Array.Empty(); + public DebugInfoVersion Version { get; set; } = new(); + public DebugInfoTable Table { get; set; } = new(); + } + + public class DebugInfoVersion + { + public string Version { get; set; } = string.Empty; + public string Revision { get; set; } = string.Empty; + + public bool IsValid() + { + return !string.IsNullOrEmpty(Version) && !string.IsNullOrEmpty(Revision); + } + + public override string ToString() + { + return JsonConvert.SerializeObject(this); + } + } + + public class DebugInfoTable + { + public DebugInfoTableNode LocalNode { get; set; } = new(); + public DebugInfoTableNode[] Nodes { get; set; } = Array.Empty(); + } + + public class DebugInfoTableNode + { + public string NodeId { get; set; } = string.Empty; + public string PeerId { get; set; } = string.Empty; + public string Record { get; set; } = string.Empty; + public string Address { get; set; } = string.Empty; + public bool Seen { get; set; } } public class DebugPeer @@ -19,12 +54,6 @@ public ContentId Cid { get; set; } = new ContentId(); } - public class DebugVersion - { - public string Version { get; internal set; } = string.Empty; - public string Revision { get; internal set; } = string.Empty; - } - public class ContentId { public ContentId() diff --git a/ProjectPlugins/CodexPlugin/Mapper.cs b/ProjectPlugins/CodexPlugin/Mapper.cs index d7905ab..7616a3c 100644 --- a/ProjectPlugins/CodexPlugin/Mapper.cs +++ b/ProjectPlugins/CodexPlugin/Mapper.cs @@ -11,8 +11,72 @@ namespace CodexPlugin Id = debugInfo.Id, Spr = debugInfo.Spr, Addrs = debugInfo.Addrs.ToArray(), - AnnounceAddresses = ((JArray) debugInfo.AdditionalProperties["announceAddresses"]).Select(x => x.ToString()).ToArray(), + AnnounceAddresses = JArray(debugInfo.AdditionalProperties, "announceAddresses").Select(x => x.ToString()).ToArray(), + Version = MapDebugInfoVersion(JObject(debugInfo.AdditionalProperties, "codex")), + Table = MapDebugInfoTable(JObject(debugInfo.AdditionalProperties, "table")) }; } + + private DebugInfoVersion MapDebugInfoVersion(JObject obj) + { + return new DebugInfoVersion + { + Version = StringOrEmpty(obj, "version"), + Revision = StringOrEmpty(obj, "revision") + }; + } + + private DebugInfoTable MapDebugInfoTable(JObject obj) + { + return new DebugInfoTable + { + LocalNode = MapDebugInfoTableNode(obj.GetValue("localNode")), + Nodes = new DebugInfoTableNode[0] + }; + } + + private DebugInfoTableNode MapDebugInfoTableNode(JToken? token) + { + var obj = token as JObject; + if (obj == null) return new DebugInfoTableNode(); + + return new DebugInfoTableNode + { + Address = StringOrEmpty(obj, "address"), + NodeId = StringOrEmpty(obj, "nodeId"), + PeerId = StringOrEmpty(obj, "peerId"), + Record = StringOrEmpty(obj, "record"), + Seen = Bool(obj, "seen") + }; + } + + private JArray JArray(IDictionary map, string name) + { + return (JArray)map[name]; + } + + private JObject JObject(IDictionary map, string name) + { + return (JObject)map[name]; + } + + private string StringOrEmpty(JObject obj, string name) + { + if (obj.TryGetValue(name, out var token)) + { + var str = (string?)token; + if (!string.IsNullOrEmpty(str)) return str; + } + return string.Empty; + } + + private bool Bool(JObject obj, string name) + { + if (obj.TryGetValue(name, out var token)) + { + return (bool)token; + } + return false; + } } } diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 5f4c300..874c071 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -103,7 +103,7 @@ namespace CodexTests.BasicTests private void CheckRoutingTables(IEnumerable nodes) { var all = nodes.ToArray(); - var allIds = all.Select(n => n.GetDebugInfo().table.localNode.nodeId).ToArray(); + var allIds = all.Select(n => n.GetDebugInfo().Table.LocalNode.NodeId).ToArray(); var errors = all.Select(n => AreAllPresent(n, allIds)).Where(s => !string.IsNullOrEmpty(s)).ToArray(); @@ -116,8 +116,8 @@ namespace CodexTests.BasicTests private string AreAllPresent(ICodexNode n, string[] allIds) { var info = n.GetDebugInfo(); - var known = info.table.nodes.Select(n => n.nodeId).ToArray(); - var expected = allIds.Where(i => i != info.table.localNode.nodeId).ToArray(); + var known = info.Table.Nodes.Select(n => n.NodeId).ToArray(); + var expected = allIds.Where(i => i != info.Table.LocalNode.NodeId).ToArray(); if (!expected.All(ex => known.Contains(ex))) { diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 53d78d2..5089788 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -72,13 +72,13 @@ namespace CodexTests.PeerDiscoveryTests private string AreAllPresent(DebugInfo info, DebugInfo[] allResponses) { - var knownIds = info.table.nodes.Select(n => n.nodeId).ToArray(); + var knownIds = info.Table.Nodes.Select(n => n.NodeId).ToArray(); var allOthers = GetAllOtherResponses(info, allResponses); - var expectedIds = allOthers.Select(i => i.table.localNode.nodeId).ToArray(); + var expectedIds = allOthers.Select(i => i.Table.LocalNode.NodeId).ToArray(); if (!expectedIds.All(ex => knownIds.Contains(ex))) { - return $"Node {info.id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'"; + return $"Node {info.Id}: Not all of '{string.Join(",", expectedIds)}' were present in routing table: '{string.Join(",", knownIds)}'"; } return string.Empty; From 8fcf3516136697b8dcf2869caf944c653de5b9c9 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 10:31:49 +0100 Subject: [PATCH 13/25] Restores debug/peer --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 21 +++++++++++----- ProjectPlugins/CodexPlugin/CodexTypes.cs | 3 ++- .../Helpers/FullConnectivityHelper.cs | 2 ++ .../Helpers/PeerConnectionTestHelpers.cs | 16 ++++++------ .../LayeredDiscoveryTests.cs | 25 +++++++++---------- 5 files changed, 39 insertions(+), 28 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index f576cfd..2d4084a 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -30,20 +30,21 @@ namespace CodexPlugin return Map(OnCodex(api => api.GetDebugInfoAsync())); } - public CodexDebugPeerResponse GetDebugPeer(string peerId) + public DebugPeer GetDebugPeer(string peerId) { - var http = Http(); - var str = http.HttpGetString($"debug/peer/{peerId}"); + // Cannot use openAPI: debug/peer endpoint is not specified there. + var endoint = GetEndpoint(); + var str = endoint.HttpGetString($"debug/peer/{peerId}"); if (str.ToLowerInvariant() == "unable to find peer!") { - return new CodexDebugPeerResponse + return new DebugPeer { IsPeerFound = false }; } - var result = http.Deserialize(str); + var result = endoint.Deserialize(str); result.IsPeerFound = true; return result; } @@ -117,7 +118,8 @@ namespace CodexPlugin private T OnCodex(Func> action) { var address = GetAddress(); - var result = tools.CreateHttp().OnClient(client => + var result = tools.CreateHttp(CheckContainerCrashed) + .OnClient(client => { var api = new CodexApi(client); api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; @@ -150,6 +152,13 @@ namespace CodexPlugin // return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); //} + private IEndpoint GetEndpoint() + { + return tools + .CreateHttp(CheckContainerCrashed) + .CreateEndpoint(GetAddress(), "/api/codex/v1/", Container.Name); + } + private Address GetAddress() { return Container.GetAddress(tools.GetLog(), CodexContainerRecipe.ApiPortTag); diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index 1226a2f..06dfc86 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -46,7 +46,8 @@ namespace CodexPlugin public class DebugPeer { public bool IsPeerFound { get; set; } - + public string PeerId { get; set; } = string.Empty; + public string[] Addresses { get; set; } = Array.Empty(); } public class LocalDataset diff --git a/Tests/CodexTests/Helpers/FullConnectivityHelper.cs b/Tests/CodexTests/Helpers/FullConnectivityHelper.cs index 8db17ed..58474d1 100644 --- a/Tests/CodexTests/Helpers/FullConnectivityHelper.cs +++ b/Tests/CodexTests/Helpers/FullConnectivityHelper.cs @@ -31,6 +31,8 @@ namespace CodexTests.Helpers private void AssertFullyConnected(ICodexNode[] nodes) { Log($"Asserting '{implementation.Description()}' for nodes: '{string.Join(",", nodes.Select(n => n.GetName()))}'..."); + Assert.That(nodes.Length, Is.GreaterThan(1)); + var entries = CreateEntries(nodes); var pairs = CreatePairs(entries); diff --git a/Tests/CodexTests/Helpers/PeerConnectionTestHelpers.cs b/Tests/CodexTests/Helpers/PeerConnectionTestHelpers.cs index d644296..f0261a7 100644 --- a/Tests/CodexTests/Helpers/PeerConnectionTestHelpers.cs +++ b/Tests/CodexTests/Helpers/PeerConnectionTestHelpers.cs @@ -26,12 +26,12 @@ namespace CodexTests.Helpers public string ValidateEntry(Entry entry, Entry[] allEntries) { var result = string.Empty; - foreach (var peer in entry.Response.table.nodes) + foreach (var peer in entry.Response.Table.Nodes) { var expected = GetExpectedDiscoveryEndpoint(allEntries, peer); - if (expected != peer.address) + if (expected != peer.Address) { - result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.address}', expected: '{expected}'. "; + result += $"Node:{entry.Node.GetName()} has incorrect peer table entry. Was: '{peer.Address}', expected: '{expected}'. "; } } return result; @@ -39,24 +39,24 @@ namespace CodexTests.Helpers public PeerConnectionState Check(Entry from, Entry to) { - var peerId = to.Response.id; + var peerId = to.Response.Id; var response = from.Node.GetDebugPeer(peerId); if (!response.IsPeerFound) { return PeerConnectionState.NoConnection; } - if (!string.IsNullOrEmpty(response.peerId) && response.addresses.Any()) + if (!string.IsNullOrEmpty(response.PeerId) && response.Addresses.Any()) { return PeerConnectionState.Connection; } return PeerConnectionState.Unknown; } - private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, CodexDebugTableNodeResponse node) + private static string GetExpectedDiscoveryEndpoint(Entry[] allEntries, DebugInfoTableNode node) { - var peer = allEntries.SingleOrDefault(e => e.Response.table.localNode.peerId == node.peerId); - if (peer == null) return $"peerId: {node.peerId} is not known."; + var peer = allEntries.SingleOrDefault(e => e.Response.Table.LocalNode.PeerId == node.PeerId); + if (peer == null) return $"peerId: {node.PeerId} is not known."; var container = peer.Node.Container; var podInfo = peer.Node.GetPodInfo(); diff --git a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs index 386c3d8..00c0cda 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/LayeredDiscoveryTests.cs @@ -1,5 +1,4 @@ -using CodexPlugin; -using NUnit.Framework; +using NUnit.Framework; namespace CodexTests.PeerDiscoveryTests { @@ -9,10 +8,10 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void TwoLayersTest() { - var root = Ci.StartCodexNode(); - var l1Source = Ci.StartCodexNode(s => s.WithBootstrapNode(root)); - var l1Node = Ci.StartCodexNode(s => s.WithBootstrapNode(root)); - var l2Target = Ci.StartCodexNode(s => s.WithBootstrapNode(l1Node)); + var root = AddCodex(); + var l1Source = AddCodex(s => s.WithBootstrapNode(root)); + var l1Node = AddCodex(s => s.WithBootstrapNode(root)); + var l2Target = AddCodex(s => s.WithBootstrapNode(l1Node)); AssertAllNodesConnected(); } @@ -20,11 +19,11 @@ namespace CodexTests.PeerDiscoveryTests [Test] public void ThreeLayersTest() { - var root = Ci.StartCodexNode(); - var l1Source = Ci.StartCodexNode(s => s.WithBootstrapNode(root)); - var l1Node = Ci.StartCodexNode(s => s.WithBootstrapNode(root)); - var l2Node = Ci.StartCodexNode(s => s.WithBootstrapNode(l1Node)); - var l3Target = Ci.StartCodexNode(s => s.WithBootstrapNode(l2Node)); + var root = AddCodex(); + var l1Source = AddCodex(s => s.WithBootstrapNode(root)); + var l1Node = AddCodex(s => s.WithBootstrapNode(root)); + var l2Node = AddCodex(s => s.WithBootstrapNode(l1Node)); + var l3Target = AddCodex(s => s.WithBootstrapNode(l2Node)); AssertAllNodesConnected(); } @@ -34,10 +33,10 @@ namespace CodexTests.PeerDiscoveryTests [TestCase(10)] public void NodeChainTest(int chainLength) { - var node = Ci.StartCodexNode(); + var node = AddCodex(); for (var i = 1; i < chainLength; i++) { - node = Ci.StartCodexNode(s => s.WithBootstrapNode(node)); + node = AddCodex(s => s.WithBootstrapNode(node)); } AssertAllNodesConnected(); From 85774847b741d69666efc08e25f455bbca3dd127 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 11:39:59 +0100 Subject: [PATCH 14/25] Restoring marketplace types --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 76 +++++-------------- ProjectPlugins/CodexPlugin/CodexNode.cs | 7 +- ProjectPlugins/CodexPlugin/CodexTypes.cs | 11 +++ ProjectPlugins/CodexPlugin/Mapper.cs | 55 +++++++++++++- .../CodexPlugin/MarketplaceTypes.cs | 53 +++---------- .../BasicTests/BlockExchangeTests.cs | 63 --------------- Tests/CodexTests/BasicTests/ExampleTests.cs | 6 +- 7 files changed, 98 insertions(+), 173 deletions(-) delete mode 100644 Tests/CodexTests/BasicTests/BlockExchangeTests.cs diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 2d4084a..c4ac8c2 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -33,8 +33,8 @@ namespace CodexPlugin public DebugPeer GetDebugPeer(string peerId) { // Cannot use openAPI: debug/peer endpoint is not specified there. - var endoint = GetEndpoint(); - var str = endoint.HttpGetString($"debug/peer/{peerId}"); + var endpoint = GetEndpoint(); + var str = endpoint.HttpGetString($"debug/peer/{peerId}"); if (str.ToLowerInvariant() == "unable to find peer!") { @@ -44,59 +44,45 @@ namespace CodexPlugin }; } - var result = endoint.Deserialize(str); + var result = endpoint.Deserialize(str); result.IsPeerFound = true; return result; } - public CodexDebugBlockExchangeResponse GetDebugBlockExchange() - { - return Http().HttpGetJson("debug/blockexchange"); - } - - public CodexDebugRepoStoreResponse[] GetDebugRepoStore() - { - return LongHttp().HttpGetJson("debug/repostore"); - } - - public CodexDebugThresholdBreaches GetDebugThresholdBreaches() - { - return Http().HttpGetJson("debug/loop"); - } - public string UploadFile(FileStream fileStream) { - return Http().HttpPostStream("data", fileStream); + return OnCodex(api => api.UploadAsync(fileStream)); } public Stream DownloadFile(string contentId) { - return Http().HttpGetStream("data/" + contentId + "/network"); + var fileResponse = OnCodex(api => api.DownloadNetworkAsync(contentId)); + if (fileResponse.StatusCode != 200) throw new Exception("Download failed with StatusCode: " + fileResponse.StatusCode); + return fileResponse.Stream; } - public CodexLocalDataResponse[] LocalFiles() + public LocalDataset[] LocalFiles() { - return Http().HttpGetJson("data"); + return Map(OnCodex(api => api.ListDataAsync())); } - public CodexSalesAvailabilityResponse SalesAvailability(CodexSalesAvailabilityRequest request) + public StorageAvailability SalesAvailability(StorageAvailability request) { - return Http().HttpPostJson("sales/availability", request); + var body = Map(request); + var read = OnCodex(api => api.OfferStorageAsync(body)); + return Map(read); } - public string RequestStorage(CodexSalesRequestStorageRequest request, string contentId) + public string RequestStorage(StoragePurchaseRequest request) { - return Http().HttpPostJson($"storage/request/{contentId}", request); + var body = Map(request); + var read = ""; /* fix incoming*/ OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); + return read; } - public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + public StoragePurchaseRequest GetPurchaseStatus(string purchaseId) { - return Http().HttpGetJson($"storage/purchases/{purchaseId}"); - } - - public string ConnectToPeer(string peerId, string peerMultiAddress) - { - return Http().HttpGetString($"connect/{peerId}?addrs={peerMultiAddress}"); + return Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); } public string GetName() @@ -128,30 +114,6 @@ namespace CodexPlugin return result; } - //private IHttp Http() - //{ - // var address = GetAddress(); - // var api = new CodexOpenApi.CodexApi(new HttpClient()); - // api.BaseUrl = $"{address.Host}:{address.Port}/api/codex/v1"; - - // var debugInfo = Time.Wait(api.GetDebugInfoAsync()); - - // using var stream = File.OpenRead("C:\\Users\\thatb\\Desktop\\Collect\\Wallpapers\\demerui_djinn_illuminatus_fullbody_full_body_view_in_the_style__86ea9491-1fe1-44ab-8577-a3636cad1b21.png"); - // var cid = Time.Wait(api.UploadAsync(stream)); - - // var file = Time.Wait(api.DownloadNetworkAsync(cid)); - // while (file.IsPartial) Thread.Sleep(100); - // using var outfile = File.OpenWrite("C:\\Users\\thatb\\Desktop\\output.png"); - // file.Stream.CopyTo(outfile); - - // return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, Container.Name); - //} - - //private IHttp LongHttp() - //{ - // return tools.CreateHttp(GetAddress(), baseUrl: "/api/codex/v1", CheckContainerCrashed, new LongTimeSet(), Container.Name); - //} - private IEndpoint GetEndpoint() { return tools diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 6698d38..244d931 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -3,6 +3,7 @@ using FileUtils; using GethPlugin; using KubernetesWorkflow; using KubernetesWorkflow.Types; +using Logging; using MetricsPlugin; using Utils; @@ -121,11 +122,7 @@ namespace CodexPlugin public LocalDataset[] LocalFiles() { - return CodexAccess.LocalFiles().Select(l => new LocalDataset() - { - Cid = new ContentId(l.cid) - //, l.manifest - }).ToArray(); + return CodexAccess.LocalFiles(); } public void ConnectToPeer(ICodexNode node) diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index 06dfc86..813c85c 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -55,6 +55,17 @@ namespace CodexPlugin public ContentId Cid { get; set; } = new ContentId(); } + public class SalesRequestStorageRequest + { + public string Duration { get; set; } = string.Empty; + public string ProofProbability { get; set; } = string.Empty; + public string Reward { get; set; } = string.Empty; + public string Collateral { get; set; } = string.Empty; + public string? Expiry { get; set; } + public uint? Nodes { get; set; } + public uint? Tolerance { get; set; } + } + public class ContentId { public ContentId() diff --git a/ProjectPlugins/CodexPlugin/Mapper.cs b/ProjectPlugins/CodexPlugin/Mapper.cs index 7616a3c..66f1393 100644 --- a/ProjectPlugins/CodexPlugin/Mapper.cs +++ b/ProjectPlugins/CodexPlugin/Mapper.cs @@ -1,4 +1,6 @@ -using Newtonsoft.Json.Linq; +using CodexContractsPlugin; +using Newtonsoft.Json.Linq; +using System.Numerics; namespace CodexPlugin { @@ -17,6 +19,45 @@ namespace CodexPlugin }; } + public LocalDataset[] Map(ICollection dataList) + { + return Array.Empty(); + } + + public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability) + { + return new CodexOpenApi.SalesAvailabilityCREATE + { + Duration = ToDecInt(availability.MaxDuration.TotalSeconds), + MinPrice = ToDecInt(availability.MinPriceForTotalSpace), + MaxCollateral = ToDecInt(availability.MaxCollateral), + TotalSize = ToDecInt(availability.TotalSpace.SizeInBytes) + }; + } + + public CodexOpenApi.StorageRequestCreation Map(StoragePurchaseRequest purchase) + { + return new CodexOpenApi.StorageRequestCreation + { + Duration = ToDecInt(purchase.Duration.TotalSeconds), + ProofProbability = ToDecInt(purchase.ProofProbability), + Reward = ToDecInt(purchase.PricePerSlotPerSecond), + Collateral = ToDecInt(purchase.RequiredCollateral), + Expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + purchase.Expiry.TotalSeconds), + Nodes = purchase.MinRequiredNumberOfNodes, + Tolerance = purchase.NodeFailureTolerance + }; + } + + public StoragePurchase Map(CodexOpenApi.Purchase purchase) + { + return new StoragePurchase + { + State = purchase.State, + Error = purchase.Error + }; + } + private DebugInfoVersion MapDebugInfoVersion(JObject obj) { return new DebugInfoVersion @@ -78,5 +119,17 @@ namespace CodexPlugin } return false; } + + private string ToDecInt(double d) + { + var i = new BigInteger(d); + return i.ToString("D"); + } + + private string ToDecInt(TestToken t) + { + var i = new BigInteger(t.Amount); + return i.ToString("D"); + } } } diff --git a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs index dc7f7e5..0adcc8f 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs @@ -1,13 +1,12 @@ using CodexContractsPlugin; using Logging; -using System.Numerics; using Utils; namespace CodexPlugin { - public class StoragePurchase : MarketplaceType + public class StoragePurchaseRequest { - public StoragePurchase(ContentId cid) + public StoragePurchaseRequest(ContentId cid) { ContentId = cid; } @@ -21,20 +20,6 @@ namespace CodexPlugin public TimeSpan Duration { get; set; } public TimeSpan Expiry { get; set; } - public CodexSalesRequestStorageRequest ToApiRequest() - { - return new CodexSalesRequestStorageRequest - { - duration = ToDecInt(Duration.TotalSeconds), - proofProbability = ToDecInt(ProofProbability), - reward = ToDecInt(PricePerSlotPerSecond), - collateral = ToDecInt(RequiredCollateral), - expiry = ToDecInt(DateTimeOffset.UtcNow.ToUnixTimeSeconds() + Expiry.TotalSeconds), - nodes = MinRequiredNumberOfNodes, - tolerance = NodeFailureTolerance - }; - } - public void Log(ILog log) { log.Log($"Requesting storage for: {ContentId.Id}... (" + @@ -48,7 +33,13 @@ namespace CodexPlugin } } - public class StorageAvailability : MarketplaceType + public class StoragePurchase + { + public string State { get; set; } = string.Empty; + public string Error { get; set; } = string.Empty; + } + + public class StorageAvailability { public StorageAvailability(ByteSize totalSpace, TimeSpan maxDuration, TestToken minPriceForTotalSpace, TestToken maxCollateral) { @@ -63,17 +54,6 @@ namespace CodexPlugin public TestToken MinPriceForTotalSpace { get; } public TestToken MaxCollateral { get; } - public CodexSalesAvailabilityRequest ToApiRequest() - { - return new CodexSalesAvailabilityRequest - { - totalSize = ToDecInt(TotalSpace.SizeInBytes), - duration = ToDecInt(MaxDuration.TotalSeconds), - maxCollateral = ToDecInt(MaxCollateral), - minPrice = ToDecInt(MinPriceForTotalSpace) - }; - } - public void Log(ILog log) { log.Log($"Making storage available... (" + @@ -83,19 +63,4 @@ namespace CodexPlugin $"maxCollateral: {MaxCollateral})"); } } - - public abstract class MarketplaceType - { - protected string ToDecInt(double d) - { - var i = new BigInteger(d); - return i.ToString("D"); - } - - protected string ToDecInt(TestToken t) - { - var i = new BigInteger(t.Amount); - return i.ToString("D"); - } - } } diff --git a/Tests/CodexTests/BasicTests/BlockExchangeTests.cs b/Tests/CodexTests/BasicTests/BlockExchangeTests.cs deleted file mode 100644 index 83f67d4..0000000 --- a/Tests/CodexTests/BasicTests/BlockExchangeTests.cs +++ /dev/null @@ -1,63 +0,0 @@ -using CodexPlugin; -using NUnit.Framework; -using Utils; - -namespace CodexTests.BasicTests -{ - [TestFixture] - public class BlockExchangeTests : CodexDistTest - { - [Test] - public void EmptyAfterExchange() - { - var bootstrap = AddCodex(s => s.WithName("bootstrap")); - var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap)); - - AssertExchangeIsEmpty(bootstrap, node); - - var file = GenerateTestFile(1.MB()); - var cid = bootstrap.UploadFile(file); - node.DownloadContent(cid); - - AssertExchangeIsEmpty(bootstrap, node); - } - - [Test] - public void EmptyAfterExchangeWithBystander() - { - var bootstrap = AddCodex(s => s.WithName("bootstrap")); - var node = AddCodex(s => s.WithName("node").WithBootstrapNode(bootstrap)); - var bystander = AddCodex(s => s.WithName("bystander").WithBootstrapNode(bootstrap)); - - AssertExchangeIsEmpty(bootstrap, node, bystander); - - var file = GenerateTestFile(1.MB()); - var cid = bootstrap.UploadFile(file); - node.DownloadContent(cid); - - AssertExchangeIsEmpty(bootstrap, node, bystander); - } - - private void AssertExchangeIsEmpty(params ICodexNode[] nodes) - { - foreach (var node in nodes) - { - // API Call not available in master-line Codex image. - //Time.Retry(() => AssertBlockExchangeIsEmpty(node), nameof(AssertExchangeIsEmpty)); - } - } - - //private void AssertBlockExchangeIsEmpty(ICodexNode node) - //{ - // var msg = $"BlockExchange for {node.GetName()}: "; - // var response = node.GetDebugBlockExchange(); - // foreach (var peer in response.peers) - // { - // var activeWants = peer.wants.Where(w => !w.cancel).ToArray(); - // Assert.That(activeWants.Length, Is.EqualTo(0), msg + "thinks a peer has active wants."); - // } - // Assert.That(response.taskQueue, Is.EqualTo(0), msg + "has tasks in queue."); - // Assert.That(response.pendingBlocks, Is.EqualTo(0), msg + "has pending blocks."); - //} - } -} diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 4973e2a..3250668 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -89,7 +89,7 @@ namespace CodexTests.BasicTests var contentId = buyer.UploadFile(testFile); - var purchase = new StoragePurchase(contentId) + var purchase = new StoragePurchaseRequest(contentId) { PricePerSlotPerSecond = 2.TestTokens(), RequiredCollateral = 10.TestTokens(), @@ -135,7 +135,7 @@ namespace CodexTests.BasicTests Assert.That(discN, Is.LessThan(bootN)); } - private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchase purchase, Request request, ICodexNode seller) + private void AssertSlotFilledEvents(ICodexContracts contracts, StoragePurchaseRequest purchase, Request request, ICodexNode seller) { // Expect 1 fulfilled event for the purchase. var requestFulfilledEvents = contracts.GetRequestFulfilledEvents(GetTestRunTimeRange()); @@ -153,7 +153,7 @@ namespace CodexTests.BasicTests } } - private void AssertStorageRequest(Request request, StoragePurchase purchase, ICodexContracts contracts, ICodexNode buyer) + private void AssertStorageRequest(Request request, StoragePurchaseRequest purchase, ICodexContracts contracts, ICodexNode buyer) { Assert.That(contracts.GetRequestState(request), Is.EqualTo(RequestState.Started)); Assert.That(request.ClientAddress, Is.EqualTo(buyer.EthAddress)); From 3bd297a22daa421fb44db15747cd15e932b6fa12 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 12:14:02 +0100 Subject: [PATCH 15/25] restores connect-to-peer --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 9 +++++++++ ProjectPlugins/CodexPlugin/CodexNode.cs | 13 +++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index c4ac8c2..4d369ee 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -49,6 +49,15 @@ namespace CodexPlugin return result; } + public void ConnectToPeer(string peerId, string[] peerMultiAddresses) + { + OnCodex(api => + { + Time.Wait(api.ConnectPeerAsync(peerId, peerMultiAddresses)); + return Task.FromResult(string.Empty); + }); + } + public string UploadFile(FileStream fileStream) { return OnCodex(api => api.UploadAsync(fileStream)); diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index 244d931..ce27632 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -28,7 +28,6 @@ namespace CodexPlugin public class CodexNode : ICodexNode { - private const string SuccessfullyConnectedMessage = "Successfully connected to peer"; private const string UploadFailedMessage = "Unable to store block"; private readonly IPluginTools tools; private readonly EthAddress? ethAddress; @@ -131,9 +130,8 @@ namespace CodexPlugin Log($"Connecting to peer {peer.GetName()}..."); var peerInfo = node.GetDebugInfo(); - var response = CodexAccess.ConnectToPeer(peerInfo.id, GetPeerMultiAddress(peer, peerInfo)); + CodexAccess.ConnectToPeer(peerInfo.Id, GetPeerMultiAddresses(peer, peerInfo)); - FrameworkAssert.That(response == SuccessfullyConnectedMessage, "Unable to connect codex nodes."); Log($"Successfully connected to peer {peer.GetName()}."); } @@ -168,17 +166,16 @@ namespace CodexPlugin Version = debugInfo.Version; } - private string GetPeerMultiAddress(CodexNode peer, DebugInfo peerInfo) + private string[] GetPeerMultiAddresses(CodexNode peer, DebugInfo peerInfo) { - var multiAddress = peerInfo.Addrs.First(); - // Todo: Is there a case where First address in list is not the way? - // The peer we want to connect is in a different pod. // We must replace the default IP with the pod IP in the multiAddress. var workflow = tools.CreateWorkflow(); var podInfo = workflow.GetPodInfo(peer.Container); - return multiAddress.Replace("0.0.0.0", podInfo.Ip); + return peerInfo.Addrs.Select(a => a + .Replace("0.0.0.0", podInfo.Ip)) + .ToArray(); } private void DownloadToFile(string contentId, TrackedFile file) From f57188914e95ecb92862496018dd6fbc02c0b6ee Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 12:23:38 +0100 Subject: [PATCH 16/25] Restores marketplace access --- ProjectPlugins/CodexPlugin/CodexAccess.cs | 3 +- ProjectPlugins/CodexPlugin/Mapper.cs | 14 +++++++++ .../CodexPlugin/MarketplaceAccess.cs | 29 +++++++++---------- .../CodexPlugin/MarketplaceTypes.cs | 1 + 4 files changed, 31 insertions(+), 16 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 4d369ee..4111af4 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -85,11 +85,12 @@ namespace CodexPlugin public string RequestStorage(StoragePurchaseRequest request) { var body = Map(request); + throw new Exception("todo"); var read = ""; /* fix incoming*/ OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); return read; } - public StoragePurchaseRequest GetPurchaseStatus(string purchaseId) + public StoragePurchase GetPurchaseStatus(string purchaseId) { return Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); } diff --git a/ProjectPlugins/CodexPlugin/Mapper.cs b/ProjectPlugins/CodexPlugin/Mapper.cs index 66f1393..76b2401 100644 --- a/ProjectPlugins/CodexPlugin/Mapper.cs +++ b/ProjectPlugins/CodexPlugin/Mapper.cs @@ -21,6 +21,7 @@ namespace CodexPlugin public LocalDataset[] Map(ICollection dataList) { + throw new Exception("todo"); return Array.Empty(); } @@ -58,6 +59,19 @@ namespace CodexPlugin }; } + public StorageAvailability Map(CodexOpenApi.SalesAvailabilityREAD read) + { + return new StorageAvailability( + totalSpace: new Utils.ByteSize(Convert.ToInt64(read.TotalSize)), + maxDuration: TimeSpan.FromSeconds(Convert.ToDouble(read.Duration)), + minPriceForTotalSpace: new TestToken(Convert.ToDecimal(read.MinPrice)), + maxCollateral: new TestToken(Convert.ToDecimal(read.MaxCollateral)) + ) + { + Id = read.Id + }; + } + private DebugInfoVersion MapDebugInfoVersion(JObject obj) { return new DebugInfoVersion diff --git a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs index 411b481..c0d7d18 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceAccess.cs @@ -7,7 +7,7 @@ namespace CodexPlugin public interface IMarketplaceAccess { string MakeStorageAvailable(StorageAvailability availability); - StoragePurchaseContract RequestStorage(StoragePurchase purchase); + StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase); } public class MarketplaceAccess : IMarketplaceAccess @@ -21,14 +21,14 @@ namespace CodexPlugin this.codexAccess = codexAccess; } - public StoragePurchaseContract RequestStorage(StoragePurchase purchase) + public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) { purchase.Log(log); - var request = purchase.ToApiRequest(); - var response = codexAccess.RequestStorage(request, purchase.ContentId.Id); + var response = codexAccess.RequestStorage(purchase); - if (response == "Purchasing not available" || + if (string.IsNullOrEmpty(response) || + response == "Purchasing not available" || response == "Expiry required" || response == "Expiry needs to be in future" || response == "Expiry has to be before the request's end (now + duration)") @@ -44,13 +44,12 @@ namespace CodexPlugin public string MakeStorageAvailable(StorageAvailability availability) { availability.Log(log); - var request = availability.ToApiRequest(); - var response = codexAccess.SalesAvailability(request); + var response = codexAccess.SalesAvailability(availability); - Log($"Storage successfully made available. Id: {response.id}"); + Log($"Storage successfully made available. Id: {response.Id}"); - return response.id; + return response.Id; } private void Log(string msg) @@ -67,7 +66,7 @@ namespace CodexPlugin throw new NotImplementedException(); } - public StoragePurchaseContract RequestStorage(StoragePurchase purchase) + public StoragePurchaseContract RequestStorage(StoragePurchaseRequest purchase) { Unavailable(); throw new NotImplementedException(); @@ -87,7 +86,7 @@ namespace CodexPlugin private readonly TimeSpan gracePeriod = TimeSpan.FromSeconds(10); private DateTime? contractStartUtc; - public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchase purchase) + public StoragePurchaseContract(ILog log, CodexAccess codexAccess, string purchaseId, StoragePurchaseRequest purchase) { this.log = log; this.codexAccess = codexAccess; @@ -96,7 +95,7 @@ namespace CodexPlugin } public string PurchaseId { get; } - public StoragePurchase Purchase { get; } + public StoragePurchaseRequest Purchase { get; } public void WaitForStorageContractStarted() { @@ -117,7 +116,7 @@ namespace CodexPlugin WaitForStorageContractState(timeout, "finished"); } - public CodexStoragePurchase GetPurchaseStatus(string purchaseId) + public StoragePurchase GetPurchaseStatus(string purchaseId) { return codexAccess.GetPurchaseStatus(purchaseId); } @@ -132,9 +131,9 @@ namespace CodexPlugin { var purchaseStatus = codexAccess.GetPurchaseStatus(PurchaseId); var statusJson = JsonConvert.SerializeObject(purchaseStatus); - if (purchaseStatus != null && purchaseStatus.state != lastState) + if (purchaseStatus != null && purchaseStatus.State != lastState) { - lastState = purchaseStatus.state; + lastState = purchaseStatus.State; log.Debug("Purchase status: " + statusJson); } diff --git a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs index 0adcc8f..5edc456 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceTypes.cs @@ -49,6 +49,7 @@ namespace CodexPlugin MaxCollateral = maxCollateral; } + public string Id { get; set; } = string.Empty; public ByteSize TotalSpace { get; } public TimeSpan MaxDuration { get; } public TestToken MinPriceForTotalSpace { get; } From bce9a2c124265b804dd9c3418c211d60adb913b3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 14:07:06 +0100 Subject: [PATCH 17/25] Lines up new openAPI.yaml --- Framework/Utils/ByteSize.cs | 4 ++- ProjectPlugins/CodexPlugin/CodexAccess.cs | 6 ++--- ProjectPlugins/CodexPlugin/CodexNode.cs | 4 +-- ProjectPlugins/CodexPlugin/CodexTypes.cs | 17 +++++++++++- ProjectPlugins/CodexPlugin/Mapper.cs | 29 ++++++++++++++++++--- ProjectPlugins/CodexPlugin/openapi.yaml | 4 +++ Tests/CodexTests/BasicTests/ExampleTests.cs | 4 +-- 7 files changed, 55 insertions(+), 13 deletions(-) diff --git a/Framework/Utils/ByteSize.cs b/Framework/Utils/ByteSize.cs index a5204c7..1184607 100644 --- a/Framework/Utils/ByteSize.cs +++ b/Framework/Utils/ByteSize.cs @@ -2,6 +2,9 @@ { public class ByteSize { + public static readonly ByteSize Zero = new ByteSize(0); + public const double DefaultSecondsPerMB = 10.0; + public ByteSize(long sizeInBytes) { if (sizeInBytes < 0) throw new ArgumentException("Cannot create ByteSize object with size less than 0. Was: " + sizeInBytes); @@ -10,7 +13,6 @@ public long SizeInBytes { get; } - public const double DefaultSecondsPerMB = 10.0; public long ToMB() { diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 4111af4..3759ae9 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -70,7 +70,7 @@ namespace CodexPlugin return fileResponse.Stream; } - public LocalDataset[] LocalFiles() + public LocalDatasetList LocalFiles() { return Map(OnCodex(api => api.ListDataAsync())); } @@ -85,9 +85,7 @@ namespace CodexPlugin public string RequestStorage(StoragePurchaseRequest request) { var body = Map(request); - throw new Exception("todo"); - var read = ""; /* fix incoming*/ OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); - return read; + return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); } public StoragePurchase GetPurchaseStatus(string purchaseId) diff --git a/ProjectPlugins/CodexPlugin/CodexNode.cs b/ProjectPlugins/CodexPlugin/CodexNode.cs index ce27632..e1e475c 100644 --- a/ProjectPlugins/CodexPlugin/CodexNode.cs +++ b/ProjectPlugins/CodexPlugin/CodexNode.cs @@ -16,7 +16,7 @@ namespace CodexPlugin DebugPeer GetDebugPeer(string peerId); ContentId UploadFile(TrackedFile file); TrackedFile? DownloadContent(ContentId contentId, string fileLabel = ""); - LocalDataset[] LocalFiles(); + LocalDatasetList LocalFiles(); void ConnectToPeer(ICodexNode node); DebugInfoVersion Version { get; } IMarketplaceAccess Marketplace { get; } @@ -119,7 +119,7 @@ namespace CodexPlugin return file; } - public LocalDataset[] LocalFiles() + public LocalDatasetList LocalFiles() { return CodexAccess.LocalFiles(); } diff --git a/ProjectPlugins/CodexPlugin/CodexTypes.cs b/ProjectPlugins/CodexPlugin/CodexTypes.cs index 813c85c..945251e 100644 --- a/ProjectPlugins/CodexPlugin/CodexTypes.cs +++ b/ProjectPlugins/CodexPlugin/CodexTypes.cs @@ -1,4 +1,5 @@ using Newtonsoft.Json; +using Utils; namespace CodexPlugin { @@ -50,9 +51,23 @@ namespace CodexPlugin public string[] Addresses { get; set; } = Array.Empty(); } + public class LocalDatasetList + { + public LocalDataset[] Content { get; set; } = Array.Empty(); + } + public class LocalDataset { - public ContentId Cid { get; set; } = new ContentId(); + public ContentId Cid { get; set; } = new(); + public Manifest Manifest { get; set; } = new(); + } + + public class Manifest + { + public string RootHash { get; set; } = string.Empty; + public ByteSize OriginalBytes { get; set; } = ByteSize.Zero; + public ByteSize BlockSize { get; set; } = ByteSize.Zero; + public bool Protected { get; set; } } public class SalesRequestStorageRequest diff --git a/ProjectPlugins/CodexPlugin/Mapper.cs b/ProjectPlugins/CodexPlugin/Mapper.cs index 76b2401..014fbc7 100644 --- a/ProjectPlugins/CodexPlugin/Mapper.cs +++ b/ProjectPlugins/CodexPlugin/Mapper.cs @@ -1,6 +1,7 @@ using CodexContractsPlugin; using Newtonsoft.Json.Linq; using System.Numerics; +using Utils; namespace CodexPlugin { @@ -19,10 +20,21 @@ namespace CodexPlugin }; } - public LocalDataset[] Map(ICollection dataList) + public LocalDatasetList Map(CodexOpenApi.DataList dataList) { - throw new Exception("todo"); - return Array.Empty(); + return new LocalDatasetList + { + Content = dataList.Content.Select(Map).ToArray() + }; + } + + public LocalDataset Map(CodexOpenApi.DataItem dataItem) + { + return new LocalDataset + { + Cid = new ContentId(dataItem.Cid), + Manifest = MapManifest(dataItem.Manifest) + }; } public CodexOpenApi.SalesAvailabilityCREATE Map(StorageAvailability availability) @@ -105,6 +117,17 @@ namespace CodexPlugin }; } + private Manifest MapManifest(CodexOpenApi.ManifestItem manifest) + { + return new Manifest + { + BlockSize = new ByteSize(Convert.ToInt64(manifest.BlockSize)), + OriginalBytes = new ByteSize(Convert.ToInt64(manifest.OriginalBytes)), + RootHash = manifest.RootHash, + Protected = manifest.Protected + }; + } + private JArray JArray(IDictionary map, string name) { return (JArray)map[name]; diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index adfc887..cfbc800 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -632,6 +632,10 @@ paths: responses: "200": description: Returns the Request ID as decimal string + content: + text/plain: + schema: + type: string "400": description: Invalid or missing Request ID "404": diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 3250668..203e1f0 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -20,8 +20,8 @@ namespace CodexTests.BasicTests var cid = primary.UploadFile(GenerateTestFile(5.MB())); - var content = primary.LocalFiles(); - CollectionAssert.Contains(content.Select(c => c.Cid), cid); + var localDatasets = primary.LocalFiles(); + CollectionAssert.Contains(localDatasets.Content.Select(c => c.Cid), cid); var log = Ci.DownloadLog(primary); From e72c1b037c00604864b36e42f96b6c77fab5f9b6 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 14:42:47 +0100 Subject: [PATCH 18/25] Implements api compatiblity checker --- ProjectPlugins/CodexPlugin/ApiChecker.cs | 91 +++++++++++++++++++ ProjectPlugins/CodexPlugin/CodexPlugin.cs | 2 - ProjectPlugins/CodexPlugin/CodexStarter.cs | 5 + ProjectPlugins/CodexPluginPrebuild/Program.cs | 49 +++++++--- 4 files changed, 131 insertions(+), 16 deletions(-) create mode 100644 ProjectPlugins/CodexPlugin/ApiChecker.cs diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs new file mode 100644 index 0000000..5d71ad0 --- /dev/null +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -0,0 +1,91 @@ +using Core; +using KubernetesWorkflow.Types; +using Logging; +using System.Security.Cryptography; +using System.Text; + +namespace CodexPlugin +{ + public class ApiChecker + { + // + private const string OpenApiYamlHash = "5E-B8-2A-E3-61-0C-D6-11-F7-F6-19-4C-F9-35-CA-8B-D1-FF-51-52-1E-E7-A3-7A-5D-0C-2A-3D-50-93-5E-55"; + private const string OpenApiFilePath = "/codex/openapi.yaml"; + private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; + + private const bool Disable = false; + + private const string Warning = + "Warning: CodexPlugin was unable to find the openapi.yaml file in the Codex container. Are you running an old version of Codex? " + + "Plugin will continue as normal, but API compatibility is not guaranteed!"; + + private const string Failure = + "Codex API compatibility check failed! " + + "openapi.yaml used by CodexPlugin does not match openapi.yaml in Codex container. Please update the openapi.yaml in " + + "'ProjectPlugins/CodexPlugin' and rebuild this project. If you wish to disable API compatibility checking, please set " + + $"the environment variable '{DisableEnvironmentVariable}' or set the disable bool in 'ProjectPlugins/CodexPlugin/ApiChecker.cs'."; + + private static bool checkPassed = false; + + private readonly IPluginTools pluginTools; + private readonly ILog log; + + public ApiChecker(IPluginTools pluginTools) + { + this.pluginTools = pluginTools; + log = pluginTools.GetLog(); + + if (string.IsNullOrEmpty(OpenApiYamlHash)) throw new Exception("OpenAPI yaml hash was not inserted by pre-build trigger."); + } + + public void CheckCompatibility(RunningContainers[] containers) + { + if (checkPassed) return; + + Log("CodexPlugin is checking API compatibility..."); + + if (Disable || !string.IsNullOrEmpty(Environment.GetEnvironmentVariable(DisableEnvironmentVariable))) + { + Log("API compatibility checking has been disabled."); + checkPassed = true; + return; + } + + var workflow = pluginTools.CreateWorkflow(); + var container = containers.First().Containers.First(); + var containerApi = workflow.ExecuteCommand(container, "cat", OpenApiFilePath); + + if (string.IsNullOrEmpty(containerApi)) + { + log.Error(Warning); + + checkPassed = true; + return; + } + + var containerHash = Hash(containerApi); + if (containerHash == OpenApiYamlHash) + { + Log("API compatibility check passed."); + checkPassed = true; + return; + } + + log.Error(Failure); + throw new Exception(Failure); + } + + private string Hash(string file) + { + var fileBytes = Encoding.ASCII.GetBytes(file); + var sha = SHA256.Create(); + var hash = sha.ComputeHash(fileBytes); + return BitConverter.ToString(hash); + } + + private void Log(string msg) + { + log.Log(msg); + } + } +} diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index a7e0f57..99a9337 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -9,8 +9,6 @@ namespace CodexPlugin private readonly IPluginTools tools; private readonly CodexLogLevel defaultLogLevel = CodexLogLevel.Trace; - private const string OpenApiYamlHash = "8B-DD-61-54-42-D7-28-8F-5A-A0-AF-C2-A4-53-A7-08-B6-C7-02-FD-59-1A-01-A9-B4-7D-E4-81-FA-84-23-7F"; - public CodexPlugin(IPluginTools tools) { codexStarter = new CodexStarter(tools); diff --git a/ProjectPlugins/CodexPlugin/CodexStarter.cs b/ProjectPlugins/CodexPlugin/CodexStarter.cs index d10f502..05db6bd 100644 --- a/ProjectPlugins/CodexPlugin/CodexStarter.cs +++ b/ProjectPlugins/CodexPlugin/CodexStarter.cs @@ -9,11 +9,14 @@ namespace CodexPlugin { private readonly IPluginTools pluginTools; private readonly CodexContainerRecipe recipe = new CodexContainerRecipe(); + private readonly ApiChecker apiChecker; private DebugInfoVersion? versionResponse; public CodexStarter(IPluginTools pluginTools) { this.pluginTools = pluginTools; + + apiChecker = new ApiChecker(pluginTools); } public RunningContainers[] BringOnline(CodexSetup codexSetup) @@ -25,6 +28,8 @@ namespace CodexPlugin var containers = StartCodexContainers(startupConfig, codexSetup.NumberOfNodes, codexSetup.Location); + apiChecker.CheckCompatibility(containers); + foreach (var rc in containers) { var podInfo = GetPodInfo(rc); diff --git a/ProjectPlugins/CodexPluginPrebuild/Program.cs b/ProjectPlugins/CodexPluginPrebuild/Program.cs index f33246d..5ad5e89 100644 --- a/ProjectPlugins/CodexPluginPrebuild/Program.cs +++ b/ProjectPlugins/CodexPluginPrebuild/Program.cs @@ -3,8 +3,30 @@ public static class Program { private const string OpenApiFile = "../CodexPlugin/openapi.yaml"; - private const string Search = ""; - private const string TargetFile = "CodexPlugin.cs"; + private const string Search = ""; + private const string TargetFile = "ApiChecker.cs"; + + public static void Main(string[] args) + { + Console.WriteLine("Injecting hash of 'openapi.yaml'..."); + + var hash = CreateHash(); + // This hash is used to verify that the Codex docker image being used is compatible + // with the openapi.yaml being used by the Codex plugin. + // If the openapi.yaml files don't match, an exception is thrown. + + SearchAndInject(hash); + + // This program runs as the pre-build trigger for "CodexPlugin". + // You might be wondering why this work isn't done by a shell script. + // This is because this project is being run on many different platforms. + // (Mac, Unix, Win, but also desktop/cloud containers.) + // In order to not go insane trying to make a shell script that works in all possible cases, + // instead we use the one tool that's definitely installed in all platforms and locations + // when you're trying to run this plugin. + + Console.WriteLine("Done!"); + } private static string CreateHash() { @@ -14,23 +36,22 @@ public static class Program return BitConverter.ToString(hash); } - private static void SearchAndReplace(string hash) + private static void SearchAndInject(string hash) { var lines = File.ReadAllLines(TargetFile); - lines = lines.Select(l => l.Replace(Search, hash)).ToArray(); + Inject(lines, hash); File.WriteAllLines(TargetFile, lines); } - public static void Main(string[] args) + private static void Inject(string[] lines, string hash) { - Console.WriteLine("Injecting hash of 'openapi.yaml'..."); - // This hash is used to verify that the Codex docker image being used is compatible - // with the openapi.yaml being used by the Codex plugin. - // If the openapi.yaml files don't match, an exception is thrown. - - var hash = CreateHash(); - SearchAndReplace(hash); - - Console.WriteLine("Done!"); + for (var i = 0; i < lines.Length; i++) + { + if (lines[i].Contains(Search)) + { + lines[i + 1] = $" private const string OpenApiYamlHash = \"{hash}\";"; + return; + } + } } } \ No newline at end of file From d9d8441b6e31d46914c960305c37fea47c1a6a24 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 15:12:28 +0100 Subject: [PATCH 19/25] compiletime checking of all map calls --- ProjectPlugins/CodexPlugin/ApiChecker.cs | 2 +- ProjectPlugins/CodexPlugin/CodexAccess.cs | 17 ++++++----------- ProjectPlugins/CodexPlugin/openapi.yaml | 5 ++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 5d71ad0..39fb22a 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -9,7 +9,7 @@ namespace CodexPlugin public class ApiChecker { // - private const string OpenApiYamlHash = "5E-B8-2A-E3-61-0C-D6-11-F7-F6-19-4C-F9-35-CA-8B-D1-FF-51-52-1E-E7-A3-7A-5D-0C-2A-3D-50-93-5E-55"; + private const string OpenApiYamlHash = "EB-31-10-0C-A5-B9-D2-4A-AD-2C-A7-BF-FD-70-BD-92-32-29-D7-FF-06-B0-52-46-8E-54-D5-EB-17-C8-DB-E3"; private const string OpenApiFilePath = "/codex/openapi.yaml"; private const string DisableEnvironmentVariable = "CODEXPLUGIN_DISABLE_APICHECK"; diff --git a/ProjectPlugins/CodexPlugin/CodexAccess.cs b/ProjectPlugins/CodexPlugin/CodexAccess.cs index 3759ae9..bd2582e 100644 --- a/ProjectPlugins/CodexPlugin/CodexAccess.cs +++ b/ProjectPlugins/CodexPlugin/CodexAccess.cs @@ -27,7 +27,7 @@ namespace CodexPlugin public DebugInfo GetDebugInfo() { - return Map(OnCodex(api => api.GetDebugInfoAsync())); + return mapper.Map(OnCodex(api => api.GetDebugInfoAsync())); } public DebugPeer GetDebugPeer(string peerId) @@ -72,25 +72,25 @@ namespace CodexPlugin public LocalDatasetList LocalFiles() { - return Map(OnCodex(api => api.ListDataAsync())); + return mapper.Map(OnCodex(api => api.ListDataAsync())); } public StorageAvailability SalesAvailability(StorageAvailability request) { - var body = Map(request); + var body = mapper.Map(request); var read = OnCodex(api => api.OfferStorageAsync(body)); - return Map(read); + return mapper.Map(read); } public string RequestStorage(StoragePurchaseRequest request) { - var body = Map(request); + var body = mapper.Map(request); return OnCodex(api => api.CreateStorageRequestAsync(request.ContentId.Id, body)); } public StoragePurchase GetPurchaseStatus(string purchaseId) { - return Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); + return mapper.Map(OnCodex(api => api.GetPurchaseAsync(purchaseId))); } public string GetName() @@ -104,11 +104,6 @@ namespace CodexPlugin return workflow.GetPodInfo(Container); } - private dynamic Map(dynamic input) - { - return mapper.Map(input); - } - private T OnCodex(Func> action) { var address = GetAddress(); diff --git a/ProjectPlugins/CodexPlugin/openapi.yaml b/ProjectPlugins/CodexPlugin/openapi.yaml index cfbc800..3a1ad70 100644 --- a/ProjectPlugins/CodexPlugin/openapi.yaml +++ b/ProjectPlugins/CodexPlugin/openapi.yaml @@ -357,9 +357,8 @@ paths: content: application/json: schema: - type: array - items: - $ref: "#/components/schemas/DataList" + $ref: "#/components/schemas/DataList" + "400": description: Invalid CID is specified "404": From a53d2de13b0ed783525045bee7d032a1832526a3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 15:35:26 +0100 Subject: [PATCH 20/25] Moves ethAccount to Geth plugin. Enables setting ethAccount when Codex starts. --- .../CodexPlugin/CodexContainerRecipe.cs | 10 ++--- .../CodexPlugin/CodexNodeFactory.cs | 6 +-- ProjectPlugins/CodexPlugin/CodexPlugin.cs | 4 +- ProjectPlugins/CodexPlugin/CodexSetup.cs | 37 +++++++++++++------ .../CodexPlugin/MarketplaceInitialConfig.cs | 6 +-- .../CodexPlugin/MarketplaceStartResults.cs | 17 --------- .../CodexPlugin/MarketplaceStarter.cs | 19 ---------- ProjectPlugins/GethPlugin/EthAccount.cs | 28 ++++++++++++++ ProjectPlugins/GethPlugin/EthAddress.cs | 1 + .../BasicTests/ContinuousSubstitute.cs | 3 +- Tests/CodexTests/BasicTests/ExampleTests.cs | 10 +++-- .../FullyConnectedDownloadTests.cs | 3 +- .../PeerDiscoveryTests/PeerDiscoveryTests.cs | 3 +- Tools/CodexNetDeployer/CodexNodeStarter.cs | 7 ++-- 14 files changed, 80 insertions(+), 74 deletions(-) delete mode 100644 ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs delete mode 100644 ProjectPlugins/CodexPlugin/MarketplaceStarter.cs create mode 100644 ProjectPlugins/GethPlugin/EthAccount.cs diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 9d041c6..7ff1bde 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,8 +7,6 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-c219a5f-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; @@ -106,13 +104,13 @@ namespace CodexPlugin AddEnvVar("CODEX_ETH_PROVIDER", $"{wsAddress.Host.Replace("http://", "ws://")}:{wsAddress.Port}"); AddEnvVar("CODEX_MARKETPLACE_ADDRESS", marketplaceAddress); + var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup; + // Custom scripting in the Codex test image will write this variable to a private-key file, // and pass the correct filename to Codex. - var mStart = marketplaceStarter.Start(); - AddEnvVar("PRIV_KEY", mStart.PrivateKey); - Additional(mStart); + AddEnvVar("PRIV_KEY", marketplaceSetup.EthAccount.PrivateKey); + Additional(marketplaceSetup.EthAccount); - var marketplaceSetup = config.MarketplaceConfig.MarketplaceSetup; SetCommandOverride(marketplaceSetup); if (marketplaceSetup.IsValidator) { diff --git a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs index f0c7166..05b80b2 100644 --- a/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs +++ b/ProjectPlugins/CodexPlugin/CodexNodeFactory.cs @@ -35,9 +35,9 @@ namespace CodexPlugin private EthAddress? GetEthAddress(CodexAccess access) { - var mStart = access.Container.Recipe.Additionals.Get(); - if (mStart == null) return null; - return mStart.EthAddress; + var ethAccount = access.Container.Recipe.Additionals.Get(); + if (ethAccount == null) return null; + return ethAccount.EthAddress; } public CrashWatcher CreateCrashWatcher(RunningContainer c) diff --git a/ProjectPlugins/CodexPlugin/CodexPlugin.cs b/ProjectPlugins/CodexPlugin/CodexPlugin.cs index 3e40a99..9c16af9 100644 --- a/ProjectPlugins/CodexPlugin/CodexPlugin.cs +++ b/ProjectPlugins/CodexPlugin/CodexPlugin.cs @@ -52,8 +52,8 @@ namespace CodexPlugin var mconfig = codexSetup.MarketplaceConfig; foreach (var node in result) { - mconfig.GethNode.SendEth(node, mconfig.InitialEth); - mconfig.CodexContracts.MintTestTokens(node, mconfig.InitialTokens); + mconfig.GethNode.SendEth(node, mconfig.MarketplaceSetup.InitialEth); + mconfig.CodexContracts.MintTestTokens(node, mconfig.MarketplaceSetup.InitialTestTokens); } } diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index 54be6a9..d2560c4 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -17,8 +17,7 @@ namespace CodexPlugin ICodexSetup WithBlockMaintenanceInterval(TimeSpan duration); ICodexSetup WithBlockMaintenanceNumber(int numberOfBlocks); ICodexSetup EnableMetrics(); - ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens); - ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, Action marketplaceSetup); + ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action marketplaceSetup); /// /// Provides an invalid proof every N proofs /// @@ -28,6 +27,8 @@ namespace CodexPlugin public interface IMarketplaceSetup { + IMarketplaceSetup WithInitial(Ether eth, TestToken tokens); + IMarketplaceSetup WithAccount(EthAccount account); IMarketplaceSetup AsStorageNode(); IMarketplaceSetup AsValidator(); } @@ -122,17 +123,12 @@ namespace CodexPlugin return this; } - public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens) - { - return EnableMarketplace(gethNode, codexContracts, initialEth, initialTokens, s => { }); - } - - public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens, Action marketplaceSetup) + public ICodexSetup EnableMarketplace(IGethNode gethNode, ICodexContracts codexContracts, Action marketplaceSetup) { var ms = new MarketplaceSetup(); marketplaceSetup(ms); - MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts, initialEth, initialTokens); + MarketplaceConfig = new MarketplaceInitialConfig(ms, gethNode, codexContracts); return this; } @@ -169,6 +165,9 @@ namespace CodexPlugin { public bool IsStorageNode { get; private set; } public bool IsValidator { get; private set; } + public Ether InitialEth { get; private set; } = 0.Eth(); + public TestToken InitialTestTokens { get; private set; } = 0.TestTokens(); + public EthAccount EthAccount { get; private set; } = EthAccount.GenerateNew(); public IMarketplaceSetup AsStorageNode() { @@ -182,14 +181,28 @@ namespace CodexPlugin return this; } + public IMarketplaceSetup WithAccount(EthAccount account) + { + EthAccount = account; + return this; + } + + public IMarketplaceSetup WithInitial(Ether eth, TestToken tokens) + { + InitialEth = eth; + InitialTestTokens = tokens; + return this; + } + public override string ToString() { var result = "[(clientNode)"; // When marketplace is enabled, being a clientNode is implicit. result += IsStorageNode ? "(storageNode)" : "()"; - result += IsValidator ? "(validator)" : "()"; - result += "]"; + result += IsValidator ? "(validator)" : "() "; + result += $"Address: '{EthAccount.EthAddress}' "; + result += $"InitialEth/TT({InitialEth.Eth}/{InitialTestTokens.Amount})"; + result += "] "; return result; } } - } diff --git a/ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs b/ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs index 6bd4999..0f55580 100644 --- a/ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs +++ b/ProjectPlugins/CodexPlugin/MarketplaceInitialConfig.cs @@ -5,19 +5,15 @@ namespace CodexPlugin { public class MarketplaceInitialConfig { - public MarketplaceInitialConfig(MarketplaceSetup marketplaceSetup, IGethNode gethNode, ICodexContracts codexContracts, Ether initialEth, TestToken initialTokens) + public MarketplaceInitialConfig(MarketplaceSetup marketplaceSetup, IGethNode gethNode, ICodexContracts codexContracts) { MarketplaceSetup = marketplaceSetup; GethNode = gethNode; CodexContracts = codexContracts; - InitialEth = initialEth; - InitialTokens = initialTokens; } public MarketplaceSetup MarketplaceSetup { get; } public IGethNode GethNode { get; } public ICodexContracts CodexContracts { get; } - public Ether InitialEth { get; } - public TestToken InitialTokens { get; } } } diff --git a/ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs b/ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs deleted file mode 100644 index 4bce517..0000000 --- a/ProjectPlugins/CodexPlugin/MarketplaceStartResults.cs +++ /dev/null @@ -1,17 +0,0 @@ -using GethPlugin; - -namespace CodexPlugin -{ - [Serializable] - public class MarketplaceStartResults - { - public MarketplaceStartResults(EthAddress ethAddress, string privateKey) - { - EthAddress = ethAddress; - PrivateKey = privateKey; - } - - public EthAddress EthAddress { get; } - public string PrivateKey { get; } - } -} diff --git a/ProjectPlugins/CodexPlugin/MarketplaceStarter.cs b/ProjectPlugins/CodexPlugin/MarketplaceStarter.cs deleted file mode 100644 index 42a5851..0000000 --- a/ProjectPlugins/CodexPlugin/MarketplaceStarter.cs +++ /dev/null @@ -1,19 +0,0 @@ -using GethPlugin; -using Nethereum.Hex.HexConvertors.Extensions; -using Nethereum.Web3.Accounts; - -namespace CodexPlugin -{ - public class MarketplaceStarter - { - public MarketplaceStartResults Start() - { - var ecKey = Nethereum.Signer.EthECKey.GenerateKey(); - var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex(); - var account = new Account(privateKey); - var ethAddress = new EthAddress(account.Address); - - return new MarketplaceStartResults(ethAddress, account.PrivateKey); - } - } -} diff --git a/ProjectPlugins/GethPlugin/EthAccount.cs b/ProjectPlugins/GethPlugin/EthAccount.cs new file mode 100644 index 0000000..9f1318b --- /dev/null +++ b/ProjectPlugins/GethPlugin/EthAccount.cs @@ -0,0 +1,28 @@ +using Nethereum.Hex.HexConvertors.Extensions; +using Nethereum.Web3.Accounts; + +namespace GethPlugin +{ + [Serializable] + public class EthAccount + { + public EthAccount(EthAddress ethAddress, string privateKey) + { + EthAddress = ethAddress; + PrivateKey = privateKey; + } + + public EthAddress EthAddress { get; } + public string PrivateKey { get; } + + public static EthAccount GenerateNew() + { + var ecKey = Nethereum.Signer.EthECKey.GenerateKey(); + var privateKey = ecKey.GetPrivateKeyAsBytes().ToHex(); + var account = new Account(privateKey); + var ethAddress = new EthAddress(account.Address); + + return new EthAccount(ethAddress, account.PrivateKey); + } + } +} diff --git a/ProjectPlugins/GethPlugin/EthAddress.cs b/ProjectPlugins/GethPlugin/EthAddress.cs index 803a1f7..9b32cd2 100644 --- a/ProjectPlugins/GethPlugin/EthAddress.cs +++ b/ProjectPlugins/GethPlugin/EthAddress.cs @@ -5,6 +5,7 @@ EthAddress EthAddress { get; } } + [Serializable] public class EthAddress { public EthAddress(string address) diff --git a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs index 5f4c300..71ce58e 100644 --- a/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs +++ b/Tests/CodexTests/BasicTests/ContinuousSubstitute.cs @@ -21,7 +21,8 @@ namespace CodexTests.BasicTests var group = AddCodex(5, o => o .EnableMetrics() - .EnableMarketplace(geth, contract, 10.Eth(), 100000.TestTokens(), s => s + .EnableMarketplace(geth, contract, s => s + .WithInitial(10.Eth(), 100000.TestTokens()) .AsStorageNode() .AsValidator()) .WithBlockTTL(TimeSpan.FromMinutes(5)) diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 4973e2a..1d4d5d1 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -64,7 +64,8 @@ namespace CodexTests.BasicTests .WithName("Seller") .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)) .WithStorageQuota(11.GB()) - .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: sellerInitialBalance, s => s + .EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), sellerInitialBalance) .AsStorageNode() .AsValidator())); @@ -81,9 +82,10 @@ namespace CodexTests.BasicTests var testFile = GenerateTestFile(fileSize); var buyer = AddCodex(s => s - .WithName("Buyer") - .WithBootstrapNode(seller) - .EnableMarketplace(geth, contracts, initialEth: 10.Eth(), initialTokens: buyerInitialBalance)); + .WithName("Buyer") + .WithBootstrapNode(seller) + .EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), buyerInitialBalance))); AssertBalance(contracts, buyer, Is.EqualTo(buyerInitialBalance)); diff --git a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs index b4762b9..7f11171 100644 --- a/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs +++ b/Tests/CodexTests/DownloadConnectivityTests/FullyConnectedDownloadTests.cs @@ -21,7 +21,8 @@ namespace CodexTests.DownloadConnectivityTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens())); + AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), 1000.TestTokens()))); AssertAllNodesConnected(); } diff --git a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs index 10c1cde..82aad30 100644 --- a/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs +++ b/Tests/CodexTests/PeerDiscoveryTests/PeerDiscoveryTests.cs @@ -31,7 +31,8 @@ namespace CodexTests.PeerDiscoveryTests { var geth = Ci.StartGethNode(s => s.IsMiner()); var contracts = Ci.StartCodexContracts(geth); - AddCodex(2, s => s.EnableMarketplace(geth, contracts, 10.Eth(), 1000.TestTokens())); + AddCodex(2, s => s.EnableMarketplace(geth, contracts, m => m + .WithInitial(10.Eth(), 1000.TestTokens()))); AssertAllNodesConnected(); } diff --git a/Tools/CodexNetDeployer/CodexNodeStarter.cs b/Tools/CodexNetDeployer/CodexNodeStarter.cs index 663659b..d25e373 100644 --- a/Tools/CodexNetDeployer/CodexNodeStarter.cs +++ b/Tools/CodexNetDeployer/CodexNodeStarter.cs @@ -41,10 +41,11 @@ namespace CodexNetDeployer if (config.ShouldMakeStorageAvailable) { - s.EnableMarketplace(gethNode, contracts, 100.Eth(), config.InitialTestTokens.TestTokens(), s => + s.EnableMarketplace(gethNode, contracts, m => { - if (validatorsLeft > 0) s.AsValidator(); - if (config.ShouldMakeStorageAvailable) s.AsStorageNode(); + m.WithInitial(100.Eth(), config.InitialTestTokens.TestTokens()); + if (validatorsLeft > 0) m.AsValidator(); + if (config.ShouldMakeStorageAvailable) m.AsStorageNode(); }); } From 9fe97f5d40673596a8ba7dcd8236fb61bc8ff3d8 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 26 Mar 2024 16:10:50 +0100 Subject: [PATCH 21/25] Debugs compatibility check. --- ProjectPlugins/CodexPlugin/ApiChecker.cs | 3 ++- ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs | 2 +- ProjectPlugins/CodexPluginPrebuild/Program.cs | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/ApiChecker.cs b/ProjectPlugins/CodexPlugin/ApiChecker.cs index 39fb22a..437ad22 100644 --- a/ProjectPlugins/CodexPlugin/ApiChecker.cs +++ b/ProjectPlugins/CodexPlugin/ApiChecker.cs @@ -77,7 +77,8 @@ namespace CodexPlugin private string Hash(string file) { - var fileBytes = Encoding.ASCII.GetBytes(file); + var fileBytes = Encoding.ASCII.GetBytes(file + .Replace(Environment.NewLine, "")); var sha = SHA256.Create(); var hash = sha.ComputeHash(fileBytes); return BitConverter.ToString(hash); diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index 9d041c6..bf5e724 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -9,7 +9,7 @@ namespace CodexPlugin { private readonly MarketplaceStarter marketplaceStarter = new MarketplaceStarter(); - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-c219a5f-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-455b95d-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; diff --git a/ProjectPlugins/CodexPluginPrebuild/Program.cs b/ProjectPlugins/CodexPluginPrebuild/Program.cs index 5ad5e89..8f4a8ec 100644 --- a/ProjectPlugins/CodexPluginPrebuild/Program.cs +++ b/ProjectPlugins/CodexPluginPrebuild/Program.cs @@ -1,4 +1,5 @@ using System.Security.Cryptography; +using System.Text; public static class Program { @@ -30,7 +31,10 @@ public static class Program private static string CreateHash() { - var fileBytes = File.ReadAllBytes(OpenApiFile); + var file = File.ReadAllText(OpenApiFile); + var fileBytes = Encoding.ASCII.GetBytes(file + .Replace(Environment.NewLine, "")); + var sha = SHA256.Create(); var hash = sha.ComputeHash(fileBytes); return BitConverter.ToString(hash); From 6597728e5ca43d9e10da6785f2964a4d912637f9 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 08:13:36 +0100 Subject: [PATCH 22/25] Makes contract-clock log topic configurable --- ProjectPlugins/CodexPlugin/CodexSetup.cs | 1 + ProjectPlugins/CodexPlugin/CodexStartupConfig.cs | 3 +-- Tests/CodexTests/BasicTests/ExampleTests.cs | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/ProjectPlugins/CodexPlugin/CodexSetup.cs b/ProjectPlugins/CodexPlugin/CodexSetup.cs index d2560c4..6fcaf87 100644 --- a/ProjectPlugins/CodexPlugin/CodexSetup.cs +++ b/ProjectPlugins/CodexPlugin/CodexSetup.cs @@ -50,6 +50,7 @@ namespace CodexPlugin public CodexLogLevel DiscV5 { get; set; } public CodexLogLevel Libp2p { get; set; } + public CodexLogLevel ContractClock { get; set; } = CodexLogLevel.Warn; public CodexLogLevel? BlockExchange { get; } } diff --git a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs index 72d154b..dbed45a 100644 --- a/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs +++ b/ProjectPlugins/CodexPlugin/CodexStartupConfig.cs @@ -77,8 +77,7 @@ namespace CodexPlugin level = $"{level};" + $"{CustomTopics.DiscV5.ToString()!.ToLowerInvariant()}:{string.Join(",", discV5Topics)};" + $"{CustomTopics.Libp2p.ToString()!.ToLowerInvariant()}:{string.Join(",", libp2pTopics)};" + - // Contract clock is always set to warn. It logs a trace every second. - $"{CodexLogLevel.Warn.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}"; + $"{CustomTopics.ContractClock.ToString().ToLowerInvariant()}:{string.Join(",", contractClockTopics)}"; if (CustomTopics.BlockExchange != null) { diff --git a/Tests/CodexTests/BasicTests/ExampleTests.cs b/Tests/CodexTests/BasicTests/ExampleTests.cs index 1d4d5d1..df32b75 100644 --- a/Tests/CodexTests/BasicTests/ExampleTests.cs +++ b/Tests/CodexTests/BasicTests/ExampleTests.cs @@ -62,7 +62,10 @@ namespace CodexTests.BasicTests var seller = AddCodex(s => s .WithName("Seller") - .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn)) + .WithLogLevel(CodexLogLevel.Trace, new CodexLogCustomTopics(CodexLogLevel.Error, CodexLogLevel.Error, CodexLogLevel.Warn) + { + ContractClock = CodexLogLevel.Trace, + }) .WithStorageQuota(11.GB()) .EnableMarketplace(geth, contracts, m => m .WithInitial(10.Eth(), sellerInitialBalance) From 127b7daecc04e6fb282935723a5e31670b5ca652 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 08:41:18 +0100 Subject: [PATCH 23/25] Improves plugin log initialization --- Framework/Core/PluginTools.cs | 6 +++--- Framework/Logging/LogPrefixer.cs | 17 ++++++++++++----- .../CodexPlugin/CodexContainerRecipe.cs | 2 +- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/Framework/Core/PluginTools.cs b/Framework/Core/PluginTools.cs index ff77bae..5e1faee 100644 --- a/Framework/Core/PluginTools.cs +++ b/Framework/Core/PluginTools.cs @@ -36,11 +36,11 @@ namespace Core private readonly ITimeSet timeSet; private readonly WorkflowCreator workflowCreator; private readonly IFileManager fileManager; - private ILog log; + private readonly LogPrefixer log; internal PluginTools(ILog log, WorkflowCreator workflowCreator, string fileManagerRootFolder, ITimeSet timeSet) { - this.log = log; + this.log = new LogPrefixer(log); this.workflowCreator = workflowCreator; this.timeSet = timeSet; fileManager = new FileManager(log, fileManagerRootFolder); @@ -48,7 +48,7 @@ namespace Core public void ApplyLogPrefix(string prefix) { - log = new LogPrefixer(log, prefix); + log.Prefix = prefix; } public IHttp CreateHttp(Action onClientCreated) diff --git a/Framework/Logging/LogPrefixer.cs b/Framework/Logging/LogPrefixer.cs index a3d2f9f..69ab105 100644 --- a/Framework/Logging/LogPrefixer.cs +++ b/Framework/Logging/LogPrefixer.cs @@ -3,14 +3,21 @@ public class LogPrefixer : ILog { private readonly ILog backingLog; - private readonly string prefix; + + public LogPrefixer(ILog backingLog) + { + this.backingLog = backingLog; + } public LogPrefixer(ILog backingLog, string prefix) { this.backingLog = backingLog; - this.prefix = prefix; + Prefix = prefix; } + public string Prefix { get; set; } = string.Empty; + + public LogFile CreateSubfile(string ext = "log") { return backingLog.CreateSubfile(ext); @@ -18,17 +25,17 @@ public void Debug(string message = "", int skipFrames = 0) { - backingLog.Debug(prefix + message, skipFrames); + backingLog.Debug(Prefix + message, skipFrames); } public void Error(string message) { - backingLog.Error(prefix + message); + backingLog.Error(Prefix + message); } public void Log(string message) { - backingLog.Log(prefix + message); + backingLog.Log(Prefix + message); } public void AddStringReplace(string from, string to) diff --git a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs index ba356b2..7e7c111 100644 --- a/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs +++ b/ProjectPlugins/CodexPlugin/CodexContainerRecipe.cs @@ -7,7 +7,7 @@ namespace CodexPlugin { public class CodexContainerRecipe : ContainerRecipeFactory { - private const string DefaultDockerImage = "codexstorage/nim-codex:sha-455b95d-dist-tests"; + private const string DefaultDockerImage = "codexstorage/nim-codex:sha-cd280d4-dist-tests"; public const string ApiPortTag = "codex_api_port"; public const string ListenPortTag = "codex_listen_port"; public const string MetricsPortTag = "codex_metrics_port"; From db6db4d38c24c6ad505a6fdb19f768173937f1f8 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 09:41:12 +0100 Subject: [PATCH 24/25] Fixes metrics tests --- ProjectPlugins/MetricsPlugin/MetricsQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs index 96d81aa..199015b 100644 --- a/ProjectPlugins/MetricsPlugin/MetricsQuery.cs +++ b/ProjectPlugins/MetricsPlugin/MetricsQuery.cs @@ -16,7 +16,7 @@ namespace MetricsPlugin log = tools.GetLog(); endpoint = tools .CreateHttp() - .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "api/v1"); + .CreateEndpoint(RunningContainer.GetAddress(log, PrometheusContainerRecipe.PortTag), "/api/v1/"); } public RunningContainer RunningContainer { get; } From f6edf6cbd59a53d912918765472cd636e0448736 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 27 Mar 2024 15:01:32 +0100 Subject: [PATCH 25/25] Overdue readme updates --- CONTRIBUTINGPLUGINS.MD | 98 +++++++++++++++++++++++++---- README.md | 15 +++-- Tests/CodexTests/README.md | 10 +-- Tests/DistTestCore/Configuration.cs | 3 + Tests/DistTestCore/DistTest.cs | 34 ++++++---- Tools/CodexNetDeployer/README.MD | 10 +-- cs-codex-dist-testing.sln | 5 ++ 7 files changed, 135 insertions(+), 40 deletions(-) diff --git a/CONTRIBUTINGPLUGINS.MD b/CONTRIBUTINGPLUGINS.MD index 231ba6c..250a839 100644 --- a/CONTRIBUTINGPLUGINS.MD +++ b/CONTRIBUTINGPLUGINS.MD @@ -1,10 +1,26 @@ # Distributed System Tests for Nim-Codex ## Contributing plugins -The testing framework was created for testing Codex. However, it's been designed such that other distributed/containerized projects can 'easily' be added. In order to add your project to the framework you must: +The testing framework was created for testing Codex. However, it's been designed such that other containerized projects can 'easily' be added. + +In this file, you'll see 'users' (in quote) mentioned once or twice. This refers to code/projects/tests which end up making use of your plugin. 'Users' come in many shapes and sizes and tend to have many differen use-cases in mind. Please consider this when reading this document and writing your plugin. + +## Checklist +Your application must pass this checklist to be compatible with the framework: +- It runs in a docker container. +- It can be configured via environment variables. (You may need to create a docker image which contains a shell script, to pass some env-vars as CLI arguments to your application. Container command overrides do work, but are not equally reliable across container platforms. When in doubt: use env-var!) +- It has network interaction: + - It exposes one or more APIs via one or more ports, OR + - It makes calls to other services. (OR both.) + +If your application's use-cases rely primarily on shell interaction, this framework might not be for you. The framework allows you to execute commands in containers AND read stdout/stderr responses. However, its focus during development has always been webservice API interactions. + +## Steps +In order to add your project to the framework you must: 1. Create a library assembly in the project plugins folder. 1. It must contain a type that implements the `IProjectPlugin` interface from the `Core` assembly. -1. If your plugin wants to expose any specific methods or objects to the code using the framework (the tests and tools), it must implement extensions for the `CoreInterface` type. +1. If your plugin wants to expose any specific methods or objects to 'users', it must implement extensions for the `CoreInterface` type. +1. If your plugin wants to run containers of its own project, it must provide a recipe. ## Constructors & Tools Your implementation of `IProjectPlugin` must have a public constructor with a single argument of type `IPluginTools`, for example: @@ -20,19 +36,34 @@ Your implementation of `IProjectPlugin` must have a public constructor with a si } ``` -`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients for containers. (Without having to figure out addresses manually.) - ## Plugin Interfaces The `IProjectPlugin` interface requires the implementation of two methods. -1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version information at this time if applicable. -1. `Decommission` - Should your plugin have any active system resources, free them in this method. +1. `Announce` - It is considered polite to use the logging functionality provided by the `IPluginTools` to announce that your plugin has been loaded. You may also want to log some manner of version and/or configuration information at this time if applicable. +1. `Decommission` - Should your plugin have any active system resources, free them in this method. Please note that resources managed by the framework (such as running containers and tracked data files) do *not* need to be manually disposed in this method. `Decommission` is to be used for resources not managed by the framework. There are a few optional interfaces your plugin may choose to implement. The framework will automatically use these interfaces. -1. `IHasLogPrefix` - Implementing this interface allows you to provide a string with will be prepended to all log statements made by your plugin. -1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by code that uses your plugin. +1. `IHasLogPrefix` - Implementing this interface allows you to provide a string which will be prepended to all log statements made by your plugin. A polite thing to do. +1. `IHasMetadata` - This allows you to provide metadata in the form of key/value pairs. This metadata can be accessed by 'users' of your plugin. Often this data finds its way into log files and container-descriptors in order to help track versions/tests/deployments, etc. + +## IPluginTools +`IPluginTools` provides your plugin access to all framework functionality, such as logging, tracked file management, container lifecycle management, and a means to create HTTP clients to make calls to containers. (Figure out addresses and ports for containers is handled by the framework.) + +It is possible and allowed for your plugin to depend on and use other plugins. (For example, maybe your project wants to interact with Ethereum and wants to use the GethPlugin to talk to a Geth node.) `IPluginTools` is *not* what is used for accessing functionality of other plugins. See 'Core Interface' section. + +ILog GetLog(); +IHttp CreateHttp(Action onClientCreated); +IHttp CreateHttp(Action onClientCreated, ITimeSet timeSet); +IHttp CreateHttp(); +IFileManager GetFileManager(); + +The plugin tools provide: +1. `Workflow` - This tool allows you to start and stop containers using "container recipes". (More on those below.) It also allows you to execute commands inside a container, access stdout/stderr, detect crashes, and access pod deployment information. The workflow tool also lets you inspect the locations available in the cluster, and decide where you want to run containers. (More on that below as well.) +1. `Log` - Good logging is priceless. Use this tool to get a log object handle, and write useful debug/info/error statements. +1. `Http` - This tool gives you a convenient way to access a standard dotnet HttpClient, and takes care of timeouts and retries (in accordance with the config). Additionally, it combos nicely with container objects created by `Workflow`, such that you never have to spend any time figuring out the addresses and ports of your containers. +1. `FileManager` - Lets you use tracked temporary files. Even if the 'user' tests/application start crashing, the framework will make sure these are cleaned up. ## Core Interface -Any functionality your plugin wants to expose to code which uses the framework will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type: +Any functionality your plugin wants to expose to 'users' will have to be added on to the `CoreInterface` type. You can accomplish this by using C# extension methods. The framework provides a `GetPlugin` method to access your plugin instance from the `CoreInterface` type: ```C# public static class CoreInterfaceExtensions { @@ -48,12 +79,14 @@ Any functionality your plugin wants to expose to code which uses the framework w } ``` +If your plugin wants to access the functionality exposed by other plugins, then you can pass the argument `CoreInterface ci` to your plugin code in order to do so. (For example, if you want to start a Geth node, the Geth plugin adds `IGethNode StartGethNode(this CoreInterface ci, Action setup)` to the core interface.) Don't forget you'll need to add a project reference to each plugin project you wish to use. + While technically you can build whatever you like on top of the `CoreInterface` and your own plugin types, I recommend that you follow the approach explained below. ## Deploying, Wrapping, and Starting When building a plugin, it is important to make as few assumptions as possible about how it will be used by whoever is going to use the framework. For this reason, I recommend you expose three kinds of methods using your `CoreInterface` extensions: -1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning containers as a result. If your project requires additional information, you can create a new class type to contain both it and the containers created. -1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. +1. Deploy - This kind of method should deploy your project, creating and configuring containers as needed and returning container objects as a result. If your project requires additional information, you can create a new class type to contain both it and the container objects created. +1. Wrap - This kind of method should, when given the previously mentioned container information, create some kind of convenient accessor or interactor object. This object should abstract away for example details of a REST API of your project, allowing users of your plugin to write their code using a set of methods and types that nicely model your project's domain. (For example, if my project has a REST API call that allows users to fetch some state information, the object returned by Wrap should have a convenient method to call that API and receive that state information.) 1. Start - This kind of method does both, simply calling a Deploy method first, then a Wrap method, and returns the result. Here's an example: @@ -69,8 +102,8 @@ public static class CoreInterfaceExtensions public static IMyProjectNode WrapMyProjectContainer(this CoreInterface ci, RunningContainers container) { - return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` tool to create an HTTP client for the container, then wrap it in an object that - // represents the API of your project. + return Plugin(ci).WrapMyContainerProject(container); // <-- This method probably will use the 'PluginTools.CreateHttp()` to create an HTTP client for the container, then wrap it in an object that + // represents the API of your project, in this case 'IMyProjectNode'. } public static IMyProjectNode StartMyProject(this CoreInterface ci, string someArgument) @@ -82,5 +115,44 @@ public static class CoreInterfaceExtensions } ``` +Should your deploy methods not return framework-types like RunningContainers, please make sure that your custom times are serializable. (Decorate them with the `[Serializable]` attribute.) Tools have been built using this framework which rely on the ability to serialize and store deployment information for later use. Please don't break this possibility. (Consider using the `SerializeGate` type to help ensure compatibility.) + The primary reason to decouple deploying and wrapping functionalities is that some use cases require these steps to be performed by separate applications, and different moments in time. For this reason, whatever is returned by the deploy methods should be serializable. After deserialization at some later time, it should then be valid input for the wrap method. The Codex continuous tests system is a clear example of this use case: The `CodexNetDeployer` tool uses deploy methods to create Codex nodes. Then it writes the returned objects to a JSON file. Some time later, the `CodexContinuousTests` application uses this JSON file to reconstruct the objects created by the deploy methods. It then uses the wrap methods to create accessors and interactors, which are used for testing. +## Container Recipes +In order to run a container of your application, the framework needs to know how to create that container. Think of a container recipe as being similar to a docker-compose.yaml file: You specify the docker image, ports, environment variables, persistent volumes, and secrets. However, container recipes are code. This allows you to add conditional behaviour to how your container is constructed. For example: The 'user' of your plugin specifies in their call input that they want to run your application in a certain mode. This would cause your container recipe to set certain environment variables, which cause the application to behave in the requested way. + +### Addresses and ports +In a docker-compose.yaml file, it is perfectly normal to specify which ports should be exposed on your container. However, in the framework there's more to consider. When your application container starts, who knows on what kind of machine it runs, and what other processes it's sharing space with? Well, Kubernetes knows. Therefore, it is recommended that container recipes *do not* specify exact port numbers. The framework allows container recipes to declare "a port" without specifying its port number. This allows the framework and Kubernetes to figure out which ports are available when it's time to deploy. In order to find out which port numbers were assigned post-deployment, you can look up the port by tag (which is just an identifying string). When you specify a port to be mapped in your container recipe, you must specify: +1. `Tag` - An identifier. +1. `Internal` or `External` - Whether this port should be accessible only inside the cluster (for other containers (k8s: "ClusterIP")) or outside the cluster as well (for external tools/applications (k8s: "NodePort")). +1. `Protocol` - TCP or UDP. Both protocols on the same port is not universally supported by all container engines, and is therefore not supported by the framework. + +If your application wants to listen for incoming traffic from inside its container, be sure to bind it to address "0.0.0.0". + +Reminder: If you don't want to worry about addresses, and internal or external ports, you don't have to! The container objects returned by the `workflow` plugin tool have a method called `GetAddress`. Given a port tag, it returns and address object. The `Http` plugin tool can use that address object to set up connections. + +## Locations +The framework is designed to allow you to control instances of your application in multiple (physical) locations. It accomplishes this by using kubernetes, and the ability to deploy containers to specific hosts (nodes) inside a kubernetes cluster. Since Kubernetes allows you to build clusters cross-site, this framework in theory enables you to deploy and interact with containers running anywhere. + +The `workflow` plugin tool provides you a list of all available locations in the cluster. When starting a container, you are able to pick one of those locations. If no location is selected, one will be chosen by kubernetes. Locations can be chosen explicitly by kubernetes node name, or, they can be picked from the array of available locations. + +Example: +```C# +{ + var location = Ci.GetKnownLocations().Get("kbnode_euwest_paris1"); + var codex = Ci.StartCodexNode(s => s.At(location)); +} +``` +In this example, 'Ci' is an instance of the core interface. The CodexPlugin exposes a function 'StartCodexNode', which allows its user to specify a location. This location is then passed to the `workflow` tool when the Codex plugin starts its container. + +The available locations array guarantees that each entry corresponds to a different kubernetes host. +```C# +{ + var knownLocations = Ci.GetKnownLocations(); + // I don't care where exactly, as long as they are different locations. + var codexAtZero = Ci.StartCodexNode(s => s.At(knownLocations.Get(0))); + var codexAtOne = Ci.StartCodexNode(s => s.At(knownLocations.Get(1))); +} +``` + diff --git a/README.md b/README.md index 0aba7b7..eaefe4f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,22 @@ -# Distributed System Tests for Nim-Codex +# Distributed System Tests -Using a common dotnet unit-test framework and a few other libraries, this project allows you to write tests that use multiple Codex node instances in various configurations to test the distributed system in a controlled, reproducible environment. +This project allows you to write tools and tests that control and interact with container-based applications to form a distributed system in a controlled, reproducible environment. -Nim-Codex: https://github.com/codex-storage/nim-codex Dotnet: v7.0 Kubernetes: v1.25.4 Dotnet-kubernetes SDK: v10.1.4 https://github.com/kubernetes-client/csharp Nethereum: v4.14.0 +Currently, this project is mainly used for distributed testing of [Nim-Codex](https://github.com/codex-storage/nim-codex). However, its plugin-structure allows for other projects to be on-boarded (relatively) easily. (See 'contribute a plugin`.) + +## Tests/DistTestCore +Library with generic distributed-testing functionality. Uses NUnit3. Reference this project to build unit-test style scenarios: setup, run test, teardown. The DistTestCore responds to the following env-vars: +- `LOGPATH` = Path where log files will be written. +- `DATAFILEPATH` = Path where (temporary) data files will be stored. +- `ALWAYS_LOGS` = When set, DistTestCore will always download all container logs at the end of a test run. By default, logs are only downloaded on test failure. + ## Tests/CodexTests and Tests/CodexLongTests -These are test assemblies that use NUnit3 to perform tests against transient Codex nodes. +These are test assemblies that use DistTestCore to perform tests against transient Codex nodes. Read more [HERE](/Tests/CodexTests/README.md) ## Tests/ContinuousTests diff --git a/Tests/CodexTests/README.md b/Tests/CodexTests/README.md index d691d97..568686c 100644 --- a/Tests/CodexTests/README.md +++ b/Tests/CodexTests/README.md @@ -10,11 +10,11 @@ The test runner will produce a folder named `CodexTestLogs` with all the test lo ## Overrides The following environment variables allow you to override specific aspects of the behaviour of the tests. -| Variable | Description | -|------------------|-------------------------------------------------------------------------------------------------------------| -| RUNID | A pod-label 'runid' is added to each pod created during the tests. Use this to set the value of that label. | -| TESTID | Similar to RUNID, except the label is 'testid'. | -| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | +| Variable | Description | +|------------------|----------------------------------------------------------------------------------------------------------------| +| DEPLOYID | A pod-label 'deployid' is added to each pod created during the tests. Use this to set the value of that label. | +| TESTID | Similar to RUNID, except the label is 'testid'. | +| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | ## Using a local Codex repository If you have a clone of the Codex git repository, and you want to run the tests using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. diff --git a/Tests/DistTestCore/Configuration.cs b/Tests/DistTestCore/Configuration.cs index 40f11de..b1a94da 100644 --- a/Tests/DistTestCore/Configuration.cs +++ b/Tests/DistTestCore/Configuration.cs @@ -14,6 +14,7 @@ namespace DistTestCore kubeConfigFile = GetNullableEnvVarOrDefault("KUBECONFIG", null); logPath = GetEnvVarOrDefault("LOGPATH", "CodexTestLogs"); dataFilesPath = GetEnvVarOrDefault("DATAFILEPATH", "TestDataFiles"); + AlwaysDownloadContainerLogs = !string.IsNullOrEmpty(GetEnvVarOrDefault("ALWAYS_LOGS", "")); } public Configuration(string? kubeConfigFile, string logPath, string dataFilesPath) @@ -23,6 +24,8 @@ namespace DistTestCore this.dataFilesPath = dataFilesPath; } + public bool AlwaysDownloadContainerLogs { get; set; } + public KubernetesWorkflow.Configuration GetK8sConfiguration(ITimeSet timeSet, string k8sNamespace) { return GetK8sConfiguration(timeSet, new DoNothingK8sHooks(), k8sNamespace); diff --git a/Tests/DistTestCore/DistTest.cs b/Tests/DistTestCore/DistTest.cs index cc71e5d..76f9d5b 100644 --- a/Tests/DistTestCore/DistTest.cs +++ b/Tests/DistTestCore/DistTest.cs @@ -3,6 +3,7 @@ using DistTestCore.Logs; using FileUtils; using Logging; using NUnit.Framework; +using NUnit.Framework.Interfaces; using System.Reflection; using Utils; using Assert = NUnit.Framework.Assert; @@ -222,7 +223,7 @@ namespace DistTestCore Log($"{result.StackTrace}"); } - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + if (result.Outcome.Status == TestStatus.Failed) { log.MarkAsFailed(); } @@ -251,21 +252,28 @@ namespace DistTestCore private void IncludeLogsOnTestFailure(TestLifecycle lifecycle) { - var result = TestContext.CurrentContext.Result; - if (result.Outcome.Status == NUnit.Framework.Interfaces.TestStatus.Failed) + var testStatus = TestContext.CurrentContext.Result.Outcome.Status; + if (testStatus == TestStatus.Failed) { fixtureLog.MarkAsFailed(); - - if (IsDownloadingLogsEnabled()) - { - lifecycle.Log.Log("Downloading all container logs because of test failure..."); - lifecycle.DownloadAllLogs(); - } - else - { - lifecycle.Log.Log("Skipping download of all container logs due to [DontDownloadLogsOnFailure] attribute."); - } } + + if (ShouldDownloadAllLogs(testStatus)) + { + lifecycle.Log.Log("Downloading all container logs..."); + lifecycle.DownloadAllLogs(); + } + } + + private bool ShouldDownloadAllLogs(TestStatus testStatus) + { + if (configuration.AlwaysDownloadContainerLogs) return true; + if (testStatus == TestStatus.Failed) + { + return IsDownloadingLogsEnabled(); + } + + return false; } private string GetCurrentTestName() diff --git a/Tools/CodexNetDeployer/README.MD b/Tools/CodexNetDeployer/README.MD index d85486a..0ff8521 100644 --- a/Tools/CodexNetDeployer/README.MD +++ b/Tools/CodexNetDeployer/README.MD @@ -9,11 +9,11 @@ After the deployment has successfully finished, a `codex-deployment.json` file w ## Overrides The arguments allow you to configure quite a bit, but not everything. Here are some environment variables the CodexNetDeployer will respond to. None of these are required. -| Variable | Description | -|------------------|--------------------------------------------------------------------------------------------------------------| -| RUNID | A pod-label 'runid' is added to each pod created during deployment. Use this to set the value of that label. | -| TESTID | Similar to RUNID, except the label is 'testid'. | -| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | +| Variable | Description | +|------------------|----------------------------------------------------------------------------------------------------------------| +| DEPLOYID | A pod-label 'deployid' is added to each pod created during the tests. Use this to set the value of that label. | +| TESTID | Similar to RUNID, except the label is 'testid'. | +| CODEXDOCKERIMAGE | If set, this will be used instead of the default Codex docker image. | ## Using a local Codex repository If you have a clone of the Codex git repository, and you want to deploy a network using your local modifications, the following environment variable options are for you. Please note that any changes made in Codex's 'vendor' directory will be discarded during the build process. diff --git a/cs-codex-dist-testing.sln b/cs-codex-dist-testing.sln index b9b5693..ada6893 100644 --- a/cs-codex-dist-testing.sln +++ b/cs-codex-dist-testing.sln @@ -61,6 +61,11 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DiscordRewards", "Framework EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CodexPluginPrebuild", "ProjectPlugins\CodexPluginPrebuild\CodexPluginPrebuild.csproj", "{88C212E9-308A-46A4-BAAD-468E8EBD8EDF}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{FD6E81BA-93E8-4BB7-94F8-98C5427E02B0}" + ProjectSection(SolutionItems) = preProject + .editorconfig = .editorconfig + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU