From c24a60667505a941827952da1b16f1584949c8f1 Mon Sep 17 00:00:00 2001 From: Slava <20563034+veaceslavdoina@users.noreply.github.com> Date: Fri, 13 Sep 2024 21:48:11 +0300 Subject: [PATCH] Initial commit --- LICENSE-APACHE-2.0 | 199 ++++++++ LICENSE-MIT | 19 + README.md | 3 + charts/codex/.helmignore | 23 + charts/codex/Chart.yaml | 32 ++ charts/codex/README.md | 371 +++++++++++++++ charts/codex/templates/_helpers.tpl | 145 ++++++ charts/codex/templates/ingress.yaml | 49 ++ .../codex/templates/poddisruptionbudjet.yaml | 17 + charts/codex/templates/rbac.yaml | 71 +++ charts/codex/templates/service.yaml | 80 ++++ charts/codex/templates/statefulset.yaml | 270 +++++++++++ .../templates/tests/test-connection.yaml | 15 + charts/codex/values.yaml | 442 ++++++++++++++++++ 14 files changed, 1736 insertions(+) create mode 100644 LICENSE-APACHE-2.0 create mode 100644 LICENSE-MIT create mode 100644 README.md create mode 100644 charts/codex/.helmignore create mode 100644 charts/codex/Chart.yaml create mode 100644 charts/codex/README.md create mode 100644 charts/codex/templates/_helpers.tpl create mode 100644 charts/codex/templates/ingress.yaml create mode 100644 charts/codex/templates/poddisruptionbudjet.yaml create mode 100644 charts/codex/templates/rbac.yaml create mode 100644 charts/codex/templates/service.yaml create mode 100644 charts/codex/templates/statefulset.yaml create mode 100644 charts/codex/templates/tests/test-connection.yaml create mode 100644 charts/codex/values.yaml diff --git a/LICENSE-APACHE-2.0 b/LICENSE-APACHE-2.0 new file mode 100644 index 0000000..cbd5c75 --- /dev/null +++ b/LICENSE-APACHE-2.0 @@ -0,0 +1,199 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..9cf1062 --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e1fd469 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Codex Helm charts + + Official Helm charts for Codex. diff --git a/charts/codex/.helmignore b/charts/codex/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/charts/codex/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/charts/codex/Chart.yaml b/charts/codex/Chart.yaml new file mode 100644 index 0000000..514704e --- /dev/null +++ b/charts/codex/Chart.yaml @@ -0,0 +1,32 @@ +apiVersion: v2 +name: codex +description: Codex is a decentralized durability engine that allows persisting data in p2p networks +type: application + +keywords: + - codex + - decentralized + - durability + - storage + - p2p + +home: https://codex.storage + +sources: + - https://github.com/codex-storage/helm-charts/treen/master/charts + - https://github.com/codex-storage/nim-codex + +# https://artifacthub.io/docs/topics/annotations/helm/ +annotations: + artifacthub.io/category: storage + artifacthub.io/license: Apache-2.0 MIT + +maintainers: + - name: Codex DevOps + email: devops@codex.storage + url: https://codex.storage + +icon: https://avatars.githubusercontent.com/u/132685952?s=400&u=92bcb78f2c1bd460e4228acd3d2c6fd5813616d6 + +version: 0.0.1 +appVersion: 0.0.3 diff --git a/charts/codex/README.md b/charts/codex/README.md new file mode 100644 index 0000000..08da689 --- /dev/null +++ b/charts/codex/README.md @@ -0,0 +1,371 @@ +# Codex Helm Chart + + 1. [Description](#description) + 2. [Installation](#installation) + 3. [Deployment strategies](#deployment-strategies) + - [Knows issues](#knows-issues) + - [Prettify](#prettify) + 4. [Development](#development) + 5. [To do](#to-do) + + +## Description + + Codex is a decentralized data storage platform that provides exceptionally strong censorship resistance and durability guarantees. + + Chart will install Codex in Kubernetes and make nodes publicly accessible in the Internet or just for in-cluster testing. For more information please read [Deployment strategies](#deployment-strategies). + + +## Installation + + > **Note:** Please read [Deployment strategies](#deployment-strategies) before the installation. + + 1. Create a namespace + ```shell + kubectl create namespace codex-ns + ``` + + 2. Create secrets if we would like to pass sensitive data + + Create `codex-eth-provider` secret, a common one for all Codex instances + ```shell + secret='apiVersion: v1 + kind: Secret + metadata: + name: codex-eth-provider + namespace: codex-ns + labels: + name: codex + type: Opaque + stringData: + CODEX_ETH_PROVIDER: https://mainnet.infura.io/v3/YOUR-API-KEY + ' + + echo $secret | kubectl create -f - + ``` + + Generate ethereum key pair for each replica + ```shell + docker run --rm gochain/web3 account create + ``` + + Create `codex-eth-private-key` secret, a common one with separate key for each Codex instance + ```shell + secret='apiVersion: v1 + kind: Secret + metadata: + name: codex-eth-private-key + namespace: codex-ns + labels: + name: codex + type: Opaque + stringData: + CODEX_ETH_PRIVATE_KEY-1-1: 0x... + CODEX_ETH_PRIVATE_KEY-2-1: 0x... + ' + + echo $secret | kubectl create -f - + ``` + > **Note:** Please note, key name should contain Pod index, like `-1-1`, `-2-1` in case of multiple replicas or single replica with `statefulSet.prettify=false` and `-1` in case of single replica. + + 3. Create a `codex-api-basic-auth` secret if we would like to expose Codex API via [Ingress NGINX Controller](https://kubernetes.github.io/ingress-nginx/) protected by [basic authentication](https://kubernetes.github.io/ingress-nginx/examples/auth/basic/) + ```shell + docker run --rm httpd htpasswd -bnB + ``` + ```shell + secret='apiVersion: v1 + kind: Secret + metadata: + name: codex-api-basic-auth + namespace: codex-ns + labels: + name: codex-api-basic-auth + type: Opaque + stringData: + auth: >- + auth file content + ' + + echo $secret | kubectl create -f - + ``` + + 4. Refer to the created secrets in the `values.yaml` +
+ values.yaml + + ```yaml + # Replica + replica: + count: 2 + + # StatefulSet + statefulSet: + ordinalsStart: 1 + + # Service account + serviceAccount: + create: true + rbac: + create: true + + # Codex + codex: + # In case we would like to pass more than one bootstrap node + args: + - codex + - persistence + - prover + - --bootstrap-node=spr:xxx + - --bootstrap-node=spr:yyy + env: + CODEX_LOG_LEVEL: TRACE + CODEX_METRICS: true + CODEX_METRICS_ADDRESS: 0.0.0.0 + CODEX_METRICS_PORT: 8008 + CODEX_DATA_DIR: /data + CODEX_API_BINDADDR: 0.0.0.0 + CODEX_API_PORT: 8080 + CODEX_STORAGE_QUOTA: 18gb + CODEX_BLOCK_TTL: 1d + CODEX_BLOCK_MI: 10m + CODEX_BLOCK_MN: 1000 + # port values will be set dynamically for each replica, base on data from service.transport + CODEX_LISTEN_ADDRS: /ip4/0.0.0.0/tcp/8070 + CODEX_DISC_IP: 0.0.0.0 + # port value will be set dynamically for each replica, base on data from service.discovery + CODEX_DISC_PORT: 8090 + # In case of single SPR, you can set it via var + # CODEX_BOOTSTRAP: "spr:xxx" + CODEX_MARKETPLACE_ADDRESS: 0x1234567890123456789012345678901234567890 + # file name will be used as a secret name to be mounted to the specified path + # unique key from `codex-eth-private-key` secret will be used to mach the unique Pod name + CODEX_ETH_PRIVATE_KEY: /opt/codex-eth-private-key + extraEnv: + - name: CODEX_ETH_PROVIDER + valueFrom: + secretKeyRef: + name: codex-eth-provider + key: CODEX_ETH_PROVIDER + + # Pod ports + ports: + api: + enabled: true + name: api + containerPort: 8080 + metrics: + enabled: true + name: metrics + containerPort: 8008 + transport: + enabled: true + name: libp2p + containerPort: 8070 + discovery: + enabled: true + name: discovery + containerPort: 8090 + + # Service + service: + type: + - service + - nodeport + api: + enabled: true + port: 8080 + metrics: + enabled: true + name: metrics + port: 8008 + transport: + enabled: true + # 30000-32767 + nodePort: 30500 + nodePortOffset: 10 + discovery: + # 30000-32767 + enabled: true + nodePort: 30600 + nodePortOffset: 10 + + # Ingress + ingress: + enabled: true + class: nginx + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-staging" + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$2 + nginx.ingress.kubernetes.io/auth-type: basic + nginx.ingress.kubernetes.io/auth-secret: codex-api-basic-auth + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - Private Area' + tls: + - secretName: api-domain-tld + hosts: + - api.domain.tld + hosts: + - host: api.domain.tld + paths: + - path: /storage + rewritePath: (/|$)(.*) + pathType: Prefix + podName: node + + # Persistence + persistence: + enabled: true + name: data + size: 20Gi + retentionPolicy: + whenDeleted: Delete + ``` +
+ + Review and update all settings and pay attention to `codex.env`, `service` and `ingress`. You also may consider to skip the values which [defaults](values.yaml) suit your needs. + + 5. Install helm chart + ```shell + helm install -f values.yaml -n codex-ns codex ./codex + ``` + + 6. Check that your Codex nodes up and running and accessible via Ingress + ```shell + # Pods + kubectl get pods -n codex-ns + + # API + # storage/node-1 + # storage/node-2 + + curl -s -k -u username:password https://api.domain.tld/storage/node-1/api/codex/v1/debug/info | jq -r + ``` + + 7. If we need more replicas, we should + - Update secrets with the keys for new replicas + - Update values.yaml with required number of replicas - `replica.count` + - Upgrade release + + +## Deployment strategies + + For P2P communication, Codex require that transport and discovery ports be accessible for direct connection. In that way we may consider two deployment strategies + - Private - Codex is accessible only inside the Kubernetes cluster + - Public - Codex is accessible to any nodes in the Internet + + **Private** + + For private deployment, Codex Pods should announce their private IP's and TCP ports. Because every Pod has unique IP, all Pods can use same TCP/UDP ports which will be directly accessible by other Pods. + + [Name resolution](https://github.com/libp2p/specs/blob/master/addressing/README.md#ip-and-name-resolution) is not yet supported and we can't use [headless service](https://kubernetes.io/docs/concepts/services-networking/service/#headless-services) for P2P communications. + + This type of deployment is mostly useful for in-cluster testing. + + **Public** + + For Public deployment, Codex Pods should announce Public IP of the Kubernetes workers node on which they are running and TCP/UDP ports should be unique, because [NodePort](https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport) is shared across all nodes in the cluster. This leads to some limitation in case we would like to use a single [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) with replicas > 1, because there is no native way in Kubernetes to assign dynamically Pods TCP/UDP ports per replica. + +**Single replica** + - Single StatefulSet is created and by default with index in the name + - `ClusterIP` service is used for API and Metrics ports + - In case of the **Public deployment** type, additionally, `NodePort` service will be created + - App configuration is done only using `values.yaml` and secrets + +**Multiple replicas** + - **Private deployment** + - Multiple StatefulSets are created to set unique TCP/UDP Pods ports + - A separate `ClusterIP` service is created for every Pod API and Metrics ports + - P2P communication is done directly via Pods IPs + - App configuration is done using `values.yaml` and secrets + + - **Public deployment** + - Multiple StatefulSets are created to set unique TCP/UDP Pods ports + - A separate `ClusterIP` service is created for every Pod API and Metrics ports + - `NodePort` service is created for every Pod with unique TCP/UDP ports + - App configuration is done using `values.yaml` and secrets + - Init container `init-env` is used to pass node `ExternalIP` to Codex container + + Variables, which would be assigned via `init-env` init container will be omitted at StatefulSet level + ```shell + # P2P + CODEX_NAT= + ``` + And to check their values, it would be required to look in to `init-env` Pod logs or Codex Pod files located in `/opt/env` folder + ```shell + # init-env container + kubectl logs -n codex -c init-env codex-1-1 + + # Codex container + kubectl exec -it -n codex-ns -c codex codex-1-1 -- bash -c "cat /opt/env/*" + ``` + + Unique data is passed via ConfigMap/Secrets + ```shell + # ConfigMap/Secrets + CODEX_ETH_PROVIDER= + CODEX_ETH_PRIVATE_KEY=<0x...> + CODEX_MARKETPLACE_ADDRESS=<0x...> + ``` + And we can use a single secrets with unique keys per Pod or multiple secrets with unique name per Pod and refer to the Pod by `-replica_index-pod_index`. This string should be added to the `values.yaml` and will be replaced by Helm with the Pod index. Please see [Installation](#installation) for an example. + +> **Note:** Keep in mind, we do not have configuration option to define **Private**/**Public deployment** type and it is defined by the service type we use - `service = private` / `service + nodeport = public`. + + +### Knows issues + 1. We can deploy just one replica per installation in case of `NodePort`, because + - In Kubernetes, we can't set different settings for replicas in StatefulSet + - Even if we can workaround that by passing environment variables via `init-env` container, Pods ports, in the manifest, also should be unique because Codex has `--listen-addrs` and `--disc-port` for P2P communication and they should be same as `NodePort` and unique for every Pod + + We can workaround that by passing unique TCP/UDP ports using init container and port forwarder sidecar and we will consider to implement that later. + + Even if we can use a single StatefulSet for **Private deployment** with `init-env` container to pass unique sensitive data, it was decided to follow same approach as we use for **Public** one. We may consider to change that later. + + This is why, for now, we have an option `replica.count` to generate multiple StatefulSets with unique settings. + + 2. When we deploy multiple nodes using single installation, multiple StatefulSets will be created. During release upgrade all of them will be upgraded/restarted almost simultaneously. + + 3. Codex erasure codding is working on the main app thread and it results of the failed liveness/readiness probes. This is why we have big values by default for these probes. + + +### Prettify + + Because we are forced to deploy multiple StatefulSets with unique settings, we add a replica index to their names. As a result we will get names which contains additionally Pod index and this is why we've introduced `prettify` key to the StatefulSet, Service and Ingress and `ordinalsStart` for StatefulSet which affect how these names will looks like. + +| Object | Accept `prettify` | Single | Single, `prettify` | Multiple | Multiple, `prettify` | Single --> Multiple | +| ----------- | ------------------ | --------------------- | ------------------ | ---------------------------------------------- | ------------------------------------------ | --------------------------------- | +| StatefulSet | :white_check_mark: | `codex-1` | `codex` | `codex-1`
`codex-2` | `codex-1`
`codex-2` | Destructive in case of `prettify` | +| Pod | :x: | `codex-1-1` | `codex-1` | `codex-1-1`
`codex-2-1` | `codex-1-1`
`codex-2-1` | Destructive in case of `prettify` | +| PVC | :x: | `data-codex-1-1` | `data-codex-1` | `data-codex-1-1`
`data-codex-2-1` | `data-codex-1-1`
`data-codex-2-1` | Destructive in case of `prettify` | +| Service | :white_check_mark: | `codex-1-1-nodeport` | `codex-nodeport` | `codex-1-1-nodeport`
`codex-2-1-nodeport` | `codex-1-nodeport`
`codex-2-nodeport` | Non destructive | +| Ingress | :white_check_mark: | `/codex-1-1` | `/codex` | `/codex-1-1`
`/codex-2-1` | `/codex-1`
`/codex-2` | Non destructive | + + +The idea is to make endpoint appropriate to the StatefulSet name by removing Pod index in case of multiple StatefulSets. + +For StatefulSet, `prettify=false` by default, in order to be able to add more replicas without destroying the fist one. With that value, a replica index will be added to the the StatefulSet, even if `replica=1`. If you would like to run just a single replica and have a name without that index, you should set `statefulSet.prettify=false`. + + +## Development + +```shell +# Render chart templates +helm template codex-bootstrap codex -n codex-ns --debug + +# Specific template +helm template codex-bootstrap codex -n codex-ns --debug -s templates/service.yaml + +# Examine a chart for possible issues +helm lint codex + +# Check the manifest +helm install codex --dry-run codex --namespace codex-ns +helm install codex --dry-run=server codex --namespace codex-ns +``` + + +## To do + 1. Make code more reusable. + 2. Check options to deploy a separate Bootstrap node and get its SPR automatically and pass it to Storage nodes, to setup a local fully working environment. + 3. Consider to use a single StatefulSet for **Private deployment** with the help of `init-env`. + 4. Consider to add a port forwarder as a sidecar to implement single StatefulSet configuration with the help of `init-env` for **Public deployment**. + 5. Consider to add an option to use Deployment instead of StatefulSet. diff --git a/charts/codex/templates/_helpers.tpl b/charts/codex/templates/_helpers.tpl new file mode 100644 index 0000000..9260669 --- /dev/null +++ b/charts/codex/templates/_helpers.tpl @@ -0,0 +1,145 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "codex.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "codex.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart namespace. +*/}} +{{- define "codex.namespace" -}} +{{ default .Release.Namespace .Values.namespaceOverride }} +{{- end }} + +{{/* +Create ingress name. +*/}} +{{- define "codex.ingress.name" -}} +{{- if .Values.ingress.fullnameOverride }} +{{- .Values.ingress.fullnameOverride }} +{{- else }} +{{- include "codex.fullname" . }} +{{- end }} +{{- end }} + +{{- define "codex.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "codex.labels" -}} +helm.sh/chart: {{ include "codex.chart" . }} +{{ include "codex.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "codex.selectorLabels" -}} +app.kubernetes.io/name: {{ include "codex.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use. +*/}} +{{- define "codex.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "codex.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Role name. +*/}} +{{- define "codex.clusterRole.name" -}} +{{ print (include "codex.namespace" .) "-" (include "codex.serviceAccountName" .) }} +{{- end }} + +{{- define "codex.role.name" -}} +{{ include "codex.serviceAccountName" . }} +{{- end }} + +{{/* +StatefulSets count. +*/}} +{{- define "codex.statefulSetCount" -}} +{{- if eq (include "codex.service.nodeport.enabled" .) "true" }} +{{- .Values.replica.count }} +{{- else }} +{{- 1 }} +{{- end }} +{{- end }} + +{{/* +Replica count. +*/}} +{{- define "codex.replica.count" -}} +{{- if eq (int (include "codex.statefulSetCount" .)) 1 }} +{{- .Values.replica.count }} +{{- else }} +{{- 1 }} +{{- end }} +{{- end }} + +{{/* +Enable NodePort service. +*/}} +{{- define "codex.service.nodeport.enabled" -}} +{{- if has "nodeport" .Values.service.type }} +{{- "true" }} +{{- else }} +{{- "false" }} +{{- end }} +{{- end }} + +{{/* +Enable initEnv container. +For single replica, initEnv container is enabled only if initEnv.enabled is set to true. +For multiple replicas, initEnv container is enabled if initEnv.enabled or we have multiple StatefulSets with NodePort service enabled. +*/}} +{{- define "codex.initEnv.enabled" -}} +{{- if or .Values.initEnv.enabled (eq (include "codex.service.nodeport.enabled" .) "true") }} +{{- "true" }} +{{- else }} +{{- "false" }} +{{- end }} +{{- end }} + +{{/* +Mount CODEX_ETH_PRIVATE_KEY. +*/}} +{{- define "codex.env.ethPrivateKey.mount" -}} +{{- if .Values.codex.env.CODEX_ETH_PRIVATE_KEY }} +{{- "true" }} +{{- else }} +{{- "false" }} +{{- end }} +{{- end }} diff --git a/charts/codex/templates/ingress.yaml b/charts/codex/templates/ingress.yaml new file mode 100644 index 0000000..aba9b1f --- /dev/null +++ b/charts/codex/templates/ingress.yaml @@ -0,0 +1,49 @@ +{{- if .Values.ingress.enabled -}} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ include "codex.ingress.name" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "codex.labels" . | nindent 4 }} + annotations: + {{- with .Values.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + ingressClassName: {{ .Values.ingress.class }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + rules: + {{- range $hosts := $.Values.ingress.hosts }} + - host: {{ $hosts.host }} + http: + paths: + {{- range $paths := $hosts.paths }} + {{- range $replica_index := until (int $.Values.replica.count) }} + {{- with $ }} + {{- $service_replica_suffix := ternary "" (print "-" (add $replica_index .Values.statefulSet.ordinalsStart)) (and (eq (int .Values.replica.count) 1) .Values.statefulSet.prettify) }} + {{- $service_suffix := print "-" .Values.statefulSet.ordinalsStart -}} + {{- $service_name := print (include "codex.fullname" .) $service_replica_suffix (ternary ($service_suffix) "" (eq (toString .Values.service.prettify) "false")) "-service" }} + {{- $pod_replica_suffix := ternary "" (printf "-%0*s" (int $paths.leadingZeros | default 1) (toString (add $replica_index .Values.statefulSet.ordinalsStart))) (and (eq (int .Values.replica.count) 1) .Values.statefulSet.prettify) }} + {{- $pod_suffix := ternary (print "-" .Values.statefulSet.ordinalsStart) "" (eq (toString $paths.prettify) "false") }} + {{- $pod_uri := print (ternary (include "codex.fullname" .) $paths.podName (or (eq $paths.podName "") (eq $paths.podName nil))) $pod_replica_suffix $pod_suffix }} + {{- $path := print $paths.path (ternary "" "/" (eq $paths.path "/")) $pod_uri $paths.rewritePath }} + - path: {{ $path }} + pathType: {{ $paths.pathType }} + backend: + service: + name: {{ $service_name }} + port: + number: {{ .Values.service.api.port }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/charts/codex/templates/poddisruptionbudjet.yaml b/charts/codex/templates/poddisruptionbudjet.yaml new file mode 100644 index 0000000..2949757 --- /dev/null +++ b/charts/codex/templates/poddisruptionbudjet.yaml @@ -0,0 +1,17 @@ +{{- if .Values.podDisruptionBudget }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "codex.fullname" .}} + labels: {{ include "codex.labels" . | nindent 4 }} + annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{ toYaml .Values.podDisruptionBudget | nindent 2 }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "codex.fullname" .}} + app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} diff --git a/charts/codex/templates/rbac.yaml b/charts/codex/templates/rbac.yaml new file mode 100644 index 0000000..2d776bc --- /dev/null +++ b/charts/codex/templates/rbac.yaml @@ -0,0 +1,71 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "codex.serviceAccountName" . }} + labels: + {{- include "codex.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +automountServiceAccountToken: {{ .Values.serviceAccount.automount }} + +{{- if and .Values.serviceAccount.rbac.create }} + +{{- if .Values.serviceAccount.rbac.rules }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "codex.role.name" . }} + labels: + {{- include "codex.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "codex.role.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "codex.serviceAccountName" . }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "codex.role.name" . }} + labels: + {{- include "codex.labels" . | nindent 4 }} +rules: +{{ toYaml .Values.serviceAccount.rbac.rules | indent 0 }} +{{- end }} + +{{- if .Values.serviceAccount.rbac.clusterRules }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "codex.clusterRole.name" . }} + labels: + {{- include "codex.labels" . | nindent 4 }} +rules: +{{ toYaml .Values.serviceAccount.rbac.clusterRules | indent 0 }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "codex.clusterRole.name" . }} + labels: + {{- include "codex.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "codex.clusterRole.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "codex.serviceAccountName" . }} + namespace: {{ include "codex.namespace" . }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/codex/templates/service.yaml b/charts/codex/templates/service.yaml new file mode 100644 index 0000000..1a1db94 --- /dev/null +++ b/charts/codex/templates/service.yaml @@ -0,0 +1,80 @@ +{{- range $replica_index := until (int .Values.replica.count) }} +{{- range $type := $.Values.service.type }} +{{- with $ }} +{{- $pod_replica_suffix := ternary "" (print "-" (add $replica_index .Values.statefulSet.ordinalsStart)) (and (eq (int .Values.replica.count) 1) .Values.statefulSet.prettify) }} +{{- $pod_suffix := print "-" .Values.statefulSet.ordinalsStart -}} +{{- $pod_name := print (include "codex.fullname" .) $pod_replica_suffix $pod_suffix }} +{{- $service_name := print (include "codex.fullname" .) $pod_replica_suffix (ternary ($pod_suffix) "" (eq (toString .Values.service.prettify) "false")) "-" $type }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $service_name }} + labels: + {{- include "codex.labels" . | nindent 4 }} + pod: {{ $pod_name }} +spec: + {{- if eq $type "nodeport" }} + type: NodePort + externalTrafficPolicy: Local + {{- else }} + type: ClusterIP + {{- end }} + ports: + {{- /* + API port + */ -}} + {{- if ne $type "nodeport" }} + {{- if .Values.service.api.enabled }} + - port: {{ .Values.service.api.port }} + targetPort: {{ .Values.ports.api.name }} + protocol: TCP + name: api + {{- end }} + {{- /* + Metrics port + */ -}} + {{- if .Values.service.metrics.enabled }} + - port: {{ .Values.service.metrics.port }} + targetPort: {{ .Values.ports.metrics.name }} + protocol: TCP + name: metrics + {{- end }} + {{- end }} + {{- /* + Transport port + */ -}} + {{- if or (eq $type "nodeport") (eq (include "codex.service.nodeport.enabled" .) "false") }} + {{- if and .Values.service.transport.enabled }} + {{- $node_port_transport := add .Values.service.transport.nodePort (mul $replica_index .Values.service.transport.nodePortOffset) }} + - port: {{ $node_port_transport }} + targetPort: {{ .Values.ports.transport.name }} + {{- if and (eq $type "nodeport") .Values.service.transport.nodePort }} + nodePort: {{ $node_port_transport }} + {{- end }} + protocol: TCP + name: transport + {{- end }} + {{- /* + Discovery port + */ -}} + {{- if .Values.service.discovery.enabled }} + {{- $node_port_discovery := add .Values.service.discovery.nodePort (mul $replica_index .Values.service.discovery.nodePortOffset) }} + - port: {{ $node_port_discovery }} + targetPort: {{ .Values.ports.discovery.name }} + {{- if and (eq $type "nodeport") .Values.service.discovery.nodePort }} + nodePort: {{ $node_port_discovery }} + {{- end }} + protocol: UDP + name: discovery + {{- end }} + {{- end }} + {{- if .Values.extraPorts }} + {{- toYaml .Values.extraPorts | nindent 4 }} + {{- end }} + selector: + {{- include "codex.selectorLabels" . | nindent 4 }} + statefulset.kubernetes.io/pod-name: {{ $pod_name }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/codex/templates/statefulset.yaml b/charts/codex/templates/statefulset.yaml new file mode 100644 index 0000000..fa8522a --- /dev/null +++ b/charts/codex/templates/statefulset.yaml @@ -0,0 +1,270 @@ +{{- range $replica_index := until (int (include "codex.statefulSetCount" .)) }} +{{- with $ }} +{{- $replica_suffix := ternary "" (print "-" (add $replica_index .Values.statefulSet.ordinalsStart)) (and (eq (int .Values.replica.count) 1) .Values.statefulSet.prettify) }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ print (include "codex.fullname" .) $replica_suffix }} + namespace: {{ include "codex.namespace" . }} + labels: {{ include "codex.labels" . | nindent 4 }} + annotations: + meta.helm.sh/release-name: {{ include "codex.fullname" . }} + meta.helm.sh/release-namespace: {{ include "codex.namespace" . }} + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ include "codex.replica.count" . }} + ordinals: + start: {{ .Values.statefulSet.ordinalsStart }} + selector: + matchLabels: + {{- include "codex.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "codex.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "codex.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + {{- range .Values.initContainers }} + - {{ tpl (toYaml .) $ | nindent 10 | trim }} + {{- end }} + {{- if eq (include "codex.initEnv.enabled" .) "true" }} + - name: init-env + image: {{ print .Values.initEnv.image.repository ":" .Values.initEnv.image.tag }} + imagePullPolicy: {{ .Values.initEnv.image.pullPolicy }} + securityContext: + runAsNonRoot: false + runAsUser: 0 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ENV_PATH + value: "{{ .Values.initEnv.envPath }}" + command: + - sh + - -c + - > + NODEPORT_FILE=${ENV_PATH}/nodeport; + {{- if ne .Values.initEnv.command "" }} + {{- .Values.initEnv.command | toYaml | nindent 14 }} + {{- else }} + CODEX_NAT=$(kubectl get nodes ${NODE_NAME} -o jsonpath='{.status.addresses[?(@.type=="ExternalIP")].address}'); + echo "CODEX_NAT=${CODEX_NAT}" >> ${NODEPORT_FILE}; + cat ${NODEPORT_FILE}; + {{- end }} + volumeMounts: + - name: env-nodeport + mountPath: {{ .Values.initEnv.envPath }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: {{ print .Values.image.repository ":" (.Values.image.tag | default .Chart.AppVersion) }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + lifecycle: + {{- if .Values.lifecycle }} + {{- toYaml .Values.lifecycle | nindent 12 }} + {{- else if .Values.codex.marketplace }} + postStart: + exec: + command: + - bash + - -c + - | + sleep 10 + + while true; do + curl --max-time 2 http://localhost:{{ .Values.ports.api.containerPort}}/api/codex/v1/debug/info && break + sleep 10 + done + + availability=$(curl http://localhost:{{ .Values.ports.api.containerPort}}/api/codex/v1/sales/availability | jq -r '.[]') + + if [[ -z "${availability}" ]]; then + availability=$(curl http://localhost:{{ .Values.ports.api.containerPort}}/api/codex/v1/sales/availability \ + --max-time 2 \ + --request POST \ + --header 'Content-Type: application/json' \ + --data '{ + "totalSize": "{{ .Values.codex.marketplace.totalSize }}", + "duration": "{{ .Values.codex.marketplace.duration }}", + "minPrice": "{{ .Values.codex.marketplace.minPrice }}", + "maxCollateral": "{{ .Values.codex.marketplace.maxCollateral }}" + }') + echo -e "Storage availability was configured \n${availability}" >/opt/storage-availability + else + echo -e "Storage availability already configured \n${availability}" >/opt/storage-availability + fi + {{- end }} + {{- if .Values.codex.command }} + command: + {{- toYaml .Values.codex.command | nindent 12 }} + {{- end }} + {{- if .Values.codex.args }} + args: + {{- toYaml .Values.codex.args | nindent 12 }} + {{- end }} + + env: + {{- if eq (include "codex.initEnv.enabled" .) "true" }} + - name: ENV_PATH + value: "{{ .Values.initEnv.envPath }}" + {{- end }} + {{- range $name, $value := .Values.codex.env }} + {{- if and (include "codex.service.nodeport.enabled" $) (eq $name "CODEX_DISC_PORT") }} + {{- $discovery_port_replica := add $.Values.service.discovery.nodePort (mul $replica_index $.Values.service.discovery.nodePortOffset) }} + - name: {{ $name }} + value: {{ $discovery_port_replica | quote }} + {{- else if and (include "codex.service.nodeport.enabled" $) (eq $name "CODEX_LISTEN_ADDRS") }} + {{- $transport_port := splitList "/" $value | last }} + {{- $transport_port_replica := add $.Values.service.transport.nodePort (mul $replica_index $.Values.service.discovery.nodePortOffset) }} + {{- $lister_addrs := $value | replace (toString $transport_port) (toString $transport_port_replica) }} + - name: {{ $name }} + value: {{ $lister_addrs | quote }} + {{- else }} + - name: {{ $name }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- range .Values.codex.extraEnv }} + - {{ toYaml . | replace "replica_index" (toString (add $replica_index $.Values.statefulSet.ordinalsStart)) | replace "pod_index" (toString $.Values.statefulSet.ordinalsStart) | nindent 14 | trim }} + {{- end }} + + ports: + {{- if .Values.ports.api }} + - name: {{ .Values.ports.api.name }} + containerPort: {{ .Values.ports.api.containerPort }} + protocol: TCP + {{- end }} + {{- if .Values.ports.metrics }} + - name: {{ .Values.ports.metrics.name }} + containerPort: {{ .Values.ports.metrics.containerPort }} + protocol: TCP + {{- end }} + {{- if .Values.ports.transport }} + - name: {{ .Values.ports.transport.name }} + containerPort: {{ if include "codex.service.nodeport.enabled" . }}{{- add .Values.service.transport.nodePort (mul $replica_index .Values.service.transport.nodePortOffset) -}}{{ end }} + protocol: TCP + {{- end }} + {{- if .Values.ports.discovery }} + - name: {{ .Values.ports.discovery.name }} + containerPort: {{ if include "codex.service.nodeport.enabled" . }}{{- add .Values.service.discovery.nodePort (mul $replica_index .Values.service.discovery.nodePortOffset) -}}{{ end }} + protocol: UDP + {{- end }} + {{- with .Values.extraContainerPorts }} + {{- toYaml . | nindent 12 }} + {{- end }} + + {{- with .Values.startupProbe }} + startupProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.livenessProbe }} + livenessProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + {{- with .Values.readinessProbe }} + readinessProbe: + {{- tpl (toYaml .) $ | nindent 12 }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 12 }} + + volumeMounts: + {{- if eq (include "codex.initEnv.enabled" .) "true" }} + - name: env-nodeport + mountPath: {{ .Values.initEnv.envPath }} + {{- end }} + {{- if include "codex.env.ethPrivateKey.mount" . }} + - name: codex-eth-private-key + mountPath: {{ .Values.codex.env.CODEX_ETH_PRIVATE_KEY }} + subPath: codex-eth-private-key + {{- end }} + {{- if .Values.persistence.enabled }} + - name: {{ .Values.persistence.name }} + mountPath: {{ .Values.codex.env.CODEX_DATA_DIR }} + {{ else if .Values.persistence.exis}} + - name: {{ .Values.persistence.name }} + emptyDir: {} + {{- end }} + {{- with .Values.volumeMounts }} + {{- toYaml . | nindent 12 }} + {{- end }} + + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 10 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 10 }} + {{- end }} + topologySpreadConstraints: + {{- toYaml .Values.topologySpreadConstraints | nindent 10 }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} + {{- range .Values.extraContainers }} + - {{- toYaml . | nindent 10 | trim }} + {{- end }} + + volumes: + {{- if eq (include "codex.initEnv.enabled" .) "true" }} + - name: env-nodeport + emptyDir: {} + {{- end }} + {{- if eq (include "codex.env.ethPrivateKey.mount" .) "true" }} + - name: codex-eth-private-key + secret: + secretName: {{ .Values.codex.env.CODEX_ETH_PRIVATE_KEY | splitList "/" | last }} + items: + - key: {{ print "CODEX_ETH_PRIVATE_KEY" $replica_suffix "-1" }} + path: codex-eth-private-key + defaultMode: 384 + {{- end }} + + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: {{ .Values.persistence.name }} + annotations: + {{- toYaml .Values.persistence.annotations | nindent 10 }} + spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 10 }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + storageClassName: {{ .Values.persistence.storageClassName }} + {{- with .Values.persistence.selector }} + selector: + {{- toYaml . | nindent 10 }} + {{- end }} + persistentVolumeClaimRetentionPolicy: + whenDeleted: {{ .Values.persistence.retentionPolicy.whenDeleted }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/codex/templates/tests/test-connection.yaml b/charts/codex/templates/tests/test-connection.yaml new file mode 100644 index 0000000..7bb5f86 --- /dev/null +++ b/charts/codex/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "codex.fullname" . }}-test-connection" + labels: + {{- include "codex.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "codex.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/charts/codex/values.yaml b/charts/codex/values.yaml new file mode 100644 index 0000000..ca8bf27 --- /dev/null +++ b/charts/codex/values.yaml @@ -0,0 +1,442 @@ +# Default values for codex. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Override chart name +nameOverride: "" + +# Override chart full name +fullnameOverride: "" + +# Replicas +replica: + # Replicas count + count: 1 + +# StatefulSet +statefulSet: + # First replica index + ordinalsStart: 1 + # For single replica we will skip replica index in the StatefulSet name + # and it is useful only when no more than one replica is planned to be deployed + # otherwise, when replica will be increased > 1, previous StatefulSet will be deleted + # and new ones with index in the names will be created + # this is why by default we set it to false, to avoid destructive transition + # name StatefulSet + # true: codex --> codex + # false: codex --> codex-1 + prettify: false + +image: + # Image repository + repository: codexstorage/nim-codex + # Image tag + tag: latest + # Image pull policy + pullPolicy: IfNotPresent + +# Image pull secrets +imagePullSecrets: [] + +# Lifecycle +lifecycle: {} + +# Namespace +namespaceOverride: "" + +# StatefulSet labels +labels: {} + # foo: bar + +# StatefulSet annotations +annotations: {} + # foo: bar + +# ServiceAccount is required for init-env container to get node External IP +serviceAccount: + # Specifies whether a service account should be created + create: true + # Automatically mount a ServiceAccount's API credentials? + automount: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + rbac: + # Specifies whether RBAC resources should be created + create: true + # rbac specific annotations + annotations: {} + # foo: bar + # rbac specific labels + labels: {} + # foo: bar + # Get node External IP + clusterRules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + # Get service NodePort - not use currently + rules: + # - apiGroups: [""] + # resources: ["services"] + # verbs: ["get", "list", "watch"] + +# Pod labels +podLabels: {} + # foo: bar + +# Pod annotations +podAnnotations: {} + # foo: bar + +podSecurityContext: {} + # fsGroup: 1000 + # runAsGroup: 1000 + # runAsUser: 1000 + # runAsNonRoot: true + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +# Codex container command with arguments +codex: + command: + # known as Docker ENTRYPOINT + # - /bin/bash + # - /docker-entrypoint.sh + args: + # known as Docker CMD + # - codex + # - --bootstrap-node=spr:xxx + # - --bootstrap-node=spr:yyy + # - --data-dir=/data + # + # Codex configuration using environment variables + env: + # Logging + CODEX_LOG_LEVEL: INFO + # TRACE;warn:discv5,providers,manager,cache;warn:libp2p,multistream,switch,transport,tcptransport,semaphore,asyncstreamwrapper,lpstream,mplex,mplexchannel,noise,bufferstream,mplexcoder,secure,chronosstream,connection,connmanager,websock,ws-session + # + # Metrics + # CODEX_METRICS: true + # CODEX_METRICS_ADDRESS: 0.0.0.0 + # CODEX_METRICS_PORT: 8008 + # + # API + # CODEX_API_BINDADDR: 0.0.0.0 + # CODEX_API_PORT: 8080 + # + # P2P communication + # Currently, only one node can be set via env variable - https://github.com/codex-storage/nim-codex/issues/525 + # and we can use codex.args to pass multiple bootstrap nodes + # CODEX_BOOTSTRAP: "" + # CODEX_NAT: "127.0.0.1" + CODEX_LISTEN_ADDRS: /ip4/0.0.0.0/tcp/8070 + CODEX_DISC_IP: 0.0.0.0 + CODEX_DISC_PORT: 8090 + # + # Storage + CODEX_DATA_DIR: /data + # b, k/kb, m/mb, g/gb, t/tb, p/pb + # CODEX_STORAGE_QUOTA: 18gb + # + # Maintenance + # s, m, h, d, w + # CODEX_BLOCK_TTL: 1d + # CODEX_BLOCK_MI: 10m + # CODEX_BLOCK_MN: 1000 + # + # Persistence + # CODEX_PERSISTENCE: false + # CODEX_ETH_PROVIDER: ws://localhost:8545 + # CODEX_ETH_ACCOUNT: 0x1234567890123456789012345678901234567890 + # file name `codex-eth-private-key` will be used as a secret name to be mounted to the Pod + # CODEX_ETH_PRIVATE_KEY: /opt/codex-eth-private-key + # CODEX_MARKETPLACE_ADDRESS: 0x1234567890123456789012345678901234567890 + + extraEnv: + - name: CODEX_ETH_PROVIDER + valueFrom: + secretKeyRef: + name: codex-eth-provider + key: CODEX_ETH_PROVIDER + - name: CODEX_ETH_ACCOUNT + valueFrom: + secretKeyRef: + name: codex_eth_account + key: CODEX_ETH_ACCOUNT_replica_index_pod_index + + # Extra environment variables + # extraEnv: + # When multiple StatefulSets are created, we can use replica_index to pass different ConfigMap/Secrets to each replica + # Value Secrets + # codex_eth_account_replica_index-pod_index --> codex_eth_account_1_1, codex_eth_account_2_1, codex_eth_account_3_1, ... + # + # - name: FOO + # value: BAR + # + # - name: FOO_CONFIG + # valueFrom: + # configMapKeyRef: + # name: configmap-name + # key: configmap-key + # + # - name: FOO_SECRET + # valueFrom: + # secretKeyRef: + # name: secret-name + # key: secret-key + # + # - name: CODEX_ETH_ACCOUNT + # valueFrom: + # secretKeyRef: + # name: codex_eth_account + # key: codex_eth_account_replica_index-pod_index + + # Marketplace storage for sale configuration + # We use postStart lifecycle hook to setup Codex node marketplace configuration via API + # mutually exclusive with the lifecycle configuration bellow + marketplace: {} + # totalSize: "8000000000" + # duration: "7200" + # minPrice: "10" + # maxCollateral: "10" + +# Init container to pass environment variables dynamically to each replica +initEnv: + image: + # Image repository + repository: bitnami/kubectl + # Image tag + tag: latest + # Image pull policy + pullPolicy: IfNotPresent + # Enable init Env container - enabled automatically when replica.count > 1 and service.type.nodeport + enabled: false + command: "" + # command: >- + # echo test + envPath: /opt/env + +# Pod ports +ports: + api: + enabled: true + name: api + containerPort: 8080 + metrics: + enabled: true + name: metrics + containerPort: 8008 + transport: + enabled: true + name: libp2p + containerPort: 8070 + discovery: + enabled: true + name: dht + containerPort: 8090 + +# Service +service: + # Service type to deploy - nodeport is required when Codex node should be reachable from the Internet + type: + - service + - nodeport + # Prettify service name + # single StatefulSet + # true: codex-service codex-nodeport + # false: codex-1-service codex-1-nodeport + # multiple StatefulSets + # true: codex-1-service codex-1-nodeport codex-2-service codex-2-nodeport + # false: codex-1-1-service codex-1-1-nodeport codex-2-1-service codex-2-1-nodeport + prettify: true + # API port + api: + enabled: true + port: 8080 + # Metrics port + metrics: + enabled: true + port: 8008 + # Transport port + transport: + enabled: true + # 30000-32767 + nodePort: 30510 + nodePortOffset: 10 + # Discovery port + discovery: + enabled: true + # 30000-32767 + nodePort: 30510 + nodePortOffset: 10 + +# Extra container ports +extraContainerPorts: [] + # - name: foo + # containerPort: 1234 + # protocol: TCP + +# Extra ports +extraPorts: [] + # - name: foo + # port: 1234 + # protocol: TCP + +# Ingress for Codex API +ingress: + # Enable ingress + enabled: false + # Override chart full name + fullnameOverride: "" + # Ingress class + class: nginx + prettify: false + # Ingress annotations + annotations: + # kubernetes.io/ingress.class: "nginx" + # cert-manager.io/cluster-issuer: "letsencrypt-staging" + # nginx.ingress.kubernetes.io/use-regex: "true" + # nginx.ingress.kubernetes.io/rewrite-target: /$2 + # nginx.ingress.kubernetes.io/auth-type: basic + # nginx.ingress.kubernetes.io/auth-secret: api-domain-tld-basic-auth + # nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - Private Area' + tls: + - secretName: api-domain-tld + hosts: + - api.domain.tld + hosts: + - host: api.domain.tld + paths: + # When podName is blank, URL will use deployed Pod name + # https://api.domain.tld/pod-name-1 + # https://api.domain.tld/pod-name-2 + - path: / + rewritePath: (/|$)(.*) + pathType: Prefix + # When podName is set, in URL we will use that value instead of the deployed Pod name + # When prettify is set to false, Pod URI will contain the StatefulSet Pod index + # leadingZeros specify the number of digits in the endpoint Pod name - pod-name-1/01/001 + # https://api.domain.tld/endpoints/node-01-1 + # https://api.domain.tld/endpoints/node-02-1 + - path: /storage + rewritePath: (/|$)(.*) + pathType: Prefix + podName: node + prettify: false + leadingZeros: 2 + +# Pod resources +resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +# Startup probe +startupProbe: {} + +# Liveness probe +livenessProbe: + httpGet: + path: /api/codex/v1/debug/info + port: '{{ .Values.ports.api.name }}' + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 1200 + +# Readiness probe +readinessProbe: + httpGet: + path: /api/codex/v1/debug/info + port: '{{ .Values.ports.api.name }}' + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 1200 + +# Persistence +persistence: + # Enable persistence using Persistent Volume Claims + enabled: true + # Name prefix for Persistent Volume Claims + name: data + # Use an existing PVC to persist data + existingClaim: "" + # Access mode of data volume + accessModes: + - ReadWriteOnce + # Size of data volume + size: 10Gi + # Storage class of data volume + storageClassName: "" + # Annotations for volume claim template + annotations: {} + # foo: bar + # Selector for volume claim template + selector: {} + # matchLabels: + # foo: bar + retentionPolicy: + whenDeleted: Retain + +# Additional volumes on the output Deployment definition. +volumes: [] +# - name: foo +# secret: +# secretName: mysecret +# optional: false + +# Additional volumeMounts on the output Deployment definition. +volumeMounts: [] +# - name: foo +# mountPath: "/etc/foo" +# readOnly: true + +initContainers: [] + # - name: datadir-cleanup + # image: busybox:latest + # command: + # - sh + # - -c + # - | + # echo "clean-up datadir" + # rm -rf {{ .Values.codex.env.CODEX_DATA_DIR }}/* + # volumeMounts: + # - name: '{{ .Values.persistence.name }}' + # mountPath: '{{ .Values.codex.env.CODEX_DATA_DIR }}' + +# Additional +extraContainers: [] +# - name: extra +# image: busybox:latest +# command: ['sh', '-c', 'echo "hello"'] + +nodeSelector: {} + +tolerations: [] + +affinity: {} + +topologySpreadConstraints: [] + +terminationGracePeriodSeconds: 120 + +podDisruptionBudget: {} + # maxUnavailable: 1 + # minAvailable: 1