diff --git a/deployment/deployment.sh b/deployment/deployment.sh index baf5f0e..37023ce 100755 --- a/deployment/deployment.sh +++ b/deployment/deployment.sh @@ -2,7 +2,9 @@ kubectl apply -f bootstrap.yaml kubectl rollout status --watch --timeout=30000s statefulset/bootstrap -n zerotesting #kubectl apply -f midstrap.yaml #kubectl rollout status --watch --timeout=30000s statefulset/midstrap -n zerotesting -kubectl apply -f nodes.yaml +kubectl apply -f nodes-nwaku.yaml +kubectl apply -f nodes-gowaku.yaml echo "We have deployed all nodes, please watch Prometheus or Grafana to see when they have reached a healthy state." echo "Please note you cannot (yet) rely on the Ready state as it does not actually indicate an unhealthy peer, just one that is not ready for bootstrapping from." #kubectl rollout status --watch --timeout=30000s statefulset/nodes -n zerotesting +kubectl apply -f publisher.yaml \ No newline at end of file diff --git a/deployment/docker/Dockerfile b/deployment/docker/getenr/Dockerfile similarity index 100% rename from deployment/docker/Dockerfile rename to deployment/docker/getenr/Dockerfile diff --git a/deployment/docker/getenr.sh b/deployment/docker/getenr/getenr.sh similarity index 100% rename from deployment/docker/getenr.sh rename to deployment/docker/getenr/getenr.sh diff --git a/deployment/docker/go-waku/Dockerfile b/deployment/docker/go-waku/Dockerfile new file mode 100644 index 0000000..d243540 --- /dev/null +++ b/deployment/docker/go-waku/Dockerfile @@ -0,0 +1,2 @@ +FROM wakuorg/go-waku:v0.9.0 +RUN apk add curl diff --git a/deployment/docker/nwaku/Dockerfile b/deployment/docker/nwaku/Dockerfile index 1b67cc3..00db715 100644 --- a/deployment/docker/nwaku/Dockerfile +++ b/deployment/docker/nwaku/Dockerfile @@ -1,2 +1,2 @@ -FROM harbor.riff.cc/localmirror/wakuorg/nwaku:wakunode_dst -RUN apk add wget +FROM wakuorg/nwaku:v0.32.0 +RUN apk add curl diff --git a/deployment/nodes-gowaku-preview.yaml b/deployment/nodes-gowaku-preview.yaml new file mode 100644 index 0000000..1287608 --- /dev/null +++ b/deployment/nodes-gowaku-preview.yaml @@ -0,0 +1,122 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nodes + namespace: zerotesting +spec: + replicas: 1000 + podManagementPolicy: "Parallel" + serviceName: zerotesting-service + selector: + matchLabels: + app: zerotenkay + template: + metadata: + labels: + app: zerotenkay + spec: + dnsConfig: + searches: + - zerotesting-service.zerotesting.svc.cluster.local + volumes: + - name: enr-data + emptyDir: {} +# - name: perflogs +# persistentVolumeClaim: +# claimName: perflogs + - name: perflogs + hostPath: + path: /mnt/swfs/services/perflogs + type: Directory + initContainers: + - name: grabenr + image: zorlin/getenr:v0.3.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: enr-data + mountPath: /etc/enr + command: + - /app/getenr.sh + args: + - "3" + terminationGracePeriodSeconds: 150 + containers: + - name: waku + image: harbor.status.im/wakuorg/go-waku:bump-golibp2-pubsub + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8545 + - containerPort: 8008 + volumeMounts: + - name: enr-data + mountPath: /etc/enr + - name: perflogs + mountPath: /mnt/perflogs + readinessProbe: + exec: + command: + - /bin/sh + - -c + - > + curl -s http://127.0.0.1:8008/metrics | + awk '!/^#/ && /^libp2p_gossipsub_healthy_peers_topics /{ + print "Found gossipsub:", $0; + if ($2 == 1.0) { + exit 0; # success, healthy state + } else { + exit 1; # failure, unhealthy state + } + }' + successThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 1 + failureThreshold: 2 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "kill -2 $(pidof wakunode) ; sleep 5 ; $(pidof perf); sleep 150"] + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + resources: + requests: + memory: "64Mi" + cpu: "150m" + limits: + memory: "600Mi" + cpu: "400m" + env: + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - sh + - -c + - | + . /etc/enr/enr.env + echo ENRs are $ENR1 $ENR2 $ENR3 + HOSTNAME=$(hostname) + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + LOGFILE="/mnt/perflogs/${HOSTNAME}-waku-${TIMESTAMP}.log" + perf record -F 10 -o ${LOGFILE} -- \ + nice -n 19 /usr/bin/wakunode \ + --relay=true \ + --max-connections=150 \ + --rest=true \ + --rest-admin=true \ + --rest-private=true \ + --rest-address=0.0.0.0 \ + --discv5-discovery=true \ + --discv5-enr-auto-update=True \ + --log-level=INFO \ + --metrics-server=True \ + --metrics-server-address=0.0.0.0 \ + --discv5-bootstrap-node=$ENR1 \ + --discv5-bootstrap-node=$ENR2 \ + --discv5-bootstrap-node=$ENR3 \ + --nat=extip:${IP} \ + --cluster-id=2 \ + --pubsub-topic="/waku/2/rs/2/0" diff --git a/deployment/nodes.yaml b/deployment/nodes-gowaku.yaml similarity index 99% rename from deployment/nodes.yaml rename to deployment/nodes-gowaku.yaml index ce0c9ec..815118d 100644 --- a/deployment/nodes.yaml +++ b/deployment/nodes-gowaku.yaml @@ -4,7 +4,7 @@ metadata: name: nodes namespace: zerotesting spec: - replicas: 400 + replicas: 1000 podManagementPolicy: "Parallel" serviceName: zerotesting-service selector: diff --git a/deployment/nodes-nwaku.yaml b/deployment/nodes-nwaku.yaml new file mode 100644 index 0000000..02cfa98 --- /dev/null +++ b/deployment/nodes-nwaku.yaml @@ -0,0 +1,121 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nodes + namespace: zerotesting +spec: + replicas: 1000 + podManagementPolicy: "Parallel" + serviceName: zerotesting-service + selector: + matchLabels: + app: zerotenkay + template: + metadata: + labels: + app: zerotenkay + spec: + dnsConfig: + searches: + - zerotesting-service.zerotesting.svc.cluster.local + volumes: + - name: enr-data + emptyDir: {} +# - name: perflogs +# persistentVolumeClaim: +# claimName: perflogs + - name: perflogs + hostPath: + path: /mnt/swfs/services/perflogs + type: Directory + initContainers: + - name: grabenr + image: zorlin/getenr:v0.3.0 + imagePullPolicy: IfNotPresent + volumeMounts: + - name: enr-data + mountPath: /etc/enr + command: + - /app/getenr.sh + args: + - "3" + terminationGracePeriodSeconds: 150 + containers: + - name: waku + image: zorlin/waku:debug-extra-nim-libp2p-logs-over-v0.31.0-with-extra-logs-mplex-perf + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8545 + - containerPort: 8008 + volumeMounts: + - name: enr-data + mountPath: /etc/enr + - name: perflogs + mountPath: /mnt/perflogs + readinessProbe: + exec: + command: + - /bin/sh + - -c + - > + curl -s http://127.0.0.1:8008/metrics | + awk '!/^#/ && /^libp2p_gossipsub_healthy_peers_topics /{ + print "Found gossipsub:", $0; + if ($2 == 1.0) { + exit 0; # success, healthy state + } else { + exit 1; # failure, unhealthy state + } + }' + successThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 1 + failureThreshold: 2 + timeoutSeconds: 5 + lifecycle: + preStop: + exec: + command: ["/bin/sh", "-c", "kill -2 $(pidof wakunode) ; sleep 5 ; $(pidof perf); sleep 150"] + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + resources: + requests: + memory: "64Mi" + cpu: "150m" + limits: + memory: "600Mi" + cpu: "400m" + env: + - name: IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - sh + - -c + - | + . /etc/enr/enr.env + echo ENRs are $ENR1 $ENR2 $ENR3 + HOSTNAME=$(hostname) + TIMESTAMP=$(date +%Y%m%d-%H%M%S) + LOGFILE="/mnt/perflogs/${HOSTNAME}-waku-${TIMESTAMP}.log" + /usr/bin/wakunode \ + --relay=true \ + --max-connections=150 \ + --rest=true \ + --rest-admin=true \ + --rest-private=true \ + --rest-address=0.0.0.0 \ + --discv5-discovery=true \ + --discv5-enr-auto-update=True \ + --log-level=INFO \ + --metrics-server=True \ + --metrics-server-address=0.0.0.0 \ + --discv5-bootstrap-node=$ENR1 \ + --discv5-bootstrap-node=$ENR2 \ + --discv5-bootstrap-node=$ENR3 \ + --nat=extip:${IP} \ + --cluster-id=2 \ + --pubsub-topic="/waku/2/rs/2/0"