working deployment of agents on minikube

2026-01-02 13:03:13 +00:00 · 2025-01-20 11:39:43 -03:00 · 2025-01-20 11:39:43 -03:00 · 94556d7a53
commit 94556d7a53
parent 27d0815be8
11 changed files with 62 additions and 51 deletions
--- a/6
+++ b/6
@ -34,8 +34,10 @@ tests: unit integration
 image-test:
 	docker build -t bittorrent-benchmarks:test -f ./docker/bittorrent-benchmarks.Dockerfile .

-image-release:
-	docker build -t bittorrent-benchmarks:test --build-arg BUILD_TYPE="release" \
+image-minikube:
+	eval $(minikube docker-env)
+	docker build -t bittorrent-benchmarks:minikube \
+		--build-arg BUILD_TYPE="release" \
 		-f ./docker/bittorrent-benchmarks.Dockerfile .

 # Runs the integration tests in a docker container.
--- a/benchmarks/deluge/config.py
+++ b/benchmarks/deluge/config.py
@ -36,7 +36,7 @@ class DelugeNodeSetConfig(BaseModel):
    listen_ports: list[int] = Field(min_length=2, max_length=2)
    first_node_index: int = 1
    nodes: List[DelugeNodeConfig] = []
-    agent_url: HttpUrl
+    agent_url: str

    @model_validator(mode="after")
    def expand_nodes(self):
@ -46,7 +46,7 @@ class DelugeNodeSetConfig(BaseModel):
                address=self.address.format(node_index=str(i)),
                daemon_port=self.daemon_port,
                listen_ports=self.listen_ports,
-                agent_url=self.agent_url,
+                agent_url=self.agent_url.format(node_index=str(i)),
            )
            for i in range(
                self.first_node_index, self.first_node_index + self.network_size
--- a/benchmarks/deluge/tests/test_config.py
+++ b/benchmarks/deluge/tests/test_config.py
@ -20,7 +20,7 @@ def test_should_expand_node_sets_into_simple_nodes():
        network_size=4,
        daemon_port=6080,
        listen_ports=[6081, 6082],
-        agent_url="http://localhost:8000",
+        agent_url="http://deluge-{node_index}.local.svc:8000",
    )

    assert nodeset.nodes == [
@ -29,28 +29,28 @@ def test_should_expand_node_sets_into_simple_nodes():
            address="deluge-1.local.svc",
            daemon_port=6080,
            listen_ports=[6081, 6082],
-            agent_url="http://localhost:8000",
+            agent_url="http://deluge-1.local.svc:8000",
        ),
        DelugeNodeConfig(
            name="custom-2",
            address="deluge-2.local.svc",
            daemon_port=6080,
            listen_ports=[6081, 6082],
-            agent_url="http://localhost:8000",
+            agent_url="http://deluge-2.local.svc:8000",
        ),
        DelugeNodeConfig(
            name="custom-3",
            address="deluge-3.local.svc",
            daemon_port=6080,
            listen_ports=[6081, 6082],
-            agent_url="http://localhost:8000",
+            agent_url="http://deluge-3.local.svc:8000",
        ),
        DelugeNodeConfig(
            name="custom-4",
            address="deluge-4.local.svc",
            daemon_port=6080,
            listen_ports=[6081, 6082],
-            agent_url="http://localhost:8000",
+            agent_url="http://deluge-4.local.svc:8000",
        ),
    ]

@ -99,7 +99,7 @@ def test_should_build_experiment_from_config():
        address: 'node-{node_index}.deluge.codexbenchmarks.svc.cluster.local'
        daemon_port: 6890
        listen_ports: [ 6891, 6892 ]
-        agent_url: http://localhost:8080
+        agent_url: 'http://node-{node_index}.deluge.codexbenchmarks.svc.cluster.local:8080'
    """)

    config = DelugeExperimentConfig.model_validate(
@ -135,7 +135,7 @@ def test_should_create_n_repetitions_per_seeder_set():
        address: 'node-{node_index}.deluge.codexbenchmarks.svc.cluster.local'
        daemon_port: 6890
        listen_ports: [ 6891, 6892 ]
-        agent_url: http://localhost:8080
+        agent_url: 'http://node-{node_index}.deluge.codexbenchmarks.svc.cluster.local:8080'
    """)

    config = DelugeExperimentConfig.model_validate(
--- a/experiments.k8s.yaml
+++ b/experiments.k8s.yaml
@ -13,5 +13,9 @@ deluge_experiment:
    # Setting the correct name is critical, and it has to match what's in deluge the StatefulSet.
    name: "${DELUGE_STATEFULSET}-{node_index}"
    address: "${DELUGE_STATEFULSET}-{node_index}.${DELUGE_SERVICE}.${NAMESPACE}.svc.cluster.local"
+    agent_url: "http://${DELUGE_STATEFULSET}-{node_index}.${DELUGE_SERVICE}.${NAMESPACE}:${DELUGE_AGENT_PORT}/"
    daemon_port: 6890
-    listen_ports: [ 6891, 6892 ]
+    listen_ports: [ 6891, 6892 ]
+
+deluge_agent:
+  torrents_path: /var/lib/deluge/downloads
--- a/k8s/charts/deluge/templates/_helpers.tpl
+++ b/k8s/charts/deluge/templates/_helpers.tpl
@ -55,3 +55,15 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/instance: {{ include "experiment.id" . }}
 app.kubernetes.io/part-of: {{ include "experiment.groupId" . }}
 {{- end }}
+
+{{/*
+Minikube env.
+*/}}
+
+{{- define "benchmark.harness.image" -}}
+{{ .Values.deployment.minikubeEnv | ternary "bittorrent-benchmarks:minikube" "codexstorage/bittorrent-benchmarks:latest" }}
+{{- end }}-}}
+
+{{- define "benchmark.harness.imagePullPolicy" -}}
+{{ .Values.deployment.minikubeEnv | ternary "Never" "Always" }}
+{{- end }}
--- a/k8s/charts/deluge/templates/deluge-pvc.yaml
+++ b/k8s/charts/deluge/templates/deluge-pvc.yaml
@ -1,17 +0,0 @@
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: {{ include "deluge.pvc" . }}
-  namespace: {{ .Release.Namespace }}
-  labels:
-    {{- include "deluge-benchmarks.labels" . | nindent 4 }}
-
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: {{ include "storage.size" . }}
-  {{- if .Values.deployment.storageClass }}
-  storageClassName: {{ .Values.deployment.storageClass }}
-  {{- end }}
--- a/k8s/charts/deluge/templates/deluge-statefulset.yaml
+++ b/k8s/charts/deluge/templates/deluge-statefulset.yaml
@ -20,7 +20,7 @@ spec:
        {{- include "deluge-benchmarks.labels" . | nindent 8 }}

    spec:
-      {{- if not .Values.experiment.allowColocation }}
+      {{- if not .Values.deployment.allowColocation }}
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
@ -57,23 +57,26 @@ spec:
          args:
            - |
              export DELUGE_NODE_ID=${CLIENT_POD_NAME}
-              # Sadly we can't mount a subpath, so need add the correct prefix
-              # into all of the config directories.
-              export DELUGE_CONFIG_DIR=/var/lib/deluge/${CLIENT_POD_NAME}
-              export DELUGE_DOWNLOAD_DIR=${DELUGE_CONFIG_DIR}/downloads
-              export DELUGE_TORRENTFILE_DIR=${DELUGE_CONFIG_DIR}/downloads
-              export DELUGE_PLUGINS_DIR=${DELUGE_CONFIG_DIR}/plugins
-              
-              # Makes sure there's no previous garbage in there.
-              rm -rf ${DELUGE_CONFIG_DIR}
-              mkdir -p ${DELUGE_CONFIG_DIR}
-              
+              # Makes sure there's no previous garbage in the config dir.
+              rm -rf ${DELUGE_CONFIG_DIR}/*
              ${DELUGE_APP}/docker/bin/start.sh
          volumeMounts:
-            - name: deluge-{{ .Release.Name }}-volume
+            - name: deluge-node-storage
+              mountPath: /var/lib/deluge
+
+        - name: deluge-agent
+          image: {{ include "benchmark.harness.image" . }}
+          imagePullPolicy: {{ include "benchmark.harness.imagePullPolicy" . }}
+          command: [
+            "poetry", "run", "bittorrent-benchmarks",
+            "agent", "experiments.k8s.yaml", "deluge_agent", "--port", "9001"
+          ]
+          ports:
+            - containerPort: 9001
+          volumeMounts:
+            - name: deluge-node-storage
              mountPath: /var/lib/deluge

      volumes:
-        - name: deluge-{{ .Release.Name }}-volume
-          persistentVolumeClaim:
-            claimName: {{ include "deluge.pvc" . }}
+        - name: deluge-node-storage
+          emptyDir: { }
--- a/k8s/charts/deluge/templates/testrunner-job.yaml
+++ b/k8s/charts/deluge/templates/testrunner-job.yaml
@ -1,3 +1,4 @@
+{{- if .Values.experiment.testRunner }}
 apiVersion: batch/v1
 kind: Job
 metadata:
@ -21,8 +22,8 @@ spec:
      {{- end }}
      containers:
        - name: deluge-experiment-runner
-          image: codexstorage/bittorrent-benchmarks:latest
-          imagePullPolicy: Always
+          image: {{ include "benchmark.harness.image" . }}
+          imagePullPolicy: {{ include "benchmark.harness.imagePullPolicy" . }}
          args: [ "run", "deluge_experiment" ]
          env:
            - name: NETWORK_SIZE
@ -58,10 +59,7 @@ spec:
              cpu: "1"
              memory: "2Gi"

-      volumes:
-        - name: benchmark-volume
-          persistentVolumeClaim:
-            claimName: {{ include "deluge.pvc" . }}
      restartPolicy: Never

  backoffLimit: 0
+{{- end }}
--- a/k8s/charts/deluge/values.yaml
+++ b/k8s/charts/deluge/values.yaml
@ -13,6 +13,9 @@ experiment:
  # Identifies the experiment group. Needs to be defined by the user.
  groupId: ""

+  # If set to false, does not deploy a test runner (useful if you just want the network).
+  testRunner: true
+
 deployment:
  pvcName: ""
  appName: ""
@ -20,6 +23,9 @@ deployment:
  # If false, Deluge nodes will not be allowed to run on the same node.
  allowColocation: true

+  # Disables pulling of images and uses :minikube tag for the test runner and agents.
+  minikubeEnv: false
+
  # Labels for setting Deluge, tracker, and experiment runner node affinity towards pools.
  # Example:
  #   delugeNodePool:
--- a/k8s/clusters/local/vector/vector-chart-values.yaml
+++ b/k8s/clusters/local/vector/vector-chart-values.yaml
@ -11,6 +11,8 @@ extraVolumeMounts:
  - name: vector-logs
    mountPath: /vector-logs

+logLevel: "debug"
+
 customConfig:
  data_dir: /var/lib/vector
  sources:
@ -28,6 +30,7 @@ customConfig:
        pod_uid: ""
        pod_ip: ""
        pod_ips: ""
+      glob_minimum_cooldown_ms: 5000

  sinks:
    output:
--- a/k8s/clusters/local/vector/vector-pv.yaml
+++ b/k8s/clusters/local/vector/vector-pv.yaml