set deluge node id explicitly

analysis/collect-logs.sh

@@ -1,5 +1,16 @@
 #!/usr/bin/env bash
 set -e
 
-kubectl logs --prefix -n codex-benchmarks -l app=deluge-nodes --tail=-1 > "${1}.log"
-grep '\[M' "${1}.log" | tr -s ' ' | cut -d ' ' -f1,7 | sed 's/ /,/' | grep -v 'metric' > "${1}.csv"
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <output-folder>"
+    exit 1
+fi
+
+# TODO build auto naming for experiment folders based on metadata
+mkdir -p "${1}"
+
+echo "Collect"
+kubectl logs --prefix -n codex-benchmarks -l "app in (deluge-nodes,testrunner)" --tail=-1 > "${1}/raw-logs.log"
+
+echo "Parse"
+python -m benchmarks.cli logs "${1}/raw-logs.log" "${1}/parsed"

benchmarks/cli.py

@@ -1,14 +1,13 @@
 import argparse
 import sys
 from pathlib import Path
-from typing import Dict, Iterable
+from typing import Dict
 
-from mypy.memprofile import defaultdict
 from pydantic_core import ValidationError
 
 from benchmarks.core.config import ConfigParser, ExperimentBuilder
 from benchmarks.core.experiments.experiments import Experiment
-from benchmarks.core.logging import basic_log_parser, LogEntry, LogSplitter
+from benchmarks.core.logging import basic_log_parser, LogSplitter
 from benchmarks.deluge.config import DelugeExperimentConfig
 from benchmarks.deluge.logging import DelugeTorrentDownload
 

docker-compose.local.yaml

@@ -13,6 +13,7 @@ services:
       - DELUGE_RPC_PORT=6890
       - DELUGE_LISTEN_PORTS=6891,6892
       - DELUGE_LOG_LEVEL=${DELUGE_LOG_LEVEL:-info}
+      - DELUGE_NODE_ID=deluge-1
     volumes:
       - ./volume/deluge-1:/var/lib/deluge
       - ./volume/deluge-1/downloads:/var/lib/deluge/downloads
@@ -27,6 +28,7 @@ services:
       - DELUGE_RPC_PORT=6893
       - DELUGE_LISTEN_PORTS=6894,6895
       - DELUGE_LOG_LEVEL=${DELUGE_LOG_LEVEL:-info}
+      - DELUGE_NODE_ID=deluge-2
     volumes:
       - ./volume/deluge-2:/var/lib/deluge
       - ./volume/deluge-2/downloads:/var/lib/deluge/downloads
@@ -41,6 +43,7 @@ services:
       - DELUGE_RPC_PORT=6896
       - DELUGE_LISTEN_PORTS=6897,6898
       - DELUGE_LOG_LEVEL=${DELUGE_LOG_LEVEL:-info}
+      - DELUGE_NODE_ID=deluge-3
     volumes:
       - ./volume/deluge-3:/var/lib/deluge
       - ./volume/deluge-3/downloads:/var/lib/deluge/downloads

k8s/deluge-statefulset.yaml

@@ -38,6 +38,7 @@ spec:
           command: [ "/bin/bash", "--login", "-c" ]
           args:
             - |
+              export DELUGE_NODE_ID=${CLIENT_POD_NAME}
               # Sadly we can't mount a subpath, so need add the correct prefix
               # into all of the config directories.
               export DELUGE_CONFIG_DIR=/var/lib/deluge/${CLIENT_POD_NAME}

k8s/testrunner-job.yaml

@@ -26,7 +26,7 @@ spec:
             - name: REPETITIONS
               value: "10"
             - name: SEEDER_SETS
-              value: "2"
+              value: "5"
             - name: FILE_SIZE
               value: "104857600"
             - name: TRACKER_ANNOUNCE_URL