Docker multiarch debug (#447)

* Uses correct string for marketplace address

* first steps towards support for arm64

* Applies multiarch ubuntu dockerfile as codex-dockerfile.

* Add `--simulate-proof-failures` env variable, update docker-compose to point to slimmed image

* Add image to CI, and update startCodex.sh

* Sets up separate docker build for arm

* Update arm64 arm of docker ci

* [docker] modify startCodex.sh

- include overridden node name in log output if specified in test
- quote `—log-level` value so that multiple log levels can be specified
- ensure any CLI parameter env vars are passed through to the codex binary, instead of conditionally including them
- add `—persistence`
- add `—validator`

* fixes load and push for amd docker build

---------

Co-authored-by: Eric Mastro <github@egonat.me>

.github/workflows/docker.yml

@@ -9,35 +9,117 @@ on:
   workflow_dispatch:
 
 jobs:
-  docker:
+  docker-amd64:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v4
         with:
-          images: thatbenbierens/nim-codex
+          images: thatbenbierens/nim-codex-amd64
           tags: |
             type=semver,pattern={{version}}
             type=sha
-      - name: Set up QEMU
+
-        uses: docker/setup-qemu-action@v2
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
       - name: Login to Docker Hub
         if: github.event_name != 'pull_request'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-      - name: Build and push
+
+      - name: Build and export to Docker
+        id: build
         uses: docker/build-push-action@v4
         with:
           context: .
           file: docker/codex.Dockerfile
-          platforms: linux/amd64,linux/arm64,linux/arm/v7
+          platforms: linux/amd64
-          push: ${{ github.event_name != 'pull_request' }}
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Minify docker image
+        uses: kitabisa/docker-slim-action@v1
+        env:
+          DSLIM_HTTP_PROBE: false
+        with:
+          target: ${{ steps.meta.outputs.tags }}
+          overwrite: true
+
+      - name: Show slim report
+        run: echo "${{ steps.slim.outputs.report }}"
+
+      - name: Push to Docker registry
+        if: github.event_name != 'pull_request'
+        id: push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/amd64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  docker-arm64:
+    runs-on: buildjet-4vcpu-ubuntu-2204-arm
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: thatbenbierens/nim-codex-arm64
+          tags: |
+            type=semver,pattern={{version}}
+            type=sha
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and export to Docker
+        id: build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/arm64
+          load: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Minify docker image
+        uses: kitabisa/docker-slim-action@v1
+        env:
+          DSLIM_HTTP_PROBE: false
+        with:
+          target: ${{ steps.meta.outputs.tags }}
+          overwrite: true
+
+      - name: Show slim report
+        run: echo "${{ steps.slim.outputs.report }}"
+
+      - name: Push to Docker registry
+        if: github.event_name != 'pull_request'
+        id: push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: docker/codex.Dockerfile
+          platforms: linux/arm64
+          push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

docker/README.md

@@ -32,9 +32,15 @@ Codex docker image supports the following environment variables:
 - ETH_PROVIDER
 - ETH_ACCOUNT
 - ETH_DEPLOYMENT
+- SIMULATE_PROOF_FAILURES
+- VALIDATOR
+- PERSISTENCE
+- CODEX_NODENAME(†)
 
 (*) These variables have default values in the docker image that are different from Codex's standard default values.
 
+(†) CODEX_NODENAME is used for logging purposes only in the docker image
+
 All environment variables are optional and will default to Codex's CLI default values.
 
 # Constants
@@ -46,3 +52,13 @@ To get the IP address of a container within a network:
 Find container Id: `docker ps`
 Open terminal in container: `docker exec -it <CONTAINER ID> sh`
 Get IP addresses: `ifconfig`
+
+# Slim
+1. Build the image using `docker build -t status-im/codexsetup:latest -f codex.Dockerfile ..`
+2. The docker image can then be minifed using [slim](https://github.com/slimtoolkit/slim). Install slim on your path and then run:
+```shell
+slim # brings up interactive prompt
+>>> build --target status-im/codexsetup --http-probe-off true
+```
+3. This should output an image with name `status-im/codexsetup.slim`
+4. We can then bring up the image using `docker-compose up -d`.

docker/codex.Dockerfile

@@ -1,14 +1,18 @@
-FROM nimlang/nim:1.6.10-alpine AS builder
+FROM ubuntu:lunar-20230415 AS builder
+RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential nim
+RUN echo 'export NIMBLE_DIR="${HOME}/.nimble"' >> "${HOME}/.bash_env"
+RUN echo 'export PATH="${NIMBLE_DIR}/bin:${PATH}"' >> "${HOME}/.bash_env"
+
 WORKDIR /src
-RUN apk update && apk add git cmake curl make git bash linux-headers
 COPY . .
 RUN make clean
 RUN make -j4 update
 RUN make -j4 NIM_PARAMS="-d:disableMarchNative -d:codex_enable_api_debug_peers=true"
 
-FROM alpine:3.17.2
+FROM ubuntu:lunar-20230415
-WORKDIR /root/
+WORKDIR /root
-RUN apk add --no-cache openssl libstdc++ libgcc libgomp
+RUN apt-get update && apt-get install -y libgomp1 bash
 COPY --from=builder /src/build/codex ./
 COPY --from=builder /src/docker/startCodex.sh ./
-CMD ["sh", "startCodex.sh"]
+RUN chmod +x ./startCodex.sh
+CMD ["/bin/bash", "-l", "-c", "./startCodex.sh"]

docker/docker-compose.yaml

@@ -1,77 +1,27 @@
 services:
   codex-node1:
-    image: clustertest-nim-codex
+    image: status-im/codexsetup.slim:latest
-    build:
-      context: ../.
-      dockerfile: docker/codex.Dockerfile
     ports:
       - 8080:8080
     # Available environment variables:
-    # environment:
-      # - LOG_LEVEL=TRACE
-      # - METRICS_ADDR=0.0.0.0
-      # - METRICS_PORT=9090
-      # - NAT_IP=2.3.4.5
-      # - API_PORT=8080
-      # - DISC_IP=3.4.5.6
-      # - DISC_PORT=8765
-      # - NET_PRIVKEY=privkey
-      # - BOOTSTRAP_SPR=bootstrap_record
-      # - MAX_PEERS=123
-      # - AGENT_STRING=agent_string
-      # - STORAGE_QUOTA=123456789
-      # - BLOCK_TTL=23456
-      # - CACHE_SIZE=6543
-      # - ETH_PROVIDER=eth
-      # - ETH_ACCOUNT=account
-      # - ETH_DEPLOYMENT=deploy
-    volumes:
-      - ./hostdatadir/node1:/datadir
-    networks:
-      - primary
-
-  # Example with metrics enabled.
-  codex-node2:
-    image: clustertest-nim-codex
-    ports:
-      - 8081:8080
-      - 9090:9090
     environment:
+      - LOG_LEVEL=TRACE
       - METRICS_ADDR=0.0.0.0
       - METRICS_PORT=9090
-    volumes:
+      - NAT_IP=2.3.4.5
-      - ./hostdatadir/node2:/datadir
+      - API_PORT=8080
-    depends_on:
+      - DISC_IP=3.4.5.6
-      - codex-node1
+      - DISC_PORT=8765
-    networks:
+      - NET_PRIVKEY=privkey
-      - primary
+      - BOOTSTRAP_SPR=bootstrap_record
-      - secondary
+      - MAX_PEERS=123
+      - AGENT_STRING=agent_string
+      - STORAGE_QUOTA=123456789
+      - BLOCK_TTL=23456
+      - CACHE_SIZE=6543
+      - ETH_PROVIDER=eth
+      - ETH_ACCOUNT=account
+      - ETH_MARKETPLACE_ADDRESS=0x59b670e9fA9D0A427751Af201D676719a970857b
+      - SIMULATE_PROOF_FAILURES=2
 
-  codex-node3:
-    image: clustertest-nim-codex
-    ports:
-      - 8082:8080
-    volumes:
-      - ./hostdatadir/node3:/datadir
-    depends_on:
-      - codex-node1
-    networks:
-      - secondary
 
-  prometheus:
-    image: prom/prometheus:v2.30.3
-    ports:
-      - 9000:9090
-    volumes:
-      - ./prometheus:/etc/prometheus
-      - ./prometheus-data:/prometheus
-    command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml
-    networks:
-      - primary
-      - secondary
-
-networks:
-  primary:
-    name: primary
-  secondary:
-    name: secondary

docker/startCodex.sh

@@ -1,4 +1,8 @@
-echo "Starting Codex..."
+NAME=""
+if [ -n "$CODEX_NODENAME" ]; then
+  NAME=" '$CODEX_NODENAME'"
+fi
+echo "Starting Codex node$NAME"
 
 args=""
 
@@ -32,7 +36,7 @@ fi
 # Log level
 if [ -n "$LOG_LEVEL" ]; then
   echo "Log level: $LOG_LEVEL"
-  args="$args --log-level=$LOG_LEVEL"
+  args="$args --log-level=\"$LOG_LEVEL\""
 fi
 
 # Metrics
@@ -115,10 +119,35 @@ if [ -n "$ETH_PROVIDER" ] && [ -n "$ETH_ACCOUNT" ] && [ -n "$ETH_MARKETPLACE_ADD
     args="$args --eth-account=$ETH_ACCOUNT"
     # args="$args --validator"
 
-    # Remove this as soon as CLI option is available:
+if [ -n "$ETH_ACCOUNT" ]; then
-    echo "{\"contracts\": { \"Marketplace\": { \"address\": \""$ACCOUNTSTR"\" } } }" > /root/marketplace_address.json
+  echo "Ethereum account: $ETH_ACCOUNT"
-    args="$args --eth-deployment=/root/marketplace_address.json"
+  args="$args --eth-account=$ETH_ACCOUNT"
 fi
 
-echo "./root/codex $args"
+if [ -n "$ETH_MARKETPLACE_ADDRESS" ]; then
-sh -c "/root/codex $args"
+  # Remove this as soon as CLI option is available:
+  echo "{\"contracts\": { \"Marketplace\": { \"address\": \""$ETH_MARKETPLACE_ADDRESS"\" } } }" > /root/marketplace_address.json
+  args="$args --eth-deployment=/root/marketplace_address.json"
+fi
+
+if [ -n "$SIMULATE_PROOF_FAILURES" ]; then
+  echo "Simulate proof failures: $SIMULATE_PROOF_FAILURES"
+  args="$args --simulate-proof-failures=$SIMULATE_PROOF_FAILURES"
+fi
+
+if [ "$PERSISTENCE" = "true" ] || [ "$PERSISTENCE" = "1" ]; then
+  echo "Persistence enabled"
+  args="$args --persistence"
+else
+  echo "Persistence disabled"
+fi
+
+if [ "$VALIDATOR" = "true" ] || [ "$VALIDATOR" = "1" ]; then
+  echo "Validator enabled"
+  args="$args --validator"
+else
+  echo "Validator disabled"
+fi
+
+echo "./codex $args"
+/bin/bash -l -c "./codex $args"