From d263ca073546dbc550261ab2aa2adde2ff685818 Mon Sep 17 00:00:00 2001 From: Ben Bierens <39762930+benbierens@users.noreply.github.com> Date: Wed, 8 Mar 2023 12:45:55 +0100 Subject: [PATCH] Docker build (#354) * Sets up working dockerized build and codex docker image creation * Making codex configurable from the docker environment * Sets up two networks with three codex nodes * enables and exposes metrics endpoint for first node * Manually performed two-client test scenario with docker containers * Sets up docker-ignore and docker github workflow * Wires up all codex CLI arguments to docker env vars * Makes API_PORT variable optional as well * Removes duplicate docker-login step * Fixes path to docker file * Switches target dockerhub for debugging * Adds git tag info to --version output * Exposes version information via debug endpoint * Debugging docker image * specifies target platforms for docker build * specifies platform for QEMU and buildx steps * Attempt to debug line endings * Disables march-native in config.nims as test * Applies make argument to disable architecture optimization during docker build * Removes subset version tags from docker build * Restore multi-arch build * Removes docker-build test branch from CI branches --- .dockerignore | 6 ++ .github/workflows/docker.yml | 42 +++++++++++++ .gitignore | 2 + codex/codex.nim | 2 +- codex/conf.nim | 30 ++++++--- codex/rest/api.nim | 6 +- docker/README.md | 43 +++++++++++++ docker/codex.Dockerfile | 14 +++++ docker/docker-compose.yaml | 77 +++++++++++++++++++++++ docker/prometheus/alert.yml | 6 ++ docker/prometheus/prometheus.yml | 15 +++++ docker/startCodex.sh | 102 +++++++++++++++++++++++++++++++ 12 files changed, 336 insertions(+), 9 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/docker.yml create mode 100644 docker/README.md create mode 100644 docker/codex.Dockerfile create mode 100644 docker/docker-compose.yaml create mode 100644 docker/prometheus/alert.yml create mode 100644 docker/prometheus/prometheus.yml create mode 100644 docker/startCodex.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..6427da1d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.github +build +docs +metrics +nimcache +tests diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..7085e172 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,42 @@ +name: docker + +on: + push: + branches: + - "main" + tags: + - "v*.*.*" + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: thatbenbierens/nim-codex + tags: | + type=semver,pattern={{version}} + type=sha + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Login to Docker Hub + if: github.event_name != 'pull_request' + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: . + file: docker/codex.Dockerfile + platforms: linux/amd64,linux/arm64 + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index 4ecde076..2fe4caa1 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ nimble.paths .update.timestamp codex.nims nimbus-build-system.paths +docker/hostdatadir +docker/prometheus-data diff --git a/codex/codex.nim b/codex/codex.nim index c6831e85..63560fe1 100644 --- a/codex/codex.nim +++ b/codex/codex.nim @@ -173,7 +173,7 @@ proc new*(T: type CodexServer, config: CodexConf, privateKey: CodexPrivateKey): codexNode = CodexNodeRef.new(switch, store, engine, erasure, discovery, contracts) restServer = RestServerRef.new( codexNode.initRestApi(config), - initTAddress("127.0.0.1" , config.apiPort), + initTAddress(config.apiBindAddress , config.apiPort), bufferSize = (1024 * 64), maxRequestBodySize = int.high) .expect("Should start rest server!") diff --git a/codex/conf.nim b/codex/conf.nim index 6999edfc..f3c94d68 100644 --- a/codex/conf.nim +++ b/codex/conf.nim @@ -138,6 +138,12 @@ type desc: "Node agent string which is used as identifier in network" name: "agent-string" }: string + apiBindAddress* {. + desc: "The REST API bind address" + defaultValue: "127.0.0.1" + name: "api-bindaddr" + }: string + apiPort* {. desc: "The REST Api port", defaultValue: 8080 @@ -195,16 +201,26 @@ type EthAddress* = ethers.Address +proc getCodexVersion(): string = + let tag = strip(staticExec("git tag")) + if tag.isEmptyOrWhitespace: + return "untagged build" + return tag + +proc getCodexRevision(): string = + strip(staticExec("git rev-parse --short HEAD"))[0..5] + +proc getNimBanner(): string = + staticExec("nim --version | grep Version") + const - gitRevision* = strip(staticExec("git rev-parse --short HEAD"))[0..5] - - nimBanner* = staticExec("nim --version | grep Version") - - #TODO add versionMajor, Minor & Fix when we switch to semver - codexVersion* = gitRevision + codexVersion* = getCodexVersion() + codexRevision* = getCodexRevision() + nimBanner* = getNimBanner() codexFullVersion* = - "Codex build " & codexVersion & "\p" & + "Codex version: " & codexVersion & "\p" & + "Codex revision: " & codexRevision & "\p" & nimBanner proc defaultDataDir*(): string = diff --git a/codex/rest/api.nim b/codex/rest/api.nim index 485d797f..1c2c43a0 100644 --- a/codex/rest/api.nim +++ b/codex/rest/api.nim @@ -234,7 +234,11 @@ proc initRestApi*(node: CodexNodeRef, conf: CodexConf): RestRouter = if node.discovery.dhtRecord.isSome: node.discovery.dhtRecord.get.toURI else: - "" + "", + "codex": { + "version": $codexVersion, + "revision": $codexRevision + } } return RestApiResponse.response($json) diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..36dfedd3 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,43 @@ +# Codex Docker Image + +Build and run using the example docker-compose file: +`docker-compose up -d` + +Stop and retain image and volume data: +`docker-compose down` + +Stop and delete image and volume data: +`docker-compose down --rmi all -v` +`rm -R hostdatadir` + +# Environment variables +Codex docker image supports the following environment variables: +- LOG_LEVEL +- METRICS_ADDR +- METRICS_PORT +- NAT_IP +- API_PORT +- DISC_IP +- DISC_PORT +- NET_PRIVKEY +- BOOTSTRAP_SPR +- MAX_PEERS +- AGENT_STRING +- STORAGE_QUOTA +- BLOCK_TTL +- CACHE_SIZE +- ETH_PROVIDER +- ETH_ACCOUNT +- ETH_DEPLOYMENT + +All environment variables are optional and will default to Codex's CLI default values. + +# Constants +Codex CLI arguments 'data-dir', 'listen-addrs', and 'api-bindaddr' cannot be configured. They are set to values required for docker in case of bind addresses. In the case of 'data-dir', the value is set to `/datadir`. It is important that you map this folder to a host volume in your container configuration. See docker-compose.yaml for examples. + +# Useful +Connect nodes with the `/connect` endpoint. +To get the IP address of a container within a network: +Find container Id: `docker ps` +Open terminal in container: `docker exec -it sh` +Get IP addresses: `ifconfig` diff --git a/docker/codex.Dockerfile b/docker/codex.Dockerfile new file mode 100644 index 00000000..112ebc49 --- /dev/null +++ b/docker/codex.Dockerfile @@ -0,0 +1,14 @@ +FROM nimlang/nim:1.6.10-alpine AS builder +WORKDIR /src +RUN apk update && apk add git cmake curl make git bash linux-headers +COPY . . +RUN make clean +RUN make update +RUN make exec NIM_PARAMS="-d:disableMarchNative" + +FROM alpine:3.17.2 +WORKDIR /root/ +RUN apk add --no-cache openssl libstdc++ libgcc libgomp +COPY --from=builder /src/build/codex ./ +COPY --from=builder /src/docker/startCodex.sh ./ +CMD ["sh", "startCodex.sh"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 00000000..76c74e8c --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,77 @@ +services: + codex-node1: + image: clustertest-nim-codex + build: + context: ../. + dockerfile: docker/codex.Dockerfile + ports: + - 8080:8080 + # Available environment variables: + # environment: + # - LOG_LEVEL=TRACE + # - METRICS_ADDR=0.0.0.0 + # - METRICS_PORT=9090 + # - NAT_IP=2.3.4.5 + # - API_PORT=8080 + # - DISC_IP=3.4.5.6 + # - DISC_PORT=8765 + # - NET_PRIVKEY=privkey + # - BOOTSTRAP_SPR=bootstrap_record + # - MAX_PEERS=123 + # - AGENT_STRING=agent_string + # - STORAGE_QUOTA=123456789 + # - BLOCK_TTL=23456 + # - CACHE_SIZE=6543 + # - ETH_PROVIDER=eth + # - ETH_ACCOUNT=account + # - ETH_DEPLOYMENT=deploy + volumes: + - ./hostdatadir/node1:/datadir + networks: + - primary + + # Example with metrics enabled. + codex-node2: + image: clustertest-nim-codex + ports: + - 8081:8080 + - 9090:9090 + environment: + - METRICS_ADDR=0.0.0.0 + - METRICS_PORT=9090 + volumes: + - ./hostdatadir/node2:/datadir + depends_on: + - codex-node1 + networks: + - primary + - secondary + + codex-node3: + image: clustertest-nim-codex + ports: + - 8082:8080 + volumes: + - ./hostdatadir/node3:/datadir + depends_on: + - codex-node1 + networks: + - secondary + + prometheus: + image: prom/prometheus:v2.30.3 + ports: + - 9000:9090 + volumes: + - ./prometheus:/etc/prometheus + - ./prometheus-data:/prometheus + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + networks: + - primary + - secondary + +networks: + primary: + name: primary + secondary: + name: secondary diff --git a/docker/prometheus/alert.yml b/docker/prometheus/alert.yml new file mode 100644 index 00000000..b415a9bc --- /dev/null +++ b/docker/prometheus/alert.yml @@ -0,0 +1,6 @@ +groups: + - name: DemoAlerts + rules: + - alert: InstanceDown + expr: up{job="services"} < 1 + for: 5m diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 00000000..84b40afd --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 30s + scrape_timeout: 10s + +rule_files: + - alert.yml + +scrape_configs: + - job_name: services + metrics_path: /metrics + static_configs: + - targets: + - 'prometheus:9090' + - 'codex-node1:9090' + - 'codex-node2:9090' diff --git a/docker/startCodex.sh b/docker/startCodex.sh new file mode 100644 index 00000000..95d14ac3 --- /dev/null +++ b/docker/startCodex.sh @@ -0,0 +1,102 @@ +echo "Starting Codex..." + +args="" + + +# Required arguments +args="$args --data-dir=/datadir" +args="$args --listen-addrs=/ip4/0.0.0.0/tcp/8071" +args="$args --api-bindaddr=0.0.0.0" + +# Optional arguments +# Log level +if [ -n "$LOG_LEVEL" ]; then + echo "Log level: $LOG_LEVEL" + args="$args --log-level=$LOG_LEVEL" +fi + +# Metrics +if [ -n "$METRICS_ADDR" ] && [ -n "$METRICS_PORT" ]; then + echo "Metrics enabled" + args="$args --metrics=true" + args="$args --metrics-address=$METRICS_ADDR" + args="$args --metrics-port=$METRICS_PORT" +fi + +# NAT +if [ -n "$NAT_IP" ]; then + echo "NAT: $NAT_IP" + args="$args --nat=$NAT_IP" +fi + +# Discovery IP +if [ -n "$DISC_IP" ]; then + echo "Discovery IP: $DISC_IP" + args="$args --disc-ip=$DISC_IP" +fi + +# Discovery Port +if [ -n "$DISC_PORT" ]; then + echo "Discovery Port: $DISC_PORT" + args="$args --disc-port=$DISC_PORT" +fi + +# Net private key +if [ -n "$NET_PRIVKEY" ]; then + echo "Network Private Key path: $NET_PRIVKEY" + args="$args --net-privkey=$NET_PRIVKEY" +fi + +# Bootstrap SPR +if [ -n "$BOOTSTRAP_SPR" ]; then + echo "Bootstrap SPR: $BOOTSTRAP_SPR" + args="$args --bootstrap-node=$BOOTSTRAP_SPR" +fi + +# Max peers +if [ -n "$MAX_PEERS" ]; then + echo "Max peers: $MAX_PEERS" + args="$args --max-peers=$MAX_PEERS" +fi + +# Agent string +if [ -n "$AGENT_STRING" ]; then + echo "Agent string: $AGENT_STRING" + args="$args --agent-string=$AGENT_STRING" +fi + +# API port +if [ -n "$API_PORT" ]; then + echo "API port: $API_PORT" + args="$args --api-port=$API_PORT" +fi + +# Storage quota +if [ -n "$STORAGE_QUOTA" ]; then + echo "Storage quote: $STORAGE_QUOTA" + args="$args --storage-quota=$STORAGE_QUOTA" +fi + +# Block TTL +if [ -n "$BLOCK_TTL" ]; then + echo "Block TTL: $BLOCK_TTL" + args="$args --block-ttl=$BLOCK_TTL" +fi + +# Cache size +if [ -n "$CACHE_SIZE" ]; then + echo "Cache size: $CACHE_SIZE" + args="$args --cache-size=$CACHE_SIZE" +fi + +# Ethereum persistence +if [ -n "$ETH_PROVIDER" ] && [ -n "$ETH_ACCOUNT" ] && [ -n "$ETH_DEPLOYMENT" ]; then + echo "Persistence enabled" + args="$args --persistence=true" + args="$args --eth-provider=$ETH_PROVIDER" + args="$args --eth-account=$ETH_ACCOUNT" + args="$args --eth-deployment=$ETH_DEPLOYMENT" +fi + +echo "./root/codex $args" +sh -c "/root/codex $args"