feat: docker image builds (#2)

* feat: add cleanup script in favor of many build trials

* feat: create docker images from specific state of spawned bee clients

* chore: add generated Docker files to git

* refactor: remove individual Docker files and create one common

* feat: remove Dockerfile creation and simplify the script

* feat: update common dockerfile on build

* refactor: remove generated Dockerfile from git

* chore: add generated Dockerfile to gitignore

* chore: ignore Dockerfile

* feat: check whether system environment variable is defined

* feat: common variable handling, new bee.sh functionalities and params

* feat: blockchain docker image build script

* feat: environment script to start the created images

* docs: amend readme

* feat: add bee image prefix env variable and its handlings

* feat: env variable handling + image prefix handling

* docs: amend readme

* feat: docker image prune

* docs: amend readme

* chore: echo for sudo password

* chore: echo for sudo password

* fix: environment queen logging

* chore: remove commented out code

* refactor: use osx compatible local variable init

* refactor: use shell check extension and make the script osx compatible

* fix: stop blockchain container as well on interrupting

* fix: rule out commented lines from env checking

* refactor: remove unnecessary echo

* refactor: remove cat command from the .env grep

* refactor: add some quotes to variable references where is necessary

README.md

@@ -7,7 +7,19 @@ All services run in `Docker` containers only.
 Currently, the repository supports running Bee nodes up to 5 by default.
 
 # Usage
-You can setup the whole environment that Bee needs by running some scripts
+The whole Bee environment (with blockchain) can be started by [running one script](###Run-Environment),
+but for that you need to have the necessary Docker images, which is possible to build yourself by [running some scripts](###Setup-the-environment)
+
+First you may want to set all global variables that the scripts will use.
+For that, there is a [.env](scripts/.env) file which contains all necessary variables that you need.
+
+```sh
+$ set -a && source ./scripts/.env && set +a
+```
+
+If you do not set these global variables, the scripts will use those which are available in the [.env](scripts/.env) file.
+
+## Setup the environment
 
 Create the common Docker network for the environment with
 
@@ -37,8 +49,32 @@ $ npm run supply
 
 and the configured accounts will get 1 ether and 100 BZZ Token.
 
-After all above went successfully you can start the Bee nodes.
+After all above went successfully you can start the Bee nodes
 
 ```sh
 $ ./scripts/bee.sh start --workers=4
 ```
+
+OR it is possible to build docker images on a desired state, so that a fresh environment can be started on each run.
+
+### Build Docker Images
+
+Basically, a full-featured Bee environment has 2 types of Docker image:
+
+- Bee images: Bee clients with pre-defined keys (and optionally including the state which you nodes have in its [data-dirs](scripts/bee-data-dirs))
+```sh
+$ ./scripts/bee-docker-build.sh
+```
+- Blockchain image: Ganache blockchain which you may want to take a snapshot of after the contracts are deployed and the pre-defined Bee client keys are funded already.
+```sh
+$ ./scripts/blockchain-docker-build.sh
+```
+
+## Run Environment
+
+If you have all Docker images that your [environment file](scripts/.env) requires,
+start the Bee cluster
+
+```sh
+$ ./scripts/environment.sh start
+```

scripts/.env

@@ -0,0 +1,3 @@
+BEE_VERSION="0.5.3"
+BEE_ENV_PREFIX="swarm-test"
+BEE_IMAGE_PREFIX=ethersphere

scripts/bee-cleanup.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+echo "Removing 'localstore' and 'statestore' folders from Bee datadirs..."
+echo "You may need to pass your password for sudo permission to remove the bee-data folders"
+
+MY_PATH=$(dirname "$0")
+MY_PATH=$( cd "$MY_PATH" && pwd )
+BEE_DIRS=$(ls "$MY_PATH/bee-data-dirs")
+for BEE_DIR in $BEE_DIRS
+do
+  echo "$BEE_DIR"
+  BEE_DIR_PATH="$MY_PATH/bee-data-dirs/$BEE_DIR"
+  sudo rm -rf "$BEE_DIR_PATH/localstore"
+  sudo rm -rf "$BEE_DIR_PATH/statestore"
+done
+
+echo "Removing built Docker images..."
+
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+BEE_VERSION=$("$MY_PATH/utils/env-variable-value.sh" BEE_VERSION)
+BEE_IMAGE_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_IMAGE_PREFIX)
+DOCKER_IMAGES=$(docker image ls -qaf reference="$BEE_IMAGE_PREFIX/$BEE_ENV_PREFIX*:$BEE_VERSION")
+for DOCKER_IMAGE in $DOCKER_IMAGES
+do
+  echo "$DOCKER_IMAGE"
+  docker image rm "$DOCKER_IMAGE"
+done

scripts/bee-data-dirs/.gitignore

@@ -0,0 +1 @@
+Dockerfile

scripts/bee-docker-build.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+dockerfile() {
+    cat << DOCKERFILE > "$1"
+FROM ethersphere/bee:$2
+
+# Sample docker file
+COPY --chown=bee:bee . /home/bee/.bee
+DOCKERFILE
+}
+
+dockerbuild() {
+  IMAGE_NAME=$(basename "$1")
+  IMAGE_NAME="$4/$IMAGE_NAME"
+  docker build "$1" --no-cache -f "$2" -t "$IMAGE_NAME:$3"
+}
+
+MY_PATH=$(dirname "$0")
+MY_PATH=$( cd "$MY_PATH" && pwd )
+BEE_DIRS=$(ls "$MY_PATH/bee-data-dirs")
+BEE_VERSION=$("$MY_PATH/utils/env-variable-value.sh" BEE_VERSION)
+BEE_IMAGE_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_IMAGE_PREFIX)
+
+# Make sure we the user has permission all the files
+echo "Build Bee Docker images..."
+echo "You may need to pass your password for sudo permission to give the right permission to the bee-data folders"
+sudo chmod 777 -R "$MY_PATH/bee-data-dirs"
+
+echo "Update common dockerfile"
+dockerfile "$MY_PATH/bee-data-dirs/Dockerfile" "$BEE_VERSION"
+
+echo "Build Dockerfiles"
+for BEE_DIR in $BEE_DIRS
+do
+  echo "$BEE_DIR"
+  dockerbuild "$MY_PATH/bee-data-dirs/$BEE_DIR" "$MY_PATH/bee-data-dirs/Dockerfile" "$BEE_VERSION" "$BEE_IMAGE_PREFIX"
+done
+
+echo "Docker image builds were successful!"

scripts/bee.sh

@@ -18,15 +18,17 @@ PARAMETERS:
                                 3. 21633:21635 (...)
                                 number represents the nodes number to map from. Default is 4.
     --password=string           password for Bee client(s).
+    --own-image                 If passed, the used Docker image names will be identical as the name of the workers.
     --version=x.y.z             used version of Bee client.
+    --detach                    It will not log the output of Queen node at the end of the process.
 USAGE
     exit 1
 }
 
 stop() {
-    echo "Stop following containers:"
+    echo "Stop Bee following containers:"
-    docker container stop $QUEEN_CONTAINER_NAME;
+    docker container stop "$QUEEN_CONTAINER_NAME";
-    WORKER_NAMES=`docker container ls -f name="$WORKER_CONTAINER_NAME*" --format "{{.Names}}"`
+    WORKER_NAMES=$(docker container ls -f name="$WORKER_CONTAINER_NAME*" --format "{{.Names}}")
     for WORKER_NAME in $WORKER_NAMES; do
         docker container stop "$WORKER_NAME"
     done
@@ -36,7 +38,7 @@ stop() {
 }
 
 fetch_queen_underlay_addr() {
-    if [[ ! -z "$QUEEN_UNDERLAY_ADDRESS" ]] ; then return; fi
+    if [[ -n "$QUEEN_UNDERLAY_ADDRESS" ]] ; then return; fi
 
     while : ; do
         QUEEN_UNDERLAY_ADDRESS=$(curl -s localhost:1635/addresses | python -mjson.tool 2>&1 | grep "/ip4/" | awk '!/127.0.0.1/' | xargs)
@@ -51,25 +53,32 @@ fetch_queen_underlay_addr() {
 
 log_queen() {
     trap stop SIGINT
-    docker logs --tail 25 -f $QUEEN_CONTAINER_NAME
+    docker logs --tail 25 -f "$QUEEN_CONTAINER_NAME"
 }
 
+MY_PATH=$(dirname "$0")              # relative
+MY_PATH=$( cd "$MY_PATH" && pwd )  # absolutized and normalized
+# Check used system variable set
+BEE_IMAGE_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_IMAGE_PREFIX)
+BEE_VERSION=$("$MY_PATH/utils/env-variable-value.sh" BEE_VERSION)
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+
 # Init variables
 EPHEMERAL=false
 WORKERS=4
-QUEEN_CONTAINER_NAME="swarm-test-queen"
+LOG=true
-WORKER_CONTAINER_NAME="swarm-test-worker"
+QUEEN_CONTAINER_NAME="$BEE_ENV_PREFIX-queen"
-QUEEN_CONTAINER_IN_DOCKER=`docker container ls -qaf name=$QUEEN_CONTAINER_NAME`
+WORKER_CONTAINER_NAME="$BEE_ENV_PREFIX-worker"
-BEE_VERSION="0.5.3"
+SWARM_BLOCKCHAIN_NAME="$BEE_ENV_PREFIX-blockchain"
-BEE_IMAGE="ethersphere/bee:$BEE_VERSION"
+NETWORK="$BEE_ENV_PREFIX-network"
+QUEEN_CONTAINER_IN_DOCKER=$(docker container ls -qaf name="$QUEEN_CONTAINER_NAME")
+BEE_BASE_IMAGE="ethersphere/bee"
+OWN_IMAGE=false
 BEE_PASSWORD="password"
 QUEEN_BOOTNODE=""
 PORT_MAPS=2
-NETWORK="swarm-test-network"
 SWAP=true
 SWAP_FACTORY_ADDRESS="0x5b1869D9A4C187F2EAa108f3062412ecf0526b24"
-MY_PATH=`dirname "$0"`              # relative
-MY_PATH=`( cd "$MY_PATH" && pwd )`  # absolutized and normalized
 INIT_ROOT_DATA_DIR="$MY_PATH/bee-data-dirs"
 
 # Decide script action
@@ -104,13 +113,20 @@ do
         ;;
         --version=*)
         BEE_VERSION="${1#*=}"
-        BEE_IMAGE="ethersphere/bee:$BEE_VERSION"
         shift 1
         ;;
         --port-maps=*)
         PORT_MAPS="${1#*=}"
         shift 1
         ;;
+        --own-image)
+        OWN_IMAGE=true
+        shift 1
+        ;;
+        --detach)
+        LOG=false
+        shift 1
+        ;;
         --help)
         usage
         ;;
@@ -121,33 +137,39 @@ do
     esac
 done
 
+BEE_IMAGE="$BEE_BASE_IMAGE:$BEE_VERSION"
+
 if $EPHEMERAL ; then
-    EXTRA_DOCKER_PARAMS=" --rm"
+    EXTRA_DOCKER_PARAMS="--rm"
 fi
 
 # Start Bee Queen
 if [ -z "$QUEEN_CONTAINER_IN_DOCKER" ] || $EPHEMERAL ; then
-    EXTRA_QUEEN_PARAMS=""
+    DOCKER_IMAGE="$BEE_IMAGE"
-    if [ $PORT_MAPS -ge 1 ] ; then
+    if $OWN_IMAGE ; then
-        EXTRA_QUEEN_PARAMS=" -p 127.0.0.1:1633-1635:1633-1635"
+        DOCKER_IMAGE="$BEE_IMAGE_PREFIX/$QUEEN_CONTAINER_NAME:$BEE_VERSION"
+    else
+        EXTRA_QUEEN_PARAMS="-v $INIT_ROOT_DATA_DIR/$QUEEN_CONTAINER_NAME:/home/bee/.bee"
+    fi
+    if [ "$PORT_MAPS" -ge 1 ] ; then
+        EXTRA_QUEEN_PARAMS="$EXTRA_QUEEN_PARAMS -p 127.0.0.1:1633-1635:1633-1635"
     fi
 
     echo "start Bee Queen process"
     docker run \
       -d \
-      --network=$NETWORK \
+      --network="$NETWORK" \
-      --name $QUEEN_CONTAINER_NAME \
+      --name="$QUEEN_CONTAINER_NAME" \
-      -v $INIT_ROOT_DATA_DIR/$QUEEN_CONTAINER_NAME:/home/bee/.bee \
       $EXTRA_DOCKER_PARAMS \
       $EXTRA_QUEEN_PARAMS \
-      $BEE_IMAGE \
+      $DOCKER_IMAGE \
         start \
-        --password $BEE_PASSWORD \
+        --password "$BEE_PASSWORD" \
-        --bootnode=$QUEEN_BOOTNODE \
+        --bootnode="$QUEEN_BOOTNODE" \
         --debug-api-enable \
         --verbosity=4 \
         --swap-enable=$SWAP \
-        --swap-endpoint="http://swarm-test-blockchain:9545" \
+        --swap-endpoint="http://$SWARM_BLOCKCHAIN_NAME:9545" \
         --swap-factory-address=$SWAP_FACTORY_ADDRESS \
         --welcome-message="You have found the queen of the beehive..." \
         --cors-allowed-origins="*"
@@ -156,37 +178,42 @@ else
 fi
 
 # Start Bee workers
-for i in $(seq 1 1 $WORKERS); do
+for i in $(seq 1 1 "$WORKERS"); do
     WORKER_NAME="$WORKER_CONTAINER_NAME-$i"
-    WORKER_CONTAINER_IN_DOCKER=`docker container ls -qaf name=$WORKER_NAME`
+    WORKER_CONTAINER_IN_DOCKER=$(docker container ls -qaf name="$WORKER_NAME")
     if [ -z "$WORKER_CONTAINER_IN_DOCKER" ] || $EPHEMERAL ; then
         # fetch queen underlay address
         fetch_queen_underlay_addr
 
         # construct additional params
         EXTRA_WORKER_PARAMS=""
+        DOCKER_IMAGE="$BEE_IMAGE"
+        if $OWN_IMAGE ; then
+            DOCKER_IMAGE="$BEE_IMAGE_PREFIX/$WORKER_NAME:$BEE_VERSION"
+        else
+            EXTRA_WORKER_PARAMS="$EXTRA_WORKER_PARAMS -v $INIT_ROOT_DATA_DIR/$WORKER_NAME:/home/bee/.bee"
+        fi
         if [ $PORT_MAPS -gt $i ] ; then
-            PORT_START=$((1633+(10000*$i)))
+            PORT_START=$((1633+(10000*i)))
-            PORT_END=$(($PORT_START + 2))
+            PORT_END=$((PORT_START + 2))
-            EXTRA_WORKER_PARAMS=" -p 127.0.0.1:$PORT_START-$PORT_END:1633-1635"
+            EXTRA_WORKER_PARAMS="$EXTRA_WORKER_PARAMS -p 127.0.0.1:$PORT_START-$PORT_END:1633-1635"
         fi
 
         # run docker container
         echo "start Bee worker $i process"
         docker run \
         -d \
-        --network=$NETWORK \
+        --network="$NETWORK" \
-        --name $WORKER_NAME \
+        --name="$WORKER_NAME" \
-        -v $INIT_ROOT_DATA_DIR/$WORKER_NAME:/home/bee/.bee \
         $EXTRA_DOCKER_PARAMS \
         $EXTRA_WORKER_PARAMS \
-        $BEE_IMAGE \
+        $DOCKER_IMAGE \
           start \
-          --password $BEE_PASSWORD \
+          --password "$BEE_PASSWORD" \
           --bootnode="$QUEEN_UNDERLAY_ADDRESS" \
           --debug-api-enable \
           --swap-enable=$SWAP \
-          --swap-endpoint="http://swarm-test-blockchain:9545" \
+          --swap-endpoint="http://$SWARM_BLOCKCHAIN_NAME:9545" \
           --swap-factory-address=$SWAP_FACTORY_ADDRESS \
           --welcome-message="I'm just Bee worker ${i} in the beehive." \
           --cors-allowed-origins="*"
@@ -196,4 +223,6 @@ for i in $(seq 1 1 $WORKERS); do
 done
 
 # log Bee Queen
-log_queen
+if $LOG ; then
+    log_queen
+fi

scripts/blockchain-docker-build.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+MY_PATH=$(dirname "$0")
+MY_PATH=$( cd "$MY_PATH" && pwd )
+# Check used system variable set
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+BEE_IMAGE_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_IMAGE_PREFIX)
+BEE_VERSION=$("$MY_PATH/utils/env-variable-value.sh" BEE_VERSION)
+
+NAME="$BEE_ENV_PREFIX-blockchain"
+
+echo "Make a snapshot from the blockchain..."
+docker commit $NAME $BEE_IMAGE_PREFIX/$NAME:$BEE_VERSION

scripts/blockchain.sh

@@ -1,13 +1,19 @@
 #!/bin/bash
-NAME=swarm-test-blockchain
+MY_PATH=$(dirname "$0")
-CONTAINER_IN_DOCKER=`docker container ls -qaf name=$NAME`
+MY_PATH=$( cd "$MY_PATH" && pwd )
+# Check used system variable set
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+
+NETWORK="$BEE_ENV_PREFIX-network"
+NAME="$BEE_ENV_PREFIX-blockchain"
+CONTAINER_IN_DOCKER=$(docker container ls -qaf name=$NAME)
 
 if [ -z "$CONTAINER_IN_DOCKER" ]; then
-  # necessary "-b 1" because anyway the Bee throws Error: waiting backend sync: Post "http://swarm-test-blockchain:9545": EOF 
+  # necessary "-b 1" because anyway the Bee throws Error: waiting backend sync: Post "http://swarm-test-blockchain:9545": EOF
   docker run \
     -p 127.0.0.1:9545:9545 \
-    --network swarm-test-network \
+    --network $NETWORK \
-    --name swarm-test-blockchain -d \
+    --name $NAME -d \
     trufflesuite/ganache-cli ganache-cli \
       -d -i 4020 -h 0.0.0.0 -p 9545 \
       -b 1 \

scripts/environment.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+usage() {
+    cat << USAGE >&2
+USAGE:
+    $ environment.sh [COMMAND] [PARAMETERS]
+COMMANDS:
+    start                       create Bee cluster with the given parameters
+    stop                        stop Bee cluster
+PARAMETERS:
+    --ephemeral                 create ephemeral container for bee-client. Data won't be persisted.
+    --workers=number            all Bee nodes in the test environment. Default is 4.
+    --detach                    It will not log the output of Queen node at the end of the process.
+USAGE
+    exit 1
+}
+
+
+stop() {
+    #Stop Bee nodes
+    docker stop "$SWARM_BLOCKCHAIN_NAME"
+    #Stop blockchain nodes
+    "$MY_PATH/bee.sh" stop
+
+    trap - SIGINT
+    exit 0;
+}
+
+MY_PATH=$(dirname "$0")              # relative
+MY_PATH=$( cd "$MY_PATH" && pwd )  # absolutized and normalized
+# Check used system variable set
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+BEE_IMAGE_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_IMAGE_PREFIX)
+BEE_VERSION=$("$MY_PATH/utils/env-variable-value.sh" BEE_VERSION)
+
+# Init variables
+EPHEMERAL=false
+WORKERS=4
+LOG=true
+SWARM_BLOCKCHAIN_NAME="$BEE_ENV_PREFIX-blockchain"
+SWARM_NETWORK="$BEE_ENV_PREFIX-network"
+
+# Decide script action
+case "$1" in
+    start)
+    shift 1
+    ;;
+    stop)
+    stop
+    ;;
+    *)
+    echoerr "Unknown command: $1"
+    usage
+    ;;
+esac
+
+
+# Alter variables from flags
+while [ $# -gt 0 ]
+do
+    case "$1" in
+        --ephemeral)
+        EPHEMERAL=true
+        shift 1
+        ;;
+        --workers=*)
+        WORKERS=${1#*=}
+        shift 1
+        ;;
+        --detach)
+        LOG=false
+        shift 1
+        ;;
+        --help)
+        usage
+        ;;
+        *)
+        echoerr "Unknown argument: $1"
+        usage
+        ;;
+    esac
+done
+
+# Start blockchain node
+echo "Start Blockchain node..."
+BLOCKCHAIN_CONTAINER=$(docker container ls -qaf name=$SWARM_BLOCKCHAIN_NAME)
+if [ -z "$BLOCKCHAIN_CONTAINER" ] ; then
+    BLOCKCHAIN_ARGUMENTS="--name $SWARM_BLOCKCHAIN_NAME --network $SWARM_NETWORK -d"
+    if $EPHEMERAL ; then
+        BLOCKCHAIN_ARGUMENTS="$BLOCKCHAIN_ARGUMENTS --rm"
+    fi
+    docker run $BLOCKCHAIN_ARGUMENTS $BEE_IMAGE_PREFIX/$SWARM_BLOCKCHAIN_NAME:$BEE_VERSION
+else
+    docker start $BLOCKCHAIN_CONTAINER
+fi
+
+# Wait for blockchain service initializes
+sleep 5
+
+# Build up bee.sh parameters
+BEE_SH_ARGUMENTS="--workers=$WORKERS --own-image"
+if $EPHEMERAL ; then
+    BEE_SH_ARGUMENTS="$BEE_SH_ARGUMENTS --ephemeral"
+fi
+if ! $LOG ; then
+    BEE_SH_ARGUMENTS="$BEE_SH_ARGUMENTS --detach"
+fi
+
+# Call bee.sh with the passed arguments
+echo "Start Bee nodes..."
+"$MY_PATH/bee.sh" start $BEE_SH_ARGUMENTS
+
+# If the code run reach this point without detach flag, 
+# then the user interrupted the log process in the bee.sh
+if $LOG ; then
+    docker stop $SWARM_BLOCKCHAIN_NAME
+fi

scripts/network.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
-NETWORK="swarm-test-network"
+MY_PATH=$(dirname "$0")
-if ! `docker network inspect $NETWORK > /dev/null` ; then
+MY_PATH=$( cd "$MY_PATH" && pwd )
+# Check used system variable set
+BEE_ENV_PREFIX=$("$MY_PATH/utils/env-variable-value.sh" BEE_ENV_PREFIX)
+
+NETWORK="$BEE_ENV_PREFIX-network"
+if ! eval "docker network inspect $NETWORK > /dev/null" ; then
   echo "Creating $NETWORK..."
-  docker network create swarm-test-network
+  docker network create $NETWORK
-fi
+fi

scripts/utils/env-variable-value.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+echo_env_variable_value() {
+  REF=$(printenv $1)
+
+  # There were no global set
+  if [ -z "$REF" ] ; then
+    MY_PATH=$(dirname "$0")
+    ENV_PATH=$( cd "$MY_PATH/.." && pwd )
+    VALUE=$(grep "^$1=" "$ENV_PATH/.env" | cut -d= -f2)
+    VALUE=${VALUE//\"/}
+    VALUE=${VALUE//\'/}
+    echo "$VALUE"
+  else
+    echo "$REF"
+  fi
+}
+
+echo_env_variable_value $1