build: generations improvements (#103)

* build: generations improvements

* fix: missing chown

* refactor: simpler wait for usable

* fix: use sudo

* chore: echo image for commit version tag

* fix: use sudo

* fix: i don't know what anymore

* build: debug

* build: fix

* fix: push only when requested

.github/workflows/publish.yaml

@@ -69,6 +69,7 @@ jobs:
       - name: Auth to Docker Hub
         if: ${{ github.event_name == 'repository_dispatch' || (github.event.inputs.buildImage == 'true' && success())  }}
         run: |
+          echo "PUSH_IMAGES=1" >> $GITHUB_ENV
           echo "${{ secrets.DOCKERHUB_PASSWORD }}" | docker login -u ${{secrets.DOCKERHUB_USERNAME}} --password-stdin
 
       - uses: actions/cache@v2
@@ -94,13 +95,6 @@ jobs:
           fi
           npm run build:env -- $BUILD_PARAMS
 
-      - name: Publish if required
-        id: publish
-        if: ${{ github.event_name == 'repository_dispatch' || (github.event.inputs.buildImage == 'true' && success()) }}
-        run: |
-          cd ./generator
-          npm run publish:env
-
       - name: Update bee version in package.json
         uses: jossef/action-set-json-field@v1
         if: ${{ github.event_name == 'repository_dispatch' }}

generator/package.json

@@ -15,7 +15,7 @@
   "dependencies": {
     "@ethersphere/bee-js": "^3.3.4",
     "@openzeppelin/contracts": "^3.1.0",
-    "truffle": "^5.3.5"
+    "truffle": "^5.5.13"
   },
   "keywords": [
     "swarm",

generator/scripts/.env

@@ -1,6 +1,6 @@
-BEE_VERSION="1.4.1"
+BEE_VERSION="1.5.1"
 BLOCKCHAIN_VERSION="1.2.0"
 BEE_ENV_PREFIX="bee-factory"
 BEE_IMAGE_PREFIX="ethersphere"
 COMMIT_VERSION_TAG="false"
-STATE_COMMIT="false"
+STATE_COMMIT="true"

generator/scripts/bee-cleanup.sh

@@ -1,16 +1,18 @@
 #!/bin/bash
+set -o errexit
+set -o pipefail
+
 echo "Removing 'localstore' and 'statestore' folders from Bee datadirs..."
 echo "You may need to pass your password for sudo permission to remove the bee-data folders"
 
 MY_PATH=$(dirname "$0")
 MY_PATH=$( cd "$MY_PATH" && pwd )
-BEE_DIRS=$(ls "$MY_PATH/bee-data-dirs")
+BEE_DIRS=$(ls -d $MY_PATH/bee-data-dirs/*/)
 for BEE_DIR in $BEE_DIRS
 do
   echo "$BEE_DIR"
-  BEE_DIR_PATH="$MY_PATH/bee-data-dirs/$BEE_DIR"
+  rm -rf "$BEE_DIR/localstore"
-  sudo rm -rf "$BEE_DIR_PATH/localstore"
+  rm -rf "$BEE_DIR/statestore"
-  sudo rm -rf "$BEE_DIR_PATH/statestore"
 done
 
 echo "Removing stopped Bee Docker containers..."

generator/scripts/bee-docker-build.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+set -o errexit
+set -o pipefail
+
 dockerfile() {
     cat << DOCKERFILE > "$1"
 FROM ethersphere/bee:$2
@@ -14,9 +17,13 @@ dockerbuild() {
   PLATFORM_FLAG=""
 
   if [ -n "$BEE_PLATFORM" ]; then
+    PLATFORM_FLAG="--platform=$BEE_PLATFORM"
+  fi
+
+  if [ -n "$PUSH_IMAGES" ]; then
     # Multiplatform build needs to push the images right away as docker buildx does not output images loaded into
     # docker itself, or it can do that but only for one platform.
-    PLATFORM_FLAG="--platform=$BEE_PLATFORM --push"
+      PLATFORM_FLAG+=" --push"
   fi
 
   IMAGE_NAME=$(basename "$1")
@@ -34,9 +41,6 @@ OFFICIAL_BEE_IMAGE="ethersphere/bee:$BEE_VERSION"
 
 # Make sure we the user has permission all the files
 echo "Build Bee Docker images..."
-echo "You may need to pass your password for sudo permission to give the right permission to the bee-data folders"
-sudo chmod 777 -R "$MY_PATH/bee-data-dirs"
-
 echo "Update common dockerfile"
 dockerfile "$MY_PATH/bee-data-dirs/Dockerfile" "$BEE_VERSION"
 
@@ -46,7 +50,7 @@ dockerfile "$MY_PATH/bee-data-dirs/Dockerfile" "$BEE_VERSION"
 # The image will be built with the tag that is the bee version string
 COMMIT_VERSION_TAG="$("$MY_PATH/utils/env-variable-value.sh" COMMIT_VERSION_TAG)"
 if [ "$COMMIT_VERSION_TAG" == "true" ] ; then
-  echo "Image version tag will be extracted from the bee version command..."
+  echo "Image version tag will be extracted from the bee version command from image $OFFICIAL_BEE_IMAGE"
   docker pull $OFFICIAL_BEE_IMAGE
   # somehow the version command's output goes to the stderr
   BEE_VERSION=$(docker run --rm $OFFICIAL_BEE_IMAGE version 2>&1)

generator/scripts/bee.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+set -o errexit
+set -o pipefail
+
 echoerr() { if [[ $QUIET -ne 1 ]] ; then echo "$@" 1>&2; fi }
 
 usage() {
@@ -22,7 +25,6 @@ PARAMETERS:
     --own-image                 If passed, the used Docker image names will be identical as the name of the workers.
     --version=x.y.z             used version of Bee client.
     --detach                    It will not log the output of Queen node at the end of the process.
-    --hostname=string           Interface to which should the nodes be bound (default 127.0.0.0).
 USAGE
     exit 1
 }
@@ -62,13 +64,15 @@ get_token() {
 }
 
 fetch_queen_underlay_addr() {
+    set +e
+
     if [[ -n "$QUEEN_UNDERLAY_ADDRESS" ]] ; then return; fi
-    check_queen_is_running
     ELAPSED_TIME=0
     WAITING_TIME=5
     # Wait 2 mins for queen start
     TIMEOUT=$((2*12*WAITING_TIME))
     while (( TIMEOUT > ELAPSED_TIME )) ; do
+        check_queen_is_running
         QUEEN_UNDERLAY_ADDRESS=$(curl -s "$HOSTNAME:1635/addresses" | python -mjson.tool | grep "/ip4/" | awk "!/127.0.0.1/" | sed 's/,$//' | xargs)
         if [[ -z "$QUEEN_UNDERLAY_ADDRESS" ]] ; then
             echo "Waiting for the Queen initialization..."
@@ -79,10 +83,12 @@ fetch_queen_underlay_addr() {
             break;
         fi
     done
+    set -e
 
     if (( TIMEOUT == ELAPSED_TIME )) ; then
         queen_failure
     fi
+
 }
 
 log_queen() {
@@ -179,10 +185,6 @@ do
         LOG=false
         shift 1
         ;;
-        --hostname=*)
-        HOSTNAME="${1#*=}"
-        shift 1
-        ;;
         --help)
         usage
         ;;
@@ -208,7 +210,7 @@ if [ -z "$QUEEN_CONTAINER_IN_DOCKER" ] || $EPHEMERAL ; then
         EXTRA_QUEEN_PARAMS="-v $INIT_ROOT_DATA_DIR/$QUEEN_CONTAINER_NAME:/home/bee/.bee"
     fi
     if [ "$PORT_MAPS" -ge 1 ] ; then
-        EXTRA_QUEEN_PARAMS="$EXTRA_QUEEN_PARAMS -p $HOSTNAME:1633-1635:1633-1635"
+        EXTRA_QUEEN_PARAMS="$EXTRA_QUEEN_PARAMS -p 1633-1635:1633-1635"
     fi
 
     echo "start Bee Queen process"
@@ -230,6 +232,8 @@ if [ -z "$QUEEN_CONTAINER_IN_DOCKER" ] || $EPHEMERAL ; then
         --bootnode="$QUEEN_BOOTNODE" \
         --debug-api-enable \
         --verbosity=4 \
+        --mainnet=false \
+        --block-time=1 \
         --swap-enable=$SWAP \
         --swap-endpoint="http://$SWARM_BLOCKCHAIN_NAME:9545" \
         --swap-factory-address=$SWAP_FACTORY_ADDRESS \
@@ -262,7 +266,7 @@ for i in $(seq 1 1 "$WORKERS"); do
         if [ $PORT_MAPS -gt $i ] ; then
             PORT_START=$((1633+(10000*i)))
             PORT_END=$((PORT_START + 2))
-            EXTRA_WORKER_PARAMS="$EXTRA_WORKER_PARAMS -p $HOSTNAME:$PORT_START-$PORT_END:1633-1635"
+            EXTRA_WORKER_PARAMS="$EXTRA_WORKER_PARAMS -p $PORT_START-$PORT_END:1633-1635"
         fi
 
         # run docker container
@@ -279,6 +283,9 @@ for i in $(seq 1 1 "$WORKERS"); do
           --password "$BEE_PASSWORD" \
           --bootnode="$QUEEN_UNDERLAY_ADDRESS" \
           --debug-api-enable \
+          --verbosity=4 \
+          --mainnet=false \
+          --block-time=1 \
           --swap-enable=$SWAP \
           --swap-endpoint="http://$SWARM_BLOCKCHAIN_NAME:9545" \
           --swap-factory-address=$SWAP_FACTORY_ADDRESS \

generator/scripts/blockchain-docker-build.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+set -o errexit
+set -o pipefail
+
 MY_PATH=$(dirname "$0")
 MY_PATH=$( cd "$MY_PATH" && pwd )
 # Check used system variable set
@@ -13,6 +16,10 @@ NAME="$BEE_ENV_PREFIX-blockchain"
 echo "Make a snapshot from the blockchain..."
 docker commit $NAME $BEE_IMAGE_PREFIX/$NAME:$BLOCKCHAIN_VERSION
 
+if [ -n "$PUSH_IMAGES" ]; then
+  docker push $BEE_IMAGE_PREFIX/$NAME:$BLOCKCHAIN_VERSION
+fi
+
 echo "Stop and remove running blockchain node that the image built on..."
 docker container stop $NAME
 docker container rm $NAME

generator/scripts/blockchain.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+set -o errexit
+set -o pipefail
+
 MY_PATH=$(dirname "$0")
 MY_PATH=$( cd "$MY_PATH" && pwd )
 # Check used system variable set
@@ -11,14 +14,14 @@ CONTAINER_IN_DOCKER=$(docker container ls -qaf name=$NAME)
 if [ -z "$CONTAINER_IN_DOCKER" ]; then
   # necessary "-b 1" because anyway the Bee throws Error: waiting backend sync: Post "http://swarm-test-blockchain:9545": EOF
   docker run \
-    -p 127.0.0.1:9545:9545 \
+    -p 9545:9545 \
     --network $NETWORK \
     --name $NAME -d \
-    trufflesuite/ganache-cli ganache-cli \
+    trufflesuite/ganache \
-      -d -i 4020 -h 0.0.0.0 -p 9545 \
+      --wallet.deterministic --chain.networkId 4020 -h 0.0.0.0 -p 9545 \
-      -b 1 \
+      --miner.blockTime 2 \
-      --chainId 4020 \
+      --chain.chainId 4020 \
-      --db swarm-testchain --gasLimit 6721975
+      --database.dbPath swarm-testchain --miner.blockGasLimit 6721975
 else
   docker start $NAME
 fi

generator/scripts/build-environment.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+set -o errexit
+set -o pipefail
+
 usage() {
     cat << USAGE >&2
 USAGE:
@@ -104,12 +108,12 @@ fi
 "$MY_PATH/blockchain.sh"
 npm run migrate:contracts
 npm run supply
+chmod -R 777 "$MY_PATH/bee-data-dirs/"
+
 if $GEN_TRAFFIC ; then
     export STATE_COMMIT='true'
     echo "Bee image with special state will be commited... traffic generation is on."
     # give the permission to the bee user
-    BEE_DIR_PATH="$MY_PATH/bee-data-dirs/"
-    sudo chown 999:999 -R "$BEE_DIR_PATH"
     echo "Start Bee nodes so that traffic can be generated and commited to the images"
     "$MY_PATH/bee.sh" start --version="$BEE_VERSION" --workers=$SUPPORTED_WORKER_N --detach
     echo "Generating traffic on Bee node $GEN_TRAFFIC_UPLOAD_NODE"
@@ -121,8 +125,8 @@ if $GEN_TRAFFIC ; then
     # is then used to trigger Bee-js PR creation
     if [ "$CI" == 'true' ]; then
       echo "Image version tag will be extracted from the bee version command..."
-      docker pull "$BEE_VERSION"
+      docker pull "ethersphere/bee:$BEE_VERSION"
-      FULL_BEE_VERSION=$(docker run --rm "$BEE_VERSION" version 2>&1)
+      FULL_BEE_VERSION=$(docker run --rm "ethersphere/bee:$BEE_VERSION" version 2>&1)
       echo "Extracted Bee version: $FULL_BEE_VERSION"
       echo "::set-output name=full-version::$FULL_BEE_VERSION"
 

generator/scripts/environment.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+set -o errexit
+set -o pipefail
+
 usage() {
     cat << USAGE >&2
 USAGE:

generator/scripts/gen-traffic.js

@@ -57,6 +57,22 @@ async function uploadRandomBytes(beePair, seed = 500, bytes = 1024 * 4 * 400) {
   console.log(`${beePair.uploaderBee.url} uploaded ${bytes} bytes to ${reference}`)
 }
 
+const DEFAULT_POLLING_FREQUENCY = 1_000
+const DEFAULT_STAMP_USABLE_TIMEOUT = 120_000
+async function waitUntilStampUsable(batchId, beeDebug, options = {}) {
+  const timeout = options?.timeout || DEFAULT_STAMP_USABLE_TIMEOUT
+  const pollingFrequency = options?.pollingFrequency || DEFAULT_POLLING_FREQUENCY
+
+  for (let i = 0; i < timeout; i += pollingFrequency) {
+    const stamp = await beeDebug.getPostageBatch(batchId)
+
+    if (stamp.usable) return stamp
+    await sleep(pollingFrequency)
+  }
+
+  throw new Error('Wait until stamp usable timeout has been reached')
+}
+
 /**
  * Generate traffic on Bee node(s)
  *
@@ -92,16 +108,16 @@ async function genTrafficLoop(hosts, minCheques) {
 
     console.log(`Creating postage stamp on ${uploaderBeeDebugUrl}...`)
     const postageBatchId = await uploaderBeeDebug.createPostageBatch(POSTAGE_STAMPS_AMOUNT, POSTAGE_STAMPS_DEPTH)
-    console.log(`Generated ${postageBatchId} postage stamp on ${uploaderBeeDebugUrl}...`)
+    console.log(`Generated ${postageBatchId} postage stamp on ${uploaderBeeDebugUrl}. Waiting until it is usable.`)
+
+    await waitUntilStampUsable(postageBatchId, uploaderBeeDebug)
+    console.log('Postage stamp usable.')
 
     return new BeePair(chequeReceiverBeeDebug, uploaderBee, uploaderBeeDebug, postageBatchId)
   })
 
   const bees = await Promise.all(promises)
 
-  console.log(`wait 11 secs (>10 block time) before postage stamp usages`)
-  await sleep(11 * 1000)
-
   while (true) {
     await genTrafficOnOpenPorts(bees)
 
@@ -125,7 +141,7 @@ async function genTrafficLoop(hosts, minCheques) {
       }
       if (beesUncashedCheques.every(uncashedCheques => uncashedCheques.length >= minCheques)) {
         console.log(`Generated at least ${minCheques} for every node on the given Debug API endpoints`)
-        break
+        process.exit()
       } else {
         console.log(
           `There is not enough uncashed cheques on Bee node(s)`,

generator/scripts/network.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+set -o errexit
+set -o pipefail
+
 MY_PATH=$(dirname "$0")
 MY_PATH=$( cd "$MY_PATH" && pwd )
 # Check used system variable set

generator/scripts/publish-environment.sh

@@ -1,4 +1,8 @@
 #!/bin/bash
+
+set -o errexit
+set -o pipefail
+
 MY_PATH=$(dirname "$0")
 MY_PATH=$( cd "$MY_PATH" && pwd )
 
@@ -33,9 +37,3 @@ done
 
 echo "Push Blockchain docker image: $BLOCKCHAIN_IMAGE_NAME"
 docker push "$BLOCKCHAIN_IMAGE_NAME"
-
-# This sets output parameter in Github Actions that
-# is then used to trigger Bee-js PR creation
-if [ $CI == 'true' ]; then
-  echo "::set-output name=bee-version::$BEE_VERSION"
-fi