logos-storage/codex-factory
1
0
Fork 0
mirror of https://github.com/logos-storage/codex-factory.git synced 2026-01-05 22:43:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: timeout on infinite loops (#18)

Browse Source 
* feat: timeout on infinite loops

* refactor: time-standing solution for timeout check

* chore: take out 2 min comment
This commit is contained in:
nugaon 2021-05-12 17:55:46 +02:00 committed by GitHub
parent 0ecd8974ce
commit fcb873ee5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 34 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
scripts/bee.sh
View File

@ -25,30 +25,50 @@ USAGE
exit 1 exit 1
} }
stop() { stop_containers() {
echo "Stop Bee following containers:" echo "Stop Bee following containers:"
docker container stop "$QUEEN_CONTAINER_NAME"; docker container stop "$QUEEN_CONTAINER_NAME";
WORKER_NAMES=$(docker container ls -f name="$WORKER_CONTAINER_NAME*" --format "{{.Names}}") WORKER_NAMES=$(docker container ls -f name="$WORKER_CONTAINER_NAME*" --format "{{.Names}}")
for WORKER_NAME in $WORKER_NAMES; do for WORKER_NAME in $WORKER_NAMES; do
docker container stop "$WORKER_NAME" docker container stop "$WORKER_NAME"
done done
}
stop() {
stop_containers
trap - SIGINT trap - SIGINT
exit 0; exit 0;
} }
queen_failure() {
docker logs "$QUEEN_CONTAINER_NAME"
stop_containers
echo "Timeout limit has been reached, exit from the process.."
exit 1
}
fetch_queen_underlay_addr() { fetch_queen_underlay_addr() {
if [[ -n "$QUEEN_UNDERLAY_ADDRESS" ]] ; then return; fi if [[ -n "$QUEEN_UNDERLAY_ADDRESS" ]] ; then return; fi
while : ; do ELAPSED_TIME=0
QUEEN_UNDERLAY_ADDRESS=$(curl -s localhost:1635/addresses | python -mjson.tool 2>&1 | grep "/ip4/" | awk '!/127.0.0.1/' | sed 's/,$//' | xargs) WAITING_TIME=5
# Wait 2 mins for queen start
TIMEOUT=$((2*12*WAITING_TIME))
while (( TIMEOUT > ELAPSED_TIME )) ; do
QUEEN_UNDERLAY_ADDRESS=$(curl -s localhost:1635/addresses | python -mjson.tool | grep "/ip4/" | awk '!/127.0.0.1/' | sed 's/,$//' | xargs)
if [[ -z "$QUEEN_UNDERLAY_ADDRESS" ]] ; then if [[ -z "$QUEEN_UNDERLAY_ADDRESS" ]] ; then
echo "Waiting for the Queen initialization..." echo "Waiting for the Queen initialization..."
sleep 5 ELAPSED_TIME=$((ELAPSED_TIME+WAITING_TIME))
sleep $WAITING_TIME
else else
echo "Queen underlay address: $QUEEN_UNDERLAY_ADDRESS"
break; break;
fi fi
done done
if (( TIMEOUT == ELAPSED_TIME )) ; then
queen_failure
fi
} }
log_queen() { log_queen() {
@ -57,7 +77,7 @@ log_queen() {
} }
count_connected_peers() { count_connected_peers() {
COUNT=$( curl -s http://localhost:1635/peers | python -c 'import json,sys; obj=json.load(sys.stdin); print (len(obj["peers"]));' ) COUNT=$( (curl -s http://localhost:1635/peers | python -c 'import json,sys; obj=json.load(sys.stdin); print (len(obj["peers"]));') || echo 0 )
echo "$COUNT" echo "$COUNT"
} }
@ -229,12 +249,19 @@ for i in $(seq 1 1 "$WORKERS"); do
done done
echo "Check whether the queen node has been connected to every worker..." echo "Check whether the queen node has been connected to every worker..."
while : ; do ELAPSED_TIME=0
WAITING_TIME=2
TIMEOUT=$((2*30*WAITING_TIME))
while (( TIMEOUT > ELAPSED_TIME )) ; do
COUNT=$(count_connected_peers) COUNT=$(count_connected_peers)
[[ $COUNT < $WORKERS ]] || break [[ $COUNT < $WORKERS ]] || break
echo "Only $COUNT peers have been connected to the Queen Bee node yet. Waiting until $WORKERS" echo "Only $COUNT peers have been connected to the Queen Bee node yet. Waiting until $WORKERS"
sleep 2 ELAPSED_TIME=$((ELAPSED_TIME+WAITING_TIME))
sleep $WAITING_TIME
done done
if (( ELAPSED_TIME >= TIMEOUT )) ; then
queen_failure
fi
# log Bee Queen # log Bee Queen
if $LOG ; then if $LOG ; then