mirror of
https://github.com/logos-storage/logos-storage-network-crawler.git
synced 2026-01-02 13:33:08 +00:00
Setting up docker file and workflow
This commit is contained in:
Ben
parent
088c160267
commit
d3469ba5c2
217
.github/workflows/docker-reusable.yml
vendored
Normal file
217
.github/workflows/docker-reusable.yml
vendored
Normal file
@ -0,0 +1,217 @@
|
||||
name: Reusable - Docker
|
||||
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
docker_file:
|
||||
default: docker/crawler.Dockerfile
|
||||
description: Dockerfile
|
||||
required: false
|
||||
type: string
|
||||
docker_repo:
|
||||
default: codexstorage/codex-network-crawler
|
||||
description: DockerHub repository
|
||||
required: false
|
||||
type: string
|
||||
make_parallel:
|
||||
default: 4
|
||||
description: Make parallel
|
||||
required: false
|
||||
type: number
|
||||
nimflags:
|
||||
default: '-d:disableMarchNative'
|
||||
description: Nim flags for builds
|
||||
required: false
|
||||
type: string
|
||||
nat_ip_auto:
|
||||
default: false
|
||||
description: Enable NAT IP auto
|
||||
required: false
|
||||
type: boolean
|
||||
tag_latest:
|
||||
default: true
|
||||
description: Set latest tag for Docker images
|
||||
required: false
|
||||
type: boolean
|
||||
tag_sha:
|
||||
default: true
|
||||
description: Set Git short commit as Docker tag
|
||||
required: false
|
||||
type: boolean
|
||||
tag_suffix:
|
||||
default: ''
|
||||
description: Suffix for Docker images tag
|
||||
required: false
|
||||
type: string
|
||||
continuous_tests_list:
|
||||
default: ''
|
||||
description: Continuous Tests list
|
||||
required: false
|
||||
type: string
|
||||
continuous_tests_duration:
|
||||
default: 48h
|
||||
description: Continuous Tests duration
|
||||
required: false
|
||||
type: string
|
||||
run_release_tests:
|
||||
description: Run Release tests
|
||||
required: false
|
||||
type: string
|
||||
default: false
|
||||
|
||||
|
||||
env:
|
||||
# Build
|
||||
DOCKER_FILE: ${{ inputs.docker_file }}
|
||||
DOCKER_REPO: ${{ inputs.docker_repo }}
|
||||
MAKE_PARALLEL: ${{ inputs.make_parallel }}
|
||||
NIMFLAGS: ${{ inputs.nimflags }}
|
||||
NAT_IP_AUTO: ${{ inputs.nat_ip_auto }}
|
||||
TAG_LATEST: ${{ inputs.tag_latest }}
|
||||
TAG_SHA: ${{ inputs.tag_sha }}
|
||||
TAG_SUFFIX: ${{ inputs.tag_suffix }}
|
||||
|
||||
jobs:
|
||||
# Build platform specific image
|
||||
build:
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
target:
|
||||
- os: linux
|
||||
arch: amd64
|
||||
- os: linux
|
||||
arch: arm64
|
||||
include:
|
||||
- target:
|
||||
os: linux
|
||||
arch: amd64
|
||||
builder: ubuntu-22.04
|
||||
- target:
|
||||
os: linux
|
||||
arch: arm64
|
||||
builder: ubuntu-22.04-arm
|
||||
|
||||
name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }}
|
||||
runs-on: ${{ matrix.builder }}
|
||||
env:
|
||||
PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Build and Push by digest
|
||||
id: build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ${{ env.DOCKER_FILE }}
|
||||
platforms: ${{ env.PLATFORM }}
|
||||
push: true
|
||||
build-args: |
|
||||
MAKE_PARALLEL=${{ env.MAKE_PARALLEL }}
|
||||
NIMFLAGS=${{ env.NIMFLAGS }}
|
||||
NAT_IP_AUTO=${{ env.NAT_IP_AUTO }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true
|
||||
|
||||
- name: Docker - Export digest
|
||||
run: |
|
||||
mkdir -p /tmp/digests
|
||||
digest="${{ steps.build.outputs.digest }}"
|
||||
touch "/tmp/digests/${digest#sha256:}"
|
||||
|
||||
- name: Docker - Upload digest
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: digests-${{ matrix.target.arch }}
|
||||
path: /tmp/digests/*
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
|
||||
# Publish multi-platform image
|
||||
publish:
|
||||
name: Publish multi-platform image
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.meta.outputs.version }}
|
||||
needs: build
|
||||
steps:
|
||||
- name: Docker - Variables
|
||||
run: |
|
||||
# Adjust custom suffix when set and
|
||||
if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
# Disable SHA tags on tagged release
|
||||
if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then
|
||||
echo "TAG_SHA=false" >>$GITHUB_ENV
|
||||
fi
|
||||
# Handle latest and latest-custom using raw
|
||||
if [[ ${{ env.TAG_SHA }} == "false" ]]; then
|
||||
echo "TAG_LATEST=false" >>$GITHUB_ENV
|
||||
echo "TAG_RAW=true" >>$GITHUB_ENV
|
||||
if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then
|
||||
echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV
|
||||
else
|
||||
echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV
|
||||
fi
|
||||
else
|
||||
echo "TAG_RAW=false" >>$GITHUB_ENV
|
||||
fi
|
||||
|
||||
- name: Docker - Download digests
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: digests-*
|
||||
merge-multiple: true
|
||||
path: /tmp/digests
|
||||
|
||||
- name: Docker - Set up Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Docker - Meta
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.DOCKER_REPO }}
|
||||
flavor: |
|
||||
latest=${{ env.TAG_LATEST }}
|
||||
suffix=${{ env.TAG_SUFFIX }},onlatest=true
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
type=raw,enable=${{ env.TAG_RAW }},value=latest
|
||||
type=sha,enable=${{ env.TAG_SHA }}
|
||||
|
||||
- name: Docker - Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Docker - Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
|
||||
$(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *)
|
||||
|
||||
- name: Docker - Inspect image
|
||||
run: |
|
||||
docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }}
|
||||
28
.github/workflows/docker.yml
vendored
Normal file
28
.github/workflows/docker.yml
vendored
Normal file
@ -0,0 +1,28 @@
|
||||
name: Docker
|
||||
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
tags:
|
||||
- 'v*.*.*'
|
||||
paths-ignore:
|
||||
- '**/*.md'
|
||||
- '.gitignore'
|
||||
- '.github/**'
|
||||
- '!.github/workflows/docker.yml'
|
||||
- '!.github/workflows/docker-reusable.yml'
|
||||
- 'docker/**'
|
||||
- '!docker/crawler.Dockerfile'
|
||||
- '!docker/docker-entrypoint.sh'
|
||||
workflow_dispatch:
|
||||
|
||||
|
||||
jobs:
|
||||
build-and-push:
|
||||
name: Build and Push
|
||||
uses: ./.github/workflows/docker-reusable.yml
|
||||
with:
|
||||
tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }}
|
||||
secrets: inherit
|
||||
@ -6,6 +6,7 @@ description = "Crawler for Codex networks"
|
||||
license = "MIT"
|
||||
skipDirs = @["tests"]
|
||||
bin = @["codexcrawler"]
|
||||
binDir = "build"
|
||||
|
||||
# Dependencies
|
||||
requires "secp256k1#2acbbdcc0e63002a013fff49f015708522875832" # >= 0.5.2 & < 0.6.0
|
||||
|
||||
36
docker/crawler.Dockerfile
Normal file
36
docker/crawler.Dockerfile
Normal file
@ -0,0 +1,36 @@
|
||||
# Variables
|
||||
ARG BUILDER=ubuntu:24.04
|
||||
ARG IMAGE=${BUILDER}
|
||||
ARG BUILD_HOME=/src
|
||||
ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4}
|
||||
ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}"
|
||||
ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1}
|
||||
ARG APP_HOME=/crawler
|
||||
|
||||
# Build
|
||||
FROM ${BUILDER} AS builder
|
||||
ARG BUILD_HOME
|
||||
ARG MAKE_PARALLEL
|
||||
ARG NIMFLAGS
|
||||
ARG USE_LIBBACKTRACE
|
||||
|
||||
RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential
|
||||
RUN curl https://nim-lang.org/choosenim/init.sh -sSf | sh
|
||||
RUN choosenim 2.0.14
|
||||
|
||||
WORKDIR ${BUILD_HOME}
|
||||
COPY . .
|
||||
RUN nimble build
|
||||
|
||||
# Create
|
||||
FROM ${IMAGE}
|
||||
ARG BUILD_HOME
|
||||
ARG APP_HOME
|
||||
ARG NAT_IP_AUTO
|
||||
|
||||
WORKDIR ${APP_HOME}
|
||||
COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin
|
||||
COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh /
|
||||
RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/*
|
||||
ENTRYPOINT ["/docker-entrypoint.sh"]
|
||||
CMD ["codexcrawler"]
|
||||
44
docker/docker-compose.yaml
Normal file
44
docker/docker-compose.yaml
Normal file
@ -0,0 +1,44 @@
|
||||
services:
|
||||
codex-node1:
|
||||
image: codexstorage/nim-codex:sha-82b0399
|
||||
environment:
|
||||
- CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE}
|
||||
- CODEX_METRICS=${CODEX_METRICS:-false}
|
||||
- CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0}
|
||||
- CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008}
|
||||
- CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir}
|
||||
- CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345}
|
||||
- CODEX_NAT=${CODEX_NAT:-10.0.0.10}
|
||||
- CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090}
|
||||
- CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key}
|
||||
# - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE}
|
||||
- CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160}
|
||||
- CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex}
|
||||
- CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0}
|
||||
- CODEX_API_PORT=${CODEX_API_PORT:-8080}
|
||||
- CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs}
|
||||
- CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592}
|
||||
- CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0}
|
||||
# - CODEX_BLOCK_MI=${CODEX_BLOCK_MI}
|
||||
- CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000}
|
||||
- CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0}
|
||||
- CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false}
|
||||
- CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545}
|
||||
# - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT}
|
||||
# - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b}
|
||||
- CODEX_VALIDATOR=${CODEX_VALIDATOR:-false}
|
||||
- CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000}
|
||||
- NAT_IP_AUTO=false
|
||||
- NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip
|
||||
ports:
|
||||
- 8080:8080/tcp # REST API
|
||||
- 8008:8008/tcp # Metrics
|
||||
- 2345:2345/tcp # libp2p
|
||||
- 8090:8090/udp # DHT discovery
|
||||
volumes:
|
||||
- ./datadir:/datadir:z
|
||||
networks:
|
||||
- codex
|
||||
networks:
|
||||
codex:
|
||||
driver: bridge
|
||||
76
docker/docker-entrypoint.sh
Normal file
76
docker/docker-entrypoint.sh
Normal file
@ -0,0 +1,76 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Environment variables from files
|
||||
# If set to file path, read the file and export the variables
|
||||
# If set to directory path, read all files in the directory and export the variables
|
||||
if [[ -n "${ENV_PATH}" ]]; then
|
||||
set -a
|
||||
[[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done
|
||||
set +a
|
||||
fi
|
||||
|
||||
# Should be passed if env variable is set.
|
||||
# public IP should fetch ip.codex.storage
|
||||
|
||||
# --logLevel=<l> Sets log level [default: TRACE]
|
||||
# --publicIp=<a> Public IP address where this instance is reachable. [default: 45.82.185.194]
|
||||
# --metricsAddress=<ip> Listen address of the metrics server [default: 0.0.0.0]
|
||||
# --metricsPort=<p> Listen HTTP port of the metrics server [default: 8008]
|
||||
# --dataDir=<dir> Directory for storing data [default: crawler_data]
|
||||
# --discoveryPort=<p> Port used for DHT [default: 8090]
|
||||
# --bootNodes=<n> Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs]
|
||||
# --stepDelay=<ms> Delay in milliseconds per crawl step [default: 3000]
|
||||
# --revisitDelay=<m> Delay in minutes after which a node can be revisited [default: 1] (24h)
|
||||
|
||||
|
||||
# Parameters
|
||||
if [[ -z "${CODEX_NAT}" ]]; then
|
||||
if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then
|
||||
export CODEX_NAT="extip:$(hostname --ip-address)"
|
||||
echo "Private: CODEX_NAT=${CODEX_NAT}"
|
||||
elif [[ -n "${NAT_PUBLIC_IP_AUTO}" ]]; then
|
||||
# Run for 60 seconds if fail
|
||||
WAIT=120
|
||||
SECONDS=0
|
||||
SLEEP=5
|
||||
while (( SECONDS < WAIT )); do
|
||||
IP=$(curl -s -f -m 5 "${NAT_PUBLIC_IP_AUTO}")
|
||||
# Check if exit code is 0 and returned value is not empty
|
||||
if [[ $? -eq 0 && -n "${IP}" ]]; then
|
||||
export CODEX_NAT="extip:${IP}"
|
||||
echo "Public: CODEX_NAT=${CODEX_NAT}"
|
||||
break
|
||||
else
|
||||
# Sleep and check again
|
||||
echo "Can't get Public IP - Retry in $SLEEP seconds / $((WAIT - SECONDS))"
|
||||
sleep $SLEEP
|
||||
fi
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
# Stop Codex run if can't get NAT IP when requested
|
||||
if [[ "${NAT_IP_AUTO}" == "true" && -z "${CODEX_NAT}" ]]; then
|
||||
echo "Can't get Private IP - Stop Codex run"
|
||||
exit 1
|
||||
elif [[ -n "${NAT_PUBLIC_IP_AUTO}" && -z "${CODEX_NAT}" ]]; then
|
||||
echo "Can't get Public IP in $WAIT seconds - Stop Codex run"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# If marketplace is enabled from the testing environment,
|
||||
# The file has to be written before Codex starts.
|
||||
for key in PRIV_KEY ETH_PRIVATE_KEY; do
|
||||
keyfile="private.key"
|
||||
if [[ -n "${!key}" ]]; then
|
||||
[[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!"
|
||||
echo "${!key}" > "${keyfile}"
|
||||
chmod 600 "${keyfile}"
|
||||
export CODEX_ETH_PRIVATE_KEY="${keyfile}"
|
||||
echo "Private key set"
|
||||
fi
|
||||
done
|
||||
|
||||
# Run
|
||||
echo "Run Codex node"
|
||||
exec "$@"
|
||||
Loading…
x
Reference in New Issue
Block a user