diff --git a/.github/workflows/docker-reusable.yml b/.github/workflows/docker-reusable.yml new file mode 100644 index 0000000..a45b538 --- /dev/null +++ b/.github/workflows/docker-reusable.yml @@ -0,0 +1,217 @@ +name: Reusable - Docker + + +on: + workflow_call: + inputs: + docker_file: + default: docker/crawler.Dockerfile + description: Dockerfile + required: false + type: string + docker_repo: + default: codexstorage/codex-network-crawler + description: DockerHub repository + required: false + type: string + make_parallel: + default: 4 + description: Make parallel + required: false + type: number + nimflags: + default: '-d:disableMarchNative' + description: Nim flags for builds + required: false + type: string + nat_ip_auto: + default: false + description: Enable NAT IP auto + required: false + type: boolean + tag_latest: + default: true + description: Set latest tag for Docker images + required: false + type: boolean + tag_sha: + default: true + description: Set Git short commit as Docker tag + required: false + type: boolean + tag_suffix: + default: '' + description: Suffix for Docker images tag + required: false + type: string + continuous_tests_list: + default: '' + description: Continuous Tests list + required: false + type: string + continuous_tests_duration: + default: 48h + description: Continuous Tests duration + required: false + type: string + run_release_tests: + description: Run Release tests + required: false + type: string + default: false + + +env: + # Build + DOCKER_FILE: ${{ inputs.docker_file }} + DOCKER_REPO: ${{ inputs.docker_repo }} + MAKE_PARALLEL: ${{ inputs.make_parallel }} + NIMFLAGS: ${{ inputs.nimflags }} + NAT_IP_AUTO: ${{ inputs.nat_ip_auto }} + TAG_LATEST: ${{ inputs.tag_latest }} + TAG_SHA: ${{ inputs.tag_sha }} + TAG_SUFFIX: ${{ inputs.tag_suffix }} + +jobs: + # Build platform specific image + build: + strategy: + fail-fast: true + matrix: + target: + - os: linux + arch: amd64 + - os: linux + arch: arm64 + include: + - target: + os: linux + arch: amd64 + builder: ubuntu-22.04 + - target: + os: linux + arch: arm64 + builder: ubuntu-22.04-arm + + name: Build ${{ matrix.target.os }}/${{ matrix.target.arch }} + runs-on: ${{ matrix.builder }} + env: + PLATFORM: ${{ format('{0}/{1}', 'linux', matrix.target.arch) }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Docker - Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REPO }} + + - name: Docker - Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker - Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker - Build and Push by digest + id: build + uses: docker/build-push-action@v5 + with: + context: . + file: ${{ env.DOCKER_FILE }} + platforms: ${{ env.PLATFORM }} + push: true + build-args: | + MAKE_PARALLEL=${{ env.MAKE_PARALLEL }} + NIMFLAGS=${{ env.NIMFLAGS }} + NAT_IP_AUTO=${{ env.NAT_IP_AUTO }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.DOCKER_REPO }},push-by-digest=true,name-canonical=true,push=true + + - name: Docker - Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Docker - Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.target.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + + # Publish multi-platform image + publish: + name: Publish multi-platform image + runs-on: ubuntu-latest + outputs: + version: ${{ steps.meta.outputs.version }} + needs: build + steps: + - name: Docker - Variables + run: | + # Adjust custom suffix when set and + if [[ -n "${{ env.TAG_SUFFIX }}" ]]; then + echo "TAG_SUFFIX=-${{ env.TAG_SUFFIX }}" >>$GITHUB_ENV + fi + # Disable SHA tags on tagged release + if [[ ${{ startsWith(github.ref, 'refs/tags/') }} == "true" ]]; then + echo "TAG_SHA=false" >>$GITHUB_ENV + fi + # Handle latest and latest-custom using raw + if [[ ${{ env.TAG_SHA }} == "false" ]]; then + echo "TAG_LATEST=false" >>$GITHUB_ENV + echo "TAG_RAW=true" >>$GITHUB_ENV + if [[ -z "${{ env.TAG_SUFFIX }}" ]]; then + echo "TAG_RAW_VALUE=latest" >>$GITHUB_ENV + else + echo "TAG_RAW_VALUE=latest-{{ env.TAG_SUFFIX }}" >>$GITHUB_ENV + fi + else + echo "TAG_RAW=false" >>$GITHUB_ENV + fi + + - name: Docker - Download digests + uses: actions/download-artifact@v4 + with: + pattern: digests-* + merge-multiple: true + path: /tmp/digests + + - name: Docker - Set up Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker - Meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_REPO }} + flavor: | + latest=${{ env.TAG_LATEST }} + suffix=${{ env.TAG_SUFFIX }},onlatest=true + tags: | + type=semver,pattern={{version}} + type=raw,enable=${{ env.TAG_RAW }},value=latest + type=sha,enable=${{ env.TAG_SHA }} + + - name: Docker - Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Docker - Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.DOCKER_REPO }}@sha256:%s ' *) + + - name: Docker - Inspect image + run: | + docker buildx imagetools inspect ${{ env.DOCKER_REPO }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000..1e5c6a4 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,28 @@ +name: Docker + + +on: + push: + branches: + - master + tags: + - 'v*.*.*' + paths-ignore: + - '**/*.md' + - '.gitignore' + - '.github/**' + - '!.github/workflows/docker.yml' + - '!.github/workflows/docker-reusable.yml' + - 'docker/**' + - '!docker/crawler.Dockerfile' + - '!docker/docker-entrypoint.sh' + workflow_dispatch: + + +jobs: + build-and-push: + name: Build and Push + uses: ./.github/workflows/docker-reusable.yml + with: + tag_latest: ${{ github.ref_name == github.event.repository.default_branch || startsWith(github.ref, 'refs/tags/') }} + secrets: inherit diff --git a/codexcrawler.nimble b/codexcrawler.nimble index a6bdcc3..4f7071e 100644 --- a/codexcrawler.nimble +++ b/codexcrawler.nimble @@ -6,6 +6,7 @@ description = "Crawler for Codex networks" license = "MIT" skipDirs = @["tests"] bin = @["codexcrawler"] +binDir = "build" # Dependencies requires "secp256k1#2acbbdcc0e63002a013fff49f015708522875832" # >= 0.5.2 & < 0.6.0 diff --git a/docker/crawler.Dockerfile b/docker/crawler.Dockerfile new file mode 100644 index 0000000..64cc0e2 --- /dev/null +++ b/docker/crawler.Dockerfile @@ -0,0 +1,36 @@ +# Variables +ARG BUILDER=ubuntu:24.04 +ARG IMAGE=${BUILDER} +ARG BUILD_HOME=/src +ARG MAKE_PARALLEL=${MAKE_PARALLEL:-4} +ARG NIMFLAGS="${NIMFLAGS:-"-d:disableMarchNative"}" +ARG USE_LIBBACKTRACE=${USE_LIBBACKTRACE:-1} +ARG APP_HOME=/crawler + +# Build +FROM ${BUILDER} AS builder +ARG BUILD_HOME +ARG MAKE_PARALLEL +ARG NIMFLAGS +ARG USE_LIBBACKTRACE + +RUN apt-get update && apt-get install -y git cmake curl make bash lcov build-essential +RUN curl https://nim-lang.org/choosenim/init.sh -sSf | sh +RUN choosenim 2.0.14 + +WORKDIR ${BUILD_HOME} +COPY . . +RUN nimble build + +# Create +FROM ${IMAGE} +ARG BUILD_HOME +ARG APP_HOME +ARG NAT_IP_AUTO + +WORKDIR ${APP_HOME} +COPY --from=builder ${BUILD_HOME}/build/* /usr/local/bin +COPY --from=builder --chmod=0755 ${BUILD_HOME}/docker/docker-entrypoint.sh / +RUN apt-get update && apt-get install -y libgomp1 curl jq && rm -rf /var/lib/apt/lists/* +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["codexcrawler"] diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 0000000..9ffdd82 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,44 @@ +services: + codex-node1: + image: codexstorage/nim-codex:sha-82b0399 + environment: + - CODEX_LOG_LEVEL=${CODEX_LOG_LEVEL:-TRACE} + - CODEX_METRICS=${CODEX_METRICS:-false} + - CODEX_METRICS_ADDRESS=${CODEX_METRICS_ADDRESS:-0.0.0.0} + - CODEX_METRICS_PORT=${CODEX_METRICS_PORT:-8008} + - CODEX_DATA_DIR=${CODEX_DATA_DIR:-/datadir} + - CODEX_LISTEN_ADDRS=${CODEX_LISTEN_ADDRS:-/ip4/0.0.0.0/tcp/2345} + - CODEX_NAT=${CODEX_NAT:-10.0.0.10} + - CODEX_DISC_PORT=${CODEX_DISC_PORT:-8090} + - CODEX_NET_PRIVKEY=${CODEX_NET_PRIVKEY:-key} + # - CODEX_BOOTSTRAP_NODE=${CODEX_BOOTSTRAP_NODE} + - CODEX_MAX_PEERS=${CODEX_MAX_PEERS:-160} + - CODEX_AGENT_STRING=${CODEX_AGENT_STRING:-Codex} + - CODEX_API_BINDADDR=${CODEX_API_BINDADDR:-0.0.0.0} + - CODEX_API_PORT=${CODEX_API_PORT:-8080} + - CODEX_REPO_KIND=${CODEX_REPO_KIND:-fs} + - CODEX_STORAGE_QUOTA=${CODEX_STORAGE_QUOTA:-8589934592} + - CODEX_BLOCK_TTL=${CODEX_BLOCK_TTL:-0} + # - CODEX_BLOCK_MI=${CODEX_BLOCK_MI} + - CODEX_BLOCK_MN=${CODEX_BLOCK_MN:-1000} + - CODEX_CACHE_SIZE=${CODEX_CACHE_SIZE:-0} + - CODEX_PERSISTENCE=${CODEX_PERSISTENCE:-false} + - CODEX_ETH_PROVIDER=${CODEX_ETH_PROVIDER:-ws://localhost:8545} + # - CODEX_ETH_ACCOUNT=${CODEX_ETH_ACCOUNT} + # - CODEX_MARKETPLACE_ADDRESS=${CODEX_MARKETPLACE_ADDRESS:-0x59b670e9fA9D0A427751Af201D676719a970857b} + - CODEX_VALIDATOR=${CODEX_VALIDATOR:-false} + - CODEX_VALIDATOR_MAX_SLOTS=${CODEX_VALIDATOR_MAX_SLOTS:-1000} + - NAT_IP_AUTO=false + - NAT_PUBLIC_IP_AUTO=https://ipinfo.io/ip + ports: + - 8080:8080/tcp # REST API + - 8008:8008/tcp # Metrics + - 2345:2345/tcp # libp2p + - 8090:8090/udp # DHT discovery + volumes: + - ./datadir:/datadir:z + networks: + - codex +networks: + codex: + driver: bridge diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh new file mode 100644 index 0000000..5618499 --- /dev/null +++ b/docker/docker-entrypoint.sh @@ -0,0 +1,76 @@ +#!/bin/bash + +# Environment variables from files +# If set to file path, read the file and export the variables +# If set to directory path, read all files in the directory and export the variables +if [[ -n "${ENV_PATH}" ]]; then + set -a + [[ -f "${ENV_PATH}" ]] && source "${ENV_PATH}" || for f in "${ENV_PATH}"/*; do source "$f"; done + set +a +fi + +# Should be passed if env variable is set. +# public IP should fetch ip.codex.storage + +# --logLevel= Sets log level [default: TRACE] +# --publicIp= Public IP address where this instance is reachable. [default: 45.82.185.194] +# --metricsAddress= Listen address of the metrics server [default: 0.0.0.0] +# --metricsPort=

Listen HTTP port of the metrics server [default: 8008] +# --dataDir=

Directory for storing data [default: crawler_data] +# --discoveryPort=

Port used for DHT [default: 8090] +# --bootNodes= Semi-colon-separated list of Codex bootstrap SPRs [default: testnet_sprs] +# --stepDelay= Delay in milliseconds per crawl step [default: 3000] +# --revisitDelay= Delay in minutes after which a node can be revisited [default: 1] (24h) + + +# Parameters +if [[ -z "${CODEX_NAT}" ]]; then + if [[ "${NAT_IP_AUTO}" == "true" && -z "${NAT_PUBLIC_IP_AUTO}" ]]; then + export CODEX_NAT="extip:$(hostname --ip-address)" + echo "Private: CODEX_NAT=${CODEX_NAT}" + elif [[ -n "${NAT_PUBLIC_IP_AUTO}" ]]; then + # Run for 60 seconds if fail + WAIT=120 + SECONDS=0 + SLEEP=5 + while (( SECONDS < WAIT )); do + IP=$(curl -s -f -m 5 "${NAT_PUBLIC_IP_AUTO}") + # Check if exit code is 0 and returned value is not empty + if [[ $? -eq 0 && -n "${IP}" ]]; then + export CODEX_NAT="extip:${IP}" + echo "Public: CODEX_NAT=${CODEX_NAT}" + break + else + # Sleep and check again + echo "Can't get Public IP - Retry in $SLEEP seconds / $((WAIT - SECONDS))" + sleep $SLEEP + fi + done + fi +fi + +# Stop Codex run if can't get NAT IP when requested +if [[ "${NAT_IP_AUTO}" == "true" && -z "${CODEX_NAT}" ]]; then + echo "Can't get Private IP - Stop Codex run" + exit 1 +elif [[ -n "${NAT_PUBLIC_IP_AUTO}" && -z "${CODEX_NAT}" ]]; then + echo "Can't get Public IP in $WAIT seconds - Stop Codex run" + exit 1 +fi + +# If marketplace is enabled from the testing environment, +# The file has to be written before Codex starts. +for key in PRIV_KEY ETH_PRIVATE_KEY; do + keyfile="private.key" + if [[ -n "${!key}" ]]; then + [[ "${key}" == "PRIV_KEY" ]] && echo "PRIV_KEY variable is deprecated and will be removed in the next releases, please use ETH_PRIVATE_KEY instead!" + echo "${!key}" > "${keyfile}" + chmod 600 "${keyfile}" + export CODEX_ETH_PRIVATE_KEY="${keyfile}" + echo "Private key set" + fi +done + +# Run +echo "Run Codex node" +exec "$@"