diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 54ebbc224..33f8295e9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -273,7 +273,7 @@ jobs: build/nimbus.exe --help mingw32-make ${DEFAULT_MAKE_FLAGS} test if [[ '${{ matrix.target.cpu }}' == 'amd64' ]]; then - mingw32-make ${DEFAULT_MAKE_FLAGS} test-reproducibility + mingw32-make ${DEFAULT_MAKE_FLAGS} fi - name: Run nimbus-eth1 tests (Linux) @@ -284,14 +284,15 @@ jobs: env CC=gcc make ${DEFAULT_MAKE_FLAGS} build/nimbus --help # CC, GOARCH, and CGO_ENABLED are needed to select correct compiler 32/64 bit - env CC=gcc GOARCH=${GOARCH} CXX=g++ CGO_ENABLED=1 make ${DEFAULT_MAKE_FLAGS} test test-reproducibility + env CC=gcc GOARCH=${GOARCH} CXX=g++ CGO_ENABLED=1 make ${DEFAULT_MAKE_FLAGS} test - name: Run nimbus-eth1 tests (Macos) if: runner.os == 'Macos' run: | + export ZERO_AR_DATE=1 # avoid timestamps in binaries DEFAULT_MAKE_FLAGS="-j${ncpu} ENABLE_EVMC=${ENABLE_EVMC}" make ${DEFAULT_MAKE_FLAGS} build/nimbus --help # "-static" option will not work for osx unless static system libraries are provided - make ${DEFAULT_MAKE_FLAGS} test test-reproducibility + make ${DEFAULT_MAKE_FLAGS} test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..36e834a1a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,262 @@ +on: + push: + tags: + - 'v*' # "v1.2.3" + +name: Upload Release Asset + +jobs: + build-amd64: + name: Linux AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-amd64 + cd dist + ARCHIVE=$(echo nimbus-eth1_Linux_amd64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus.sha512sum + retention-days: 2 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push a Docker image for end users + run: | + cd dist + tar -xzf ${{ steps.make_dist.outputs.archive }} + mv ${{ steps.make_dist.outputs.archive_dir }} ../docker/dist/binaries/nimbus-eth1 + cd ../docker/dist/binaries + REFNAME="${{ github.ref }}" + TAG="${REFNAME#refs/tags/}" + DOCKER_BUILDKIT=1 docker build -f Dockerfile.amd64 -t statusim/nimbus-eth1:amd64-${TAG} -t statusim/nimbus-eth1:amd64-latest . + docker push statusim/nimbus-eth1:amd64-${TAG} + docker push statusim/nimbus-eth1:amd64-latest + build-arm64: + name: Linux ARM64 release asset + runs-on: ubuntu-latest + steps: + - name: Install packages + env: + DEBIAN_FRONTEND: "noninteractive" + TZ: "Etc/UTC" + run: | + sudo apt-get -qq update + sudo apt-get -qq -y install binfmt-support qemu-user-static + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-arm64 + cd dist + ARCHIVE=$(echo nimbus-eth1_Linux_arm64v8_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Linux_arm64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus.sha512sum + retention-days: 2 + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build and push a Docker image for end users + run: | + cd dist + tar -xzf ${{ steps.make_dist.outputs.archive }} + mv ${{ steps.make_dist.outputs.archive_dir }} ../docker/dist/binaries/nimbus-eth1 + cd ../docker/dist/binaries + REFNAME="${{ github.ref }}" + TAG="${REFNAME#refs/tags/}" + cp -a /usr/bin/qemu-aarch64-static . + DOCKER_BUILDKIT=1 docker build -f Dockerfile.arm64 -t statusim/nimbus-eth1:arm64-${TAG} -t statusim/nimbus-eth1:arm64-latest . + docker push statusim/nimbus-eth1:arm64-${TAG} + docker push statusim/nimbus-eth1:arm64-latest + build-win64: + name: Windows AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-win64 + cd dist + ARCHIVE=$(echo nimbus-eth1_Windows_amd64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: Windows_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: Windows_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus.sha512sum + retention-days: 2 + build-macos-amd64: + name: macOS AMD64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-macos + cd dist + ARCHIVE=$(echo nimbus-eth1_macOS_amd64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_amd64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus.sha512sum + retention-days: 2 + build-macos-arm64: + name: macOS ARM64 release asset + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Build project + id: make_dist + run: | + make dist-macos-arm64 + cd dist + ARCHIVE=$(echo nimbus-eth1_macOS_arm64_*.tar.gz) + echo "::set-output name=archive::"${ARCHIVE} + echo "::set-output name=archive_dir::"${ARCHIVE%.tar.gz} + tar -xzf ${ARCHIVE} ${ARCHIVE%.tar.gz}/build/nimbus.sha512sum + - name: Upload archive artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_archive + path: ./dist/${{ steps.make_dist.outputs.archive }} + retention-days: 2 + - name: Upload checksum artefact + uses: actions/upload-artifact@v2 + with: + name: macOS_arm64_checksum + path: ./dist/${{ steps.make_dist.outputs.archive_dir }}/build/nimbus.sha512sum + retention-days: 2 + prepare-release: + name: Prepare release draft + needs: [build-amd64, build-arm64, build-win64, build-macos-amd64, build-macos-arm64] + runs-on: ubuntu-latest + steps: + - name: Download artefacts + uses: actions/download-artifact@v2 + - name: Create release notes + run: | + cat > release_notes.md <> release_notes.md + cat Linux_amd64_checksum/* >> release_notes.md + echo '# Linux ARM64' >> release_notes.md + cat Linux_arm64_checksum/* >> release_notes.md + echo '# Windows AMD64' >> release_notes.md + cat Windows_amd64_checksum/* >> release_notes.md + echo '# macOS AMD64' >> release_notes.md + cat macOS_amd64_checksum/* >> release_notes.md + echo '# macOS ARM64' >> release_notes.md + cat macOS_arm64_checksum/* >> release_notes.md + echo '```' >> release_notes.md + - name: Create release + id: create_release + uses: softprops/action-gh-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + draft: true + prerelease: false + body_path: release_notes.md + files: | + Linux_amd64_archive/* + Linux_arm64_archive/* + Windows_amd64_archive/* + macOS_amd64_archive/* + macOS_arm64_archive/* + - name: Delete artefacts + uses: geekyeggo/delete-artifact@v1 + with: + failOnError: false + name: | + Linux_amd64_archive + Linux_amd64_checksum + Linux_arm64_archive + Linux_arm64_checksum + Windows_amd64_archive + Windows_amd64_checksum + macOS_amd64_archive + macOS_amd64_checksum + macOS_arm64_archive + macOS_arm64_checksum + + - name: Login to Docker Hub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build multi-arch Docker image + run: | + REFNAME="${{ github.ref }}" + TAG="${REFNAME#refs/tags/}" + docker manifest create \ + statusim/nimbus-eth1:multiarch-${TAG} \ + --amend statusim/nimbus-eth1:amd64-${TAG} \ + --amend statusim/nimbus-eth1:arm64-${TAG} + docker manifest push statusim/nimbus-eth1:multiarch-${TAG} + docker manifest create \ + statusim/nimbus-eth1:multiarch-latest \ + --amend statusim/nimbus-eth1:amd64-latest \ + --amend statusim/nimbus-eth1:arm64-latest + docker manifest push statusim/nimbus-eth1:multiarch-latest + diff --git a/.gitignore b/.gitignore index 7f0c48cb0..4f1ce2069 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ nimcache /.update.timestamp *.generated.nim + +/dist + diff --git a/.gitmodules b/.gitmodules index 63a2b5dd8..8c02c60ee 100644 --- a/.gitmodules +++ b/.gitmodules @@ -68,11 +68,6 @@ url = https://github.com/status-im/nim-confutils.git ignore = dirty branch = master -[submodule "vendor/nim-beacon-chain"] - path = vendor/nim-beacon-chain - url = https://github.com/status-im/nim-beacon-chain.git - ignore = dirty - branch = master [submodule "vendor/nim-blscurve"] path = vendor/nim-blscurve url = https://github.com/status-im/nim-blscurve @@ -88,16 +83,6 @@ url = https://github.com/OpenSystemsLab/tempfile.nim.git ignore = dirty branch = master -[submodule "vendor/go/src/github.com/libp2p/go-libp2p-daemon"] - path = vendor/go/src/github.com/libp2p/go-libp2p-daemon - url = https://github.com/libp2p/go-libp2p-daemon.git - ignore = dirty - branch = master -[submodule "vendor/nim-libp2p"] - path = vendor/nim-libp2p - url = https://github.com/status-im/nim-libp2p.git - ignore = dirty - branch = master [submodule "vendor/nim-nat-traversal"] path = vendor/nim-nat-traversal url = https://github.com/status-im/nim-nat-traversal.git @@ -118,21 +103,11 @@ url = https://github.com/status-im/nim-web3.git ignore = dirty branch = master -[submodule "vendor/nim-byteutils"] - path = vendor/nim-byteutils - url = https://github.com/status-im/nim-byteutils.git - ignore = dirty - branch = master [submodule "vendor/nim-snappy"] path = vendor/nim-snappy url = https://github.com/status-im/nim-snappy.git ignore = dirty branch = master -[submodule "vendor/NimYAML"] - path = vendor/NimYAML - url = https://github.com/flyx/NimYAML.git - ignore = dirty - branch = master [submodule "vendor/nimbus-build-system"] path = vendor/nimbus-build-system url = https://github.com/status-im/nimbus-build-system.git diff --git a/Makefile b/Makefile index 5ec036b4e..88a4b2058 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -# Copyright (c) 2018-2021 Status Research & Development GmbH. Licensed under +# Copyright (c) 2018-2022 Status Research & Development GmbH. Licensed under # either of: # - Apache License, version 2.0 # - MIT license @@ -35,7 +35,14 @@ TOOLS_CSV := $(subst $(SPACE),$(COMMA),$(TOOLS)) clean \ libnimbus.so \ libnimbus.a \ - libbacktrace + libbacktrace \ + dist-amd64 \ + dist-arm64 \ + dist-arm \ + dist-win64 \ + dist-macos \ + dist-macos-arm64 \ + dist ifeq ($(NIM_PARAMS),) # "variables.mk" was not included, so we update the submodules. @@ -64,29 +71,11 @@ USE_MIRACL := 0 # default: use nim native evm ENABLE_EVMC := 0 -# "-d:release" implies "--stacktrace:off" and it cannot be added to config.nims +# "-d:release" cannot be added to config.nims +NIM_PARAMS += -d:release + ifeq ($(USE_LIBBACKTRACE), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:debug -d:disable_libbacktrace -else -NIM_PARAMS := $(NIM_PARAMS) -d:release -endif - -ifneq ($(USE_MIRACL), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:BLS_FORCE_BACKEND=miracl -endif - -ifneq ($(ENABLE_EVMC), 0) -NIM_PARAMS := $(NIM_PARAMS) -d:evmc_enabled -endif - -# disabled by default, enable with ENABLE_VM2SLOW=1 -ifneq ($(if $(ENABLE_VM2LOWMEM),$(ENABLE_VM2LOWMEM),0),0) -NIM_PARAMS := $(NIM_PARAMS) -d:vm2_enabled -d:lowmem:1 -else -# disabled by default, enable with ENABLE_VM2=1 -ifneq ($(if $(ENABLE_VM2),$(ENABLE_VM2),0),0) -NIM_PARAMS := $(NIM_PARAMS) -d:vm2_enabled -endif + NIM_PARAMS += -d:disable_libbacktrace endif deps: | deps-common nat-libs nimbus.nims @@ -94,6 +83,24 @@ ifneq ($(USE_LIBBACKTRACE), 0) deps: | libbacktrace endif +ifneq ($(USE_MIRACL), 0) + NIM_PARAMS += -d:BLS_FORCE_BACKEND=miracl +endif + +ifneq ($(ENABLE_EVMC), 0) + NIM_PARAMS += -d:evmc_enabled +endif + +# disabled by default, enable with ENABLE_VM2LOWMEM=1 +ifneq ($(ENABLE_VM2LOWMEM), 0) + NIM_PARAMS += -d:vm2_enabled -d:lowmem:1 +else + # disabled by default, enable with ENABLE_VM2=1 + ifneq ($(ENABLE_VM2), 0) + NIM_PARAMS += -d:vm2_enabled + endif +endif + #- deletes and recreates "nimbus.nims" which on Windows is a copy instead of a proper symlink update: | update-common rm -rf nimbus.nims && \ @@ -108,7 +115,7 @@ $(TOOLS): | build deps # a phony target, because teaching `make` how to do conditional recompilation of Nim projects is too complicated nimbus: | build deps echo -e $(BUILD_MSG) "build/$@" && \ - $(ENV_SCRIPT) nim nimbus $(NIM_PARAMS) nimbus.nims + $(ENV_SCRIPT) nim c $(NIM_PARAMS) -d:chronicles_log_level=TRACE -o:build/$@ "nimbus/$@.nim" # symlink nimbus.nims: @@ -122,7 +129,11 @@ libbacktrace: test: | build deps $(ENV_SCRIPT) nim test $(NIM_PARAMS) nimbus.nims -# primitive reproducibility test +# Primitive reproducibility test. +# +# On some platforms, with some GCC versions, it may not be possible to get a +# deterministic order for debugging info sections - even with +# "-frandom-seed=...". Striping the binaries should make them identical, though. test-reproducibility: + [ -e build/nimbus ] || $(MAKE) V=0 nimbus; \ MD5SUM1=$$($(MD5SUM) build/nimbus | cut -d ' ' -f 1) && \ @@ -170,7 +181,7 @@ fluffy-test-portal-testnet: | build deps # usual cleaning clean: | clean-common - rm -rf build/{nimbus,fluffy,$(TOOLS_CSV),all_tests,test_rpc,all_fluffy_tests,portalcli} + rm -rf build/{nimbus,fluffy,$(TOOLS_CSV),all_tests,test_rpc,all_fluffy_tests,portalcli,*.dSYM} ifneq ($(USE_LIBBACKTRACE), 0) + $(MAKE) -C vendor/nim-libbacktrace clean $(HANDLE_OUTPUT) endif @@ -187,4 +198,38 @@ endif # can be found in Git history. Look for the `nimbus-eth1` commit that adds # this comment and removes `wrappers/*`. +dist-amd64: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh amd64 + +dist-arm64: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh arm64 + +# We get an ICE on RocksDB-7.0.2 with "arm-linux-gnueabihf-g++ (Ubuntu 9.4.0-1ubuntu1~20.04.1) 9.4.0" +# and with "arm-linux-gnueabihf-g++ (Ubuntu 10.3.0-1ubuntu1) 10.3.0". +#dist-arm: + #+ MAKE="$(MAKE)" \ + #scripts/make_dist.sh arm + +dist-win64: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh win64 + +dist-macos: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh macos + +dist-macos-arm64: + + MAKE="$(MAKE)" \ + scripts/make_dist.sh macos-arm64 + +dist: + + $(MAKE) --no-print-directory dist-amd64 + + $(MAKE) --no-print-directory dist-arm64 + #+ $(MAKE) --no-print-directory dist-arm + + $(MAKE) --no-print-directory dist-win64 + + $(MAKE) --no-print-directory dist-macos + + $(MAKE) --no-print-directory dist-macos-arm64 + endif # "variables.mk" was not included diff --git a/README.md b/README.md index a39455d8b..6474756ee 100644 --- a/README.md +++ b/README.md @@ -6,12 +6,11 @@ ![GH action-fluffy](https://github.com/status-im/nimbus-eth1/workflows/fluffy%20CI/badge.svg) [![Discord: Nimbus](https://img.shields.io/badge/discord-nimbus-orange.svg)](https://discord.gg/XRxWahP) -[![Gitter: #status-im/nimbus](https://img.shields.io/badge/gitter-status--im%2Fnimbus-orange.svg)](https://gitter.im/status-im/nimbus) [![Status: #nimbus-general](https://img.shields.io/badge/status-nimbus--general-orange.svg)](https://get.status.im/chat/public/nimbus-general) ## Introduction -This repository contains our development work on our execution-layer client to pair with [our consensus-layer client](https://github.com/status-im/nimbus-eth2). This client focuses on efficiency and security and strives to be as light-weight as possible in terms of resources used. +This repository contains development work on an execution-layer client to pair with [our consensus-layer client](https://github.com/status-im/nimbus-eth2). This client focuses on efficiency and security and strives to be as light-weight as possible in terms of resources used. This repository is also home to [fluffy](./fluffy/README.md), a [Portal Network](https://github.com/ethereum/stateless-ethereum-specs/blob/master/portal-network.md) @@ -28,7 +27,7 @@ Monthly development updates are shared Some recent highlights include: - Renewed funding from the EF to accelerate development - Completed Berlin and London fork compatibility (EIP-1559). It now passes nearly all the EF Hive testsuite, and 100% of contract execution tests (47,951 tests) -- New GraphQL and WebSocket APIs, complementing JSON-RPC +- New GraphQL and WebSocket APIs, complementing JSON-RPC - EVMC compatibility, supporting third-party optimised EVM plugins - Up to 100x memory saving during contract executions - Asynchronous EVM to execute many contracts in parallel, while they wait for data from the network @@ -42,8 +41,6 @@ For more detailed write-ups on the development progress, follow the ## Building & Testing -_We currently do not guarantee that Nimbus will work on Windows._ - ### Prerequisites * [RocksDB](https://github.com/facebook/rocksdb/) @@ -84,7 +81,9 @@ nix-shell default.nix ```bash # The first `make` invocation will update all Git submodules. # You'll run `make update` after each `git pull`, in the future, to keep those submodules up to date. -make nimbus +# Assuming you have 4 CPU cores available, you can ask Make to run 4 parallel jobs, with "-j4". + +make -j4 nimbus # See available command line options build/nimbus --help @@ -94,7 +93,7 @@ build/nimbus # Update to latest version git pull -make update +make -j4 update # Run tests make test @@ -212,22 +211,22 @@ build/nimbus ``` ### Experimental make variables -Apart from standard make flags (see link in the next [chapter](#devel-tips)), -the following make variables can be set to control which version of a virtual +Apart from standard Make flags (see link in the next [chapter](#devel-tips)), +the following Make variables can be set to control which version of a virtual engine is compiled. The variables are listed with decreasing priority (in case of doubt, the lower prioritised variable is ignored when the higher on is available.) * ENABLE_EVMC=1
- Enable mostly EVMC compliant wrapper around the native nim VM + Enable mostly EVMC compliant wrapper around the native Nim VM * ENABLE_VM2LOWMEM=1
- Enable new re-factored version of the native nim VM. This version is not + Enable new re-factored version of the native Nim VM. This version is not optimised and coded in a way so that low memory compilers can handle it (observed on 32 bit windows 7.) * ENABLE_VM2=1
- Enable new re-factored version of the native nim VM. + Enable new re-factored version of the native Nim VM. For these variables, using <variable>=0 is ignored and <variable>=2 has the same effect as <variable>=1 (ditto for other numbers.) diff --git a/TransactionTests.md b/TransactionTests.md index d74a25d19..27d59f35e 100644 --- a/TransactionTests.md +++ b/TransactionTests.md @@ -16,10 +16,11 @@ OK: 4/4 Fail: 0/4 Skip: 0/4 + DataTestNotEnoughGAS.json OK + DataTestZeroBytes.json OK + String10MbData.json OK ++ String10MbDataNotEnoughGAS.json OK + dataTx_bcValidBlockTest.json OK + dataTx_bcValidBlockTestFrontier.json OK ``` -OK: 8/8 Fail: 0/8 Skip: 0/8 +OK: 9/9 Fail: 0/9 Skip: 0/9 ## ttEIP2028 ```diff + DataTestInsufficientGas2028.json OK @@ -29,30 +30,39 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 ## ttGasLimit ```diff + NotEnoughGasLimit.json OK -+ TransactionWithGasLimitOverflow.json OK -+ TransactionWithGasLimitOverflow2.json OK -+ TransactionWithGasLimitOverflow63.json OK -+ TransactionWithGasLimitOverflow63_1.json OK ++ TransactionWithGasLimitOverflow256.json OK ++ TransactionWithGasLimitOverflow64.json OK ++ TransactionWithGasLimitOverflowZeros64.json OK + TransactionWithGasLimitxPriceOverflow.json OK -+ TransactionWithGasLimitxPriceOverflow2.json OK -+ TransactionWithHighGas.json OK -+ TransactionWithHihghGasLimit63m1.json OK ++ TransactionWithHighGasLimit63.json OK ++ TransactionWithHighGasLimit63Minus1.json OK ++ TransactionWithHighGasLimit63Plus1.json OK ++ TransactionWithHighGasLimit64Minus1.json OK ++ TransactionWithLeadingZerosGasLimit.json OK ``` -OK: 9/9 Fail: 0/9 Skip: 0/9 +OK: 10/10 Fail: 0/10 Skip: 0/10 ## ttGasPrice ```diff + TransactionWithGasPriceOverflow.json OK + TransactionWithHighGasPrice.json OK + TransactionWithHighGasPrice2.json OK ++ TransactionWithLeadingZerosGasPrice.json OK ``` -OK: 3/3 Fail: 0/3 Skip: 0/3 +OK: 4/4 Fail: 0/4 Skip: 0/4 ## ttNonce ```diff ++ TransactionWithEmptyBigInt.json OK + TransactionWithHighNonce256.json OK + TransactionWithHighNonce32.json OK ++ TransactionWithHighNonce64.json OK ++ TransactionWithHighNonce64Minus1.json OK ++ TransactionWithHighNonce64Minus2.json OK ++ TransactionWithHighNonce64Plus1.json OK ++ TransactionWithLeadingZerosNonce.json OK + TransactionWithNonceOverflow.json OK ++ TransactionWithZerosBigInt.json OK ``` -OK: 3/3 Fail: 0/3 Skip: 0/3 +OK: 10/10 Fail: 0/10 Skip: 0/10 ## ttRSValue ```diff + RightVRSTestF0000000a.json OK @@ -71,6 +81,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + TransactionWithRvalueHigh.json OK + TransactionWithRvalueOverflow.json OK + TransactionWithRvaluePrefixed00.json OK ++ TransactionWithRvaluePrefixed00BigInt.json OK + TransactionWithRvalueTooHigh.json OK + TransactionWithSvalue0.json OK + TransactionWithSvalue1.json OK @@ -80,13 +91,15 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + TransactionWithSvalueLessThan_c_secp256k1n_x05.json OK + TransactionWithSvalueOverflow.json OK + TransactionWithSvaluePrefixed00.json OK ++ TransactionWithSvaluePrefixed00BigInt.json OK + TransactionWithSvalueTooHigh.json OK + unpadedRValue.json OK ``` -OK: 27/27 Fail: 0/27 Skip: 0/27 +OK: 29/29 Fail: 0/29 Skip: 0/29 ## ttSignature ```diff + EmptyTransaction.json OK ++ PointAtInfinity.json OK + RSsecp256k1.json OK + RightVRSTest.json OK + SenderTest.json OK @@ -120,9 +133,11 @@ OK: 27/27 Fail: 0/27 Skip: 0/27 + invalidSignature.json OK + libsecp256k1test.json OK ``` -OK: 33/33 Fail: 0/33 Skip: 0/33 +OK: 34/34 Fail: 0/34 Skip: 0/34 ## ttVValue ```diff ++ InvalidChainID0ValidV0.json OK ++ InvalidChainID0ValidV1.json OK + V_equals37.json OK + V_equals38.json OK + V_overflow32bit.json OK @@ -137,6 +152,12 @@ OK: 33/33 Fail: 0/33 Skip: 0/33 + V_wrongvalue_124.json OK + V_wrongvalue_ff.json OK + V_wrongvalue_ffff.json OK ++ ValidChainID1InvalidV0.json OK ++ ValidChainID1InvalidV00.json OK ++ ValidChainID1InvalidV01.json OK ++ ValidChainID1InvalidV1.json OK ++ ValidChainID1ValidV0.json OK ++ ValidChainID1ValidV1.json OK + WrongVRSTestVEqual26.json OK + WrongVRSTestVEqual29.json OK + WrongVRSTestVEqual31.json OK @@ -144,13 +165,14 @@ OK: 33/33 Fail: 0/33 Skip: 0/33 + WrongVRSTestVEqual39.json OK + WrongVRSTestVEqual41.json OK ``` -OK: 20/20 Fail: 0/20 Skip: 0/20 +OK: 28/28 Fail: 0/28 Skip: 0/28 ## ttValue ```diff + TransactionWithHighValue.json OK + TransactionWithHighValueOverflow.json OK ++ TransactionWithLeadingZerosValue.json OK ``` -OK: 2/2 Fail: 0/2 Skip: 0/2 +OK: 3/3 Fail: 0/3 Skip: 0/3 ## ttWrongRLP ```diff + RLPAddressWithFirstZeros.json OK @@ -167,8 +189,6 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + RLPNonceWithFirstZeros.json OK + RLPTransactionGivenAsArray.json OK + RLPValueWithFirstZeros.json OK -+ RLPWrongAddress.json OK -+ RLPWrongData.json OK + RLPgasLimitWithFirstZeros.json OK + RLPgasPriceWithFirstZeros.json OK + TRANSCT_HeaderGivenAsArray_0.json OK @@ -184,16 +204,6 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + TRANSCT__RandomByteAtRLP_8.json OK + TRANSCT__RandomByteAtRLP_9.json OK + TRANSCT__RandomByteAtTheEnd.json OK -+ TRANSCT__WrongCharAtRLP_0.json OK -+ TRANSCT__WrongCharAtRLP_1.json OK -+ TRANSCT__WrongCharAtRLP_2.json OK -+ TRANSCT__WrongCharAtRLP_3.json OK -+ TRANSCT__WrongCharAtRLP_4.json OK -+ TRANSCT__WrongCharAtRLP_5.json OK -+ TRANSCT__WrongCharAtRLP_6.json OK -+ TRANSCT__WrongCharAtRLP_7.json OK -+ TRANSCT__WrongCharAtRLP_8.json OK -+ TRANSCT__WrongCharAtRLP_9.json OK + TRANSCT__ZeroByteAtRLP_0.json OK + TRANSCT__ZeroByteAtRLP_1.json OK + TRANSCT__ZeroByteAtRLP_2.json OK @@ -204,7 +214,6 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + TRANSCT__ZeroByteAtRLP_7.json OK + TRANSCT__ZeroByteAtRLP_8.json OK + TRANSCT__ZeroByteAtRLP_9.json OK -+ TRANSCT__ZeroByteAtTheEnd.json OK + TRANSCT_data_GivenAsList.json OK + TRANSCT_gasLimit_GivenAsList.json OK + TRANSCT_gasLimit_Prefixed0000.json OK @@ -221,10 +230,10 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + TRANSCT_to_TooLarge.json OK + TRANSCT_to_TooShort.json OK + aCrashingRLP.json OK -+ aMalicousRLP.json OK ++ aMaliciousRLP.json OK + tr201506052141PYTHON.json OK ``` -OK: 70/70 Fail: 0/70 Skip: 0/70 +OK: 57/57 Fail: 0/57 Skip: 0/57 ---TOTAL--- -OK: 181/181 Fail: 0/181 Skip: 0/181 +OK: 190/190 Fail: 0/190 Skip: 0/190 diff --git a/config.nims b/config.nims index f2d94747b..495eda013 100644 --- a/config.nims +++ b/config.nims @@ -1,3 +1,5 @@ +import strutils + if defined(release): switch("nimcache", "nimcache/release/$projectName") else: @@ -15,6 +17,10 @@ if defined(windows): # set the IMAGE_FILE_LARGE_ADDRESS_AWARE flag so we can use PAE, if enabled, and access more than 2 GiB of RAM switch("passL", "-Wl,--large-address-aware") + # Avoid some rare stack corruption while using exceptions with a SEH-enabled + # toolchain: https://github.com/status-im/nimbus-eth2/issues/3121 + switch("define", "nimRawSetjmp") + # This helps especially for 32-bit x86, which sans SSE2 and newer instructions # requires quite roundabout code generation for cryptography, and other 64-bit # and larger arithmetic use cases, along with register starvation issues. When @@ -28,8 +34,17 @@ if defined(windows): # if defined(disableMarchNative): if defined(i386) or defined(amd64): - switch("passC", "-mssse3") - switch("passL", "-mssse3") + if defined(macosx): + # https://support.apple.com/kb/SP777 + # "macOS Mojave - Technical Specifications": EOL as of 2021-10 so macOS + # users on pre-Nehalem must be running either some Hackintosh, or using + # an unsupported macOS version beyond that most recently EOL'd. Nehalem + # supports instruction set extensions through SSE4.2 and POPCNT. + switch("passC", "-march=nehalem") + switch("passL", "-march=nehalem") + else: + switch("passC", "-mssse3") + switch("passL", "-mssse3") elif defined(macosx) and defined(arm64): # Apple's Clang can't handle "-march=native" on M1: https://github.com/status-im/nimbus-eth2/issues/2758 switch("passC", "-mcpu=apple-a14") @@ -63,19 +78,39 @@ if not defined(windows): switch("define", "withoutPCRE") -# the default open files limit is too low on macOS (512), breaking the -# "--debugger:native" build. It can be increased with `ulimit -n 1024`. -if not defined(macosx): +when not defined(disable_libbacktrace): + --define:nimStackTraceOverride + switch("import", "libbacktrace") +else: + --stacktrace:on + --linetrace:on + +var canEnableDebuggingSymbols = true +if defined(macosx): + # The default open files limit is too low on macOS (512), breaking the + # "--debugger:native" build. It can be increased with `ulimit -n 1024`. + let openFilesLimitTarget = 1024 + var openFilesLimit = 0 + try: + openFilesLimit = staticExec("ulimit -n").strip(chars = Whitespace + Newlines).parseInt() + if openFilesLimit < openFilesLimitTarget: + echo "Open files limit too low to enable debugging symbols and lightweight stack traces." + echo "Increase it with \"ulimit -n " & $openFilesLimitTarget & "\"" + canEnableDebuggingSymbols = false + except: + echo "ulimit error" +# We ignore this resource limit on Windows, where a default `ulimit -n` of 256 +# in Git Bash is apparently ignored by the OS, and on Linux where the default of +# 1024 is good enough for us. + +if canEnableDebuggingSymbols: # add debugging symbols and original files and line numbers --debugger:native - if not (defined(windows) and defined(i386)) and not defined(disable_libbacktrace): - # light-weight stack traces using libbacktrace and libunwind - --define:nimStackTraceOverride - switch("import", "libbacktrace") --define:nimOldCaseObjects # https://github.com/status-im/nim-confutils/issues/9 -# libnimbus.so needs position-independent code -switch("passC", "-fPIC") + +# `switch("warning[CaseTransition]", "off")` fails with "Error: invalid command line option: '--warning[CaseTransition]'" +switch("warning", "CaseTransition:off") # The compiler doth protest too much, methinks, about all these cases where it can't # do its (N)RVO pass: https://github.com/nim-lang/RFCs/issues/230 diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index c3de21f36..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM debian:buster-slim AS build - -RUN apt-get update \ - && apt-get install -y --fix-missing build-essential make git libpcre3-dev librocksdb-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ARG GIT_REVISION -ENV NPROC=2 - -RUN git clone https://github.com/status-im/nimbus-eth1.git \ - && cd nimbus-eth1 \ - && git reset --hard ${GIT_REVISION} \ - && make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update - -RUN cd nimbus-eth1 && \ - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" nimbus && \ - mv build/nimbus /usr/bin/ - -# --------------------------------- # -# Starting new image to reduce size # -# --------------------------------- # -FROM debian:buster-slim AS deploy - -RUN apt-get update \ - && apt-get install -y librocksdb-dev \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -COPY --from=build /usr/bin/nimbus /usr/bin/nimbus - -MAINTAINER Zahary Karadjov -LABEL description="Nimbus: an Ethereum 2.0 Sharding Client for Resource-Restricted Devices" - -ENTRYPOINT ["/usr/bin/nimbus"] diff --git a/docker/Makefile b/docker/Makefile deleted file mode 100644 index e322538d2..000000000 --- a/docker/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -# These default settings can be overriden by exporting env variables - -GIT_REVISION ?= $(git rev-parse HEAD) - -IMAGE_TAG ?= nimbus_latest -IMAGE_NAME ?= statusteam/nimbus_beacon_node:$(IMAGE_TAG) - -build: $(NIX_INSTALL) - docker build \ - --build-arg="GIT_REVISION=$(GIT_REVISION)" \ - -t $(IMAGE_NAME) . - -push: build - docker push $(IMAGE_NAME) - diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..5da0821ef --- /dev/null +++ b/docker/README.md @@ -0,0 +1,23 @@ +## directory layout + +### dist/base_image/ + +Base Docker images for building distributable binaries. Uploaded to +Docker Hub (we need them to reproduce officially released builds). + +### dist/ + +Dockerfiles used to build local Docker images based on the base images +described above. Only used for generating distributable binaries. Not uploaded +to Docker Hub. + +### dist/binaries/ + +Docker images for end-users, obtained by copying distributable binaries inside +official Debian images. Uploaded to Docker Hub as part of the CI release process. + +Also contains some example `docker-compose` configuration files. + +## more details + +See the ["Binary distribution internals"](https://nimbus.guide/distribution_internals.html) page of the Nimbus book. diff --git a/docker/dist/0001-Makefile-support-Mingw-more-cross-compilation.patch b/docker/dist/0001-Makefile-support-Mingw-more-cross-compilation.patch new file mode 100644 index 000000000..a6c6c5ecc --- /dev/null +++ b/docker/dist/0001-Makefile-support-Mingw-more-cross-compilation.patch @@ -0,0 +1,248 @@ +From d35dfeba58ea107b25ea73f9aa1f873461bac169 Mon Sep 17 00:00:00 2001 +From: =?UTF-8?q?=C8=98tefan=20Talpalaru?= +Date: Thu, 24 Mar 2022 10:39:21 +0100 +Subject: [PATCH] Makefile: support Mingw + more cross-compilation + +Tested on an Ubuntu 20.04 host, targeting: +- Linux ARM64, using aarch64-linux-gnu-g++ +- macOS AMD64 and ARM64, using https://github.com/tpoechtrager/osxcross +- Windows AMD64, using Mingw-w64 from https://github.com/mxe/mxe + +"build_tools/build_detect_platform" and "Makefile" have been partially +cleaned, in the process. There is more stuff left to clean there. + +The Makefile now passes more command line arguments to the detection +script via environment variables. +--- + HISTORY.md | 4 ++ + INSTALL.md | 16 ++++++ + Makefile | 87 +++++++++++++++++-------------- + build_tools/build_detect_platform | 39 +++++++------- + 4 files changed, 90 insertions(+), 56 deletions(-) + +diff --git a/Makefile b/Makefile +index a1d0a3303..befb279e2 100644 +--- a/Makefile ++++ b/Makefile +@@ -118,16 +118,19 @@ OPT += -momit-leaf-frame-pointer + endif + endif + +-ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1)) +-CXXFLAGS += -DHAS_ALTIVEC +-CFLAGS += -DHAS_ALTIVEC +-HAS_ALTIVEC=1 +-endif ++# A cross-compiler can support PPC, even though that's not our target. ++ifneq ($(CROSS_COMPILE), true) ++ ifeq (,$(shell $(CXX) -fsyntax-only -maltivec -xc /dev/null 2>&1)) ++ CXXFLAGS += -DHAS_ALTIVEC ++ CFLAGS += -DHAS_ALTIVEC ++ HAS_ALTIVEC=1 ++ endif + +-ifeq (,$(shell $(CXX) -fsyntax-only -mcpu=power8 -xc /dev/null 2>&1)) +-CXXFLAGS += -DHAVE_POWER8 +-CFLAGS += -DHAVE_POWER8 +-HAVE_POWER8=1 ++ ifeq (,$(shell $(CXX) -fsyntax-only -mcpu=power8 -xc /dev/null 2>&1)) ++ CXXFLAGS += -DHAVE_POWER8 ++ CFLAGS += -DHAVE_POWER8 ++ HAVE_POWER8=1 ++ endif + endif + + # if we're compiling for shared libraries, add the shared flags +@@ -136,34 +139,31 @@ CXXFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL + CFLAGS += $(PLATFORM_SHARED_CFLAGS) -DROCKSDB_DLL + endif + +-# if we're compiling for release, compile without debug code (-DNDEBUG) +-ifeq ($(DEBUG_LEVEL),0) +-OPT += -DNDEBUG +- +-ifneq ($(USE_RTTI), 1) +- CXXFLAGS += -fno-rtti +-else +- CXXFLAGS += -DROCKSDB_USE_RTTI +-endif +-else +-ifneq ($(USE_RTTI), 0) +- CXXFLAGS += -DROCKSDB_USE_RTTI ++ifeq ($(DEBUG_LEVEL),0) # release build ++ # if we're compiling for release, compile without debug code (-DNDEBUG) ++ OPT += -DNDEBUG ++ USE_RTTI := 0 ++else # debug build ++ USE_RTTI := 1 ++ ++ ifdef ASSERT_STATUS_CHECKED ++ # For ASC, turn off constructor elision, preventing the case where a constructor returned ++ # by a method may pass the ASC check if the status is checked in the inner method. Forcing ++ # the copy constructor to be invoked disables the optimization and will cause the calling method ++ # to check the status in order to prevent an error from being raised. ++ PLATFORM_CXXFLAGS += -fno-elide-constructors ++ ifeq ($(filter -DROCKSDB_ASSERT_STATUS_CHECKED,$(OPT)),) ++ OPT += -DROCKSDB_ASSERT_STATUS_CHECKED ++ endif ++ endif ++ ++ $(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) ++endif # DEBUG_LEVEL ++ ++ifeq ($(USE_RTTI), 0) ++ CXXFLAGS += -fno-rtti + else +- CXXFLAGS += -fno-rtti +-endif +- +-ifdef ASSERT_STATUS_CHECKED +-# For ASC, turn off constructor elision, preventing the case where a constructor returned +-# by a method may pass the ASC check if the status is checked in the inner method. Forcing +-# the copy constructor to be invoked disables the optimization and will cause the calling method +-# to check the status in order to prevent an error from being raised. +-PLATFORM_CXXFLAGS += -fno-elide-constructors +-ifeq ($(filter -DROCKSDB_ASSERT_STATUS_CHECKED,$(OPT)),) +- OPT += -DROCKSDB_ASSERT_STATUS_CHECKED +-endif +-endif +- +-$(warning Warning: Compiling in debug mode. Don't use the resulting binary in production) ++ CXXFLAGS += -DROCKSDB_USE_RTTI + endif + + # `USE_LTO=1` enables link-time optimizations. Among other things, this enables +@@ -217,12 +217,18 @@ AM_SHARE = $(AM_V_CCLD) $(CXX) $(PLATFORM_SHARED_LDFLAGS)$@ -L. $(patsubst lib%. + # Export some common variables that might have been passed as Make variables + # instead of environment variables. + dummy := $(shell (export ROCKSDB_ROOT="$(CURDIR)"; \ +- export CXXFLAGS="$(EXTRA_CXXFLAGS)"; \ +- export LDFLAGS="$(EXTRA_LDFLAGS)"; \ ++ export CC="$(CC)"; \ ++ export CXX="$(CXX)"; \ ++ export AR="$(AR)"; \ ++ export CFLAGS="$(CFLAGS)"; \ ++ export CXXFLAGS="$(CXXFLAGS)"; \ + export COMPILE_WITH_ASAN="$(COMPILE_WITH_ASAN)"; \ + export COMPILE_WITH_TSAN="$(COMPILE_WITH_TSAN)"; \ + export COMPILE_WITH_UBSAN="$(COMPILE_WITH_UBSAN)"; \ + export PORTABLE="$(PORTABLE)"; \ ++ export CROSS_COMPILE="$(CROSS_COMPILE)"; \ ++ export TARGET_OS="$(TARGET_OS)"; \ ++ export TARGET_ARCHITECTURE="$(TARGET_ARCHITECTURE)"; \ + export ROCKSDB_NO_FBCODE="$(ROCKSDB_NO_FBCODE)"; \ + export USE_CLANG="$(USE_CLANG)"; \ + export LIB_MODE="$(LIB_MODE)"; \ +@@ -504,6 +510,11 @@ ifeq ($(NO_THREEWAY_CRC32C), 1) + CXXFLAGS += -DNO_THREEWAY_CRC32C + endif + ++# The original CFLAGS and CXXFLAGS have already been included in ++# PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS, but we can't avoid the duplication ++# here because more parameters have been appended in the mean time. ++# TODO: move all the new flags to PLATFORM_* after "make_config.mk" is ++# included, so we can := instead of += here. + CFLAGS += $(C_WARNING_FLAGS) $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT) + CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers + +diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform +index 4dc9dd31b..da4b50c69 100755 +--- a/build_tools/build_detect_platform ++++ b/build_tools/build_detect_platform +@@ -45,16 +45,15 @@ if test -z "$OUTPUT"; then + exit 1 + fi + ++PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}" ++ + # we depend on C++17, but should be compatible with newer standards + if [ "$ROCKSDB_CXX_STANDARD" ]; then +- PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD" ++ PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -std=$ROCKSDB_CXX_STANDARD" + else +- PLATFORM_CXXFLAGS="-std=c++17" ++ PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -std=c++17" + fi + +-# we currently depend on POSIX platform +-COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" +- + # Default to fbcode gcc on internal fb machines + if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then + FBCODE_BUILD="true" +@@ -136,10 +135,11 @@ if test -z "$WATCH"; then + WATCH=watch + fi + +-COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}" +-CROSS_COMPILE= +-PLATFORM_CCFLAGS= +-PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS" ++if [[ ! "$TARGET_OS" =~ MINGW ]]; then ++ COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX" ++fi ++ ++PLATFORM_CCFLAGS="${CFLAGS}" + PLATFORM_SHARED_EXT="so" + PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl," + PLATFORM_SHARED_CFLAGS="-fPIC" +@@ -249,7 +249,7 @@ EOF + Cygwin) + PLATFORM=CYGWIN + PLATFORM_SHARED_CFLAGS="" +- PLATFORM_CXXFLAGS="-std=gnu++11" ++ #PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -std=gnu++11" + COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN" + if [ -z "$USE_CLANG" ]; then + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp" +@@ -259,6 +259,10 @@ EOF + PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt" + # PORT_FILES=port/linux/linux_specific.cc + ;; ++ MINGW*) ++ PLATFORM=MINGW ++ COMMON_FLAGS="$COMMON_FLAGS -DWIN32 -DOS_WIN -D_MBCS -DWIN64 -DNOMINMAX -D_WIN32_WINNT=_WIN32_WINNT_VISTA -D_POSIX_C_SOURCE=1" ++ ;; + OS_ANDROID_CROSSCOMPILE) + PLATFORM=OS_ANDROID + COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_ANDROID -DROCKSDB_PLATFORM_POSIX" +@@ -271,7 +275,6 @@ EOF + exit 1 + esac + +-PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}" + JAVA_LDFLAGS="$PLATFORM_LDFLAGS" + JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS" + JAVAC_ARGS="-source 8" +@@ -416,7 +419,7 @@ EOF + + if ! test $ROCKSDB_DISABLE_TBB; then + # Test whether tbb is available +- $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ltbb 2>/dev/null </dev/null < + int main() {} + EOF +@@ -667,13 +670,13 @@ else + fi + + if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then +- # For portability compile for macOS 10.12 (2016) or newer +- COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12" +- PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.12" ++ # For portability compile for macOS 10.14 (2018) or newer ++ COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14" ++ PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14" + # -mmacosx-version-min must come first here. +- PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.12 $PLATFORM_SHARED_LDFLAGS" +- PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12" +- JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.12" ++ PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.14 $PLATFORM_SHARED_LDFLAGS" ++ PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.14" ++ JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.14" + JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" + JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" + JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS" +-- +2.35.1 + diff --git a/docker/dist/Dockerfile.amd64 b/docker/dist/Dockerfile.amd64 new file mode 100644 index 000000000..873ac0d20 --- /dev/null +++ b/docker/dist/Dockerfile.amd64 @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus-eth1:dist_base_eth1_20220326074403@sha256:5c1d2e982d404c08f6bba1f0cc1a678bffae3f6cee80a679edaa0b5e985dc05d + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "Linux_amd64"] + diff --git a/docker/dist/Dockerfile.arm b/docker/dist/Dockerfile.arm new file mode 100644 index 000000000..9b740a8d7 --- /dev/null +++ b/docker/dist/Dockerfile.arm @@ -0,0 +1,19 @@ +# The build is reproducible only if this base image stays the same. +# FROM statusteam/nimbus_beacon_node:dist_base_20210310012752_arm_v2@sha256:65919842dc7e17386399ae12b175e9996f5ef038ad6e228000392a1ff6465082 +FROM statusteam/nimbus-eth1:dist_base_eth1_20220324084342_arm + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "Linux_arm32v7"] + diff --git a/docker/dist/Dockerfile.arm64 b/docker/dist/Dockerfile.arm64 new file mode 100644 index 000000000..c8cdffd9b --- /dev/null +++ b/docker/dist/Dockerfile.arm64 @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus-eth1:dist_base_eth1_20220326080423_arm64@sha256:8701b2994faa5cb03bcca69bc521ec128d3da02e3443843df272c7f1855ccac7 + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "Linux_arm64v8"] + diff --git a/docker/dist/Dockerfile.macos b/docker/dist/Dockerfile.macos new file mode 100644 index 000000000..4a5b36dc3 --- /dev/null +++ b/docker/dist/Dockerfile.macos @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus-eth1:dist_base_eth1_20220326081110_macos@sha256:229eb12e0c1ce94fde18916c4620bb83ef906d24443d1dc02ce86f2a55af4eb8 + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "macOS_amd64"] + diff --git a/docker/dist/Dockerfile.macos-arm64 b/docker/dist/Dockerfile.macos-arm64 new file mode 100644 index 000000000..5b409e912 --- /dev/null +++ b/docker/dist/Dockerfile.macos-arm64 @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus-eth1:dist_base_eth1_20220326081110_macos@sha256:229eb12e0c1ce94fde18916c4620bb83ef906d24443d1dc02ce86f2a55af4eb8 + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "macOS_arm64"] + diff --git a/docker/dist/Dockerfile.win64 b/docker/dist/Dockerfile.win64 new file mode 100644 index 000000000..1f010ad4a --- /dev/null +++ b/docker/dist/Dockerfile.win64 @@ -0,0 +1,18 @@ +# The build is reproducible only if this base image stays the same. +FROM statusteam/nimbus-eth1:dist_base_eth1_20220326081622_win64@sha256:d88d198a98a163bae6d76245b0359c8860198f63e32c431c99c3dbb8dc1de28a + +SHELL ["/bin/bash", "-c"] + +ARG USER_ID +ARG GROUP_ID + +RUN addgroup --gid ${GROUP_ID} user; \ + adduser --disabled-password --gecos '' --uid ${USER_ID} --gid ${GROUP_ID} user; + +USER user + +STOPSIGNAL SIGINT + +COPY "entry_point.sh" "/home/user/" +ENTRYPOINT ["/home/user/entry_point.sh", "Windows_amd64"] + diff --git a/docker/dist/README-Windows.md.tpl b/docker/dist/README-Windows.md.tpl new file mode 100644 index 000000000..e2583bc24 --- /dev/null +++ b/docker/dist/README-Windows.md.tpl @@ -0,0 +1,8 @@ +# Windows-specific requirements + +Run the wrapper scripts from a [Git for Windows](https://gitforwindows.org/) +Bash shell or an MSYS2 terminal emulator. + +If you run the Nimbus binary directly, prefix it with "winpty -- ". It +will increase the chance of Ctrl+C working inside that "mintty" terminal emulator. + diff --git a/docker/dist/README.md.tpl b/docker/dist/README.md.tpl new file mode 100644 index 000000000..fbc3c6e12 --- /dev/null +++ b/docker/dist/README.md.tpl @@ -0,0 +1,28 @@ +# Binary Nimbus distribution + +This binary distribution of the Nimbus eth1 package is compiled +in a [reproducible way](https://reproducible-builds.org/) from source files +hosted at https://github.com/status-im/nimbus-eth1. + +The tarball containing this README uses the following naming scheme: + +```bash +nimbus-eth1____.tar.gz +``` + +## Reproducing the build + +Besides the generic build requirements, you also need [Docker](https://www.docker.com/). + +```bash +git clone https://github.com/status-im/nimbus-eth1.git +cd nimbus-eth1 +git checkout GIT_COMMIT +make update +make dist +``` + +## Significant differences from self-built binaries + +No `-march=native`. + diff --git a/docker/dist/base_image/Dockerfile.amd64 b/docker/dist/base_image/Dockerfile.amd64 new file mode 100644 index 000000000..64b258e76 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.amd64 @@ -0,0 +1,18 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential git curl &>/dev/null \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + diff --git a/docker/dist/base_image/Dockerfile.arm b/docker/dist/base_image/Dockerfile.arm new file mode 100644 index 000000000..1aebaf662 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.arm @@ -0,0 +1,20 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:21.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential git curl \ + libc6-armhf-armel-cross libc6-dev-armel-armhf-cross binutils-arm-linux-gnueabihf \ + gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf &>/dev/null \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + diff --git a/docker/dist/base_image/Dockerfile.arm64 b/docker/dist/base_image/Dockerfile.arm64 new file mode 100644 index 000000000..d6ec6ddb6 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.arm64 @@ -0,0 +1,19 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential git curl \ + binutils-aarch64-linux-gnu gcc-aarch64-linux-gnu g++-aarch64-linux-gnu &>/dev/null \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + diff --git a/docker/dist/base_image/Dockerfile.macos b/docker/dist/base_image/Dockerfile.macos new file mode 100644 index 000000000..8b606a547 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.macos @@ -0,0 +1,24 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN apt-get -qq update \ + && apt-get -qq -y install build-essential git clang-11 llvm-11-dev cmake curl libssl-dev lzma-dev libxml2-dev &>/dev/null \ + && update-alternatives --install /usr/bin/clang clang /usr/bin/clang-11 100 \ + && update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-11 100 \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +COPY "build_osxcross.sh" "/root/" +RUN cd /root \ + && ./build_osxcross.sh + diff --git a/docker/dist/base_image/Dockerfile.win64 b/docker/dist/base_image/Dockerfile.win64 new file mode 100644 index 000000000..eda9f1fb8 --- /dev/null +++ b/docker/dist/base_image/Dockerfile.win64 @@ -0,0 +1,51 @@ +# This Docker image can change from one build to another, because the upstream +# Debian/Ubuntu package index is continuously updated and we have to run +# `apt-get update` in here. +# +# The only way to make this a part of our reproducible build system is to build +# it once, upload it to Docker Hub and make sure it's being pulled regularly so +# it's not deleted after 6 months of inactivity. + +FROM ubuntu:20.04 + +SHELL ["/bin/bash", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive TZ="Etc/UTC" +RUN \ + apt-get -qq update \ + && apt-get -qq -y install git gnupg software-properties-common lsb curl cmake &>/dev/null \ + && apt-get -qq -y install \ + autoconf \ + automake \ + autopoint \ + bison \ + bzip2 \ + flex \ + g++ \ + g++-multilib \ + gettext \ + gperf \ + intltool \ + libc6-dev-i386 \ + libltdl-dev \ + libssl-dev \ + libtool-bin \ + lzip \ + make \ + openssl \ + p7zip-full \ + patch \ + perl \ + python \ + ruby \ + sed \ + unzip \ + wget \ + xz-utils &>/dev/null \ + && apt-get -qq clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && git clone --depth 1 https://github.com/mxe/mxe.git /opt/mxe \ + && cd /opt/mxe \ + && make -j $(nproc) MXE_TARGETS='x86_64-w64-mingw32.static' MXE_USE_CCACHE='' DONT_CHECK_REQUIREMENTS=1 cc \ + && rm -rf /opt/mxe/.log /opt/mxe/pkg + diff --git a/docker/dist/base_image/Makefile b/docker/dist/base_image/Makefile new file mode 100644 index 000000000..0efc1f546 --- /dev/null +++ b/docker/dist/base_image/Makefile @@ -0,0 +1,54 @@ +SHELL := bash + +IMAGE_TAG := dist_base_eth1_$(shell date --utc +"%Y%m%d%H%M%S") +IMAGE_NAME := statusteam/nimbus-eth1:$(IMAGE_TAG) + +.PHONY: \ + build-amd64 \ + build-arm64 \ + build-arm \ + build-win64 \ + build-macos \ + push-amd64 \ + push-arm64 \ + push-arm \ + push-win64 \ + push-macos + +build-amd64: + $(CURDIR)/make_base_image.sh amd64 "$(IMAGE_NAME)" + +build-arm64: + $(CURDIR)/make_base_image.sh arm64 "$(IMAGE_NAME)_arm64" + +build-arm: + $(CURDIR)/make_base_image.sh arm "$(IMAGE_NAME)_arm" + +build-win64: + $(CURDIR)/make_base_image.sh win64 "$(IMAGE_NAME)_win64" + +build-macos: + $(CURDIR)/make_base_image.sh macos "$(IMAGE_NAME)_macos" + +# You probably don't want to recreate and push these base images to Docker Hub, +# because when older images expire and get deleted, it will no longer be possible +# to reproduce old releases. +# +# When you really have to, change the Docker tags by appending "_v2", "_v3", etc. to them. +# This way you won't overwrite the old image. + +#push-amd64: build-amd64 + #docker push $(IMAGE_NAME) + +#push-arm64: build-arm64 + #docker push $(IMAGE_NAME)_arm64 + +#push-arm: build-arm + #docker push $(IMAGE_NAME)_arm + +#push-win64: build-win64 + #docker push $(IMAGE_NAME)_win64 + +#push-macos: build-macos + #docker push $(IMAGE_NAME)_macos + diff --git a/docker/dist/base_image/README.md b/docker/dist/base_image/README.md new file mode 100644 index 000000000..d675059ef --- /dev/null +++ b/docker/dist/base_image/README.md @@ -0,0 +1,4 @@ +You probably don't want to recreate and push these base images to Docker Hub, +because when older images expire and get deleted, it will no longer be possible +to reproduce old releases. + diff --git a/docker/dist/base_image/build_osxcross.sh b/docker/dist/base_image/build_osxcross.sh new file mode 100755 index 000000000..e65384a15 --- /dev/null +++ b/docker/dist/base_image/build_osxcross.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -e + + +git clone https://github.com/tpoechtrager/osxcross.git + +# macOS SDK +cd osxcross/tarballs +MACOS_SDK_VER="11.3" +MACOS_SDK_TARBALL="MacOSX${MACOS_SDK_VER}.sdk.tar.xz" +curl -OLsS https://github.com/phracker/MacOSX-SDKs/releases/download/${MACOS_SDK_VER}/${MACOS_SDK_TARBALL} +cd .. + +# build OSXCross toolchain +export TARGET_DIR="/opt/osxcross" +UNATTENDED=1 ./build.sh +# "tools/osxcross_conf.sh" ignores TARGET_DIR and uses "target" instead, so do a symlink +ln -s ${TARGET_DIR} target +./build_llvm_dsymutil.sh +# ridiculous amount of uncompressed man pages +rm -rf ${TARGET_DIR}/SDK/MacOSX${MACOS_SDK_VER}.sdk/usr/share + +# cleanup +cd .. +rm -rf osxcross + diff --git a/docker/dist/base_image/make_base_image.sh b/docker/dist/base_image/make_base_image.sh new file mode 100755 index 000000000..05e54f65d --- /dev/null +++ b/docker/dist/base_image/make_base_image.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Copyright (c) 2020-2021 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# Build base Docker images for making distributable binaries. +# Should be used from "build-*" Make targets, passing the target architecture's +# name and Docker image tag as parameters. + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")" + +if [[ -z "${2}" ]]; then + echo "Usage: $(basename ${0}) ARCH DOCKER_TAG" + exit 1 +fi +ARCH="${1}" +DOCKER_TAG="${2}" + +DOCKER_BUILDKIT=1 \ + docker build \ + -t ${DOCKER_TAG} \ + --progress=plain \ + -f Dockerfile.${ARCH} . + diff --git a/docker/dist/binaries/Dockerfile.amd64 b/docker/dist/binaries/Dockerfile.amd64 new file mode 100644 index 000000000..ef389c596 --- /dev/null +++ b/docker/dist/binaries/Dockerfile.amd64 @@ -0,0 +1,17 @@ +FROM debian:bullseye-slim + +SHELL ["/bin/bash", "-c"] + +# Likely to match the first regular user:group created on the host. +RUN addgroup --gid 1000 user; \ + adduser --disabled-password --gecos '' --uid 1000 --gid 1000 user; + +USER user + +STOPSIGNAL SIGINT + +# Docker refuses to copy the source directory here, so read it as "nimbus-eth1/*" +COPY "nimbus-eth1" "/home/user/nimbus-eth1/" +WORKDIR "/home/user/nimbus-eth1/" +ENTRYPOINT ["/home/user/nimbus-eth1/build/nimbus"] + diff --git a/docker/dist/binaries/Dockerfile.arm b/docker/dist/binaries/Dockerfile.arm new file mode 100644 index 000000000..989f4150f --- /dev/null +++ b/docker/dist/binaries/Dockerfile.arm @@ -0,0 +1,21 @@ +FROM arm32v7/debian:bullseye-slim + +SHELL ["/bin/bash", "-c"] + +# We need the host's registered binfmt_misc "interpreter" inside the container, +# for that transparent virtualisation to work. +COPY "qemu-arm-static" "/usr/bin/" + +# Likely to match the first regular user:group created on the host. +RUN addgroup --gid 1000 user; \ + adduser --disabled-password --gecos '' --uid 1000 --gid 1000 user; + +USER user + +STOPSIGNAL SIGINT + +# Docker refuses to copy the source directory here, so read it as "nimbus-eth1/*" +COPY "nimbus-eth1" "/home/user/nimbus-eth1/" +WORKDIR "/home/user/nimbus-eth1/" +ENTRYPOINT ["/home/user/nimbus-eth1/build/nimbus"] + diff --git a/docker/dist/binaries/Dockerfile.arm64 b/docker/dist/binaries/Dockerfile.arm64 new file mode 100644 index 000000000..ead9d0f1a --- /dev/null +++ b/docker/dist/binaries/Dockerfile.arm64 @@ -0,0 +1,21 @@ +FROM arm64v8/debian:bullseye-slim + +SHELL ["/bin/bash", "-c"] + +# We need the host's registered binfmt_misc "interpreter" inside the container, +# for that transparent virtualisation to work. +COPY "qemu-aarch64-static" "/usr/bin/" + +# Likely to match the first regular user:group created on the host. +RUN addgroup --gid 1000 user; \ + adduser --disabled-password --gecos '' --uid 1000 --gid 1000 user; + +USER user + +STOPSIGNAL SIGINT + +# Docker refuses to copy the source directory here, so read it as "nimbus-eth1/*" +COPY "nimbus-eth1" "/home/user/nimbus-eth1/" +WORKDIR "/home/user/nimbus-eth1/" +ENTRYPOINT ["/home/user/nimbus-eth1/build/nimbus"] + diff --git a/docker/dist/binaries/README.md b/docker/dist/binaries/README.md new file mode 100644 index 000000000..da2011a86 --- /dev/null +++ b/docker/dist/binaries/README.md @@ -0,0 +1,5 @@ +Simple Docker image with the contents of a release tarball inside it. +It's being built and published to Docker Hub from a GitHub action, in CI. + +Suitable for end users. Example `docker-compose` configuration file included. + diff --git a/docker/dist/binaries/docker-compose-example1.yml b/docker/dist/binaries/docker-compose-example1.yml new file mode 100644 index 000000000..6be7cd728 --- /dev/null +++ b/docker/dist/binaries/docker-compose-example1.yml @@ -0,0 +1,34 @@ +# Example usage: +# +# mkdir data +# docker-compose -f docker-compose-example1.yml up --quiet-pull --no-color --detach + +version: "2.4" +services: + nimbus: + image: statusim/nimbus-eth1:amd64-latest + container_name: nimbus-eth1-amd64-latest + restart: unless-stopped + stop_grace_period: 1m + ports: + - 9000:9000/tcp + - 9000:9000/udp + - 127.0.0.1:9190:9190/tcp + - 127.0.0.1:8008:8008/tcp + volumes: + - ./data:/home/user/nimbus-eth1/build/data + # you need to make sure that port 9000 is accesible from outside; no automagic port forwarding here + command: >- + --network=mainnet + --data-dir=/home/user/nimbus-eth1/build/data/mainnet + --nat=extip:YOUR_EXTERNAL_IP + --log-level=info + --tcp-port=9000 + --udp-port=9000 + --rpc + --rpc-address=0.0.0.0 + --rpc-port=9190 + --metrics + --metrics-address=0.0.0.0 + --metrics-port=8008 + diff --git a/docker/dist/entry_point.sh b/docker/dist/entry_point.sh new file mode 100755 index 000000000..1055432d9 --- /dev/null +++ b/docker/dist/entry_point.sh @@ -0,0 +1,322 @@ +#!/usr/bin/env bash + +# Copyright (c) 2020-2022 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +set -e + +cd /home/user/nimbus-eth1 +git config --global core.abbrev 8 + +if [[ -z "${1}" ]]; then + echo "Usage: $(basename ${0}) PLATFORM" + exit 1 +fi +PLATFORM="${1}" +BINARIES="nimbus" +# RocksDB can be upgraded for new nimbus-eth1 versions, since it's built +# on-the-fly, as long as our fixes still apply (or are accepted upstream). +ROCKSDBVER="7.0.3" + +build_rocksdb() { + echo -e "\nBuilding: RocksDB" + ROCKSDB_ARCHIVE="rocksdb-v${ROCKSDBVER}.tar.gz" + ROCKSDB_DIR="rocksdb-${ROCKSDBVER}" + + pushd build >/dev/null + rm -rf "${ROCKSDB_DIR}" + if [[ ! -e "${ROCKSDB_ARCHIVE}" ]]; then + curl -L -s -S https://github.com/facebook/rocksdb/archive/v${ROCKSDBVER}.tar.gz -o "${ROCKSDB_ARCHIVE}" + fi + tar -xzf "${ROCKSDB_ARCHIVE}" + + pushd "${ROCKSDB_DIR}" >/dev/null + + # MINGW & cross-compilation support: https://github.com/facebook/rocksdb/pull/9752 + patch -p1 -i ../../docker/dist/0001-Makefile-support-Mingw-more-cross-compilation.patch + # ARM support: https://github.com/facebook/rocksdb/issues/8609#issuecomment-1009572506 + #patch -p1 -i ../../docker/dist/rocksdb-7.0.2-arm.patch + + # This seems the best way to get rid of those huge debugging symbols. + sed -i \ + -e '/ -g$/d' \ + Makefile + + # Avoid random symbol names for global vars. + sed -i \ + -e 's/$(CXXFLAGS) -c $build_log.txt + + popd >/dev/null + popd >/dev/null +} + +echo -e "\nPLATFORM=${PLATFORM}" +echo "ROCKSDBVER=${ROCKSDBVER}" + +#- we need to build everything against libraries available inside this container, including the Nim compiler +#- "librocksdb.a" is a C++ library so we need to link it with the C++ profile +make clean +NIMFLAGS_COMMON="-d:disableMarchNative --gcc.options.debug:'-g1' --clang.options.debug:'-gline-tables-only' -d:LibrocksbStaticArgs='/home/user/nimbus-eth1/build/rocksdb-${ROCKSDBVER}/librocksdb.a'" + +if [[ "${PLATFORM}" == "Windows_amd64" ]]; then + # Cross-compilation using the MXE distribution of Mingw-w64 + export PATH="/opt/mxe/usr/bin:${PATH}" + CC=x86_64-w64-mingw32.static-gcc + CXX=x86_64-w64-mingw32.static-g++ + ${CXX} --version + + build_rocksdb TARGET_OS=MINGW CXX="${CXX}" + + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common + #deps-common build/generate_makefile + make \ + -j$(nproc) \ + -C vendor/nim-nat-traversal/vendor/miniupnp/miniupnpc \ + -f Makefile.mingw \ + CC="${CC}" \ + libminiupnpc.a &>/dev/null + make \ + -j$(nproc) \ + -C vendor/nim-nat-traversal/vendor/libnatpmp-upstream \ + CC="${CC}" \ + CFLAGS="-Wall -Os -DWIN32 -DNATPMP_STATICLIB -DENABLE_STRNATPMPERR -DNATPMP_MAX_RETRIES=4 ${CFLAGS}" \ + libnatpmp.a &>/dev/null + # We set CXX and add CXXFLAGS for libunwind's C++ code, even though we don't + # use those C++ objects. I don't see an easy way of disabling the C++ parts in + # libunwind itself. + # + # "libunwind.a" combines objects produced from C and C++ code. Even though we + # don't link any C++-generated objects, the linker still checks them for + # undefined symbols, so we're forced to use g++ as a linker wrapper. + # For some reason, macOS's Clang doesn't need this trick, nor do native (and + # newer) Mingw-w64 toolchains on Windows. + # + # nim-blscurve's Windows SSSE3 detection doesn't work when cross-compiling, + # so we enable it here. + make \ + -j$(nproc) \ + CC="${CC}" \ + CXX="${CXX}" \ + CXXFLAGS="${CXXFLAGS} -D__STDC_FORMAT_MACROS -D_WIN32_WINNT=0x0600" \ + USE_VENDORED_LIBUNWIND=1 \ + LOG_LEVEL="TRACE" \ + NIMFLAGS="${NIMFLAGS_COMMON} --os:windows --gcc.exe=${CC} --gcc.linkerexe=${CXX} --passL:'-static' -d:BLSTuseSSSE3=1" \ + ${BINARIES} +elif [[ "${PLATFORM}" == "Linux_arm32v7" ]]; then + CC="arm-linux-gnueabihf-gcc" + CXX="arm-linux-gnueabihf-g++" + ${CXX} --version + + build_rocksdb TARGET_ARCHITECTURE=arm CXX="${CXX}" + + env CFLAGS="" make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common + #deps-common build/generate_makefile + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + NIMFLAGS="${NIMFLAGS_COMMON} --cpu:arm --gcc.exe=${CC} --gcc.linkerexe=${CXX} --passL:'-static'" \ + ${BINARIES} +elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then + CC="aarch64-linux-gnu-gcc" + CXX="aarch64-linux-gnu-g++" + ${CXX} --version + + build_rocksdb TARGET_ARCHITECTURE=arm64 CXX="${CXX}" + + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common + #deps-common build/generate_makefile + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + NIMFLAGS="${NIMFLAGS_COMMON} --cpu:arm64 --gcc.exe=${CC} --gcc.linkerexe=${CXX} --passL:'-static-libstdc++'" \ + PARTIAL_STATIC_LINKING=1 \ + ${BINARIES} +elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then + export PATH="/opt/osxcross/bin:${PATH}" + export OSXCROSS_MP_INC=1 # sets up include and library paths + export ZERO_AR_DATE=1 # avoid timestamps in binaries + DARWIN_VER="20.4" + CC="o64-clang" + CXX="o64-clang++" + AR="x86_64-apple-darwin${DARWIN_VER}-ar" + RANLIB="x86_64-apple-darwin${DARWIN_VER}-ranlib" + DSYMUTIL="x86_64-apple-darwin${DARWIN_VER}-dsymutil" + ${CXX} --version + + build_rocksdb TARGET_OS=Darwin CXX="${CXX}" AR="${AR}" + + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common + #deps-common build/generate_makefile + make \ + -j$(nproc) \ + CC="${CC}" \ + LIBTOOL="x86_64-apple-darwin${DARWIN_VER}-libtool" \ + OS="darwin" \ + NIMFLAGS="${NIMFLAGS_COMMON} --os:macosx --clang.exe=${CC}" \ + nat-libs + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + AR="${AR}" \ + RANLIB="${RANLIB}" \ + CMAKE="x86_64-apple-darwin${DARWIN_VER}-cmake" \ + CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=/opt/osxcross/toolchain.cmake" \ + DSYMUTIL="${DSYMUTIL}" \ + FORCE_DSYMUTIL=1 \ + USE_VENDORED_LIBUNWIND=1 \ + NIMFLAGS="${NIMFLAGS_COMMON} --os:macosx --clang.exe=${CC} --clang.linkerexe=${CXX} --passL:'-static-libstdc++ -mmacosx-version-min=10.14'" \ + ${BINARIES} +elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then + export PATH="/opt/osxcross/bin:${PATH}" + export OSXCROSS_MP_INC=1 # sets up include and library paths + export ZERO_AR_DATE=1 # avoid timestamps in binaries + DARWIN_VER="20.4" + CC="oa64-clang" + CXX="oa64-clang++" + AR="arm64-apple-darwin${DARWIN_VER}-ar" + RANLIB="arm64-apple-darwin${DARWIN_VER}-ranlib" + DSYMUTIL="arm64-apple-darwin${DARWIN_VER}-dsymutil" + ${CXX} --version + + build_rocksdb TARGET_OS=Darwin TARGET_ARCHITECTURE=arm64 CXX="${CXX}" AR="${AR}" + + make \ + -j$(nproc) \ + USE_LIBBACKTRACE=0 \ + QUICK_AND_DIRTY_COMPILER=1 \ + deps-common + #deps-common build/generate_makefile + make \ + -j$(nproc) \ + CC="${CC}" \ + LIBTOOL="arm64-apple-darwin${DARWIN_VER}-libtool" \ + OS="darwin" \ + NIMFLAGS="${NIMFLAGS_COMMON} --os:macosx --cpu:arm64 --passC:'-mcpu=apple-a14' --clang.exe=${CC}" \ + nat-libs + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + CC="${CC}" \ + AR="${AR}" \ + RANLIB="${RANLIB}" \ + CMAKE="arm64-apple-darwin${DARWIN_VER}-cmake" \ + CMAKE_ARGS="-DCMAKE_TOOLCHAIN_FILE=/opt/osxcross/toolchain.cmake" \ + DSYMUTIL="${DSYMUTIL}" \ + FORCE_DSYMUTIL=1 \ + USE_VENDORED_LIBUNWIND=1 \ + NIMFLAGS="${NIMFLAGS_COMMON} --os:macosx --cpu:arm64 --passC:'-mcpu=apple-a13' --passL:'-mcpu=apple-a14 -static-libstdc++' --clang.exe=${CC} --clang.linkerexe=${CXX}" \ + ${BINARIES} +else + # Linux AMD64 + g++ --version + + build_rocksdb + + make \ + -j$(nproc) \ + LOG_LEVEL="TRACE" \ + NIMFLAGS="${NIMFLAGS_COMMON} --gcc.linkerexe=g++ --passL:'-static-libstdc++'" \ + PARTIAL_STATIC_LINKING=1 \ + QUICK_AND_DIRTY_COMPILER=1 \ + ${BINARIES} +fi + +# archive directory (we need the Nim compiler in here) +PREFIX="nimbus-eth1_${PLATFORM}_" +GIT_COMMIT="$(git rev-parse --short HEAD)" +VERSION="$(./env.sh nim --verbosity:0 --hints:off --warnings:off scripts/print_version.nims)" +DIR="${PREFIX}${VERSION}_${GIT_COMMIT}" +DIST_PATH="dist/${DIR}" +# delete old artefacts +rm -rf "dist/${PREFIX}"*.tar.gz +if [[ -d "${DIST_PATH}" ]]; then + rm -rf "${DIST_PATH}" +fi + +mkdir -p "${DIST_PATH}" +mkdir "${DIST_PATH}/build" + +# copy and checksum binaries, copy docs +EXT="" +if [[ "${PLATFORM}" == "Windows_amd64" ]]; then + EXT=".exe" +fi +for BINARY in ${BINARIES}; do + cp -a "./build/${BINARY}${EXT}" "${DIST_PATH}/build/" + if [[ "${PLATFORM}" =~ macOS ]]; then + # Collect debugging info and filter out warnings. + # + # First two also happen with a native "dsymutil", while the next two only + # with the "llvm-dsymutil" we use when cross-compiling. + "${DSYMUTIL}" build/${BINARY} 2>&1 \ + | grep -v "failed to insert symbol" \ + | grep -v "could not find object file symbol for symbol" \ + | grep -v "while processing" \ + | grep -v "warning: line table paramters mismatch. Cannot emit." \ + || true + cp -a "./build/${BINARY}.dSYM" "${DIST_PATH}/build/" + fi + cd "${DIST_PATH}/build" + sha512sum "${BINARY}${EXT}" > "${BINARY}.sha512sum" + cd - >/dev/null +done +sed -e "s/GIT_COMMIT/${GIT_COMMIT}/" docker/dist/README.md.tpl > "${DIST_PATH}/README.md" + +if [[ "${PLATFORM}" == "Linux_amd64" ]]; then + sed -i -e 's/^make dist$/make dist-amd64/' "${DIST_PATH}/README.md" +elif [[ "${PLATFORM}" == "Linux_arm32v7" ]]; then + sed -i -e 's/^make dist$/make dist-arm/' "${DIST_PATH}/README.md" +elif [[ "${PLATFORM}" == "Linux_arm64v8" ]]; then + sed -i -e 's/^make dist$/make dist-arm64/' "${DIST_PATH}/README.md" +elif [[ "${PLATFORM}" == "Windows_amd64" ]]; then + sed -i -e 's/^make dist$/make dist-win64/' "${DIST_PATH}/README.md" + cp -a docker/dist/README-Windows.md.tpl "${DIST_PATH}/README-Windows.md" +elif [[ "${PLATFORM}" == "macOS_amd64" ]]; then + sed -i -e 's/^make dist$/make dist-macos/' "${DIST_PATH}/README.md" +elif [[ "${PLATFORM}" == "macOS_arm64" ]]; then + sed -i -e 's/^make dist$/make dist-macos-arm64/' "${DIST_PATH}/README.md" +fi + +# create the tarball +cd dist +tar -czf "${DIR}.tar.gz" "${DIR}" +# don't leave the directory hanging around +rm -rf "${DIR}" +cd - >/dev/null + diff --git a/docker/dist/rocksdb-7.0.2-arm.patch b/docker/dist/rocksdb-7.0.2-arm.patch new file mode 100644 index 000000000..1eb64ed5c --- /dev/null +++ b/docker/dist/rocksdb-7.0.2-arm.patch @@ -0,0 +1,15 @@ +diff --git a/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h b/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +index 225e3fa72..cd5f935f1 100644 +--- a/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h ++++ b/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +@@ -131,6 +131,10 @@ static inline tokutime_t toku_time_now(void) { + uint64_t result; + __asm __volatile__("mrs %[rt], cntvct_el0" : [ rt ] "=r"(result)); + return result; ++#elif defined(__arm__) ++ uint32_t lo, hi; ++ __asm __volatile__("mrrc p15, 1, %[lo], %[hi], c14" : [ lo ] "=r" (lo), [hi] "=r" (hi)); ++ return (uint64_t)hi << 32 | lo; + #elif defined(__powerpc__) + return __ppc_get_timebase(); + #elif defined(__s390x__) diff --git a/nimbus.nimble b/nimbus.nimble index 6d2f49c62..a1f892686 100644 --- a/nimbus.nimble +++ b/nimbus.nimble @@ -48,9 +48,6 @@ proc test(path: string, name: string, params = "", lang = "c") = task test, "Run tests": test "tests", "all_tests", "-d:chronicles_log_level=ERROR -d:unittest2DisableParamFiltering" -task nimbus, "Build Nimbus": - buildBinary "nimbus", "nimbus/", "-d:chronicles_log_level=TRACE" - task fluffy, "Build fluffy": buildBinary "fluffy", "fluffy/", "-d:chronicles_log_level=TRACE -d:chronosStrictException" diff --git a/nimbus/config.nim b/nimbus/config.nim index fd81abec3..b84dd7675 100644 --- a/nimbus/config.nim +++ b/nimbus/config.nim @@ -1,5 +1,4 @@ -# Nimbus -# Copyright (c) 2018-2021 Status Research & Development GmbH +# Copyright (c) 2018-2022 Status Research & Development GmbH # Licensed under either of # * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) # * MIT license ([LICENSE-MIT](LICENSE-MIT)) @@ -19,31 +18,12 @@ import stew/shims/net as stewNet, eth/[p2p, common, net/nat, p2p/bootnodes], "."/[db/select_backend, chain_config, - constants, vm_compile_info + constants, vm_compile_info, version ] export stewNet const - NimbusName* = "nimbus-eth1" - ## project name string - - NimbusMajor*: int = 0 - ## is the major number of Nimbus' version. - - NimbusMinor*: int = 1 - ## is the minor number of Nimbus' version. - - NimbusPatch*: int = 0 - ## is the patch number of Nimbus' version. - - NimbusVersion* = $NimbusMajor & "." & $NimbusMinor & "." & $NimbusPatch - ## is the version of Nimbus as a string. - - GitRevision = staticExec("git rev-parse --short HEAD").replace("\n") # remove CR - - NimVersion = staticExec("nim --version").strip() - # TODO: fix this agent-string format to match other # eth clients format NimbusIdent* = "$# v$# [$#: $#, $#, $#, $#]" % [ @@ -55,7 +35,6 @@ const VmName, GitRevision ] - ## project ident name for networking services let # e.g.: Copyright (c) 2018-2021 Status Research & Development GmbH @@ -73,7 +52,7 @@ let NimbusHeader* = "$#\p\p$#" % [ NimbusBuild, - NimVersion + version.NimVersion ] proc defaultDataDir*(): string = diff --git a/nimbus/nimbus.nim b/nimbus/nimbus.nim index b8b6cbd50..c0604813e 100644 --- a/nimbus/nimbus.nim +++ b/nimbus/nimbus.nim @@ -23,7 +23,7 @@ import config, genesis, rpc/[common, p2p, debug, engine_api], p2p/chain, eth/trie/db, metrics, metrics/[chronos_httpserver, chronicles_support], graphql/ethapi, context, utils/tx_pool, - "."/[conf_utils, sealer, constants, utils] + "."/[conf_utils, sealer, constants, utils, version] when defined(evmc_enabled): import transaction/evmc_dynamic_loader @@ -99,7 +99,7 @@ proc setupP2P(nimbus: NimbusNode, conf: NimbusConf, address.ip = extIP.get() let extPorts = redirectPorts(tcpPort = address.tcpPort, udpPort = address.udpPort, - description = NIMBUS_NAME & " " & NIMBUS_VERSION) + description = NimbusName & " " & NimbusVersion) if extPorts.isSome: (address.tcpPort, address.udpPort) = extPorts.get() diff --git a/nimbus/version.nim b/nimbus/version.nim new file mode 100644 index 000000000..8fc2fd8e6 --- /dev/null +++ b/nimbus/version.nim @@ -0,0 +1,30 @@ +# Copyright (c) 2018-2022 Status Research & Development GmbH +# Licensed under either of +# * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE)) +# * MIT license ([LICENSE-MIT](LICENSE-MIT)) +# at your option. +# This file may not be copied, modified, or distributed except according to +# those terms. + +import strutils + +const + NimbusName* = "nimbus-eth1" + ## project name string + + NimbusMajor*: int = 0 + ## is the major number of Nimbus' version. + + NimbusMinor*: int = 1 + ## is the minor number of Nimbus' version. + + NimbusPatch*: int = 0 + ## is the patch number of Nimbus' version. + + NimbusVersion* = $NimbusMajor & "." & $NimbusMinor & "." & $NimbusPatch + ## is the version of Nimbus as a string. + + GitRevision* = strip(staticExec("git rev-parse --short HEAD"))[0..5] + + NimVersion* = staticExec("nim --version | grep Version") + diff --git a/scripts/make_dist.sh b/scripts/make_dist.sh new file mode 100755 index 000000000..634cb76ab --- /dev/null +++ b/scripts/make_dist.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright (c) 2020-2022 Status Research & Development GmbH. Licensed under +# either of: +# - Apache License, version 2.0 +# - MIT license +# at your option. This file may not be copied, modified, or distributed except +# according to those terms. + +# Build release binaries fit for public distribution, using Docker. +# Should be used from "dist-*" Make targets, passing the target architecture's name as a parameter. + +set -e + +cd "$(dirname "${BASH_SOURCE[0]}")"/.. +REPO_DIR="${PWD}" + +ARCH="${1:-amd64}" +DOCKER_TAG="nimbus-eth1-dist-${ARCH}" + +docker rm ${DOCKER_TAG} &>/dev/null || true + +cd docker/dist + +DOCKER_BUILDKIT=1 \ + docker build \ + -t ${DOCKER_TAG} \ + --progress=plain \ + --build-arg USER_ID=$(id -u) \ + --build-arg GROUP_ID=$(id -g) \ + -f Dockerfile.${ARCH} . + +# seccomp can have some serious overhead, so we disable it with "--privileged" - https://pythonspeed.com/articles/docker-performance-overhead/ +docker run --privileged --rm --name ${DOCKER_TAG} -v ${REPO_DIR}:/home/user/nimbus-eth1 ${DOCKER_TAG} + +cd - &>/dev/null + +ls -l dist + +# We rebuild everything inside the container, so we need to clean up afterwards. +${MAKE} --no-print-directory clean + diff --git a/scripts/print_version.nims b/scripts/print_version.nims new file mode 100644 index 000000000..ffebc925a --- /dev/null +++ b/scripts/print_version.nims @@ -0,0 +1,4 @@ +import ../nimbus/version + +echo NimbusVersion + diff --git a/tests/test_rpc.nim b/tests/test_rpc.nim index 2da1c9e45..b2440a0d9 100644 --- a/tests/test_rpc.nim +++ b/tests/test_rpc.nim @@ -12,7 +12,7 @@ import eth/[rlp, keys, trie/db, p2p/private/p2p_types], ../nimbus/rpc/[common, p2p, rpc_utils], ../nimbus/[constants, config, genesis, utils, transaction, - vm_state, vm_types], + vm_state, vm_types, version], ../nimbus/db/[accounts_cache, db_chain], ../nimbus/sync/protocol_ethxx, ../nimbus/p2p/[chain, executor, executor/executor_helpers], diff --git a/vendor/nim-libbacktrace b/vendor/nim-libbacktrace index 284b3aac0..2ea147a71 160000 --- a/vendor/nim-libbacktrace +++ b/vendor/nim-libbacktrace @@ -1 +1 @@ -Subproject commit 284b3aac05a9d96c27044c389a5d27a84d8e8f4b +Subproject commit 2ea147a71c4c05d64fdff5e7b8d8990eb5821399 diff --git a/vendor/nim-nat-traversal b/vendor/nim-nat-traversal index 8994b67b0..11df74552 160000 --- a/vendor/nim-nat-traversal +++ b/vendor/nim-nat-traversal @@ -1 +1 @@ -Subproject commit 8994b67b07813955c61bebddf4bd2325439c3535 +Subproject commit 11df74552d3a3abe2c722c536c8075ef6814d5fa diff --git a/vendor/nim-rocksdb b/vendor/nim-rocksdb index 6aab1e0dd..fb3f2c30b 160000 --- a/vendor/nim-rocksdb +++ b/vendor/nim-rocksdb @@ -1 +1 @@ -Subproject commit 6aab1e0dd674b262f538d12b0458f3e7eb835b19 +Subproject commit fb3f2c30b0224466932ee132e3f853e6c35cff7c diff --git a/vendor/nimbus-build-system b/vendor/nimbus-build-system index 8d226580a..a8ab2dc39 160000 --- a/vendor/nimbus-build-system +++ b/vendor/nimbus-build-system @@ -1 +1 @@ -Subproject commit 8d226580a089b3113f63b267ab6e1bbb18977dca +Subproject commit a8ab2dc39aad4d69ba3be72868772d851b4b9741 diff --git a/witnessBuilderBC.md b/witnessBuilderBC.md index a5ed00619..a6a909021 100644 --- a/witnessBuilderBC.md +++ b/witnessBuilderBC.md @@ -341,9 +341,10 @@ OK: 96/96 Fail: 0/96 Skip: 0/96 + testOpcode_E0.json OK + testOpcode_F0.json OK + transactionFromNotExistingAccount.json OK ++ transactionFromSelfDestructedContract.json OK + txCost-sec73.json OK ``` -OK: 87/87 Fail: 0/87 Skip: 0/87 +OK: 88/88 Fail: 0/88 Skip: 0/88 ## bcTotalDifficultyTest ```diff + lotsOfBranchesOverrideAtTheEnd.json OK @@ -919,8 +920,10 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + codesizeInit.json OK + codesizeOOGInvalidSize.json OK + codesizeValid.json OK ++ create2CodeSizeLimit.json OK ++ createCodeSizeLimit.json OK ``` -OK: 3/3 Fail: 0/3 Skip: 0/3 +OK: 5/5 Fail: 0/5 Skip: 0/5 ## stCreate2 ```diff + CREATE2_Bounds.json OK @@ -929,7 +932,11 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn.json OK + CREATE2_EOF1.json OK + CREATE2_FirstByte_loop.json OK ++ CREATE2_HighNonce.json OK ++ CREATE2_HighNonceDelegatecall.json OK ++ CREATE2_HighNonceMinus1.json OK + CREATE2_Suicide.json OK ++ Create2OOGFromCallRefunds.json OK + Create2OOGafterInitCode.json OK + Create2OOGafterInitCodeReturndata.json OK + Create2OOGafterInitCodeReturndata2.json OK @@ -972,7 +979,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + returndatacopy_following_successful_create.json OK + returndatasize_following_successful_create.json OK ``` -OK: 48/48 Fail: 0/48 Skip: 0/48 +OK: 52/52 Fail: 0/52 Skip: 0/52 ## stCreateTest ```diff + CREATE_AcreateB_BSuicide_BStore.json OK @@ -995,10 +1002,14 @@ OK: 48/48 Fail: 0/48 Skip: 0/48 + CREATE_EmptyContractWithStorageAndCallIt_0wei.json OK + CREATE_EmptyContractWithStorageAndCallIt_1wei.json OK + CREATE_FirstByte_loop.json OK ++ CREATE_HighNonce.json OK ++ CREATE_HighNonceMinus1.json OK + CREATE_empty000CreateinInitCode_Transaction.json OK + CodeInConstructor.json OK + CreateCollisionResults.json OK + CreateCollisionToEmpty.json OK ++ CreateOOGFromCallRefunds.json OK ++ CreateOOGFromEOARefunds.json OK + CreateOOGafterInitCode.json OK + CreateOOGafterInitCodeReturndata.json OK + CreateOOGafterInitCodeReturndata2.json OK @@ -1006,13 +1017,14 @@ OK: 48/48 Fail: 0/48 Skip: 0/48 + CreateOOGafterInitCodeReturndataSize.json OK + CreateOOGafterInitCodeRevert.json OK + CreateOOGafterInitCodeRevert2.json OK ++ CreateOOGafterMaxCodesize.json OK + CreateResults.json OK + TransactionCollisionToEmpty.json OK + TransactionCollisionToEmptyButCode.json OK + TransactionCollisionToEmptyButNonce.json OK + createFailResult.json OK ``` -OK: 36/36 Fail: 0/36 Skip: 0/36 +OK: 41/41 Fail: 0/41 Skip: 0/41 ## stDelegatecallTestHomestead ```diff + Call1024BalanceTooLow.json OK @@ -1144,6 +1156,15 @@ OK: 7/7 Fail: 0/7 Skip: 0/7 + variedContext.json OK ``` OK: 7/7 Fail: 0/7 Skip: 0/7 +## stEIP3607 +```diff ++ initCollidingWithNonEmptyAccount.json OK ++ transactionCollidingWithNonEmptyAccount_calls.json OK ++ transactionCollidingWithNonEmptyAccount_callsItself.json OK ++ transactionCollidingWithNonEmptyAccount_init.json OK ++ transactionCollidingWithNonEmptyAccount_send.json OK +``` +OK: 5/5 Fail: 0/5 Skip: 0/5 ## stExample ```diff + accessListExample.json OK @@ -1151,13 +1172,14 @@ OK: 7/7 Fail: 0/7 Skip: 0/7 + add11_yml.json OK + basefeeExample.json OK + eip1559.json OK ++ indexesOmitExample.json OK + invalidTr.json OK + labelsExample.json OK + rangesExample.json OK + solidityExample.json OK + yulExample.json OK ``` -OK: 10/10 Fail: 0/10 Skip: 0/10 +OK: 11/11 Fail: 0/11 Skip: 0/11 ## stExtCodeHash ```diff + callToNonExistent.json OK @@ -1434,6 +1456,7 @@ OK: 71/71 Fail: 0/71 Skip: 0/71 OK: 24/24 Fail: 0/24 Skip: 0/24 ## stPreCompiledContracts ```diff ++ blake2B.json OK + idPrecomps.json OK + identity_to_bigger.json OK + identity_to_smaller.json OK @@ -1442,7 +1465,7 @@ OK: 24/24 Fail: 0/24 Skip: 0/24 + precompsEIP2929.json OK + sec80.json OK ``` -OK: 7/7 Fail: 0/7 Skip: 0/7 +OK: 8/8 Fail: 0/8 Skip: 0/8 ## stPreCompiledContracts2 ```diff + CALLBlake2f.json OK @@ -1831,6 +1854,7 @@ OK: 16/16 Fail: 0/16 Skip: 0/16 + randomStatetest381.json OK + randomStatetest382.json OK + randomStatetest383.json OK ++ randomStatetest384.json OK + randomStatetest39.json OK + randomStatetest4.json OK + randomStatetest41.json OK @@ -1882,7 +1906,7 @@ OK: 16/16 Fail: 0/16 Skip: 0/16 + randomStatetest97.json OK + randomStatetest98.json OK ``` -OK: 313/313 Fail: 0/313 Skip: 0/313 +OK: 314/314 Fail: 0/314 Skip: 0/314 ## stRandom2 ```diff + 201503110226PYTHON_DUP6.json OK @@ -2150,6 +2174,7 @@ OK: 22/22 Fail: 0/22 Skip: 0/22 + call_outsize_then_create_successful_then_returndatasize.json OK + call_then_call_value_fail_then_returndatasize.json OK + call_then_create_successful_then_returndatasize.json OK ++ clearReturnBuffer.json OK + create_callprecompile_returndatasize.json OK + modexp_modsize0_returndatasize.json OK + returndatacopy_0_0_following_successful_create.json OK @@ -2185,8 +2210,9 @@ OK: 22/22 Fail: 0/22 Skip: 0/22 + returndatasize_initial_zero_read.json OK + revertRetDataSize.json OK + subcallReturnMoreThenExpected.json OK ++ tooLongReturnDataCopy.json OK ``` -OK: 39/39 Fail: 0/39 Skip: 0/39 +OK: 41/41 Fail: 0/41 Skip: 0/41 ## stRevertTest ```diff + LoopCallsDepthThenRevert.json OK @@ -2305,6 +2331,7 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 + sar_2^256-1_255.json OK + sar_2^256-1_256.json OK + shiftCombinations.json OK ++ shiftSignedCombinations.json OK + shl01-0100.json OK + shl01-0101.json OK + shl01-ff.json OK @@ -2328,7 +2355,7 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 + shr_2^255_256.json OK + shr_2^255_257.json OK ``` -OK: 41/41 Fail: 0/41 Skip: 0/41 +OK: 42/42 Fail: 0/42 Skip: 0/42 ## stSolidityTest ```diff + AmbiguousMethod.json OK @@ -2788,6 +2815,7 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + CreateTransactionSuccess.json OK + EmptyTransaction3.json OK + HighGasLimit.json OK ++ HighGasPrice.json OK + InternalCallHittingGasLimit.json OK + InternalCallHittingGasLimit2.json OK + InternalCallHittingGasLimitSuccess.json OK @@ -2795,6 +2823,7 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + InternlCallStoreClearsSucces.json OK + Opcodes_TransactionInit.json OK + OverflowGasRequire2.json OK ++ PointAtInfinityECRecover.json OK + StoreClearsAndInternlCallStoreClearsOOG.json OK + StoreClearsAndInternlCallStoreClearsSuccess.json OK + StoreGasOnCreate.json OK @@ -2809,8 +2838,9 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + TransactionSendingToZero.json OK + TransactionToAddressh160minusOne.json OK + TransactionToItself.json OK ++ ValueOverflow.json OK ``` -OK: 29/29 Fail: 0/29 Skip: 0/29 +OK: 32/32 Fail: 0/32 Skip: 0/32 ## stTransitionTest ```diff + createNameRegistratorPerTxsAfter.json OK @@ -3229,6 +3259,7 @@ OK: 11/11 Fail: 0/11 Skip: 0/11 + codecopy.json OK + gas.json OK + jump.json OK ++ jumpToPush.json OK + jumpi.json OK + loop_stacklimit.json OK + loopsConditionals.json OK @@ -3241,7 +3272,7 @@ OK: 11/11 Fail: 0/11 Skip: 0/11 + return.json OK + sstore_sload.json OK ``` -OK: 14/14 Fail: 0/14 Skip: 0/14 +OK: 15/15 Fail: 0/15 Skip: 0/15 ## vmLogTest ```diff + log0.json OK @@ -3275,4 +3306,4 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 OK: 11/11 Fail: 0/11 Skip: 0/11 ---TOTAL--- -OK: 2937/2937 Fail: 0/2937 Skip: 0/2937 +OK: 2964/2964 Fail: 0/2964 Skip: 0/2964 diff --git a/witnessBuilderGST.md b/witnessBuilderGST.md index 2b1b95051..603af4611 100644 --- a/witnessBuilderGST.md +++ b/witnessBuilderGST.md @@ -461,8 +461,10 @@ OK: 2/2 Fail: 0/2 Skip: 0/2 + codesizeInit.json OK + codesizeOOGInvalidSize.json OK + codesizeValid.json OK ++ create2CodeSizeLimit.json OK ++ createCodeSizeLimit.json OK ``` -OK: 3/3 Fail: 0/3 Skip: 0/3 +OK: 5/5 Fail: 0/5 Skip: 0/5 ## stCreate2 ```diff + CREATE2_Bounds.json OK @@ -471,7 +473,11 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + CREATE2_ContractSuicideDuringInit_ThenStoreThenReturn.json OK + CREATE2_EOF1.json OK + CREATE2_FirstByte_loop.json OK ++ CREATE2_HighNonce.json OK ++ CREATE2_HighNonceDelegatecall.json OK ++ CREATE2_HighNonceMinus1.json OK + CREATE2_Suicide.json OK ++ Create2OOGFromCallRefunds.json OK + Create2OOGafterInitCode.json OK + Create2OOGafterInitCodeReturndata.json OK + Create2OOGafterInitCodeReturndata2.json OK @@ -514,7 +520,7 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 + returndatacopy_following_successful_create.json OK + returndatasize_following_successful_create.json OK ``` -OK: 48/48 Fail: 0/48 Skip: 0/48 +OK: 52/52 Fail: 0/52 Skip: 0/52 ## stCreateTest ```diff + CREATE_AcreateB_BSuicide_BStore.json OK @@ -537,10 +543,14 @@ OK: 48/48 Fail: 0/48 Skip: 0/48 + CREATE_EmptyContractWithStorageAndCallIt_0wei.json OK + CREATE_EmptyContractWithStorageAndCallIt_1wei.json OK + CREATE_FirstByte_loop.json OK ++ CREATE_HighNonce.json OK ++ CREATE_HighNonceMinus1.json OK + CREATE_empty000CreateinInitCode_Transaction.json OK + CodeInConstructor.json OK + CreateCollisionResults.json OK + CreateCollisionToEmpty.json OK ++ CreateOOGFromCallRefunds.json OK ++ CreateOOGFromEOARefunds.json OK + CreateOOGafterInitCode.json OK + CreateOOGafterInitCodeReturndata.json OK + CreateOOGafterInitCodeReturndata2.json OK @@ -548,13 +558,14 @@ OK: 48/48 Fail: 0/48 Skip: 0/48 + CreateOOGafterInitCodeReturndataSize.json OK + CreateOOGafterInitCodeRevert.json OK + CreateOOGafterInitCodeRevert2.json OK ++ CreateOOGafterMaxCodesize.json OK + CreateResults.json OK + TransactionCollisionToEmpty.json OK + TransactionCollisionToEmptyButCode.json OK + TransactionCollisionToEmptyButNonce.json OK + createFailResult.json OK ``` -OK: 36/36 Fail: 0/36 Skip: 0/36 +OK: 41/41 Fail: 0/41 Skip: 0/41 ## stDelegatecallTestHomestead ```diff + Call1024BalanceTooLow.json OK @@ -686,6 +697,15 @@ OK: 7/7 Fail: 0/7 Skip: 0/7 + variedContext.json OK ``` OK: 7/7 Fail: 0/7 Skip: 0/7 +## stEIP3607 +```diff ++ initCollidingWithNonEmptyAccount.json OK ++ transactionCollidingWithNonEmptyAccount_calls.json OK ++ transactionCollidingWithNonEmptyAccount_callsItself.json OK ++ transactionCollidingWithNonEmptyAccount_init.json OK ++ transactionCollidingWithNonEmptyAccount_send.json OK +``` +OK: 5/5 Fail: 0/5 Skip: 0/5 ## stExample ```diff + accessListExample.json OK @@ -693,13 +713,14 @@ OK: 7/7 Fail: 0/7 Skip: 0/7 + add11_yml.json OK + basefeeExample.json OK + eip1559.json OK ++ indexesOmitExample.json OK + invalidTr.json OK + labelsExample.json OK + rangesExample.json OK + solidityExample.json OK + yulExample.json OK ``` -OK: 10/10 Fail: 0/10 Skip: 0/10 +OK: 11/11 Fail: 0/11 Skip: 0/11 ## stExtCodeHash ```diff + callToNonExistent.json OK @@ -976,6 +997,7 @@ OK: 71/71 Fail: 0/71 Skip: 0/71 OK: 24/24 Fail: 0/24 Skip: 0/24 ## stPreCompiledContracts ```diff ++ blake2B.json OK + idPrecomps.json OK + identity_to_bigger.json OK + identity_to_smaller.json OK @@ -984,7 +1006,7 @@ OK: 24/24 Fail: 0/24 Skip: 0/24 + precompsEIP2929.json OK + sec80.json OK ``` -OK: 7/7 Fail: 0/7 Skip: 0/7 +OK: 8/8 Fail: 0/8 Skip: 0/8 ## stPreCompiledContracts2 ```diff + CALLBlake2f.json OK @@ -1373,6 +1395,7 @@ OK: 16/16 Fail: 0/16 Skip: 0/16 + randomStatetest381.json OK + randomStatetest382.json OK + randomStatetest383.json OK ++ randomStatetest384.json OK + randomStatetest39.json OK + randomStatetest4.json OK + randomStatetest41.json OK @@ -1424,7 +1447,7 @@ OK: 16/16 Fail: 0/16 Skip: 0/16 + randomStatetest97.json OK + randomStatetest98.json OK ``` -OK: 313/313 Fail: 0/313 Skip: 0/313 +OK: 314/314 Fail: 0/314 Skip: 0/314 ## stRandom2 ```diff + 201503110226PYTHON_DUP6.json OK @@ -1692,6 +1715,7 @@ OK: 22/22 Fail: 0/22 Skip: 0/22 + call_outsize_then_create_successful_then_returndatasize.json OK + call_then_call_value_fail_then_returndatasize.json OK + call_then_create_successful_then_returndatasize.json OK ++ clearReturnBuffer.json OK + create_callprecompile_returndatasize.json OK + modexp_modsize0_returndatasize.json OK + returndatacopy_0_0_following_successful_create.json OK @@ -1727,8 +1751,9 @@ OK: 22/22 Fail: 0/22 Skip: 0/22 + returndatasize_initial_zero_read.json OK + revertRetDataSize.json OK + subcallReturnMoreThenExpected.json OK ++ tooLongReturnDataCopy.json OK ``` -OK: 39/39 Fail: 0/39 Skip: 0/39 +OK: 41/41 Fail: 0/41 Skip: 0/41 ## stRevertTest ```diff + LoopCallsDepthThenRevert.json OK @@ -1847,6 +1872,7 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 + sar_2^256-1_255.json OK + sar_2^256-1_256.json OK + shiftCombinations.json OK ++ shiftSignedCombinations.json OK + shl01-0100.json OK + shl01-0101.json OK + shl01-ff.json OK @@ -1870,7 +1896,7 @@ OK: 6/6 Fail: 0/6 Skip: 0/6 + shr_2^255_256.json OK + shr_2^255_257.json OK ``` -OK: 41/41 Fail: 0/41 Skip: 0/41 +OK: 42/42 Fail: 0/42 Skip: 0/42 ## stSolidityTest ```diff + AmbiguousMethod.json OK @@ -2330,6 +2356,7 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + CreateTransactionSuccess.json OK + EmptyTransaction3.json OK + HighGasLimit.json OK ++ HighGasPrice.json OK + InternalCallHittingGasLimit.json OK + InternalCallHittingGasLimit2.json OK + InternalCallHittingGasLimitSuccess.json OK @@ -2337,6 +2364,7 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + InternlCallStoreClearsSucces.json OK + Opcodes_TransactionInit.json OK + OverflowGasRequire2.json OK ++ PointAtInfinityECRecover.json OK + StoreClearsAndInternlCallStoreClearsOOG.json OK + StoreClearsAndInternlCallStoreClearsSuccess.json OK + StoreGasOnCreate.json OK @@ -2351,8 +2379,9 @@ OK: 14/14 Fail: 0/14 Skip: 0/14 + TransactionSendingToZero.json OK + TransactionToAddressh160minusOne.json OK + TransactionToItself.json OK ++ ValueOverflow.json OK ``` -OK: 29/29 Fail: 0/29 Skip: 0/29 +OK: 32/32 Fail: 0/32 Skip: 0/32 ## stTransitionTest ```diff + createNameRegistratorPerTxsAfter.json OK @@ -2771,6 +2800,7 @@ OK: 11/11 Fail: 0/11 Skip: 0/11 + codecopy.json OK + gas.json OK + jump.json OK ++ jumpToPush.json OK + jumpi.json OK + loop_stacklimit.json OK + loopsConditionals.json OK @@ -2783,7 +2813,7 @@ OK: 11/11 Fail: 0/11 Skip: 0/11 + return.json OK + sstore_sload.json OK ``` -OK: 14/14 Fail: 0/14 Skip: 0/14 +OK: 15/15 Fail: 0/15 Skip: 0/15 ## vmLogTest ```diff + log0.json OK @@ -2817,4 +2847,4 @@ OK: 3/3 Fail: 0/3 Skip: 0/3 OK: 11/11 Fail: 0/11 Skip: 0/11 ---TOTAL--- -OK: 2571/2571 Fail: 0/2571 Skip: 0/2571 +OK: 2597/2597 Fail: 0/2597 Skip: 0/2597