diff --git a/.github/actions/install-risc0/action.yml b/.github/actions/install-risc0/action.yml deleted file mode 100644 index fef3a467..00000000 --- a/.github/actions/install-risc0/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Install risc0 -description: Installs risc0 in the environment -runs: - using: "composite" - steps: - - name: Install risc0 - run: | - curl -L https://risczero.com/install | bash - /home/runner/.risc0/bin/rzup install - shell: bash diff --git a/.github/actions/install-system-deps/action.yml b/.github/actions/install-system-deps/action.yml deleted file mode 100644 index 28c4b41c..00000000 --- a/.github/actions/install-system-deps/action.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: Install system dependencies -description: Installs system dependencies in the environment -runs: - using: "composite" - steps: - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang libclang-dev libssl-dev pkg-config - shell: bash diff --git a/.github/actions/run-in-ci-image/action.yml b/.github/actions/run-in-ci-image/action.yml new file mode 100644 index 00000000..3fafe38c --- /dev/null +++ b/.github/actions/run-in-ci-image/action.yml @@ -0,0 +1,54 @@ +name: Run in CI image +description: > + Runs a command inside the CI image on a plain runner, for jobs that need the + host's Docker daemon. `container:` cannot work for those: it puts the + workspace on a path the host daemon cannot resolve, and it forbids + `--network host`. + +inputs: + image: + description: CI image reference, from the ci-image workflow's output. + required: true + run: + description: Command to run inside the image. + required: true + env: + description: Newline-separated NAME=VALUE pairs to pass into the container. + required: false + default: "" + +runs: + using: "composite" + steps: + - name: Run in CI image + shell: bash + env: + INPUT_IMAGE: ${{ inputs.image }} + INPUT_RUN: ${{ inputs.run }} + INPUT_ENV: ${{ inputs.env }} + run: | + set -euo pipefail + + env_args=() + while IFS= read -r pair; do + [[ -z "$pair" ]] && continue + env_args+=(--env "$pair") + done <<< "$INPUT_ENV" + + # Cargo's registry is the only part of CARGO_HOME worth sharing with the + # host: mounting all of it would shadow the tools baked into the image. + mkdir -p "$HOME/.cargo/registry" + + # Mounting the workspace on its own path is what makes this work. The + # daemon resolves compose bind mounts against the host, and the test + # binaries bake CARGO_MANIFEST_DIR in at compile time, so the path has + # to mean the same thing inside and outside the container. + docker run --rm \ + --network host \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + --volume "$PWD:$PWD" \ + --volume "$HOME/.cargo/registry:/usr/local/cargo/registry" \ + --workdir "$PWD" \ + "${env_args[@]}" \ + "$INPUT_IMAGE" \ + bash -e -o pipefail -c "$INPUT_RUN" diff --git a/.github/docker/ci.Dockerfile b/.github/docker/ci.Dockerfile new file mode 100644 index 00000000..a79be15f --- /dev/null +++ b/.github/docker/ci.Dockerfile @@ -0,0 +1,97 @@ +# Image behind the `container:` of the CI jobs. Everything the jobs used to +# install per-run is baked in here instead, so a CI run no longer asks the risc0 +# install servers for anything. +# +# The image tag is a hash of this file plus rust-toolchain.toml, so editing +# either rebuilds the image on the PR that changes it. See ci-image.yml. +# +# Keep the base tag in sync with rust-toolchain.toml. +FROM rust:1.94.0-trixie + +# GitHub sets HOME=/github/home inside container jobs, which hides anything we +# bake into the image's ~. rzup defaults to $HOME/.risc0, so pin it to a fixed +# path; RUSTUP_HOME and CARGO_HOME already point at /usr/local from the base +# image, and rzup honours all three. +ENV RISC0_HOME=/usr/local/risc0 +ENV PATH=/usr/local/risc0/bin:$PATH + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + clang \ + libclang-dev \ + libssl-dev \ + pkg-config \ + curl \ + git \ + jq \ + python3-dev \ + && rm -rf /var/lib/apt/lists/* + +# The runner pre-creates the workspace as another uid, so git in a container job +# calls it dubious. checkout's own fix is --global, which only holds while HOME +# still points at the gitconfig it wrote; --system holds regardless. +RUN git config --system --add safe.directory '*' + +# The base image already has this toolchain at the minimal profile, so the +# `profile = "default"` in rust-toolchain.toml is a no-op here: rustup sees the +# toolchain as installed and never applies it. Add what the jobs need by hand. +COPY rust-toolchain.toml /tmp/toolchain/rust-toolchain.toml +RUN cd /tmp/toolchain \ + && rustup toolchain install \ + && rustup component add clippy rustfmt \ + && rm -rf /tmp/toolchain + +# For `cargo +nightly fmt` in the fmt-rs job. +RUN rustup toolchain install nightly --profile minimal --component rustfmt + +# The bootstrap script only ever drops rzup in $HOME/.risc0/bin, so run it +# against the build-time HOME and keep just the binary. rzup itself then honours +# RISC0_HOME and installs each component under /usr/local. +# +# Versions pinned to the set currently validated in local dev (`rzup show`); +# bare `rzup install` would float all four default components to latest. r0vm +# and cargo-risczero track each other and the risc0-zkvm crate version. +RUN curl -L https://risczero.com/install | bash \ + && mv /root/.risc0/bin/rzup /usr/local/bin/rzup \ + && rm -rf /root/.risc0 \ + && rzup install rust 1.94.1 \ + && rzup install cpp 2024.1.5 \ + && rzup install r0vm 3.0.5 \ + && rzup install cargo-risczero 3.0.5 + +# Copying docker. Static Go binaries, so the alpine-built ones run fine here. +COPY --from=docker:29.6.1-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-compose \ + /usr/local/libexec/docker/cli-plugins/docker-compose +COPY --from=docker:29.6.1-cli /usr/local/libexec/docker/cli-plugins/docker-buildx \ + /usr/local/libexec/docker/cli-plugins/docker-buildx + +# Prebuilt binaries; compiling these from source would dominate the image build. +# Versions pinned so a rebuild of the same image ships the same tools. +ENV BINSTALL_VERSION=1.21.0 +RUN curl -L --proto '=https' --tlsv1.2 -sSf \ + https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash \ + && cargo binstall --no-confirm \ + cargo-nextest@0.9.140 \ + taplo-cli@0.10.0 \ + cargo-machete@0.9.2 \ + cargo-deny@0.20.2 \ + just@1.56.0 + +# Smoke-test every tool the jobs use. A bad install should fail here, not later +# in some CI job that reuses this cached image. pyo3's `auto-initialize` links +# libpython, and the base image ships python3 without it, so check the .so too. +RUN cargo --version \ + && cargo +nightly fmt --version \ + && cargo clippy --version \ + && ls /usr/lib/*/libpython3*.so \ + && r0vm --version \ + && cargo risczero --version \ + && cargo nextest --version \ + && taplo --version \ + && cargo machete --version \ + && cargo deny --version \ + && just --version \ + && docker --version \ + && docker compose version \ + && docker buildx version diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index d82c0f11..89c18e32 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -9,12 +9,25 @@ on: permissions: contents: read pull-requests: write + packages: read name: bench-regression jobs: + ci-image: + permissions: + contents: read + packages: write + uses: ./.github/workflows/ci-image.yml + crypto-primitives: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 60 steps: - uses: actions/checkout@v5 @@ -24,13 +37,6 @@ jobs: # working tree, so we need the full history. fetch-depth: 0 - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install - - name: Run criterion-compare against base branch uses: boa-dev/criterion-compare-action@v3 with: diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml new file mode 100644 index 00000000..c93782de --- /dev/null +++ b/.github/workflows/ci-image.yml @@ -0,0 +1,75 @@ +name: CI image + +concurrency: + group: ci-image-${{ github.sha }} + cancel-in-progress: false + +# Resolves the CI image the calling workflow's jobs run in, building and pushing +# it only when it isn't published yet. +on: + workflow_call: + outputs: + image: + description: Image reference to pass to a job's `container:`. + value: ${{ jobs.ci-image.outputs.image }} + +jobs: + ci-image: + runs-on: ubuntu-latest + timeout-minutes: 60 + outputs: + image: ${{ steps.image-ref.outputs.image }} + steps: + - uses: actions/checkout@v5 + with: + ref: ${{ github.event.pull_request.head.sha || github.head_ref }} + + # Tagging by content hash means a PR that touches the Dockerfile or the + # toolchain builds and tests against its own image, while every other PR + # reuses the one already in the registry. + - name: Compute image reference + id: image-ref + run: | + repo="$(echo "${{ github.repository }}" | tr '[:upper:]' '[:lower:]')" + tag="$(cat .github/docker/ci.Dockerfile rust-toolchain.toml | sha256sum | cut -c1-16)" + echo "image=ghcr.io/${repo}/ci:${tag}" >> "$GITHUB_OUTPUT" + echo "CI image: ghcr.io/${repo}/ci:${tag}" + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Check whether the image is already published + id: probe + run: | + if docker manifest inspect "${{ steps.image-ref.outputs.image }}" > /dev/null 2>&1; then + echo "exists=true" >> "$GITHUB_OUTPUT" + echo "Image already published, skipping build." + else + echo "exists=false" >> "$GITHUB_OUTPUT" + echo "Image not published yet, building it." + fi + + - name: Set up Docker Buildx + if: steps.probe.outputs.exists == 'false' + uses: docker/setup-buildx-action@v3 + + # A pull_request from a fork gets a read-only GITHUB_TOKEN, so this step + # fails there. It only runs when the fork changed the image, which is the + # case that needs a maintainer's eyes anyway. + - name: Build and push CI image + if: steps.probe.outputs.exists == 'false' + uses: docker/build-push-action@v5 + with: + context: . + file: ./.github/docker/ci.Dockerfile + push: true + tags: ${{ steps.image-ref.outputs.image }} + # Links the package to the repo, so it shows up there and inherits its + # access rules. + labels: org.opencontainers.image.source=https://github.com/${{ github.repository }} + cache-from: type=gha,scope=ci-image + cache-to: type=gha,mode=max,scope=ci-image diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 194646f3..ff9b4a8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,67 +15,91 @@ on: permissions: contents: read pull-requests: read + packages: read name: General jobs: + ci-image: + permissions: + contents: read + packages: write + uses: ./.github/workflows/ci-image.yml + fmt-rs: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - name: Install nightly toolchain for rustfmt - run: rustup install nightly --profile minimal --component rustfmt - - name: Check Rust files are formatted run: cargo +nightly fmt --check fmt-toml: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - name: Install taplo-cli - run: cargo install --locked taplo-cli - + # No path argument: taplo reads a `.` as a literal file, excludes it for + # not being a .toml, and checks nothing. - name: Check TOML files are formatted - run: taplo fmt --check . + run: taplo fmt --check machete: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - name: Install active toolchain - run: rustup install - - - name: Install cargo-machete - run: cargo install cargo-machete - - name: Check for unused dependencies run: cargo machete deny: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} steps: - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - name: Install cargo-deny - run: cargo install --locked cargo-deny - - name: Check licenses and advisories run: cargo deny check lint: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 60 name: lint @@ -84,13 +108,6 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install - - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: @@ -108,36 +125,35 @@ jobs: run: cargo clippy -p "*program" -- -D warnings unit-tests: + needs: ci-image runs-on: ubuntu-latest + container: + image: ${{ needs.ci-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} timeout-minutes: 60 steps: - uses: actions/checkout@v5 with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install - - name: Restore Rust cache uses: Swatinem/rust-cache@v2 with: shared-key: ci-rust-cache save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Install nextest - run: cargo install --locked cargo-nextest - - name: Run tests env: RISC0_DEV_MODE: "1" RUST_LOG: "info" run: cargo nextest run --workspace --exclude integration_tests --exclude test_fixtures --all-features + # Not a `container:` job: the tests drive the host's Docker daemon through + # testcontainers, so they run in the CI image via `run-in-ci-image` instead. test-fixtures-tests: + needs: ci-image runs-on: ubuntu-latest timeout-minutes: 60 steps: @@ -145,12 +161,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Restore Rust cache uses: Swatinem/rust-cache@v2 @@ -158,16 +174,20 @@ jobs: shared-key: ci-rust-cache save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Install nextest - run: cargo install --locked cargo-nextest - - name: Run test_fixtures tests - env: - RISC0_DEV_MODE: "1" - RUST_LOG: "info" - run: cargo nextest run -p test_fixtures + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + env: | + RISC0_DEV_MODE=1 + RUST_LOG=info + run: cargo nextest run -p test_fixtures + # Not a `container:` job: the test binaries bake CARGO_MANIFEST_DIR in at + # compile time, so the archive has to be built on the same path the matrix + # jobs later run it from. integration-tests-prebuild: + needs: ci-image runs-on: ubuntu-latest outputs: targets: ${{ steps.discover-targets.outputs.targets }} @@ -176,12 +196,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Restore Rust cache uses: Swatinem/rust-cache@v2 @@ -189,13 +209,12 @@ jobs: shared-key: ci-rust-cache save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Install nextest - run: cargo install --locked cargo-nextest - - name: Build integration test archive - env: - RISC0_DEV_MODE: "1" - run: cargo nextest archive -p integration_tests --archive-file integration-tests.tar.zst --no-pager + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + env: RISC0_DEV_MODE=1 + run: cargo nextest archive -p integration_tests --archive-file integration-tests.tar.zst --no-pager - name: Upload integration test archive uses: actions/upload-artifact@v4 @@ -203,15 +222,22 @@ jobs: name: integration-tests-archive path: integration-tests.tar.zst + - name: List integration test binaries + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + run: | + cargo nextest list \ + --archive-file integration-tests.tar.zst \ + --list-type binaries-only \ + --message-format json \ + --no-pager > integration-tests-binaries.json + + # $GITHUB_OUTPUT is outside the mounted workspace, so the container cannot + # write to it. jq is on the runner, so do this pass here. - name: Discover integration test targets from archive id: discover-targets run: | - cargo nextest list \ - --archive-file integration-tests.tar.zst \ - --list-type binaries-only \ - --message-format json \ - --no-pager > integration-tests-binaries.json - targets_json="$(jq -c '[."rust-binaries" | to_entries[] | select(.value.kind == "test" and .value."binary-name" != "tps") | .value."binary-name"] | sort | unique' integration-tests-binaries.json)" if [[ "$targets_json" == "[]" ]]; then @@ -223,7 +249,7 @@ jobs: echo "Discovered integration targets: $targets_json" integration-tests: - needs: [test-fixtures-tests, integration-tests-prebuild] + needs: [ci-image, test-fixtures-tests, integration-tests-prebuild] runs-on: ubuntu-latest timeout-minutes: 90 strategy: @@ -236,28 +262,29 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Download integration test archive uses: actions/download-artifact@v4 with: name: integration-tests-archive - - name: Install nextest - run: cargo install --locked cargo-nextest - - name: Run tests - env: - RISC0_DEV_MODE: "1" - RUST_LOG: "info" - run: cargo nextest run --archive-file integration-tests.tar.zst -E "binary(${{ matrix.target }})" + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + env: | + RISC0_DEV_MODE=1 + RUST_LOG=info + run: cargo nextest run --archive-file integration-tests.tar.zst -E "binary(${{ matrix.target }})" valid-proof-test: + needs: ci-image runs-on: ubuntu-latest timeout-minutes: 90 steps: @@ -265,12 +292,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-system-deps - - - uses: ./.github/actions/install-risc0 - - - name: Install active toolchain - run: rustup install + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Restore Rust cache uses: Swatinem/rust-cache@v2 @@ -279,11 +306,16 @@ jobs: save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - name: Test valid proof - env: - RUST_LOG: "info" - run: cargo test -p integration_tests -- --exact private::private_transfer_to_owned_account + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + env: RUST_LOG=info + run: cargo test -p integration_tests -- --exact private::private_transfer_to_owned_account + # `just build-artifacts` drives the host's Docker daemon via `cargo risczero + # build`, so it goes through the image rather than `container:`. artifacts: + needs: ci-image runs-on: ubuntu-latest timeout-minutes: 60 @@ -293,7 +325,12 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.head_ref }} - - uses: ./.github/actions/install-risc0 + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Restore Rust cache uses: Swatinem/rust-cache@v2 @@ -301,11 +338,11 @@ jobs: shared-key: ci-rust-cache save-if: ${{ github.ref == 'refs/heads/main' || github.ref == 'refs/heads/dev' }} - - name: Install just - run: cargo install --locked just - - name: Build artifacts - run: just build-artifacts + uses: ./.github/actions/run-in-ci-image + with: + image: ${{ needs.ci-image.outputs.image }} + run: just build-artifacts - name: Check if artifacts match repository run: | diff --git a/Cargo.lock b/Cargo.lock index 3b5714b8..ea1c7782 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1368,6 +1368,25 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chain_consistency" +version = "0.1.0" +dependencies = [ + "anyhow", + "borsh", + "common", + "futures", + "lee", + "lee_core", + "log", + "logos-blockchain-core", + "logos-blockchain-zone-sdk", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", +] + [[package]] name = "chkstk_stub" version = "0.1.0" @@ -3941,6 +3960,7 @@ dependencies = [ "arc-swap", "async-stream", "borsh", + "chain_consistency", "common", "cross_zone", "cross_zone_inbox_core", @@ -3960,7 +3980,6 @@ dependencies = [ "storage", "tempfile", "testnet_initial_state", - "thiserror 2.0.18", "tokio", "url", ] @@ -9025,6 +9044,7 @@ dependencies = [ "borsh", "bridge_core", "bytesize", + "chain_consistency", "chrono", "common", "cross_zone", @@ -11020,13 +11040,11 @@ version = "0.1.0" dependencies = [ "bip39", "cbindgen", - "common", "key_protocol", "lee", "lee_core", "programs", "risc0-zkvm", - "sequencer_service_rpc", "serde_json", "tokio", "vault_core", diff --git a/Cargo.toml b/Cargo.toml index 164b64cc..bac0ea4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "lez/wallet", "lez/wallet-ffi", "lez/common", + "lez/chain_consistency", "lez/programs", "lez/programs/amm", "lez/programs/associated_token_account", @@ -72,6 +73,7 @@ members = [ lee = { path = "lee/state_machine" } lee_core = { path = "lee/state_machine/core" } common = { path = "lez/common" } +chain_consistency = { path = "lez/chain_consistency" } mempool = { path = "lez/mempool" } storage = { path = "lez/storage" } key_protocol = { path = "lee/key_protocol" } diff --git a/Justfile b/Justfile index cf7c833e..b838f4fc 100644 --- a/Justfile +++ b/Justfile @@ -11,13 +11,19 @@ ARTIFACTS := "artifacts" # Linux/CI, which is unaffected. DEMO_ENV := if os() == "macos" { "DYLD_FALLBACK_FRAMEWORK_PATH=/Library/Developer/CommandLineTools/Library/Frameworks" } else { "" } -# Build risc0 program artifacts. +# Build risc0 program artifacts and test fixture. build-artifacts: @echo "๐Ÿ”จ Building artifacts" @rm -rf {{ARTIFACTS}} @just build-artifact lee/privacy_preserving_circuit @just build-artifact lez/programs programs + @if [ "${GITHUB_ACTIONS:-}" = "true" ]; then \ + echo "Skipping test fixture regeneration because CI doesn't need it"; \ + else \ + just regenerate-test-fixture; \ + fi + build-artifact methods_path features="": @echo "Building artifacts for {{methods_path}}" @rm -rf target/{{methods_path}}/riscv32im-risc0-zkvm-elf/docker/*.bin @@ -42,7 +48,7 @@ test: # Regenerate the prebuilt sequencer db dump for fast TestContext::new() (needs Docker; commit the dump). regenerate-test-fixture: - @echo "๐Ÿงช Regenerating test fixtures" + @echo "๐Ÿงช Regenerating test fixture" RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture # Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup). @@ -131,5 +137,6 @@ clean: rm -rf lez/sequencer/service/rocksdb rm -rf lez/indexer/service/rocksdb* rm -rf lez/wallet/configs/debug/storage.json + rm -rf lez/wallet/configs/debug/statistics.json rm -rf rocksdb* cd bedrock && docker compose down -v diff --git a/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin b/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin index f54520a8..4eafd160 100644 Binary files a/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin and b/artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin differ diff --git a/examples/program_deployment/src/bin/run_hello_world.rs b/examples/program_deployment/src/bin/run_hello_world.rs index 3f2223a1..a43bd7f1 100644 --- a/examples/program_deployment/src/bin/run_hello_world.rs +++ b/examples/program_deployment/src/bin/run_hello_world.rs @@ -27,7 +27,7 @@ use wallet::WalletCore; #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the program binary @@ -59,7 +59,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); diff --git a/examples/program_deployment/src/bin/run_hello_world_private.rs b/examples/program_deployment/src/bin/run_hello_world_private.rs index f6202433..64fc39c2 100644 --- a/examples/program_deployment/src/bin/run_hello_world_private.rs +++ b/examples/program_deployment/src/bin/run_hello_world_private.rs @@ -23,7 +23,7 @@ use wallet::{AccountIdentity, WalletCore}; #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the program binary diff --git a/examples/program_deployment/src/bin/run_hello_world_through_tail_call.rs b/examples/program_deployment/src/bin/run_hello_world_through_tail_call.rs index 6ebba70f..18620220 100644 --- a/examples/program_deployment/src/bin/run_hello_world_through_tail_call.rs +++ b/examples/program_deployment/src/bin/run_hello_world_through_tail_call.rs @@ -27,7 +27,7 @@ use wallet::WalletCore; #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the program binary @@ -55,7 +55,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); diff --git a/examples/program_deployment/src/bin/run_hello_world_through_tail_call_private.rs b/examples/program_deployment/src/bin/run_hello_world_through_tail_call_private.rs index d35e6521..c1282dae 100644 --- a/examples/program_deployment/src/bin/run_hello_world_through_tail_call_private.rs +++ b/examples/program_deployment/src/bin/run_hello_world_through_tail_call_private.rs @@ -26,7 +26,7 @@ use wallet::{AccountIdentity, WalletCore}; #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the simple_tail_call program binary diff --git a/examples/program_deployment/src/bin/run_hello_world_with_authorization.rs b/examples/program_deployment/src/bin/run_hello_world_with_authorization.rs index 0d257db2..563f0877 100644 --- a/examples/program_deployment/src/bin/run_hello_world_with_authorization.rs +++ b/examples/program_deployment/src/bin/run_hello_world_with_authorization.rs @@ -29,7 +29,7 @@ use wallet::WalletCore; #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the program binary @@ -52,7 +52,8 @@ async fn main() { .storage() .key_chain() .pub_account_signing_key(account_id) - .expect("Input account should be a self owned public account"); + .expect("Input account should be a self owned public account") + .clone(); // Define the desired greeting in ASCII let greeting: Vec = vec![72, 111, 108, 97, 32, 109, 117, 110, 100, 111, 33]; @@ -60,10 +61,10 @@ async fn main() { // Construct the public transaction // Query the current nonce from the node let nonces = wallet_core - .get_accounts_nonces(vec![account_id]) + .get_accounts_nonces(&[account_id]) .await .expect("Node should be reachable to query account data"); - let signing_keys = [signing_key]; + let signing_keys = [&signing_key]; let message = Message::try_new(program.id(), vec![account_id], nonces, greeting).unwrap(); // Pass the signing key to sign the message. This will be used by the node // to flag the pre_state as `is_authorized` when executing the program @@ -72,7 +73,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); diff --git a/examples/program_deployment/src/bin/run_hello_world_with_authorization_through_tail_call_with_pda.rs b/examples/program_deployment/src/bin/run_hello_world_with_authorization_through_tail_call_with_pda.rs index 70688cfd..9139189e 100644 --- a/examples/program_deployment/src/bin/run_hello_world_with_authorization_through_tail_call_with_pda.rs +++ b/examples/program_deployment/src/bin/run_hello_world_with_authorization_through_tail_call_with_pda.rs @@ -35,7 +35,7 @@ const PDA_SEED: PdaSeed = PdaSeed::new([37; 32]); #[tokio::main] async fn main() { // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); // Parse arguments // First argument is the path to the program binary @@ -57,7 +57,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); diff --git a/examples/program_deployment/src/bin/run_hello_world_with_move_function.rs b/examples/program_deployment/src/bin/run_hello_world_with_move_function.rs index a77fe2e6..b99f41f4 100644 --- a/examples/program_deployment/src/bin/run_hello_world_with_move_function.rs +++ b/examples/program_deployment/src/bin/run_hello_world_with_move_function.rs @@ -66,7 +66,7 @@ async fn main() { let program = Program::new(bytecode.into()).unwrap(); // Initialize wallet - let wallet_core = WalletCore::from_env().unwrap(); + let wallet_core = WalletCore::from_env().await.unwrap(); match cli.command { Command::WritePublic { @@ -88,7 +88,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); @@ -127,7 +127,7 @@ async fn main() { // Submit the transaction let _response = wallet_core - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::Public(tx)) .await .unwrap(); diff --git a/flake.nix b/flake.nix index 0aaad40c..d0058bcb 100644 --- a/flake.nix +++ b/flake.nix @@ -117,6 +117,7 @@ done if [ -n "$tool" ]; then unset DEVELOPER_DIR SDKROOT + export xcrun_nocache=1 fi exec /usr/bin/xcrun "$@" ''; diff --git a/integration_tests/tests/auth_transfer/private.rs b/integration_tests/tests/auth_transfer/private.rs index ddea3ab8..655ca1a9 100644 --- a/integration_tests/tests/auth_transfer/private.rs +++ b/integration_tests/tests/auth_transfer/private.rs @@ -248,11 +248,7 @@ async fn shielded_transfer_to_foreign_account() -> Result<()> { let acc_1_balance = account_balance(&ctx, from).await?; assert!( - verify_commitment_is_in_state( - tx.message.new_commitments[0].clone(), - ctx.sequencer_client() - ) - .await + verify_commitment_is_in_state(tx.message.new_commitments[0], ctx.sequencer_client()).await ); assert_eq!(acc_1_balance, 9900); diff --git a/integration_tests/tests/bridge.rs b/integration_tests/tests/bridge.rs index 9c241ebb..8da35f4d 100644 --- a/integration_tests/tests/bridge.rs +++ b/integration_tests/tests/bridge.rs @@ -441,7 +441,7 @@ async fn bedrock_deposit_claim_and_withdraw_round_trip_succeeds() -> anyhow::Res // Now claim funds from vault back to recipient let nonces = ctx .wallet() - .get_accounts_nonces(vec![recipient_id]) + .get_accounts_nonces(&[recipient_id]) .await .context("Failed to get nonce for vault claim")?; diff --git a/integration_tests/tests/cross_zone_bridge.rs b/integration_tests/tests/cross_zone_bridge.rs index 63ce8cc4..f68301c3 100644 --- a/integration_tests/tests/cross_zone_bridge.rs +++ b/integration_tests/tests/cross_zone_bridge.rs @@ -24,7 +24,7 @@ use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, indexer_client::IndexerClient, - setup::{setup_bedrock_node, setup_indexer, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, }; use lee::{ AccountId, PrivateKey, PublicKey, PublicTransaction, @@ -69,18 +69,19 @@ async fn lock_on_zone_a_mints_wrapped_token_on_zone_b() -> Result<()> { holder: holder_id, amount: INITIAL_BALANCE, }]; - let (seq_a, _seq_a_home) = setup_sequencer(partial, bedrock_addr, genesis_a, channel_a, None) + let (seq_a, _seq_a_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_a) + .with_genesis(genesis_a) + .setup() .await .context("Failed to set up zone A sequencer")?; - let (_seq_b, _seq_b_home) = setup_sequencer( - partial, - bedrock_addr, - vec![], - channel_b, - Some(cross_zone.clone()), - ) - .await - .context("Failed to set up zone B sequencer")?; + let (_seq_b, _seq_b_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_b) + .with_genesis(vec![]) + .with_cross_zone(cross_zone.clone()) + .setup() + .await + .context("Failed to set up zone B sequencer")?; let (idx_b, _idx_b_home) = setup_indexer(bedrock_addr, channel_b, Some(cross_zone)) .await .context("Failed to set up zone B indexer")?; diff --git a/integration_tests/tests/cross_zone_ingress_guard.rs b/integration_tests/tests/cross_zone_ingress_guard.rs index 5f401869..fe5fe7fc 100644 --- a/integration_tests/tests/cross_zone_ingress_guard.rs +++ b/integration_tests/tests/cross_zone_ingress_guard.rs @@ -18,7 +18,7 @@ use cross_zone_inbox_core::{ }; use integration_tests::{ config::{self, SequencerPartialConfig}, - setup::{setup_bedrock_node, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node}, }; use lee::{ PublicTransaction, @@ -34,7 +34,10 @@ async fn user_origin_inbox_call_rejected() -> Result<()> { .context("Failed to set up Bedrock node")?; let partial = SequencerPartialConfig::default(); let channel = config::bedrock_channel_id(); - let (seq, _seq_home) = setup_sequencer(partial, bedrock_addr, vec![], channel, None) + let (seq, _seq_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel) + .with_genesis(vec![]) + .setup() .await .context("Failed to set up sequencer")?; diff --git a/integration_tests/tests/cross_zone_ping.rs b/integration_tests/tests/cross_zone_ping.rs index 19223b5f..a3ed0c0c 100644 --- a/integration_tests/tests/cross_zone_ping.rs +++ b/integration_tests/tests/cross_zone_ping.rs @@ -18,7 +18,7 @@ use common::transaction::LeeTransaction; use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, - setup::{setup_bedrock_node, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node}, }; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::program::ProgramId; @@ -54,13 +54,19 @@ async fn ping_crosses_from_zone_a_to_zone_b() -> Result<()> { }], }; - let (seq_a, _seq_a_home) = setup_sequencer(partial, bedrock_addr, vec![], channel_a, None) + let (seq_a, _seq_a_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_a) + .with_genesis(vec![]) + .setup() .await .context("Failed to set up zone A sequencer")?; - let (seq_b, _seq_b_home) = - setup_sequencer(partial, bedrock_addr, vec![], channel_b, Some(cross_zone)) - .await - .context("Failed to set up zone B sequencer")?; + let (seq_b, _seq_b_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_b) + .with_genesis(vec![]) + .with_cross_zone(cross_zone) + .setup() + .await + .context("Failed to set up zone B sequencer")?; // Submit the ping on zone A, addressed to ping_receiver on zone B. let ping = build_ping_tx(zone_b, receiver_id); diff --git a/integration_tests/tests/cross_zone_verified.rs b/integration_tests/tests/cross_zone_verified.rs index e2f016c0..aa2ac06a 100644 --- a/integration_tests/tests/cross_zone_verified.rs +++ b/integration_tests/tests/cross_zone_verified.rs @@ -17,7 +17,7 @@ use cross_zone_outbox_core::outbox_pda; use integration_tests::{ config::{self, SequencerPartialConfig}, indexer_client::IndexerClient, - setup::{setup_bedrock_node, setup_indexer, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, }; use lee::{AccountId, PublicTransaction, public_transaction::Message}; use lee_core::program::ProgramId; @@ -53,21 +53,22 @@ async fn indexer_verifies_and_delivers_cross_zone_ping() -> Result<()> { // Zone A: source. Zone B: destination, with the watcher on its sequencer and // the verifier on its indexer. - let (seq_a, _seq_a_home) = setup_sequencer(partial, bedrock_addr, vec![], channel_a, None) + let (seq_a, _seq_a_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_a) + .with_genesis(vec![]) + .setup() .await .context("Failed to set up zone A sequencer")?; let (_idx_a, _idx_a_home) = setup_indexer(bedrock_addr, channel_a, None) .await .context("Failed to set up zone A indexer")?; - let (_seq_b, _seq_b_home) = setup_sequencer( - partial, - bedrock_addr, - vec![], - channel_b, - Some(cross_zone.clone()), - ) - .await - .context("Failed to set up zone B sequencer")?; + let (_seq_b, _seq_b_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_b) + .with_genesis(vec![]) + .with_cross_zone(cross_zone.clone()) + .setup() + .await + .context("Failed to set up zone B sequencer")?; let (idx_b, _idx_b_home) = setup_indexer(bedrock_addr, channel_b, Some(cross_zone)) .await .context("Failed to set up zone B indexer")?; diff --git a/integration_tests/tests/private_pda.rs b/integration_tests/tests/private_pda.rs index 0c67049f..0b187283 100644 --- a/integration_tests/tests/private_pda.rs +++ b/integration_tests/tests/private_pda.rs @@ -91,7 +91,7 @@ async fn fund_private_pda( let tx = PrivacyPreservingTransaction::new(message, witness_set); wallet - .sequencer_client + .leader_owned() .send_transaction(LeeTransaction::PrivacyPreserving(tx)) .await .map_err(|e| anyhow::anyhow!("send transaction failed: {e}"))?; @@ -180,7 +180,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { info!("Sending to alice_pda_0 (identifier=0)"); fund_private_pda( - ctx.wallet(), + ctx.wallet_mut(), sender_0, alice_npk, alice_vpk.clone(), @@ -194,7 +194,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { info!("Sending to alice_pda_1 (identifier=1)"); fund_private_pda( - ctx.wallet(), + ctx.wallet_mut(), sender_1, alice_npk, alice_vpk.clone(), @@ -231,7 +231,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { .get_private_account_commitment(alice_pda_0_id) .context("commitment for alice_pda_0 missing")?; assert!( - verify_commitment_is_in_state(commitment_0.clone(), ctx.sequencer_client()).await, + verify_commitment_is_in_state(commitment_0, ctx.sequencer_client()).await, "alice_pda_0 commitment not in state after receive" ); @@ -240,7 +240,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { .get_private_account_commitment(alice_pda_1_id) .context("commitment for alice_pda_1 missing")?; assert!( - verify_commitment_is_in_state(commitment_1.clone(), ctx.sequencer_client()).await, + verify_commitment_is_in_state(commitment_1, ctx.sequencer_client()).await, "alice_pda_1 commitment not in state after receive" ); assert_ne!( @@ -262,7 +262,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { info!("Alice spending from alice_pda_0"); spend_private_pda( - ctx.wallet(), + ctx.wallet_mut(), alice_pda_0_id, recipient_npk_0, recipient_vpk_0, @@ -275,7 +275,7 @@ async fn private_pda_family_members_receive_and_spend() -> Result<()> { info!("Alice spending from alice_pda_1"); spend_private_pda( - ctx.wallet(), + ctx.wallet_mut(), alice_pda_1_id, recipient_npk_1, recipient_vpk_1, diff --git a/integration_tests/tests/program_deployment.rs b/integration_tests/tests/program_deployment.rs index c92daeb3..3c620168 100644 --- a/integration_tests/tests/program_deployment.rs +++ b/integration_tests/tests/program_deployment.rs @@ -31,7 +31,7 @@ async fn deploy_and_execute_program() -> Result<()> { let account_id = new_account(&mut ctx, false, None).await?; - let nonces = ctx.wallet().get_accounts_nonces(vec![account_id]).await?; + let nonces = ctx.wallet_mut().get_accounts_nonces(&[account_id]).await?; let private_key = ctx .wallet() .get_account_public_signing_key(account_id) diff --git a/integration_tests/tests/sequencer_bootstrap.rs b/integration_tests/tests/sequencer_bootstrap.rs new file mode 100644 index 00000000..5261c412 --- /dev/null +++ b/integration_tests/tests/sequencer_bootstrap.rs @@ -0,0 +1,545 @@ +//! End-to-end tests for the sequencer's startup bootstrap/reconstruction from +//! Bedrock (verify-and-reconstruct). Each test drives a real Bedrock node and one +//! or more sequencer instances sharing the same channel, exercising how a +//! sequencer reconciles its local store against what the channel serves. + +#![expect( + clippy::tests_outside_test_module, + reason = "Integration tests live at crate root and don't care about these lints" +)] + +use std::{net::SocketAddr, path::Path, time::Duration}; + +use anyhow::{Context as _, Result, bail}; +use indexer_service_rpc::RpcClient as _; +use integration_tests::L2_TO_L1_TIMEOUT; +use lee::{AccountId, PrivateKey, PublicKey}; +use logos_blockchain_core::mantle::ops::channel::ChannelId; +use sequencer_core::config::GenesisAction; +use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use test_fixtures::{ + config::{SequencerPartialConfig, UrlProtocol, addr_to_url}, + indexer_client::IndexerClient, + setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, +}; +use tokio::test; + +/// Block cadence for the tests: short so we don't wait long for local production. +fn fast_blocks() -> SequencerPartialConfig { + SequencerPartialConfig { + block_create_timeout: Duration::from_secs(2), + ..SequencerPartialConfig::default() + } +} + +/// Block cadence for a sequencer we don't want producing during the inspection +/// window, so its post-reconstruction tip stays put while we read it. +fn slow_blocks() -> SequencerPartialConfig { + SequencerPartialConfig { + block_create_timeout: Duration::from_secs(30), + ..SequencerPartialConfig::default() + } +} + +fn sequencer_client(addr: SocketAddr) -> Result { + let url = addr_to_url(UrlProtocol::Http, addr).context("Failed to build sequencer URL")?; + SequencerClientBuilder::default() + .build(url) + .context("Failed to build sequencer client") +} + +/// Polls the indexer's last finalized block id until it reaches `target`. The +/// indexer reads finalized channel history, so this is our oracle for "block +/// `target` is finalized on Bedrock" โ€” exactly what a reconstructing sequencer +/// can read back. +async fn wait_for_finalized( + indexer: &IndexerClient, + target: u64, + timeout: Duration, +) -> Result { + let poll = async { + loop { + let finalized = indexer + .get_last_finalized_block_id() + .await + .context("Failed to read indexer last finalized block id")? + .unwrap_or(0); + if finalized >= target { + return Ok::<_, anyhow::Error>(finalized); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + }; + tokio::time::timeout(timeout, poll) + .await + .with_context(|| format!("Timed out waiting for indexer to finalize block {target}"))? +} + +/// Polls the sequencer's last block id until it reaches `target` or `timeout` elapses. +async fn wait_for_block_id( + client: &SequencerClient, + target: u64, + timeout: Duration, +) -> Result { + let poll = async { + loop { + let id = client + .get_last_block_id() + .await + .context("Failed to read sequencer last block id")?; + if id >= target { + return Ok::<_, anyhow::Error>(id); + } + tokio::time::sleep(Duration::from_millis(500)).await; + } + }; + tokio::time::timeout(timeout, poll) + .await + .with_context(|| format!("Timed out waiting for block id {target}"))? +} + +/// Best-effort extraction of a panic payload's message (panics carry a `String` +/// or `&str`), for asserting a startup aborted for the *expected* reason. +fn panic_message(payload: &(dyn std::any::Any + Send)) -> String { + payload + .downcast_ref::() + .cloned() + .or_else(|| payload.downcast_ref::<&str>().map(|s| (*s).to_owned())) + .unwrap_or_else(|| "".to_owned()) +} + +/// A `SupplyAccount` genesis action for a fresh account, returning the vault +/// account id the funds land in (genesis supply goes into a claimable vault, not +/// the account directly), so tests can assert genesis state is present. +fn supplied_account(balance: u128) -> (AccountId, GenesisAction) { + let account_id = AccountId::from(&PublicKey::new_from_private_key( + &PrivateKey::new_os_random(), + )); + let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), account_id); + ( + vault_id, + GenesisAction::SupplyAccount { + account_id, + balance, + }, + ) +} + +/// Recursively copies the contents of `src` into `dst` (used to snapshot/restore a +/// sequencer's rocksdb directory while it is stopped). +fn copy_dir_recursive(src: &Path, dst: &Path) -> Result<()> { + std::fs::create_dir_all(dst) + .with_context(|| format!("Failed to create dir {}", dst.display()))?; + for entry in + std::fs::read_dir(src).with_context(|| format!("Failed to read dir {}", src.display()))? + { + let entry = entry.context("Failed to read dir entry")?; + let target = dst.join(entry.file_name()); + if entry + .file_type() + .context("Failed to read file type")? + .is_dir() + { + copy_dir_recursive(&entry.path(), &target)?; + } else { + std::fs::copy(entry.path(), &target) + .with_context(|| format!("Failed to copy into {}", target.display()))?; + } + } + Ok(()) +} + +/// Case 1: local store is empty and the Bedrock channel is empty. +/// +/// The sequencer bootstraps genesis state, finds nothing to reconstruct, publishes +/// its own genesis to open the channel, and starts producing. +#[test] +async fn empty_local_and_empty_bedrock_bootstraps_from_genesis() -> Result<()> { + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock")?; + let home = tempfile::tempdir().context("Failed to create sequencer home")?; + + let (vault_id, supply) = supplied_account(12_345); + let genesis = vec![ + supply, + GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }, + ]; + + let handle = SequencerSetup::new(fast_blocks(), bedrock_addr) + .with_genesis(genesis) + .setup_at(home.path()) + .await + .context("Failed to start sequencer")?; + let client = sequencer_client(handle.addr())?; + + // Fresh store + empty channel: startup bootstrapped genesis state directly. + assert_eq!( + client.get_account_balance(vault_id).await?, + 12_345, + "genesis-supplied vault balance must be present after bootstrap" + ); + + // The sequencer is live and producing on the freshly opened channel. + let last = wait_for_block_id(&client, 3, Duration::from_secs(60)).await?; + assert!( + last >= 3, + "sequencer should keep producing blocks, last={last}" + ); + assert!(handle.is_healthy(), "sequencer must stay healthy"); + + Ok(()) +} + +/// Case 2: local store is empty, but the Bedrock channel already has blocks. +/// +/// A first sequencer opens the channel and produces blocks; a second sequencer +/// starts from an empty store on the same channel (reusing the first's bedrock +/// signing key) and reconstructs the finalized history into its state. +#[test] +async fn empty_local_reconstructs_from_populated_bedrock() -> Result<()> { + const PRODUCED_TARGET: u64 = 3; + + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock")?; + let (indexer_handle, _indexer_dir) = setup_indexer( + bedrock_addr, + test_fixtures::config::bedrock_channel_id(), + None, + ) + .await + .context("Failed to setup indexer")?; + let indexer_url = addr_to_url(UrlProtocol::Ws, indexer_handle.addr()) + .context("Failed to build indexer URL")?; + let indexer = IndexerClient::new(&indexer_url) + .await + .context("Failed to build indexer client")?; + + let (vault_id, supply) = supplied_account(7_777); + let genesis = vec![ + supply, + GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }, + ]; + + // Sequencer A opens the channel and produces a few blocks. + let home_a = tempfile::tempdir().context("Failed to create sequencer A home")?; + let handle_a = SequencerSetup::new(fast_blocks(), bedrock_addr) + .with_genesis(genesis.clone()) + .setup_at(home_a.path()) + .await + .context("Failed to start sequencer A")?; + let client_a = sequencer_client(handle_a.addr())?; + wait_for_block_id(&client_a, PRODUCED_TARGET, Duration::from_secs(60)).await?; + + // Wait until those blocks are finalized on Bedrock โ€” reconstruction only + // reads finalized history. A stays alive so its publish task keeps flushing. + let finalized = wait_for_finalized(&indexer, PRODUCED_TARGET, L2_TO_L1_TIMEOUT).await?; + + // Stop A, then wipe just its L2 store (keeping the bedrock signing key) so it + // restarts from an empty store on the same channel/identity โ€” a sequencer that + // lost its local DB. + drop(handle_a); + tokio::time::sleep(Duration::from_secs(2)).await; + std::fs::remove_dir_all(home_a.path().join("rocksdb")) + .context("Failed to wipe sequencer L2 store")?; + + // Sequencer B restarts on the same home from that empty store and reconstructs. + let handle_b = SequencerSetup::new(slow_blocks(), bedrock_addr) + .with_genesis(genesis) + .setup_at(home_a.path()) + .await + .context("Failed to start sequencer B")?; + let client_b = sequencer_client(handle_b.addr())?; + + // Reconstruction ran synchronously during B's startup: even though its local + // store was empty, its tip is past genesis, matching the finalized channel. + let tip_b = client_b.get_last_block_id().await?; + assert!( + tip_b >= finalized, + "B should reconstruct at least the finalized blocks; tip_b={tip_b}, finalized={finalized}" + ); + assert!( + tip_b > 1, + "B should have reconstructed blocks beyond genesis; tip_b={tip_b}" + ); + + // Genesis state was rebuilt as part of the reconstruction. + assert_eq!( + client_b.get_account_balance(vault_id).await?, + 7_777, + "reconstructed genesis vault balance must be present" + ); + assert!(handle_b.is_healthy(), "sequencer B must stay healthy"); + + Ok(()) +} + +/// Case 3: local store is not empty, but the Bedrock channel is empty. +/// +/// A sequencer produces blocks (committing to a channel), is stopped, and is +/// restarted against a fresh/empty channel โ€” i.e. the channel it committed to +/// was wiped or the node points at a different chain. Startup must fail rather +/// than silently resume onto a foreign channel. Crucially this must hold even +/// though the sequencer only ever *produced* (so it never recorded a per-block +/// anchor): the committed-but-missing-channel invariant catches it. +#[test] +async fn nonempty_local_against_empty_channel_fails_startup() -> Result<()> { + const PRODUCED_TARGET: u64 = 3; + + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock")?; + + let (_vault_id, supply) = supplied_account(1); + let genesis = vec![ + supply, + GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }, + ]; + + // A opens the channel and produces blocks. They land in its local store + // immediately, so no need to wait for finalization. + let home_a = tempfile::tempdir().context("Failed to create sequencer A home")?; + let handle_a = SequencerSetup::new(fast_blocks(), bedrock_addr) + .with_genesis(genesis.clone()) + .setup_at(home_a.path()) + .await + .context("Failed to start sequencer A")?; + wait_for_block_id( + &sequencer_client(handle_a.addr())?, + PRODUCED_TARGET, + Duration::from_secs(60), + ) + .await?; + drop(handle_a); + tokio::time::sleep(Duration::from_secs(2)).await; + + // Restart on the SAME home (A's committed store: blocks + checkpoint) but + // pointed at a fresh, never-used channel โ€” the channel it committed to is gone. + let empty_channel = ChannelId::from([0x5a_u8; 32]); + + // Startup aborts on the missing-channel invariant (a panic in + // `start_from_config`). Run it on a dedicated OS thread with its own runtime + // so the panic is isolated to `join()` instead of failing the test thread. + // The `timeout` future must be created *inside* `block_on` (it needs a running + // reactor), so build it in an `async` block rather than as an eager argument. + let home_a_path = home_a.path().to_owned(); + let outcome = std::thread::spawn(move || { + let runtime = tokio::runtime::Runtime::new().expect("Failed to build runtime"); + runtime.block_on(async { + tokio::time::timeout( + Duration::from_secs(90), + SequencerSetup::new(slow_blocks(), bedrock_addr) + .with_channel_id(empty_channel) + .with_genesis(genesis) + .setup_at(&home_a_path), + ) + .await + }) + }) + .join(); + + match outcome { + // Expected: `start_from_config` panicked on the missing-channel invariant. + // Assert the *reason*, so an unrelated panic fails the test rather than + // masquerading as success. + Err(panic) => { + let message = panic_message(&*panic); + assert!( + message.contains("Refusing to resume onto a foreign channel"), + "startup panicked for an unexpected reason: {message}" + ); + } + Ok(Err(_elapsed)) => { + bail!("Sequencer startup hung instead of failing against an empty channel") + } + Ok(Ok(Err(err))) => { + bail!("Sequencer expected to panic, but it failed with error: {err:#?}") + } + Ok(Ok(Ok(_handle))) => { + bail!("Sequencer startup unexpectedly succeeded against an empty channel") + } + } + + Ok(()) +} + +/// Case 4: both non-empty, but the local store is *ahead* of the finalized channel. +/// +/// A sequencer produces blocks faster than Bedrock finalizes them (the normal +/// steady state), so on restart its local tip leads the channel's finalized tip. +/// Startup must succeed: reconstruction re-verifies the finalized blocks it already +/// holds and leaves the extra, not-yet-finalized local blocks untouched. +#[test] +async fn local_ahead_of_channel_resumes() -> Result<()> { + const FINALIZED_TARGET: u64 = 2; + + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock")?; + let (indexer_handle, _indexer_dir) = setup_indexer( + bedrock_addr, + test_fixtures::config::bedrock_channel_id(), + None, + ) + .await + .context("Failed to setup indexer")?; + let indexer_url = addr_to_url(UrlProtocol::Ws, indexer_handle.addr()) + .context("Failed to build indexer URL")?; + let indexer = IndexerClient::new(&indexer_url) + .await + .context("Failed to build indexer client")?; + + let (vault_id, supply) = supplied_account(4_242); + let genesis = vec![ + supply, + GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }, + ]; + + // A produces continuously (fast cadence) while Bedrock finalizes slowly, so + // its local tip runs well ahead of the channel's finalized tip. + let home = tempfile::tempdir().context("Failed to create sequencer home")?; + let handle_a = SequencerSetup::new(fast_blocks(), bedrock_addr) + .with_genesis(genesis.clone()) + .setup_at(home.path()) + .await + .context("Failed to start sequencer A")?; + let client_a = sequencer_client(handle_a.addr())?; + let finalized = wait_for_finalized(&indexer, FINALIZED_TARGET, L2_TO_L1_TIMEOUT).await?; + let tip_before = client_a.get_last_block_id().await?; + assert!( + tip_before > finalized, + "local tip {tip_before} should lead the finalized tip {finalized}" + ); + + // Restart on the same home; slow cadence so its tip stays put while we inspect. + drop(handle_a); + tokio::time::sleep(Duration::from_secs(2)).await; + let handle_b = SequencerSetup::new(slow_blocks(), bedrock_addr) + .with_genesis(genesis) + .setup_at(home.path()) + .await + .context("Failed to restart sequencer")?; + let client_b = sequencer_client(handle_b.addr())?; + + // Reconstruction verified the finalized prefix and preserved the extra blocks. + let tip_b = client_b.get_last_block_id().await?; + assert!( + tip_b >= tip_before, + "restart must not lose locally-produced blocks; tip_b={tip_b}, before={tip_before}" + ); + assert_eq!( + client_b.get_account_balance(vault_id).await?, + 4_242, + "genesis state must survive the restart" + ); + assert!(handle_b.is_healthy(), "sequencer must stay healthy"); + + Ok(()) +} + +/// Case 5: both non-empty, but the local store is *behind* the finalized channel. +/// +/// We snapshot a sequencer's store at an early tip, let it keep extending and +/// finalizing the channel, then restore the early snapshot and restart. Startup +/// must reconstruct forward โ€” replay the finalized blocks the local store is +/// missing โ€” catching the local tip up to the channel. +/// +/// The snapshot/restore is essential here (not a gratuitous copy): a live +/// sequencer's local tip always leads finalization, so the only way to obtain a +/// local store that *lags* the finalized channel is to preserve an earlier state +/// while the same channel advances past it. +#[test] +async fn local_behind_channel_reconstructs_forward() -> Result<()> { + const SNAPSHOT_TIP: u64 = 2; + const FINALIZED_TARGET: u64 = 4; + + let (_bedrock, bedrock_addr) = setup_bedrock_node() + .await + .context("Failed to setup Bedrock")?; + let (indexer_handle, _indexer_dir) = setup_indexer( + bedrock_addr, + test_fixtures::config::bedrock_channel_id(), + None, + ) + .await + .context("Failed to setup indexer")?; + let indexer_url = addr_to_url(UrlProtocol::Ws, indexer_handle.addr()) + .context("Failed to build indexer URL")?; + let indexer = IndexerClient::new(&indexer_url) + .await + .context("Failed to build indexer client")?; + + let (vault_id, supply) = supplied_account(5_005); + let genesis = vec![ + supply, + GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }, + ]; + + let home = tempfile::tempdir().context("Failed to create sequencer home")?; + let rocksdb = home.path().join("rocksdb"); + + // Bring the sequencer up to an early tip, then stop it so its store is at rest. + { + let handle = SequencerSetup::new(slow_blocks(), bedrock_addr) + .with_genesis(genesis.clone()) + .setup_at(home.path()) + .await + .context("Failed to start sequencer")?; + wait_for_block_id( + &sequencer_client(handle.addr())?, + SNAPSHOT_TIP, + Duration::from_secs(120), + ) + .await?; + drop(handle); + tokio::time::sleep(Duration::from_secs(2)).await; + } + + // Snapshot the early store (tip == SNAPSHOT_TIP), safe because it is at rest. + let snapshot = tempfile::tempdir().context("Failed to create snapshot dir")?; + copy_dir_recursive(&rocksdb, snapshot.path()).context("Failed to snapshot store")?; + + // Resume the sequencer (fast cadence) so it extends and finalizes the channel + // beyond the snapshot. + let finalized = { + let handle = SequencerSetup::new(fast_blocks(), bedrock_addr) + .with_genesis(genesis.clone()) + .setup_at(home.path()) + .await + .context("Failed to resume sequencer")?; + let finalized = wait_for_finalized(&indexer, FINALIZED_TARGET, L2_TO_L1_TIMEOUT).await?; + drop(handle); + tokio::time::sleep(Duration::from_secs(2)).await; + finalized + }; + + // Restore the early snapshot: the local store now lags the finalized channel. + std::fs::remove_dir_all(&rocksdb).context("Failed to remove store before restore")?; + copy_dir_recursive(snapshot.path(), &rocksdb).context("Failed to restore snapshot")?; + + // Restart: reconstruction must catch the lagging store up to the channel. + let handle = SequencerSetup::new(slow_blocks(), bedrock_addr) + .with_genesis(genesis) + .setup_at(home.path()) + .await + .context("Failed to restart sequencer from a lagging store")?; + let client = sequencer_client(handle.addr())?; + let tip = client.get_last_block_id().await?; + assert!( + tip >= finalized, + "lagging store must reconstruct forward to the finalized tip; tip={tip}, finalized={finalized}" + ); + assert!( + tip > SNAPSHOT_TIP, + "reconstruction must advance beyond the snapshot tip; tip={tip}" + ); + assert_eq!( + client.get_account_balance(vault_id).await?, + 5_005, + "genesis state must be intact after reconstruction" + ); + assert!(handle.is_healthy(), "sequencer must stay healthy"); + + Ok(()) +} diff --git a/integration_tests/tests/two_zone.rs b/integration_tests/tests/two_zone.rs index c895acd1..8f4b2697 100644 --- a/integration_tests/tests/two_zone.rs +++ b/integration_tests/tests/two_zone.rs @@ -13,7 +13,7 @@ use indexer_service_rpc::RpcClient as _; use integration_tests::{ config::{self, SequencerPartialConfig}, indexer_client::IndexerClient, - setup::{setup_bedrock_node, setup_indexer, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node, setup_indexer}, }; use sequencer_service_rpc::{RpcClient as _, SequencerClientBuilder}; use tokio::test; @@ -35,13 +35,19 @@ async fn two_zones_share_one_bedrock_and_both_advance() -> Result<()> { let channel_b = config::bedrock_channel_id_b(); // Empty genesis is enough: the clock transaction drives block production. - let (seq_a, _seq_a_home) = setup_sequencer(partial, bedrock_addr, vec![], channel_a, None) + let (seq_a, _seq_a_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_a) + .with_genesis(vec![]) + .setup() .await .context("Failed to set up zone A sequencer")?; let (idx_a, _idx_a_home) = setup_indexer(bedrock_addr, channel_a, None) .await .context("Failed to set up zone A indexer")?; - let (seq_b, _seq_b_home) = setup_sequencer(partial, bedrock_addr, vec![], channel_b, None) + let (seq_b, _seq_b_home) = SequencerSetup::new(partial, bedrock_addr) + .with_channel_id(channel_b) + .with_genesis(vec![]) + .setup() .await .context("Failed to set up zone B sequencer")?; let (idx_b, _idx_b_home) = setup_indexer(bedrock_addr, channel_b, None) diff --git a/integration_tests/tests/wallet_ffi.rs b/integration_tests/tests/wallet_ffi.rs index 321c874d..a36eb069 100644 --- a/integration_tests/tests/wallet_ffi.rs +++ b/integration_tests/tests/wallet_ffi.rs @@ -28,7 +28,6 @@ use lee::{ }; use lee_core::program::DEFAULT_PROGRAM_ID; use log::info; -use tempfile::tempdir; use wallet::{account::HumanReadableAccount, program_facades::vault::Vault}; use wallet_ffi::{ FfiAccount, FfiAccountIdWithPrivacy, FfiAccountIdentity, FfiAccountList, FfiBytes32, @@ -43,12 +42,14 @@ unsafe extern "C" { fn wallet_ffi_create_new( config_path: *const c_char, storage_path: *const c_char, + metrics_path: *const c_char, password: *const c_char, ) -> FfiCreateWalletOutput; fn wallet_ffi_open( config_path: *const c_char, storage_path: *const c_char, + metrics_path: *const c_char, ) -> *mut WalletHandle; fn wallet_ffi_destroy(handle: *mut WalletHandle); @@ -291,6 +292,7 @@ fn new_wallet_ffi_with_test_context_config( ) -> Result { let config_path = home.join("wallet_config.json"); let storage_path = home.join("storage.json"); + let metrics_path = home.join("metrics.json"); let mut config = ctx.ctx().wallet().config().to_owned(); if let Some(config_overrides) = ctx.ctx().wallet().config_overrides().clone() { config.apply_overrides(config_overrides); @@ -307,12 +309,14 @@ fn new_wallet_ffi_with_test_context_config( let config_path = CString::new(config_path.to_str().unwrap())?; let storage_path = CString::new(storage_path.to_str().unwrap())?; + let metrics_path = CString::new(metrics_path.to_str().unwrap())?; let password = CString::new(ctx.ctx().wallet_password())?; let create_wallet_result = unsafe { wallet_ffi_create_new( config_path.as_ptr(), storage_path.as_ptr(), + metrics_path.as_ptr(), password.as_ptr(), ) }; @@ -366,47 +370,37 @@ fn new_wallet_ffi_with_test_context_config( Ok(create_wallet_result) } -fn new_wallet_ffi_with_default_config(password: &str) -> Result { - let tempdir = tempdir()?; - let config_path = tempdir.path().join("wallet_config.json"); - let storage_path = tempdir.path().join("storage.json"); - let config_path_c = CString::new(config_path.to_str().unwrap())?; - let storage_path_c = CString::new(storage_path.to_str().unwrap())?; - let password = CString::new(password)?; - - let create_wallet_result = unsafe { - wallet_ffi_create_new( - config_path_c.as_ptr(), - storage_path_c.as_ptr(), - password.as_ptr(), - ) - }; - - Ok(create_wallet_result) -} - fn load_existing_ffi_wallet(home: &Path) -> Result<*mut WalletHandle> { let config_path = home.join("wallet_config.json"); let storage_path = home.join("storage.json"); + let metrics_path = home.join("metrics.json"); let config_path = CString::new(config_path.to_str().unwrap())?; let storage_path = CString::new(storage_path.to_str().unwrap())?; + let metrics_path = CString::new(metrics_path.to_str().unwrap())?; - Ok(unsafe { wallet_ffi_open(config_path.as_ptr(), storage_path.as_ptr()) }) + Ok(unsafe { + wallet_ffi_open( + config_path.as_ptr(), + storage_path.as_ptr(), + metrics_path.as_ptr(), + ) + }) } #[test] fn wallet_ffi_create_public_accounts() -> Result<()> { - let password = "password_for_tests"; + let ctx = BlockingTestContext::new()?; let n_accounts = 10; // Create `n_accounts` public accounts with wallet FFI let new_public_account_ids_ffi = unsafe { let mut account_ids = Vec::new(); + let home = tempfile::tempdir()?; let FfiCreateWalletOutput { wallet: wallet_ffi_handle, mnemonic: _, - } = new_wallet_ffi_with_default_config(password)?; + } = new_wallet_ffi_with_test_context_config(&ctx, home.path())?; for _ in 0..n_accounts { let mut out_account_id = FfiBytes32::from_bytes([0; 32]); wallet_ffi_create_account_public(wallet_ffi_handle, &raw mut out_account_id).unwrap(); @@ -436,16 +430,17 @@ fn wallet_ffi_create_public_accounts() -> Result<()> { #[test] fn wallet_ffi_create_private_accounts() -> Result<()> { - let password = "password_for_tests"; + let ctx = BlockingTestContext::new()?; let n_accounts = 10; // Create `n_accounts` receiving keys with wallet FFI let new_npks_ffi = unsafe { let mut npks = Vec::new(); + let home = tempfile::tempdir()?; let FfiCreateWalletOutput { wallet: wallet_ffi_handle, mnemonic: _, - } = new_wallet_ffi_with_default_config(password)?; + } = new_wallet_ffi_with_test_context_config(&ctx, home.path())?; for _ in 0..n_accounts { let mut out_keys = FfiPrivateAccountKeys::default(); wallet_ffi_create_private_accounts_key(wallet_ffi_handle, &raw mut out_keys).unwrap(); @@ -510,14 +505,14 @@ fn wallet_ffi_save_and_load_persistent_storage() -> Result<()> { #[test] fn test_wallet_ffi_list_accounts() -> Result<()> { - let password = "password_for_tests"; - + let ctx = BlockingTestContext::new()?; // Create the wallet FFI and track which account IDs were created as public/private let (wallet_ffi_handle, created_public_ids) = unsafe { + let home = tempfile::tempdir()?; let FfiCreateWalletOutput { wallet: handle, mnemonic: _, - } = new_wallet_ffi_with_default_config(password)?; + } = new_wallet_ffi_with_test_context_config(&ctx, home.path())?; let mut public_ids: Vec<[u8; 32]> = Vec::new(); // Create 5 public accounts and 5 receiving keys diff --git a/lee/state_machine/core/src/commitment.rs b/lee/state_machine/core/src/commitment.rs index 92085d7d..537a9643 100644 --- a/lee/state_machine/core/src/commitment.rs +++ b/lee/state_machine/core/src/commitment.rs @@ -32,7 +32,7 @@ pub const DUMMY_COMMITMENT_HASH: [u8; 32] = [ #[derive(Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[cfg_attr( any(feature = "host", test), - derive(Clone, PartialEq, Eq, Hash, PartialOrd, Ord) + derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord) )] pub struct Commitment(pub(super) [u8; 32]); diff --git a/lee/state_machine/src/state/mod.rs b/lee/state_machine/src/state/mod.rs index d9f7299e..d4e67e38 100644 --- a/lee/state_machine/src/state/mod.rs +++ b/lee/state_machine/src/state/mod.rs @@ -44,7 +44,7 @@ impl CommitmentSet { /// Inserts a list of commitments to the `CommitmentSet`. pub(crate) fn extend(&mut self, commitments: &[Commitment]) { - for commitment in commitments.iter().cloned() { + for commitment in commitments.iter().copied() { let index = self.merkle_tree.insert(commitment.to_byte_array()); self.commitments.insert(commitment, index); } diff --git a/lee/state_machine/src/state/tests/claiming.rs b/lee/state_machine/src/state/tests/claiming.rs index acfe9834..d45f39bd 100644 --- a/lee/state_machine/src/state/tests/claiming.rs +++ b/lee/state_machine/src/state/tests/claiming.rs @@ -308,7 +308,7 @@ fn authorized_public_account_claiming_succeeds_when_executed_privately() { let sender_commitment = Commitment::new(&sender_account_id, &sender_private_account); let sender_init_nullifier = Nullifier::for_account_initialization(&sender_account_id); let mut state = - V03State::new().with_private_accounts([(sender_commitment.clone(), sender_init_nullifier)]); + V03State::new().with_private_accounts([(sender_commitment, sender_init_nullifier)]); let sender_pre = AccountWithMetadata::new( sender_private_account, true, @@ -401,8 +401,8 @@ fn private_chained_call(number_of_calls: u32) { let to_init_nullifier = Nullifier::for_account_initialization(&to_account_id); let mut state = V03State::new() .with_private_accounts([ - (from_commitment.clone(), from_init_nullifier), - (to_commitment.clone(), to_init_nullifier), + (from_commitment, from_init_nullifier), + (to_commitment, to_init_nullifier), ]) .with_test_programs(); let amount: u128 = 37; diff --git a/lee/state_machine/src/state/tests/privacy_preserving.rs b/lee/state_machine/src/state/tests/privacy_preserving.rs index 8c33e202..174aa1b3 100644 --- a/lee/state_machine/src/state/tests/privacy_preserving.rs +++ b/lee/state_machine/src/state/tests/privacy_preserving.rs @@ -505,7 +505,7 @@ fn malicious_authorization_changer_should_fail_in_privacy_preserving_circuit() { sender_account.account_id, sender_account.account.balance, )])) - .with_private_accounts([(recipient_commitment.clone(), recipient_init_nullifier)]) + .with_private_accounts([(recipient_commitment, recipient_init_nullifier)]) .with_test_programs(); let balance_to_transfer = 10_u128; diff --git a/lez/chain_consistency/Cargo.toml b/lez/chain_consistency/Cargo.toml new file mode 100644 index 00000000..02bf0ea0 --- /dev/null +++ b/lez/chain_consistency/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "chain_consistency" +version = "0.1.0" +edition = "2024" +license.workspace = true + +[lints] +workspace = true + +[dependencies] +common.workspace = true +lee.workspace = true +lee_core.workspace = true + +logos-blockchain-zone-sdk.workspace = true +logos-blockchain-core.workspace = true +anyhow.workspace = true +borsh.workspace = true +futures.workspace = true +log.workspace = true +serde.workspace = true +thiserror.workspace = true +tokio.workspace = true + +[dev-dependencies] +serde_json.workspace = true diff --git a/lez/chain_consistency/src/apply.rs b/lez/chain_consistency/src/apply.rs new file mode 100644 index 00000000..28961502 --- /dev/null +++ b/lez/chain_consistency/src/apply.rs @@ -0,0 +1,196 @@ +//! Validating and applying channel L2 blocks to a `V03State`. +//! +//! These primitives are shared by the consumers that reconstruct L2 state from +//! the Bedrock channel. + +use common::{ + HashType, + block::Block, + transaction::{LeeTransaction, clock_invocation}, +}; +use lee::{GENESIS_BLOCK_ID, V03State}; +use lee_core::BlockId; +use serde::{Deserialize, Serialize}; + +/// Why a channel L2 block could not be validated or applied. +/// +/// Persisted in `RocksDB` (via the Indexer's stall reason), so every variant +/// must be `Clone + Serialize + Deserialize`. +#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] +pub enum BlockIngestError { + #[error("Failed to deserialize L2 block: {0}")] + /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. + Deserialize(String), + #[error("Unexpected block id: expected {expected}, got {got}")] + UnexpectedBlockId { expected: BlockId, got: BlockId }, + #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] + BrokenChainLink { + expected_prev: HashType, + got_prev: HashType, + }, + #[error("Block hash mismatch: computed {computed}, header {header}")] + HashMismatch { + computed: HashType, + header: HashType, + }, + #[error("Block has no transactions")] + EmptyBlock, + #[error("Last transaction must be the public clock invocation for the block timestamp")] + InvalidClockTransaction, + #[error("Genesis block must contain only public transactions")] + NonPublicGenesisTransaction, + #[error("State transition failed at transaction {tx_index}: {reason}")] + StateTransition { + /// Index of the failing transaction within the block body. + tx_index: u64, + /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. + /// + /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we + /// cannot store it directly. + reason: String, + }, +} + +impl BlockIngestError { + /// Whether the failure may be transient rather than a property of the block. + /// + /// FIXME: `StateTransition` is too coarse โ€” its `reason` string mixes genuine + /// state-transition rejections with infra failures (risc0 executor teardown, + /// storage errors). Once it carries a structured cause, narrow this so only + /// infra failures retry. + #[must_use] + pub const fn is_retryable(&self) -> bool { + matches!(self, Self::StateTransition { .. }) + } +} + +/// The last successfully applied block a candidate must extend. +#[derive(Debug, Copy, Clone)] +pub struct Tip { + pub block_id: BlockId, + pub hash: HashType, +} + +/// Checks that `block` is the valid continuation of `tip`: hash integrity, +/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip +/// (cold store) expects the genesis block. +pub fn validate_against_tip(tip: Option, block: &Block) -> Result<(), BlockIngestError> { + let computed = block.recompute_hash(); + if computed != block.header.hash { + return Err(BlockIngestError::HashMismatch { + computed, + header: block.header.hash, + }); + } + + match tip { + None => { + if block.header.block_id != GENESIS_BLOCK_ID { + return Err(BlockIngestError::UnexpectedBlockId { + expected: GENESIS_BLOCK_ID, + got: block.header.block_id, + }); + } + } + Some(tip) => { + let expected = tip + .block_id + .checked_add(1) + .expect("block id should not overflow"); + if block.header.block_id != expected { + return Err(BlockIngestError::UnexpectedBlockId { + expected, + got: block.header.block_id, + }); + } + if block.header.prev_block_hash != tip.hash { + return Err(BlockIngestError::BrokenChainLink { + expected_prev: tip.hash, + got_prev: block.header.prev_block_hash, + }); + } + } + } + Ok(()) +} + +/// Applies a block's transactions to `state`. +pub fn apply_block(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { + let (clock_tx, user_txs) = block + .body + .transactions + .split_last() + .ok_or(BlockIngestError::EmptyBlock)?; + + let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); + if *clock_tx != expected_clock { + return Err(BlockIngestError::InvalidClockTransaction); + } + + let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; + for (tx_index, transaction) in user_txs.iter().enumerate() { + let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { + tx_index: tx_index.try_into().expect("tx index fits in u64"), + reason: format!("{err:#}"), + }; + if is_genesis { + let LeeTransaction::Public(public_tx) = transaction else { + return Err(BlockIngestError::NonPublicGenesisTransaction); + }; + state + .transition_from_public_transaction( + public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| state_transition(err.into()))?; + } else { + transaction + .clone() + .execute_on_state(state, block.header.block_id, block.header.timestamp) + .map_err(|err| state_transition(err.into()))?; + } + } + + let LeeTransaction::Public(clock_public_tx) = clock_tx else { + return Err(BlockIngestError::InvalidClockTransaction); + }; + state + .transition_from_public_transaction( + clock_public_tx, + block.header.block_id, + block.header.timestamp, + ) + .map_err(|err| BlockIngestError::StateTransition { + tx_index: user_txs.len().try_into().expect("tx index fits in u64"), + reason: format!("{:#}", anyhow::Error::from(err)), + })?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn serializes_and_round_trips_externally_tagged() { + let err = BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7, + }; + let value = serde_json::to_value(&err).expect("serialize"); + assert_eq!( + value, + serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) + ); + let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); + assert!(matches!( + back, + BlockIngestError::UnexpectedBlockId { + expected: 5, + got: 7 + } + )); + } +} diff --git a/lez/indexer/core/src/chain_consistency.rs b/lez/chain_consistency/src/consistency.rs similarity index 53% rename from lez/indexer/core/src/chain_consistency.rs rename to lez/chain_consistency/src/consistency.rs index df4b5ba6..5c87fcad 100644 --- a/lez/indexer/core/src/chain_consistency.rs +++ b/lez/chain_consistency/src/consistency.rs @@ -1,18 +1,18 @@ -//! Startup check that the local store still belongs to the chain the -//! connected channel serves. +//! Startup check that a local store still belongs to the chain the connected +//! channel serves. use anyhow::Result; use common::{HashType, block::Block}; use futures::StreamExt as _; +use lee_core::BlockId; use log::warn; -use logos_blockchain_zone_sdk::{Slot, ZoneMessage, adapter::Node as _}; - -use crate::IndexerCore; +use logos_blockchain_core::mantle::ops::channel::ChannelId; +use logos_blockchain_zone_sdk::{Slot, ZoneMessage, adapter, indexer::ZoneIndexer}; /// Upper bound on the channel reads of the startup consistency check. const CHANNEL_READ_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60); -/// Result of comparing the indexer's stored chain against the channel. +/// Result of comparing a caller's stored chain against the channel. pub enum ChainConsistency { /// Channel still serves our anchor block (the stored tip position, or the /// parked block while stalled). @@ -24,7 +24,7 @@ pub enum ChainConsistency { /// - or the channel read was inconclusive (timeout / error / empty stream) /// /// NOTE: None of these prove a reset, so the caller proceeds. - /// A genuine divergence is still caught later when the ingest loop tries to apply and parks. + /// A genuine divergence is still caught later when applying the channel history. Inconclusive, /// Positive evidence that the channel is a different chain than the store. /// @@ -36,13 +36,13 @@ pub enum ChainConsistency { pub enum ChainMismatch { /// The channel serves a different block at the anchor's id. Block { - ours: (u64, HashType), - channel: (u64, HashType), + ours: (BlockId, HashType), + channel: (BlockId, HashType), }, /// The channel serves a block at/below the anchor's id past the anchor /// slot; on the same chain those ids live at earlier slots. ReinscribedBlock { - channel: (u64, HashType), + channel: (BlockId, HashType), slot: Slot, anchor_slot: Slot, }, @@ -97,18 +97,26 @@ impl std::fmt::Display for ChainMismatch { /// A block that must still be inscribed at `slot` if the channel is the chain /// the store was built from: the tip at the read cursor, or the recorded /// parked block while stalled. -struct Anchor { +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Anchor { slot: Slot, /// The anchor block's `(id, hash)`. /// - /// `None` when parked on an undeserializable inscription (no header was recorded). - block: Option<(u64, HashType)>, + /// `None` when anchored on an undeserializable inscription (no header was recorded). + block: Option<(BlockId, HashType)>, } impl Anchor { + /// Builds an anchor at `slot` on the block `(id, hash)`, or a headerless + /// anchor (`None`) when only the slot is known. + #[must_use] + pub const fn new(slot: Slot, block: Option<(BlockId, HashType)>) -> Self { + Self { slot, block } + } + /// Probes a channel message read at/after the anchor slot. - /// See [`IndexerCore::verify_chain_at_anchor`]. - pub fn probe_anchor_slot(&self, msg: &ZoneMessage, slot: Slot) -> AnchorProbe { + /// See [`verify_chain_consistency`]. + fn probe_anchor_slot(&self, msg: &ZoneMessage, slot: Slot) -> AnchorProbe { if slot < self.slot { return AnchorProbe::KeepLooking; } @@ -157,6 +165,66 @@ impl Anchor { } } +/// Classifies a stored chain against the channel one message at a time. +/// +/// The shared driver behind both consistency consumers: +/// [`verify_chain_consistency`] runs it over a channel it reads itself, while a +/// caller that already streams the channel history (e.g. a reconstructing +/// sequencer) feeds it the same messages it replays. Run [`Self::check_frontier`] +/// once the channel tip is known, then feed messages in slot order via +/// [`Self::observe`] until a verdict is reached. +pub struct AnchorConsistencyCheck { + anchor: Anchor, + verdict: Option, +} + +impl AnchorConsistencyCheck { + /// New checker for `anchor` with an undetermined verdict. + #[must_use] + pub const fn new(anchor: Anchor) -> Self { + Self { + anchor, + verdict: None, + } + } + + /// Applies the frontier check against a known channel tip. Skip it when the + /// tip could not be read, leaving the verdict to the message scan. + pub fn check_frontier(&mut self, channel_tip_slot: Option) { + if self.verdict.is_none() { + self.verdict = frontier_verdict(self.anchor.slot, channel_tip_slot) + .map(ChainConsistency::Inconsistent); + } + } + + /// Feeds the next channel message in slot order. Returns the verdict once it + /// is known so the caller can stop, or `None` while still undetermined. + pub fn observe(&mut self, msg: &ZoneMessage, slot: Slot) -> Option<&ChainConsistency> { + if self.verdict.is_none() { + self.verdict = match self.anchor.probe_anchor_slot(msg, slot) { + AnchorProbe::SameChain => Some(ChainConsistency::Consistent), + AnchorProbe::Mismatch(mismatch) => Some(ChainConsistency::Inconsistent(mismatch)), + AnchorProbe::Bail => Some(ChainConsistency::Inconclusive), + AnchorProbe::KeepLooking => None, + }; + } + self.verdict.as_ref() + } + + /// The verdict reached so far, or `None` while undetermined. + #[must_use] + pub const fn verdict(&self) -> Option<&ChainConsistency> { + self.verdict.as_ref() + } + + /// Consumes the checker, defaulting an undetermined verdict (the anchor slot + /// was never observed) to [`ChainConsistency::Inconclusive`]. + #[must_use] + pub fn finish(self) -> ChainConsistency { + self.verdict.unwrap_or(ChainConsistency::Inconclusive) + } +} + /// What a single channel message tells the anchored consistency check. enum AnchorProbe { /// The anchor is still in place: same chain. @@ -168,114 +236,65 @@ enum AnchorProbe { KeepLooking, } -#[expect( - clippy::multiple_inherent_impl, - reason = "split for clarity & isolation of relevant code" -)] -impl IndexerCore { - /// Verifies whether the channel still serves the same chain the store was built from. - /// This may change frequently during development where we reset the chain from time to - /// time in devnet/testnet, but we do not expect [`ChainConsistency::Inconsistent`] in - /// production. - /// - /// To compare the chains, we use an [`Anchor`] block that is either the parked L2 block - /// while stalled, or the tip L2 block at its own inscription L1 slot. - pub(crate) async fn verify_chain_consistency(&self) -> Result { - let Some(anchor) = self.get_startup_anchor()? else { - // empty or cold store: nothing to compare - return Ok(ChainConsistency::Inconclusive); - }; - - self.verify_chain_at_anchor(&anchor).await +/// Detects when a local store belongs to a different chain than the connected +/// L1 (e.g. a wiped/restarted Bedrock) so startup can react instead of silently +/// diverging. +/// +/// Verifies the channel still carries the anchor block at its slot. The anchor +/// was finalized at `anchor.slot`, so the same chain must still serve it there, +/// while a reset chain re-inscribes its content only at later wall-clock slots. +/// Only positive evidence of a different chain yields +/// [`ChainConsistency::Inconsistent`]; absence of data stays +/// [`ChainConsistency::Inconclusive`]. +/// +/// `node` need only implement the zone-sdk [`adapter::Node`] trait; a throwaway +/// [`ZoneIndexer`] is built internally for the channel read. +pub async fn verify_chain_consistency( + node: &N, + channel_id: ChannelId, + anchor: &Anchor, +) -> Result +where + N: adapter::Node + Clone + Sync, +{ + let mut check = AnchorConsistencyCheck::new(anchor.clone()); + match node.channel_state(channel_id).await { + Ok(state) => check.check_frontier(state.map(|s| s.tip_slot)), + Err(err) => warn!("Failed to read channel state for the consistency check: {err:#}"), + } + if check.verdict().is_some() { + return Ok(check.finish()); } - /// Builds the anchor for the startup check. - /// - /// - If stalled, returns the recorded _parked_ block - /// - If not stalled, returns the validated tip at its _own_ inscription slot. - /// - If the store is empty, returns `None`. - fn get_startup_anchor(&self) -> Result> { - if let Some(stall) = self.store.get_stall_reason()? { - return Ok(Some(Anchor { - slot: stall.l1_slot, - block: stall.block_id.zip(stall.block_hash), - })); - } + // `next_messages` is exclusive, so `slot - 1` includes the anchor slot. + let Some(from_slot) = anchor.slot.into_inner().checked_sub(1) else { + return Ok(ChainConsistency::Inconclusive); + }; - // not stalled, so anchor on the tip at its own inscription slot - let Some(slot) = self.store.get_tip_slot()?.or(self.store.get_zone_cursor()?) else { - return Ok(None); - }; - let Some(tip_id) = self.store.get_last_block_id()? else { - return Ok(None); - }; - let Some(tip) = self.store.get_block_at_id(tip_id)? else { - return Ok(None); - }; - Ok(Some(Anchor { - slot, - block: Some((tip_id, tip.header.hash)), - })) - } + let zone_indexer = ZoneIndexer::new(channel_id, node.clone()); + let scan = async { + let stream = zone_indexer + .next_messages(Some(Slot::from(from_slot))) + .await?; + let mut stream = std::pin::pin!(stream); - /// Verifies the channel still carries the anchor block at its slot. - /// - /// The anchor was finalized at `anchor.slot`, so the same chain must still - /// serve it there, while a reset chain re-inscribes its content only at - /// later wall-clock slots. - /// - /// Only positive evidence of a different chain yields `Inconsistent`. - /// Absence of data stays `Inconclusive`. - async fn verify_chain_at_anchor(&self, anchor: &Anchor) -> Result { - match self.node.channel_state(self.config.channel_id).await { - Ok(state) => { - if let Some(mismatch) = frontier_verdict(anchor.slot, state.map(|s| s.tip_slot)) { - return Ok(ChainConsistency::Inconsistent(mismatch)); - } - } - Err(err) => { - warn!("Failed to read channel state for the consistency check: {err:#}"); + while let Some((msg, slot)) = stream.next().await { + if check.observe(&msg, slot).is_some() { + break; } } + Ok::<_, anyhow::Error>(()) + }; - // `next_messages` is exclusive, so `slot - 1` includes the anchor slot. - let Some(from_slot) = anchor.slot.into_inner().checked_sub(1) else { - return Ok(ChainConsistency::Inconclusive); - }; - - let scan = async { - let stream = self - .zone_indexer - .next_messages(Some(Slot::from(from_slot))) - .await?; - let mut stream = std::pin::pin!(stream); - - while let Some((msg, slot)) = stream.next().await { - match anchor.probe_anchor_slot(&msg, slot) { - AnchorProbe::SameChain => return Ok(Some(ChainConsistency::Consistent)), - AnchorProbe::Mismatch(mismatch) => { - return Ok(Some(ChainConsistency::Inconsistent(mismatch))); - } - AnchorProbe::Bail => return Ok(Some(ChainConsistency::Inconclusive)), - AnchorProbe::KeepLooking => { /* dont do anything */ } - } - } - Ok::<_, anyhow::Error>(None) - }; - - match tokio::time::timeout(CHANNEL_READ_TIMEOUT, scan).await { - Ok(Ok(Some(outcome))) => Ok(outcome), - Ok(Ok(None)) => Ok(ChainConsistency::Inconclusive), - Ok(Err(err)) => { - warn!( - "Failed to read the anchor slot for the consistency check; proceeding: {err:#}" - ); - Ok(ChainConsistency::Inconclusive) - } - Err(_elapsed) => { - warn!("Timed out reading the anchor slot for the consistency check; proceeding"); - Ok(ChainConsistency::Inconclusive) - } + match tokio::time::timeout(CHANNEL_READ_TIMEOUT, scan).await { + Ok(Ok(())) => Ok(check.finish()), + Ok(Err(err)) => { + warn!("Failed to read the anchor slot for the consistency check; proceeding: {err:#}"); + Ok(ChainConsistency::Inconclusive) + } + Err(_elapsed) => { + warn!("Timed out reading the anchor slot for the consistency check; proceeding"); + Ok(ChainConsistency::Inconclusive) } } } @@ -297,35 +316,13 @@ fn frontier_verdict(anchor_slot: Slot, channel_tip_slot: Option) -> Option #[cfg(test)] mod tests { - use std::time::Duration; - use common::block::HashableBlockData; use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; use logos_blockchain_zone_sdk::ZoneBlock; use super::*; - use crate::{ - BlockIngestError, - block_store::AcceptOutcome, - config::{ChannelId, ClientConfig, IndexerConfig}, - }; - fn unreachable_core(dir: &std::path::Path) -> IndexerCore { - let config = IndexerConfig { - consensus_info_polling_interval: Duration::from_secs(1), - bedrock_config: ClientConfig { - addr: "http://localhost:1".parse().expect("url"), - auth: None, - }, - channel_id: ChannelId::from([1; 32]), - allow_chain_reset: false, - cross_zone: None, - bridge_lock_holdings: Vec::new(), - }; - IndexerCore::open(config, dir).expect("open core") - } - - fn test_block(block_id: u64, timestamp: u64) -> Block { + fn test_block(block_id: BlockId, timestamp: u64) -> Block { HashableBlockData { block_id, prev_block_hash: HashType([0; 32]), @@ -344,98 +341,7 @@ mod tests { } fn anchor_for(block: &Block, slot: Slot) -> Anchor { - Anchor { - slot, - block: Some((block.header.block_id, block.header.hash)), - } - } - - #[tokio::test] - async fn cold_store_is_inconclusive() { - // An empty store has no cursor, so there is nothing to compare: the check - // must be Inconclusive (not Consistent), and it returns before any L1 read. - let dir = tempfile::tempdir().expect("tempdir"); - let core = unreachable_core(dir.path()); - assert!(matches!( - core.verify_chain_consistency().await.expect("verify"), - ChainConsistency::Inconclusive - )); - } - - #[tokio::test] - async fn parked_store_with_unreachable_node_is_inconclusive() { - // Network failure is not evidence of a reset: a parked store must stay - // parked (Inconclusive), not error out or trip the wipe path. - let dir = tempfile::tempdir().expect("tempdir"); - let core = unreachable_core(dir.path()); - let parked = test_block(5, 42); - core.store - .record_stall( - Some(&parked.header), - Slot::from(1_000), - BlockIngestError::EmptyBlock, - ) - .expect("record stall"); - assert!(matches!( - core.verify_chain_consistency().await.expect("verify"), - ChainConsistency::Inconclusive - )); - } - - #[tokio::test] - async fn caught_up_store_with_unreachable_node_is_inconclusive() { - let dir = tempfile::tempdir().expect("tempdir"); - let core = unreachable_core(dir.path()); - let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); - assert!(matches!( - core.store - .accept_block(&genesis, Slot::from(1_000)) - .await - .expect("accept"), - AcceptOutcome::Applied - )); - core.store - .set_zone_cursor(&Slot::from(1_000)) - .expect("set cursor"); - assert!(matches!( - core.verify_chain_consistency().await.expect("verify"), - ChainConsistency::Inconclusive - )); - } - - #[tokio::test] - async fn startup_anchor_prefers_tip_slot_over_lagging_cursor() { - // Cursor persist failures are warn-only, so the read cursor can lag the - // tip by several blocks. The anchor must pair the tip with its own - // inscription slot; pairing it with the stale cursor would make the scan - // misread the chain's intermediate blocks as re-inscriptions. - let dir = tempfile::tempdir().expect("tempdir"); - let core = unreachable_core(dir.path()); - - let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); - core.store - .accept_block(&genesis, Slot::from(1_000)) - .await - .expect("accept"); - let block2 = common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); - core.store - .accept_block(&block2, Slot::from(1_005)) - .await - .expect("accept"); - let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); - core.store - .accept_block(&block3, Slot::from(1_010)) - .await - .expect("accept"); - - // Cursor last persisted at the genesis slot: two blocks behind the tip. - core.store - .set_zone_cursor(&Slot::from(1_000)) - .expect("set cursor"); - - let anchor = core.get_startup_anchor().expect("anchor").expect("present"); - assert_eq!(anchor.slot, Slot::from(1_010)); - assert_eq!(anchor.block, Some((3, block3.header.hash))); + Anchor::new(slot, Some((block.header.block_id, block.header.hash))) } #[test] @@ -516,10 +422,7 @@ mod tests { fn probe_accepts_any_message_for_a_headerless_anchor() { // A deserialize park records no header: any message still present at // the anchor slot means the history is intact. - let anchor = Anchor { - slot: Slot::from(1_000), - block: None, - }; + let anchor = Anchor::new(Slot::from(1_000), None); let garbage = ZoneMessage::Block(ZoneBlock { id: MsgId::from([0_u8; 32]), data: Inscription::try_from(&[1_u8, 2, 3][..]).expect("inscription"), diff --git a/lez/chain_consistency/src/lib.rs b/lez/chain_consistency/src/lib.rs new file mode 100644 index 00000000..d2b45ca9 --- /dev/null +++ b/lez/chain_consistency/src/lib.rs @@ -0,0 +1,9 @@ +//! Reconstructing and verifying L2 chain state from a Bedrock (L1) channel. + +pub use apply::{BlockIngestError, Tip, apply_block, validate_against_tip}; +pub use consistency::{ + Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, verify_chain_consistency, +}; + +pub mod apply; +pub mod consistency; diff --git a/lez/indexer/core/Cargo.toml b/lez/indexer/core/Cargo.toml index 14941773..cbe0b7e8 100644 --- a/lez/indexer/core/Cargo.toml +++ b/lez/indexer/core/Cargo.toml @@ -13,6 +13,7 @@ testnet = [] [dependencies] common.workspace = true +chain_consistency.workspace = true logos-blockchain-zone-sdk.workspace = true lee.workspace = true lee_core.workspace = true @@ -32,7 +33,6 @@ futures.workspace = true url.workspace = true logos-blockchain-core.workspace = true serde_json.workspace = true -thiserror.workspace = true async-stream.workspace = true tokio.workspace = true risc0-zkvm.workspace = true diff --git a/lez/indexer/core/src/block_store.rs b/lez/indexer/core/src/block_store.rs index cf1a77e6..16dc266d 100644 --- a/lez/indexer/core/src/block_store.rs +++ b/lez/indexer/core/src/block_store.rs @@ -1,25 +1,20 @@ use std::{path::Path, sync::Arc}; use anyhow::{Context as _, Result}; +use chain_consistency::{BlockIngestError, Tip}; use common::{ - HashType, block::{BedrockStatus, Block, BlockHeader}, - transaction::{LeeTransaction, clock_invocation}, + transaction::LeeTransaction, }; -use lee::{Account, AccountId, GENESIS_BLOCK_ID, V03State, program::Program}; +use lee::{Account, AccountId, V03State}; use lee_core::BlockId; -use log::{info, warn}; +use log::warn; use logos_blockchain_core::header::HeaderId; use logos_blockchain_zone_sdk::Slot; use storage::indexer::RocksDBIO; use tokio::sync::RwLock; -use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; - -struct Tip { - block_id: u64, - hash: HashType, -} +use crate::status::StallReason; /// Outcome of feeding a parsed L2 block to the validated tip. pub enum AcceptOutcome { @@ -45,28 +40,18 @@ pub struct IndexerStore { impl IndexerStore { /// Starting database at the start of new chain. /// Creates files if necessary. - pub fn open_db( - location: &Path, - genesis_programs: Vec, - genesis_accounts: Vec<(AccountId, Account)>, - ) -> Result { + pub fn open_db(location: &Path, genesis_seed: Vec<(AccountId, Account)>) -> Result { #[cfg(not(feature = "testnet"))] let base = testnet_initial_state::initial_state(); #[cfg(feature = "testnet")] let base = testnet_initial_state::initial_state_testnet(); - // Extend with this zone's cross-zone genesis (programs + accounts) through - // the state constructor, matching how the sequencer builds genesis so the - // replayed states stay byte-identical. - let initial_state = base - .with_programs(genesis_programs) - .with_public_accounts(genesis_accounts); - // Same fingerprint the sequencer logs; a mismatch means a divergent deploy set. - info!( - "Genesis fingerprint: {}", - hex::encode(initial_state.genesis_fingerprint()) - ); + // Seed any zone-specific genesis accounts (the bridge-lock holdings) so the + // indexer's replayed state matches the sequencer's; none are produced by a + // transaction. Cross-zone programs are base builtins, and their config + // accounts are reconstructed by replaying the genesis block's InitConfig txs. + let initial_state = base.with_public_accounts(genesis_seed); let dbio = RocksDBIO::open_or_create(location, &initial_state)?; let current_state = dbio.final_state()?; @@ -267,14 +252,14 @@ impl IndexerStore { return Ok(AcceptOutcome::AlreadyApplied); } - if let Err(err) = validate_against_tip(tip.as_ref(), block) { + if let Err(err) = chain_consistency::validate_against_tip(tip, block) { self.record_stall(Some(&block.header), l1_slot, err.clone())?; return Ok(AcceptOutcome::Parked(err)); } // TODO: we use scratch state to be atomic, but need to revisit how expensive a clone is let mut scratch = self.current_state.read().await.clone(); - if let Err(err) = apply_block_to_scratch(block, &mut scratch) { + if let Err(err) = chain_consistency::apply_block(block, &mut scratch) { if err.is_retryable() { return Ok(AcceptOutcome::RetryableFailure(err)); } @@ -299,120 +284,16 @@ impl IndexerStore { } } -/// Checks that `block` is the valid continuation of `tip`: hash integrity, -/// then block-id continuity, then `prev_block_hash` linkage. A `None` tip -/// (cold store) expects the genesis block. -fn validate_against_tip(tip: Option<&Tip>, block: &Block) -> Result<(), BlockIngestError> { - let computed = block.recompute_hash(); - if computed != block.header.hash { - return Err(BlockIngestError::HashMismatch { - computed, - header: block.header.hash, - }); - } - - match tip { - None => { - if block.header.block_id != GENESIS_BLOCK_ID { - return Err(BlockIngestError::UnexpectedBlockId { - expected: GENESIS_BLOCK_ID, - got: block.header.block_id, - }); - } - } - Some(tip) => { - let expected = tip - .block_id - .checked_add(1) - .expect("block id should not overflow"); - if block.header.block_id != expected { - return Err(BlockIngestError::UnexpectedBlockId { - expected, - got: block.header.block_id, - }); - } - if block.header.prev_block_hash != tip.hash { - return Err(BlockIngestError::BrokenChainLink { - expected_prev: tip.hash, - got_prev: block.header.prev_block_hash, - }); - } - } - } - Ok(()) -} - -/// Applies a block's transactions to `state`, mapping every failure to a -/// [`BlockIngestError`] so the caller can park rather than crash. Operates on a -/// scratch state; the caller commits only on `Ok`. -fn apply_block_to_scratch(block: &Block, state: &mut V03State) -> Result<(), BlockIngestError> { - let (clock_tx, user_txs) = block - .body - .transactions - .split_last() - .ok_or(BlockIngestError::EmptyBlock)?; - - let expected_clock = LeeTransaction::Public(clock_invocation(block.header.timestamp)); - if *clock_tx != expected_clock { - return Err(BlockIngestError::InvalidClockTransaction); - } - - let is_genesis = block.header.block_id == GENESIS_BLOCK_ID; - for (tx_index, transaction) in user_txs.iter().enumerate() { - let state_transition = |err: anyhow::Error| BlockIngestError::StateTransition { - tx_index: tx_index.try_into().expect("tx index fits in u64"), - reason: format!("{err:#}"), - }; - if is_genesis { - // Every genesis transaction is public (program config, supplies, clock); - // a non-public one is unexpected and parks the indexer rather than - // silently diverging. - let LeeTransaction::Public(public_tx) = transaction else { - return Err(BlockIngestError::NonPublicGenesisTransaction); - }; - state - .transition_from_public_transaction( - public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| state_transition(err.into()))?; - } else { - transaction - .clone() - .execute_on_state(state, block.header.block_id, block.header.timestamp) - .map_err(|err| state_transition(err.into()))?; - } - } - - let LeeTransaction::Public(clock_public_tx) = clock_tx else { - return Err(BlockIngestError::InvalidClockTransaction); - }; - state - .transition_from_public_transaction( - clock_public_tx, - block.header.block_id, - block.header.timestamp, - ) - .map_err(|err| BlockIngestError::StateTransition { - tx_index: user_txs.len().try_into().expect("tx index fits in u64"), - reason: format!("{:#}", anyhow::Error::from(err)), - })?; - - Ok(()) -} - #[cfg(test)] mod stall_reason_tests { use common::HashType; use super::*; - use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; #[tokio::test] async fn stall_reason_roundtrips_and_clears() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); assert!(store.get_stall_reason().expect("get").is_none()); @@ -459,7 +340,7 @@ mod tests { fn correct_startup() { let home = tempdir().unwrap(); - let storage = IndexerStore::open_db(home.as_ref(), Vec::new(), Vec::new()).unwrap(); + let storage = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); let final_id = storage.get_last_block_id().unwrap(); @@ -469,7 +350,7 @@ mod tests { #[tokio::test] async fn accept_block_applies_transfers_and_advances_tip() { let home = tempdir().unwrap(); - let store = IndexerStore::open_db(home.as_ref(), Vec::new(), Vec::new()).unwrap(); + let store = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); let initial_accounts = initial_pub_accounts_private_keys(); let from = initial_accounts[0].account_id; @@ -511,7 +392,7 @@ mod tests { #[tokio::test] async fn account_state_at_block_reflects_history() { let home = tempdir().unwrap(); - let store = IndexerStore::open_db(home.as_ref(), Vec::new(), Vec::new()).unwrap(); + let store = IndexerStore::open_db(home.as_ref(), Vec::new()).unwrap(); let initial_accounts = initial_pub_accounts_private_keys(); let from = initial_accounts[0].account_id; @@ -553,10 +434,10 @@ mod tests { #[cfg(test)] mod accept_tests { + use chain_consistency::BlockIngestError; use common::{HashType, block::HashableBlockData, test_utils::produce_dummy_block}; use super::*; - use crate::ingest_error::BlockIngestError; fn signing_key() -> lee::PrivateKey { lee::PrivateKey::try_new([7_u8; 32]).expect("valid key") @@ -577,7 +458,7 @@ mod accept_tests { #[tokio::test] async fn non_genesis_first_block_parks_with_unexpected_id() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let block = valid_hash_block(2, HashType([0_u8; 32])); let outcome = store @@ -600,7 +481,7 @@ mod accept_tests { #[tokio::test] async fn hash_mismatch_parks() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let mut block = valid_hash_block(1, HashType([0_u8; 32])); block.header.timestamp = 999; // invalidates the stored hash @@ -618,7 +499,7 @@ mod accept_tests { #[tokio::test] async fn second_break_bumps_orphan_count_and_keeps_first() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let first = valid_hash_block(2, HashType([0_u8; 32])); store @@ -639,7 +520,7 @@ mod accept_tests { #[tokio::test] async fn deserialize_break_records_stall_without_header() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); store .record_stall( @@ -657,7 +538,7 @@ mod accept_tests { #[tokio::test] async fn parks_then_recovers_on_valid_continuation() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); // Genesis (block 1, clock-only) applies and advances the tip. let genesis = produce_dummy_block(1, None, vec![]); @@ -705,7 +586,7 @@ mod accept_tests { #[tokio::test] async fn accept_block_records_tip_inscription_slot() { let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); assert_eq!(store.get_tip_slot().expect("get"), None); @@ -747,7 +628,7 @@ mod accept_tests { use testnet_initial_state::initial_pub_accounts_private_keys; let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let accounts = initial_pub_accounts_private_keys(); let from = accounts[0].account_id; @@ -797,7 +678,7 @@ mod accept_tests { use testnet_initial_state::initial_pub_accounts_private_keys; let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let accounts = initial_pub_accounts_private_keys(); let from = accounts[0].account_id; @@ -857,7 +738,7 @@ mod accept_tests { use testnet_initial_state::initial_pub_accounts_private_keys; let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let accounts = initial_pub_accounts_private_keys(); let from = accounts[0].account_id; @@ -894,7 +775,7 @@ mod accept_tests { // The #605 restart: reopening past the boundary must work. drop(store); - let reopened = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("reopen"); + let reopened = IndexerStore::open_db(dir.path(), Vec::new()).expect("reopen"); assert_eq!(reopened.last_block().unwrap(), Some(101)); } @@ -903,7 +784,7 @@ mod accept_tests { use testnet_initial_state::initial_pub_accounts_private_keys; let dir = tempfile::tempdir().expect("tempdir"); - let store = IndexerStore::open_db(dir.path(), Vec::new(), Vec::new()).expect("open store"); + let store = IndexerStore::open_db(dir.path(), Vec::new()).expect("open store"); let accounts = initial_pub_accounts_private_keys(); let from = accounts[0].account_id; diff --git a/lez/indexer/core/src/ingest_error.rs b/lez/indexer/core/src/ingest_error.rs deleted file mode 100644 index 299013d7..00000000 --- a/lez/indexer/core/src/ingest_error.rs +++ /dev/null @@ -1,80 +0,0 @@ -use common::HashType; -use serde::{Deserialize, Serialize}; - -/// Why the indexer could not apply an L2 block from the channel. -/// -/// Persisted in `RocksDB`, so every variant must have the following -/// traits: `Clone + Serialize + Deserialize`. -#[derive(Debug, Clone, Serialize, Deserialize, thiserror::Error)] -pub enum BlockIngestError { - #[error("Failed to deserialize L2 block: {0}")] - /// Here we store the error string that is derived from [`borsh::from_slice`]'s [`Err`]. - Deserialize(String), - #[error("Unexpected block id: expected {expected}, got {got}")] - UnexpectedBlockId { expected: u64, got: u64 }, - #[error("Broken chain link: expected prev {expected_prev}, got {got_prev}")] - BrokenChainLink { - expected_prev: HashType, - got_prev: HashType, - }, - #[error("Block hash mismatch: computed {computed}, header {header}")] - HashMismatch { - computed: HashType, - header: HashType, - }, - #[error("Block has no transactions")] - EmptyBlock, - #[error("Last transaction must be the public clock invocation for the block timestamp")] - InvalidClockTransaction, - #[error("Genesis block must contain only public transactions")] - NonPublicGenesisTransaction, - #[error("State transition failed at transaction {tx_index}: {reason}")] - StateTransition { - /// Index of the failing transaction within the block body. - tx_index: u64, - /// Reason string from `lee::Error` to `anyhow::Error` to `{:#}`. - /// - /// This is required because `lee::Error` is not `Clone + Serialize + Deserialize`, so we - /// cannot store it directly. - reason: String, - }, -} - -impl BlockIngestError { - /// Whether the failure may be transient rather than a property of the block. - /// - /// FIXME: `StateTransition` is too coarse โ€” its `reason` string mixes genuine - /// state-transition rejections with infra failures (risc0 executor teardown, - /// storage errors). Once it carries a structured cause, narrow this so only - /// infra failures retry. - #[must_use] - pub const fn is_retryable(&self) -> bool { - matches!(self, Self::StateTransition { .. }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn serializes_and_round_trips_externally_tagged() { - let err = BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7, - }; - let value = serde_json::to_value(&err).expect("serialize"); - assert_eq!( - value, - serde_json::json!({ "UnexpectedBlockId": { "expected": 5, "got": 7 } }) - ); - let back: BlockIngestError = serde_json::from_value(value).expect("deserialize"); - assert!(matches!( - back, - BlockIngestError::UnexpectedBlockId { - expected: 5, - got: 7 - } - )); - } -} diff --git a/lez/indexer/core/src/lib.rs b/lez/indexer/core/src/lib.rs index 3df23e08..e6cd9191 100644 --- a/lez/indexer/core/src/lib.rs +++ b/lez/indexer/core/src/lib.rs @@ -2,31 +2,29 @@ use std::{path::Path, sync::Arc}; use anyhow::Result; use arc_swap::ArcSwap; +pub use chain_consistency::BlockIngestError; +use chain_consistency::{Anchor, ChainConsistency}; use common::block::Block; // TODO: Remove after testnet use futures::StreamExt as _; -pub use ingest_error::BlockIngestError; use log::{error, info, warn}; use logos_blockchain_zone_sdk::{ CommonHttpClient, Slot, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer, }; use retry::ApplyRetryGate; -pub use stall_reason::StallReason; +pub use status::StallReason; use crate::{ block_store::{AcceptOutcome, IndexerStore}, - chain_consistency::ChainConsistency, config::IndexerConfig, cross_zone_verifier::CrossZoneVerifier, status::{IndexerStatus, IndexerSyncStatus}, }; + pub mod block_store; -pub mod chain_consistency; pub mod config; pub mod cross_zone_verifier; -pub mod ingest_error; mod retry; -pub mod stall_reason; pub mod status; /// Consecutive failed apply attempts of the same block before parking. @@ -109,7 +107,7 @@ impl IndexerCore { Ok(Self { zone_indexer: Arc::new(zone_indexer), - store: IndexerStore::open_db(&home, Vec::new(), genesis_accounts)?, + store: IndexerStore::open_db(&home, genesis_accounts)?, node, config, status: Arc::new(ArcSwap::from_pointee(IndexerSyncStatus::starting())), @@ -117,6 +115,53 @@ impl IndexerCore { }) } + /// Verifies whether the channel still serves the same chain the store was built from. + /// This may change frequently during development where we reset the chain from time to + /// time in devnet/testnet, but we do not expect [`ChainConsistency::Inconsistent`] in + /// production. + /// + /// To compare the chains, we use an [`Anchor`] block that is either the parked L2 block + /// while stalled, or the tip L2 block at its own inscription L1 slot. + pub(crate) async fn verify_chain_consistency(&self) -> Result { + let Some(anchor) = self.get_startup_anchor()? else { + // empty or cold store: nothing to compare + return Ok(ChainConsistency::Inconclusive); + }; + + chain_consistency::verify_chain_consistency(&self.node, self.config.channel_id, &anchor) + .await + } + + /// Builds the anchor for the startup check. + /// + /// - If stalled, returns the recorded _parked_ block + /// - If not stalled, returns the validated tip at its _own_ inscription slot. + /// - If the store is empty, returns `None`. + fn get_startup_anchor(&self) -> Result> { + if let Some(stall) = self.store.get_stall_reason()? { + return Ok(Some(Anchor::new( + stall.l1_slot, + stall.block_id.zip(stall.block_hash), + ))); + } + + // not stalled, so anchor on the tip at its own inscription slot + let Some(slot) = self + .store + .get_tip_slot()? + .map_or_else(|| self.store.get_zone_cursor(), |slot| Ok(Some(slot)))? + else { + return Ok(None); + }; + let Some(tip_id) = self.store.get_last_block_id()? else { + return Ok(None); + }; + let Some(tip) = self.store.get_block_at_id(tip_id)? else { + return Ok(None); + }; + Ok(Some(Anchor::new(slot, Some((tip_id, tip.header.hash))))) + } + /// Snapshot of the current ingestion status (sync state + indexed tip). /// /// Combines the ingest loop's live status with the L2 tip read fresh from the @@ -199,8 +244,8 @@ impl IndexerCore { let mut cursor = initial_cursor; let mut retry_gate = ApplyRetryGate::new(); - if cursor.is_some() { - info!("Resuming indexer from cursor {cursor:?}"); + if let Some(slot) = &cursor { + info!("Resuming indexer from cursor {slot:?}"); } else { info!("Starting indexer from beginning of channel"); } @@ -372,3 +417,127 @@ impl IndexerCore { } } } + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use common::{HashType, block::HashableBlockData}; + use logos_blockchain_zone_sdk::Slot; + + use super::*; + use crate::config::{ChannelId, ClientConfig, IndexerConfig}; + + fn unreachable_core(dir: &std::path::Path) -> IndexerCore { + let config = IndexerConfig { + consensus_info_polling_interval: Duration::from_secs(1), + bedrock_config: ClientConfig { + addr: "http://localhost:1".parse().expect("url"), + auth: None, + }, + channel_id: ChannelId::from([1; 32]), + allow_chain_reset: false, + cross_zone: None, + bridge_lock_holdings: Vec::new(), + }; + IndexerCore::open(config, dir).expect("open core") + } + + fn test_block(block_id: u64, timestamp: u64) -> Block { + HashableBlockData { + block_id, + prev_block_hash: HashType([0; 32]), + timestamp, + transactions: vec![], + } + .into_pending_block(&lee::PrivateKey::try_new([7; 32]).expect("valid key")) + } + + #[tokio::test] + async fn cold_store_is_inconclusive() { + // An empty store has no cursor, so there is nothing to compare: the check + // must be Inconclusive (not Consistent), and it returns before any L1 read. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn parked_store_with_unreachable_node_is_inconclusive() { + // Network failure is not evidence of a reset: a parked store must stay + // parked (Inconclusive), not error out or trip the wipe path. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let parked = test_block(5, 42); + core.store + .record_stall( + Some(&parked.header), + Slot::from(1_000), + BlockIngestError::EmptyBlock, + ) + .expect("record stall"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn caught_up_store_with_unreachable_node_is_inconclusive() { + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + assert!(matches!( + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"), + AcceptOutcome::Applied + )); + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + assert!(matches!( + core.verify_chain_consistency().await.expect("verify"), + ChainConsistency::Inconclusive + )); + } + + #[tokio::test] + async fn startup_anchor_prefers_tip_slot_over_lagging_cursor() { + // Cursor persist failures are warn-only, so the read cursor can lag the + // tip by several blocks. The anchor must pair the tip with its own + // inscription slot; pairing it with the stale cursor would make the scan + // misread the chain's intermediate blocks as re-inscriptions. + let dir = tempfile::tempdir().expect("tempdir"); + let core = unreachable_core(dir.path()); + + let genesis = common::test_utils::produce_dummy_block(1, None, vec![]); + core.store + .accept_block(&genesis, Slot::from(1_000)) + .await + .expect("accept"); + let block2 = common::test_utils::produce_dummy_block(2, Some(genesis.header.hash), vec![]); + core.store + .accept_block(&block2, Slot::from(1_005)) + .await + .expect("accept"); + let block3 = common::test_utils::produce_dummy_block(3, Some(block2.header.hash), vec![]); + core.store + .accept_block(&block3, Slot::from(1_010)) + .await + .expect("accept"); + + // Cursor last persisted at the genesis slot: two blocks behind the tip. + core.store + .set_zone_cursor(&Slot::from(1_000)) + .expect("set cursor"); + + let anchor = core.get_startup_anchor().expect("anchor").expect("present"); + let expected = Anchor::new(Slot::from(1_010), Some((3, block3.header.hash))); + assert_eq!(anchor, expected); + } +} diff --git a/lez/indexer/core/src/stall_reason.rs b/lez/indexer/core/src/stall_reason.rs deleted file mode 100644 index c5118fcc..00000000 --- a/lez/indexer/core/src/stall_reason.rs +++ /dev/null @@ -1,25 +0,0 @@ -use common::HashType; -use logos_blockchain_zone_sdk::Slot; -use serde::{Deserialize, Serialize}; - -use crate::ingest_error::BlockIngestError; - -/// Diagnostic record of the first block that broke the L2 chain. -/// -/// The block-derived fields are `None` for a deserialize break (no header was -/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. -/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StallReason { - pub block_id: Option, - pub block_hash: Option, - pub prev_block_hash: Option, - pub l1_slot: Slot, - pub error: BlockIngestError, - pub first_seen: Option, - /// Number of later non-chaining blocks (orphans, since the tip is frozen). - /// - /// TODO: We could store a different "branch" of blocks following this break, but for now we - /// just count them. - pub orphans_since: u64, -} diff --git a/lez/indexer/core/src/status.rs b/lez/indexer/core/src/status.rs index a483fde6..1307238b 100644 --- a/lez/indexer/core/src/status.rs +++ b/lez/indexer/core/src/status.rs @@ -1,6 +1,27 @@ -use serde::Serialize; +use chain_consistency::BlockIngestError; +use common::HashType; +use logos_blockchain_zone_sdk::Slot; +use serde::{Deserialize, Serialize}; -use crate::stall_reason::StallReason; +/// Diagnostic record of the first block that broke the L2 chain. +/// +/// The block-derived fields are `None` for a deserialize break (no header was +/// ever parsed). `l1_slot` is the L1 slot the breaking inscription was read at. +/// `first_seen` is the breaking block's L2 timestamp (`None` for a deserialize break). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StallReason { + pub block_id: Option, + pub block_hash: Option, + pub prev_block_hash: Option, + pub l1_slot: Slot, + pub error: BlockIngestError, + pub first_seen: Option, + /// Number of later non-chaining blocks (orphans, since the tip is frozen). + /// + /// TODO: We could store a different "branch" of blocks following this break, but for now we + /// just count them. + pub orphans_since: u64, +} /// Coarse lifecycle state of the indexer's ingestion loop, so a client can tell /// "still catching up" apart from "something went wrong". @@ -119,8 +140,6 @@ mod tests { fn stalled_status_serializes_with_stall_reason() { use logos_blockchain_zone_sdk::Slot; - use crate::{ingest_error::BlockIngestError, stall_reason::StallReason}; - let status = IndexerStatus { sync: IndexerSyncStatus::stalled("broken chain link".to_owned()), indexed_block_id: Some(41), diff --git a/lez/sequencer/core/Cargo.toml b/lez/sequencer/core/Cargo.toml index cd89f223..1fa7494d 100644 --- a/lez/sequencer/core/Cargo.toml +++ b/lez/sequencer/core/Cargo.toml @@ -11,6 +11,7 @@ workspace = true lee.workspace = true lee_core.workspace = true common.workspace = true +chain_consistency.workspace = true storage.workspace = true mempool.workspace = true logos-blockchain-zone-sdk.workspace = true diff --git a/lez/sequencer/core/src/block_publisher.rs b/lez/sequencer/core/src/block_publisher.rs index 21551131..3f2b2a62 100644 --- a/lez/sequencer/core/src/block_publisher.rs +++ b/lez/sequencer/core/src/block_publisher.rs @@ -2,14 +2,16 @@ use std::{pin::Pin, sync::Arc, time::Duration}; use anyhow::{Context as _, Result, anyhow}; use common::block::Block; +use futures::Stream; use log::{info, warn}; pub use logos_blockchain_core::mantle::ops::channel::MsgId; use logos_blockchain_core::mantle::ops::channel::{ChannelId, inscribe::Inscription}; pub use logos_blockchain_key_management_system_service::keys::{Ed25519Key, ZkKey}; pub use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; use logos_blockchain_zone_sdk::{ - CommonHttpClient, - adapter::NodeHttpClient, + CommonHttpClient, Slot, ZoneMessage, + adapter::{Node as _, NodeHttpClient}, + indexer::ZoneIndexer, sequencer::{ DepositInfo, Event, FinalizedOp, InscriptionInfo, SequencerConfig as ZoneSdkSequencerConfig, WithdrawArg, WithdrawInfo, ZoneSequencer, @@ -42,7 +44,7 @@ pub type OnWithdrawEventSink = Box Pin + Send>> + Send + 'static>; #[expect(async_fn_in_trait, reason = "We don't care about Send/Sync here")] -pub trait BlockPublisherTrait: Clone { +pub trait BlockPublisherTrait: Sized { #[expect( clippy::too_many_arguments, reason = "Looks better than bundling all those callbacks into a struct" @@ -63,15 +65,30 @@ pub trait BlockPublisherTrait: Clone { async fn publish_block(&self, block: &Block, withdrawals: Vec) -> Result<()>; fn channel_id(&self) -> ChannelId; + + /// Current channel frontier slot on the connected chain, or `None` if the + /// channel does not exist there. Drives the startup frontier check. + async fn channel_tip_slot(&self) -> Result>; + + /// Finalized channel messages from `after_slot` (exclusive) up to LIB, used + /// for the startup consistency check and reconstruction. Pass `None` to read + /// from the channel's genesis. + async fn read_channel_after( + &self, + after_slot: Option, + ) -> Result + '_>; } /// Real block publisher backed by zone-sdk's `ZoneSequencer`. -#[derive(Clone)] pub struct ZoneSdkPublisher { channel_id: ChannelId, + /// Direct node handle retained for channel reads (startup consistency check + /// and reconstruction); the sequencer itself lives in the drive task. + node: NodeHttpClient, publish_tx: mpsc::Sender<(Inscription, Vec)>, // Aborts the drive task when the last clone is dropped. _drive_task: Arc, + indexer: ZoneIndexer, } struct DriveTaskGuard(JoinHandle<()>); @@ -104,7 +121,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher { let mut sequencer = ZoneSequencer::init_with_config( config.channel_id, bedrock_signing_key, - node, + node.clone(), zone_sdk_config, initial_checkpoint, ); @@ -196,6 +213,8 @@ impl BlockPublisherTrait for ZoneSdkPublisher { Ok(Self { channel_id: config.channel_id, + indexer: ZoneIndexer::new(config.channel_id, node.clone()), + node, publish_tx, _drive_task: Arc::new(DriveTaskGuard(drive_task)), }) @@ -218,6 +237,27 @@ impl BlockPublisherTrait for ZoneSdkPublisher { fn channel_id(&self) -> ChannelId { self.channel_id } + + async fn channel_tip_slot(&self) -> Result> { + Ok(self + .node + .channel_state(self.channel_id) + .await + .context("Failed to read channel state")? + .map(|state| state.tip_slot)) + } + + async fn read_channel_after( + &self, + after_slot: Option, + ) -> Result + '_> { + let stream = self + .indexer + .next_messages(after_slot) + .await + .context("Failed to start channel read stream")?; + Ok(stream) + } } /// Deserialize inscription payload as a `Block` and return it's`block_id`. diff --git a/lez/sequencer/core/src/block_store.rs b/lez/sequencer/core/src/block_store.rs index 77b57661..3a2de375 100644 --- a/lez/sequencer/core/src/block_store.rs +++ b/lez/sequencer/core/src/block_store.rs @@ -12,7 +12,7 @@ use log::info; use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint; use storage::sequencer::{ RocksDBIO, - sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey}, + sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord}, }; pub use storage::{DbResult, sequencer::DbDump}; @@ -28,29 +28,17 @@ impl SequencerStore { /// Open existing database at the given location. Fails if no database is found. pub fn open_db(location: &Path, signing_key: lee::PrivateKey) -> DbResult { let dbio = Arc::new(RocksDBIO::open(location)?); - let genesis_id = dbio.get_meta_first_block_in_db()?; - let last_id = dbio.latest_block_meta()?.id; + Self::from_dbio_and_signing_key(dbio, signing_key) + } - info!("Preparing block cache"); - let mut tx_hash_to_block_map = HashMap::new(); - for i in genesis_id..=last_id { - let block = dbio - .get_block(i)? - .expect("Block should be present in the database"); - - tx_hash_to_block_map.extend(block_to_transactions_map(&block)); - } - info!( - "Block cache prepared. Total blocks in cache: {}", - tx_hash_to_block_map.len() - ); - - Ok(Self { - dbio, - tx_hash_to_block_map, - genesis_id, - signing_key, - }) + /// Create a fresh rocksdb at `location` from `dump`. + pub fn restore_db_from_dump( + location: &Path, + dump: &DbDump, + signing_key: lee::PrivateKey, + ) -> DbResult { + let dbio = Arc::new(RocksDBIO::restore_from_dump(location, dump)?); + Self::from_dbio_and_signing_key(dbio, signing_key) } /// Starting database at the start of new chain. @@ -75,6 +63,38 @@ impl SequencerStore { }) } + fn from_dbio_and_signing_key( + dbio: Arc, + signing_key: lee::PrivateKey, + ) -> DbResult { + let genesis_id = dbio.get_meta_first_block_in_db()?; + let last_id = dbio.latest_block_meta()?.map(|meta| meta.id); + + let mut tx_hash_to_block_map = HashMap::new(); + + if let Some(last_id) = last_id { + info!("Preparing block cache"); + for i in genesis_id..=last_id { + let block = dbio + .get_block(i)? + .expect("Block should be present in the database"); + + tx_hash_to_block_map.extend(block_to_transactions_map(&block)); + } + info!( + "Block cache prepared. Total blocks in cache: {}", + tx_hash_to_block_map.len() + ); + } + + Ok(Self { + dbio, + tx_hash_to_block_map, + genesis_id, + signing_key, + }) + } + /// Shared handle to the underlying rocksdb. Used to persist the zone-sdk /// checkpoint from the sequencer's drive task without needing &mut to the /// store. @@ -114,7 +134,7 @@ impl SequencerStore { ); } - pub fn latest_block_meta(&self) -> DbResult { + pub fn latest_block_meta(&self) -> DbResult> { self.dbio.latest_block_meta() } @@ -165,13 +185,6 @@ impl SequencerStore { self.dbio.dump_all() } - /// Create a fresh rocksdb at `location` from `dump`, closing it before returning so a sequencer - /// can open it normally afterwards. - pub fn restore_db_from_dump(location: &Path, dump: &DbDump) -> DbResult<()> { - RocksDBIO::restore_from_dump(location, dump)?; - Ok(()) - } - pub fn get_zone_checkpoint(&self) -> Result> { let Some(bytes) = self.dbio.get_zone_sdk_checkpoint_bytes()? else { return Ok(None); @@ -188,9 +201,23 @@ impl SequencerStore { Ok(()) } + /// The last channel block read back and verified from Bedrock (L1 slot + + /// `id`/`hash`), or `None` before any block has been read from the channel. + pub fn get_zone_anchor(&self) -> DbResult> { + self.dbio.get_zone_anchor() + } + + pub fn set_zone_anchor(&self, anchor: &ZoneAnchorRecord) -> DbResult<()> { + self.dbio.put_zone_anchor(anchor) + } + pub fn get_unfulfilled_deposit_events(&self) -> DbResult> { self.dbio.get_pending_deposit_events() } + + pub fn is_deposit_event_submitted(&self, deposit_op_id: HashType) -> DbResult { + self.dbio.is_deposit_event_submitted(deposit_op_id) + } } pub(crate) fn block_to_transactions_map(block: &Block) -> HashMap { @@ -275,7 +302,7 @@ mod tests { .unwrap(); // Verify that initially the latest block hash equals genesis hash - let latest_meta = node_store.latest_block_meta().unwrap(); + let latest_meta = node_store.latest_block_meta().unwrap().unwrap(); assert_eq!(latest_meta.hash, genesis_hash); } @@ -313,7 +340,7 @@ mod tests { .unwrap(); // Verify that the latest block meta now equals the new block's hash - let latest_meta = node_store.latest_block_meta().unwrap(); + let latest_meta = node_store.latest_block_meta().unwrap().unwrap(); assert_eq!(latest_meta.hash, block_hash); } diff --git a/lez/sequencer/core/src/lib.rs b/lez/sequencer/core/src/lib.rs index 43667528..c276686c 100644 --- a/lez/sequencer/core/src/lib.rs +++ b/lez/sequencer/core/src/lib.rs @@ -2,18 +2,23 @@ use std::{path::Path, sync::Arc, time::Instant}; use anyhow::{Context as _, Result, anyhow}; use borsh::BorshDeserialize; +use chain_consistency::{Anchor, AnchorConsistencyCheck, ChainConsistency, ChainMismatch, Tip}; use common::{ HashType, block::{BedrockStatus, Block, HashableBlockData}, transaction::{LeeTransaction, clock_invocation}, }; use config::{GenesisAction, SequencerConfig}; +use futures::StreamExt as _; use itertools::Itertools as _; -use lee::{AccountId, PublicTransaction, public_transaction::Message}; +use lee::{AccountId, PublicTransaction, V03State, public_transaction::Message}; use lee_core::GENESIS_BLOCK_ID; use log::{error, info, warn}; use logos_blockchain_key_management_system_service::keys::{ED25519_SECRET_KEY_SIZE, Ed25519Key}; -use logos_blockchain_zone_sdk::sequencer::{DepositInfo, WithdrawArg}; +use logos_blockchain_zone_sdk::{ + Slot, ZoneMessage, + sequencer::{DepositInfo, WithdrawArg}, +}; use mempool::{MemPool, MemPoolHandle}; #[cfg(feature = "mock")] pub use mock::SequencerCoreWithMockClients; @@ -21,7 +26,7 @@ use num_bigint::BigUint; pub use storage::error::DbError; use storage::sequencer::{ RocksDBIO, - sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey}, + sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord}, }; use crate::{ @@ -72,7 +77,7 @@ impl SequencerCore { /// assumed to represent the correct latest state consistent with Bedrock-finalized data. /// If no database is found, the sequencer performs a fresh start from genesis, /// initializing its state with the accounts defined in the configuration file. - fn open_or_create_store(config: &SequencerConfig) -> (SequencerStore, lee::V03State, Block) { + fn open_or_create_store(config: &SequencerConfig) -> (SequencerStore, lee::V03State) { let signing_key = lee::PrivateKey::try_new(config.signing_key).unwrap(); let db_path = config.home.join("rocksdb"); @@ -86,11 +91,7 @@ impl SequencerCore { let state = store .get_lee_state() .expect("Failed to read state from store"); - let genesis_block = store - .get_block_at_id(store.genesis_id()) - .expect("Failed to read genesis block from store") - .expect("Genesis block not found in store"); - (store, state, genesis_block) + (store, state) } else { warn!( "Database not found at {}, starting from genesis", @@ -115,7 +116,7 @@ impl SequencerCore { ) .expect("Failed to create database with genesis block"); - (store, genesis_state, genesis_block) + (store, genesis_state) } } @@ -126,11 +127,7 @@ impl SequencerCore { load_or_create_signing_key(&config.home.join("bedrock_signing_key")) .expect("Failed to load or create bedrock signing key"); - let (store, state, _genesis_block) = Self::open_or_create_store(&config); - - let latest_block_meta = store - .latest_block_meta() - .expect("Failed to read latest block meta from store"); + let (mut store, mut state) = Self::open_or_create_store(&config); let initial_checkpoint = store .get_zone_checkpoint() @@ -153,8 +150,28 @@ impl SequencerCore { .await .expect("Failed to initialize Block Publisher"); - // Fresh start (no checkpoint): republish all pending blocks - if is_fresh_start { + // Cross-zone messaging: start a watcher per configured peer. The inbox + // config account is seeded into genesis state in `build_genesis_state`. + if let Some(cross_zone) = &config.cross_zone { + cross_zone_watcher::spawn_watchers( + &config.bedrock_config, + cross_zone, + config.block_create_timeout, + &mempool_handle, + ); + } + // Before producing, verify our local state still belongs to the chain + // the channel serves and replay any channel blocks we are missing + // (e.g. from other sequencers). + let chanel_was_empty = + Self::verify_and_reconstruct(&block_publisher, &mut store, &mut state, is_fresh_start) + .await + .expect("Failed to verify/reconstruct sequencer state from Bedrock"); + + // Publish our blocks only when bootstrapping an empty channel. If the + // channel already has blocks (another sequencer bootstrapped it), we + // adopted them during reconstruction instead. + if is_fresh_start && chanel_was_empty { let mut pending_blocks = store .get_all_blocks() .filter_ok(|block| matches!(block.bedrock_status, BedrockStatus::Pending)) @@ -182,16 +199,10 @@ impl SequencerCore { } } - // Cross-zone messaging: start a watcher per configured peer. The inbox - // config account is seeded into genesis state in `build_genesis_state`. - if let Some(cross_zone) = &config.cross_zone { - cross_zone_watcher::spawn_watchers( - &config.bedrock_config, - cross_zone, - config.block_create_timeout, - &mempool_handle, - ); - } + let latest_block_meta = store + .latest_block_meta() + .expect("Failed to read latest block meta from store") + .expect("Sequencer store should have at least the genesis block after reconstruction"); let sequencer_core = Self { state, @@ -205,6 +216,207 @@ impl SequencerCore { (sequencer_core, mempool_handle) } + /// Verifies the local store still belongs to the chain the connected channel + /// serves and replays any finalized channel blocks missing locally into + /// `state`/`store`, recording each block's L1 inscription slot as the new + /// anchor. Fails (never parks) on any divergence. + /// + /// Returns whatever channel was empty or not. + async fn verify_and_reconstruct( + publisher: &BP, + store: &mut SequencerStore, + state: &mut V03State, + is_fresh_start: bool, + ) -> Result { + let anchor_record = store + .get_zone_anchor() + .context("Failed to read zone anchor")?; + + let after_slot = anchor_record + .and_then(|record| record.slot.checked_sub(1)) + .map(Slot::from); + let channel_tip_slot = publisher + .channel_tip_slot() + .await + .context("Failed to read channel tip slot")?; + + // If this sequencer has already committed blocks to the channel, that + // channel must still exist. A missing channel then means a wiped/rewound + // Bedrock or a node pointing at a different chain, so refuse to resume + // onto a foreign channel. + // + // "Committed" requires *both* a non-genesis tip and a checkpoint that was + // persisted before this startup: the tip alone is set the moment we produce + // (before the channel confirms it), while a checkpoint alone is written by + // zone-sdk's cold-start backfill even on a brand-new empty channel before we + // publish genesis. We must read the checkpoint presence from before `BP::new` + // ran (`!is_fresh_start`), because its cold-start backfill re-persists a + // checkpoint by the time we reach here โ€” reading the store now would always + // see one. Together they mean we produced blocks and zone-sdk processed + // channel activity in a prior run. + let local_tip = store + .latest_block_meta() + .context("Failed to read latest block meta")? + .map(|meta| meta.id); + let had_checkpoint_before_start = !is_fresh_start; + if let Some(local_tip) = local_tip + && had_checkpoint_before_start + && channel_tip_slot.is_none() + { + return Err(anyhow!( + "Sequencer holds committed blocks (tip {local_tip}) but the Bedrock channel \ + no longer exists on the connected chain โ€” the channel was wiped or the node \ + points at a different chain. Refusing to resume onto a foreign channel." + )); + } + + let divergence_error = |mismatch: &ChainMismatch| { + anyhow!( + "Sequencer store diverges from the Bedrock channel ({mismatch}). \ + Delete the sequencer storage directory or point at the correct channel." + ) + }; + + // With a recorded anchor, probe the channel for positive evidence of a + // different chain: the frontier upfront (a missing/behind channel serves + // no messages to scan), then the anchor block as messages stream in. + let mut consistency_check = anchor_record.map(|record| { + let anchor = Anchor::new( + Slot::from(record.slot), + Some((record.block_id, record.hash)), + ); + let mut check = AnchorConsistencyCheck::new(anchor); + check.check_frontier(channel_tip_slot); + check + }); + if let Some(ChainConsistency::Inconsistent(mismatch)) = consistency_check + .as_ref() + .and_then(AnchorConsistencyCheck::verdict) + { + return Err(divergence_error(mismatch)); + } + + // Verify each message against the anchor and replay the + // blocks (applying the ones we miss, checking the ones we hold). + let messages = publisher + .read_channel_after(after_slot) + .await + .context("Failed to read channel history for reconstruction")?; + let mut messages = std::pin::pin!(messages); + let mut channel_is_empty = true; + while let Some((message, slot)) = messages.next().await { + if let Some(check) = &mut consistency_check + && let Some(ChainConsistency::Inconsistent(mismatch)) = + check.observe(&message, slot) + { + return Err(divergence_error(mismatch)); + } + + let ZoneMessage::Block(zone_block) = message else { + continue; + }; + let block: Block = borsh::from_slice(&zone_block.data).map_err(|err| { + anyhow!( + "Failed to deserialize channel block at slot {}: {err}", + slot.into_inner() + ) + })?; + channel_is_empty = false; + Self::apply_reconstructed_block(store, state, &block, slot)?; + } + + Ok(channel_is_empty) + } + + /// Applies a single channel block during reconstruction: idempotent for + /// blocks we already hold (verifying their hash), a validated continuation + /// for new ones. Advances the persisted anchor to the block's slot. + fn apply_reconstructed_block( + store: &mut SequencerStore, + state: &mut V03State, + block: &Block, + slot: Slot, + ) -> Result<()> { + let tip = store + .latest_block_meta() + .context("Failed to read latest block meta")?; + let block_id = block.header.block_id; + let block_hash = block.header.hash; + + let record = ZoneAnchorRecord { + slot: slot.into_inner(), + block_id, + hash: block_hash, + }; + + // A block at/below the tip must match what we already stored, otherwise + // the channel is a different chain. + if let Some(tip) = &tip + && block_id <= tip.id + { + match store + .get_block_at_id(block_id) + .context("Failed to read stored block")? + { + Some(stored) if stored.header.hash == block_hash => { + store + .set_zone_anchor(&record) + .context("Failed to persist zone anchor")?; + return Ok(()); + } + Some(stored) => { + return Err(anyhow!( + "Channel block {block_id} hash {block_hash} does not match stored hash {}", + stored.header.hash + )); + } + None => { + return Err(anyhow!( + "Channel block {block_id} is at/below local tip {} but is missing locally", + tip.id + )); + } + } + } + + // New continuation: validate it chains onto the tip, then apply. + let cc_tip = tip.as_ref().map(|tip| Tip { + block_id: tip.id, + hash: tip.hash, + }); + chain_consistency::validate_against_tip(cc_tip, block).map_err(|err| { + anyhow!( + "Channel block {block_id} does not extend local tip {:?}: {err}", + tip.map(|tip| tip.id) + ) + })?; + + let mut scratch = state.clone(); + chain_consistency::apply_block(block, &mut scratch) + .map_err(|err| anyhow!("Failed to apply channel block {block_id}: {err}"))?; + + // Mark the deposits' pending records submitted so the production-time + // guard drops the mints cold-start backfill re-queued for them. Withdraw + // intents are deliberately not counted: backfill already re-delivered and + // dropped their finalized L1 events, so an increment here would never be + // consumed and would leave a phantom count. + let deposit_event_ids: Vec<_> = block + .body + .transactions + .iter() + .filter_map(extract_bridge_deposit_id) + .collect(); + + store + .update(block, &deposit_event_ids, Vec::new(), &scratch) + .context("Failed to persist reconstructed block")?; + *state = scratch; + store + .set_zone_anchor(&record) + .context("Failed to persist zone anchor")?; + Ok(()) + } + fn on_checkpoint(dbio: Arc) -> block_publisher::CheckpointSink { Box::new(move |cp| { let bytes = match serde_json::to_vec(&cp) { @@ -407,6 +619,14 @@ impl SequencerCore { }; if let Some(deposit_op_id) = extract_bridge_deposit_id(tx) { + if self + .store + .is_deposit_event_submitted(deposit_op_id) + .context("Failed to check whether deposit was already submitted")? + { + info!("Skipping already-submitted bridge deposit {deposit_op_id}"); + return Ok(false); + } deposit_event_ids.push(deposit_op_id); } @@ -435,7 +655,8 @@ impl SequencerCore { let latest_block_meta = self .store .latest_block_meta() - .context("Failed to get latest block meta from store")?; + .context("Failed to get latest block meta from store")? + .context("Sequencer store should have at least the genesis block")?; let new_block_timestamp = u64::try_from(chrono::Utc::now().timestamp_millis()) .expect("Timestamp must be positive"); @@ -561,8 +782,8 @@ impl SequencerCore { .collect()) } - pub fn block_publisher(&self) -> BP { - self.block_publisher.clone() + pub const fn block_publisher(&self) -> &BP { + &self.block_publisher } fn next_block_id(&self) -> u64 { diff --git a/lez/sequencer/core/src/mock.rs b/lez/sequencer/core/src/mock.rs index 39f635f9..f8349fac 100644 --- a/lez/sequencer/core/src/mock.rs +++ b/lez/sequencer/core/src/mock.rs @@ -2,9 +2,10 @@ use std::time::Duration; use anyhow::Result; use common::block::Block; +use futures::Stream; use logos_blockchain_core::mantle::ops::channel::ChannelId; use logos_blockchain_key_management_system_service::keys::Ed25519Key; -use logos_blockchain_zone_sdk::sequencer::WithdrawArg; +use logos_blockchain_zone_sdk::{Slot, ZoneMessage, sequencer::WithdrawArg}; use crate::{ block_publisher::{ @@ -19,6 +20,28 @@ pub type SequencerCoreWithMockClients = crate::SequencerCore #[derive(Clone)] pub struct MockBlockPublisher { channel_id: ChannelId, + /// Canned channel frontier returned by [`Self::channel_tip_slot`]. + tip_slot: Option, + /// Canned finalized channel history returned by [`Self::read_channel_after`]. + messages: Vec<(ZoneMessage, Slot)>, +} + +impl MockBlockPublisher { + /// Builds a mock publisher backed by a canned channel, for reconstruction + /// and consistency tests. The default (via [`BlockPublisherTrait::new`]) + /// serves an empty channel. + #[must_use] + pub const fn with_canned_channel( + channel_id: ChannelId, + tip_slot: Option, + messages: Vec<(ZoneMessage, Slot)>, + ) -> Self { + Self { + channel_id, + tip_slot, + messages, + } + } } impl BlockPublisherTrait for MockBlockPublisher { @@ -34,6 +57,8 @@ impl BlockPublisherTrait for MockBlockPublisher { ) -> Result { Ok(Self { channel_id: config.channel_id, + tip_slot: None, + messages: Vec::new(), }) } @@ -48,4 +73,21 @@ impl BlockPublisherTrait for MockBlockPublisher { fn channel_id(&self) -> ChannelId { self.channel_id } + + async fn channel_tip_slot(&self) -> Result> { + Ok(self.tip_slot) + } + + async fn read_channel_after( + &self, + after_slot: Option, + ) -> Result + '_> { + // Mirror `next_messages`: `after_slot` is exclusive. + let messages = self + .messages + .iter() + .filter(move |(_, slot)| after_slot.is_none_or(|after| *slot > after)) + .cloned(); + Ok(futures::stream::iter(messages)) + } } diff --git a/lez/sequencer/core/src/tests.rs b/lez/sequencer/core/src/tests.rs index 05e2ec4c..b398e160 100644 --- a/lez/sequencer/core/src/tests.rs +++ b/lez/sequencer/core/src/tests.rs @@ -37,6 +37,8 @@ use crate::{ mock::SequencerCoreWithMockClients, }; +mod reconstruction; + #[derive(borsh::BorshSerialize)] struct DepositMetadataForEncoding { recipient_id: lee::AccountId, @@ -625,7 +627,7 @@ async fn produce_block_with_correct_prev_meta_after_restart() { sequencer.produce_new_block().await.unwrap(); // Get the metadata of the last block produced - sequencer.store.latest_block_meta().unwrap() + sequencer.store.latest_block_meta().unwrap().unwrap() }; // Step 2: Restart sequencer from the same storage diff --git a/lez/sequencer/core/src/tests/reconstruction.rs b/lez/sequencer/core/src/tests/reconstruction.rs new file mode 100644 index 00000000..40cbaa6c --- /dev/null +++ b/lez/sequencer/core/src/tests/reconstruction.rs @@ -0,0 +1,617 @@ +#![expect( + clippy::arithmetic_side_effects, + clippy::as_conversions, + reason = "We don't care about it in tests" +)] + +use common::block::Block; +use logos_blockchain_core::mantle::ops::channel::{MsgId, inscribe::Inscription}; +use logos_blockchain_zone_sdk::{Slot, ZoneBlock, ZoneMessage}; +use storage::sequencer::sequencer_cells::ZoneAnchorRecord; + +use super::*; +use crate::{ + SequencerCore, block_store::SequencerStore, config::GenesisAction, mock::MockBlockPublisher, +}; + +fn block_to_channel_message(block: &Block, slot: u64) -> (ZoneMessage, Slot) { + let bytes = borsh::to_vec(block).expect("serialize block"); + let message = ZoneMessage::Block(ZoneBlock { + id: MsgId::from([0_u8; 32]), + data: Inscription::try_from(bytes.as_slice()).expect("inscription"), + }); + (message, Slot::from(slot)) +} + +/// Collects a sequencer's whole chain (genesis..=tip) into a canned channel, +/// one block per slot at `slot_step` spacing. +fn channel_from_store(store: &SequencerStore, slot_step: u64) -> Vec<(ZoneMessage, Slot)> { + let genesis_id = store.genesis_id(); + let tip_id = store.latest_block_meta().expect("tip").expect("present").id; + (genesis_id..=tip_id) + .enumerate() + .map(|(index, id)| { + let block = store.get_block_at_id(id).expect("read").expect("present"); + block_to_channel_message(&block, (index as u64 + 1) * slot_step) + }) + .collect() +} + +#[tokio::test] +async fn reconstructs_missing_channel_blocks_into_fresh_store() { + // Sequencer A produces a few blocks; treat its chain as the channel. + let config_a = setup_sequencer_config(); + let (mut seq_a, _handle_a) = + SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + seq_a.produce_new_block().await.unwrap(); + seq_a.produce_new_block().await.unwrap(); + let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); + + let messages = channel_from_store(seq_a.block_store(), 10); + let tip_slot = messages.last().unwrap().1; + let channel_id = config_a.bedrock_config.channel_id; + + // Sequencer B starts from a fresh store and reconstructs A's chain. + let config_b = setup_sequencer_config(); + let (mut store_b, mut state_b) = + SequencerCore::::open_or_create_store(&config_b); + let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); + + let channel_was_empty = SequencerCore::::verify_and_reconstruct( + &mock_b, + &mut store_b, + &mut state_b, + true, + ) + .await + .expect("reconstruct"); + assert!(!channel_was_empty); + + let tip_b = store_b.latest_block_meta().unwrap().unwrap(); + assert_eq!(tip_b.id, tip_a.id); + assert_eq!(tip_b.hash, tip_a.hash); + + // State matches: initial account balances agree with sequencer A. + for account in initial_public_user_accounts() { + assert_eq!( + state_b.get_account_by_id(account.account_id).balance, + seq_a.state().get_account_by_id(account.account_id).balance, + ); + } + + let anchor = store_b.get_zone_anchor().unwrap().expect("anchor recorded"); + assert_eq!(anchor.block_id, tip_a.id); + assert_eq!(anchor.slot, tip_slot.into_inner()); + + // Re-running is idempotent: everything is already applied, no error. + let channel_was_empty = SequencerCore::::verify_and_reconstruct( + &mock_b, + &mut store_b, + &mut state_b, + true, + ) + .await + .expect("reconstruct idempotent"); + assert!(!channel_was_empty); + assert_eq!(store_b.latest_block_meta().unwrap().unwrap().id, tip_a.id); +} + +#[tokio::test] +async fn fails_when_channel_serves_a_divergent_block() { + let config = setup_sequencer_config(); + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + + // Anchor on the local genesis at some slot. + let genesis_id = store.genesis_id(); + let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap(); + let anchor_slot = 100_u64; + store + .set_zone_anchor(&ZoneAnchorRecord { + slot: anchor_slot, + block_id: genesis_id, + hash: genesis.header.hash, + }) + .unwrap(); + + // The channel serves a different block at the anchor id/slot. + let mut tampered = genesis.clone(); + tampered.header.hash = HashType([9_u8; 32]); + let messages = vec![block_to_channel_message(&tampered, anchor_slot)]; + let mock = MockBlockPublisher::with_canned_channel( + config.bedrock_config.channel_id, + Some(Slot::from(anchor_slot)), + messages, + ); + + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, true, + ) + .await; + assert!(result.is_err(), "divergent channel must abort startup"); +} + +#[tokio::test] +async fn fails_when_channel_is_missing() { + let config = setup_sequencer_config(); + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + let genesis_id = store.genesis_id(); + let genesis = store.get_block_at_id(genesis_id).unwrap().unwrap(); + store + .set_zone_anchor(&ZoneAnchorRecord { + slot: 100, + block_id: genesis_id, + hash: genesis.header.hash, + }) + .unwrap(); + + // Anchor present, but the channel does not exist on the connected chain. + let mock = + MockBlockPublisher::with_canned_channel(config.bedrock_config.channel_id, None, vec![]); + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, true, + ) + .await; + assert!(result.is_err(), "missing channel must abort startup"); +} + +// The following cases exercise the divergence branches of +// `apply_reconstructed_block` reached with no recorded anchor, so the block's own +// validation fires rather than the up-front `AnchorConsistencyCheck`. + +#[tokio::test] +async fn fails_when_channel_reinscribes_genesis_with_a_different_hash() { + let config = setup_sequencer_config(); + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + + // Fresh store, no anchor. The channel serves a genesis at the same id but a + // different hash โ€” a foreign chain reinscribing genesis. + let mut reinscribed = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + reinscribed.header.hash = HashType([0xAB_u8; 32]); + + let messages = vec![block_to_channel_message(&reinscribed, 10)]; + let mock = MockBlockPublisher::with_canned_channel( + config.bedrock_config.channel_id, + Some(Slot::from(10)), + messages, + ); + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, true, + ) + .await; + assert!( + result.is_err(), + "a reinscribed genesis with a different hash must abort startup" + ); +} + +#[tokio::test] +async fn fails_when_a_stored_block_hash_diverges_from_the_channel() { + // A sequencer that committed blocks past genesis but never recorded an anchor. + let config = setup_sequencer_config(); + let (mut seq, _handle) = SequencerCoreWithMockClients::start_from_config(config.clone()).await; + seq.produce_new_block().await.unwrap(); + seq.produce_new_block().await.unwrap(); + + // A below-tip block re-served with a corrupted hash: we already hold this id + // with a different hash, so the channel is a different chain. + let below_tip_id = seq.block_store().genesis_id() + 1; + let mut block = seq + .block_store() + .get_block_at_id(below_tip_id) + .unwrap() + .unwrap(); + block.header.hash = HashType([0xCD_u8; 32]); + + let messages = vec![block_to_channel_message(&block, 10)]; + let mock = MockBlockPublisher::with_canned_channel( + config.bedrock_config.channel_id, + Some(Slot::from(10)), + messages, + ); + let result = SequencerCore::::verify_and_reconstruct( + &mock, + &mut seq.store, + &mut seq.state, + true, + ) + .await; + assert!( + result.is_err(), + "a diverging below-tip block hash must abort startup" + ); +} + +#[tokio::test] +async fn fails_when_a_channel_block_is_missing_locally() { + let config = setup_sequencer_config(); + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + + // A block numbered below our genesis is at/below the local tip yet absent from + // the store โ€” a foreign chain with a lower numbering. + let mut foreign = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + foreign.header.block_id = store.genesis_id() - 1; + + let messages = vec![block_to_channel_message(&foreign, 10)]; + let mock = MockBlockPublisher::with_canned_channel( + config.bedrock_config.channel_id, + Some(Slot::from(10)), + messages, + ); + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, true, + ) + .await; + assert!( + result.is_err(), + "a channel block below the local range must abort startup" + ); +} + +#[tokio::test] +async fn fails_when_a_channel_block_does_not_extend_the_tip() { + let config = setup_sequencer_config(); + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + + // A block claiming an id far past genesis does not chain onto the local tip. + let mut orphan = store.get_block_at_id(store.genesis_id()).unwrap().unwrap(); + orphan.header.block_id = store.genesis_id() + 5; + + let messages = vec![block_to_channel_message(&orphan, 10)]; + let mock = MockBlockPublisher::with_canned_channel( + config.bedrock_config.channel_id, + Some(Slot::from(10)), + messages, + ); + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, true, + ) + .await; + assert!( + result.is_err(), + "a non-contiguous channel block must abort startup" + ); +} + +/// A sequencer config whose genesis funds the bridge account, so replayed bridge +/// deposit transactions have a source balance to mint from. +fn bridge_funded_config() -> SequencerConfig { + let mut config = setup_sequencer_config(); + config.genesis = vec![GenesisAction::SupplyBridgeAccount { balance: 1_000_000 }]; + config +} + +/// Builds an unfulfilled pending deposit event for `recipient`, matching the +/// encoding `build_bridge_deposit_tx_from_event` expects. +fn deposit_event_record( + op_id: [u8; 32], + amount: u64, + recipient: lee::AccountId, +) -> PendingDepositEventRecord { + PendingDepositEventRecord { + deposit_op_id: HashType(op_id), + source_tx_hash: HashType([0_u8; 32]), + amount, + metadata: borsh::to_vec(&DepositMetadataForEncoding { + recipient_id: recipient, + }) + .unwrap(), + submitted_in_block_id: None, + } +} + +/// Builds a signed public bridge `Withdraw` transaction (the normal user path). +fn build_public_withdraw_tx( + sender: lee::AccountId, + nonce: u128, + amount: u64, + bedrock_account_pk: [u8; 32], + signing_key: &lee::PrivateKey, +) -> LeeTransaction { + let message = lee::public_transaction::Message::try_new( + programs::bridge().id(), + vec![sender, system_accounts::bridge_account_id()], + vec![nonce.into()], + bridge_core::Instruction::Withdraw { + amount, + bedrock_account_pk, + }, + ) + .unwrap(); + let witness_set = lee::public_transaction::WitnessSet::for_message(&message, &[signing_key]); + LeeTransaction::Public(lee::PublicTransaction::new(message, witness_set)) +} + +/// The cold-start backfill re-delivers an already-finalized deposit into the +/// mempool before reconstruction applies the same deposit block, and the queued +/// mint cannot be pulled back out. Since the bridge program does not dedup on +/// `l1_deposit_op_id`, block production must skip the already-submitted deposit +/// so the vault is minted exactly once. +#[tokio::test] +async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() { + let recipient = initial_public_user_accounts()[0].account_id; + let deposit_amount = 500_u64; + let withdraw_amount = 100_u64; + let bedrock_account_pk = [0x22_u8; 32]; + let deposit_op_id = [0x0d_u8; 32]; + + // Sequencer A produces a deposit block then a withdraw block. + let config_a = bridge_funded_config(); + let (mut seq_a, mempool_a) = + SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + + let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient); + let deposit_tx = + crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx"); + mempool_a + .push((TransactionOrigin::Sequencer, deposit_tx.clone())) + .await + .unwrap(); + seq_a.produce_new_block().await.unwrap(); + + let withdraw_tx = build_public_withdraw_tx( + recipient, + 0, + withdraw_amount, + bedrock_account_pk, + &create_signing_key_for_account1(), + ); + mempool_a + .push((TransactionOrigin::User, withdraw_tx.clone())) + .await + .unwrap(); + seq_a.produce_new_block().await.unwrap(); + + let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap(); + let messages = channel_from_store(seq_a.block_store(), 10); + let tip_slot = messages.last().unwrap().1; + let channel_id = config_a.bedrock_config.channel_id; + + let config_b = bridge_funded_config(); + let (mut seq_b, mempool_b) = SequencerCoreWithMockClients::start_from_config(config_b).await; + + // Backfill re-delivery: persist the pending record and queue the mint, as + // `on_deposit_event` does, before reconstruction runs. + assert!( + seq_b + .block_store() + .dbio() + .add_pending_deposit_event(deposit_record.clone()) + .unwrap() + ); + mempool_b + .push((TransactionOrigin::Sequencer, deposit_tx)) + .await + .unwrap(); + + let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); + SequencerCore::::verify_and_reconstruct( + &mock_b, + &mut seq_b.store, + &mut seq_b.state, + true, + ) + .await + .expect("reconstruct"); + seq_b.chain_height = seq_b.store.latest_block_meta().unwrap().unwrap().id; + + let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap(); + assert_eq!(tip_b.id, tip_a.id); + assert_eq!(tip_b.hash, tip_a.hash); + + seq_b.produce_new_block().await.unwrap(); + + let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient); + let bridge_id = system_accounts::bridge_account_id(); + for account in [vault_id, bridge_id, recipient] { + assert_eq!( + seq_b.state().get_account_by_id(account).balance, + seq_a.state().get_account_by_id(account).balance, + "reconstructed balance mismatch for {account:?}", + ); + } + assert_eq!( + seq_b.state().get_account_by_id(vault_id).balance, + u128::from(deposit_amount), + "deposit must mint into the recipient vault exactly once, not twice" + ); + + let produced = seq_b + .block_store() + .get_block_at_id(tip_b.id + 1) + .unwrap() + .expect("produced block present"); + assert!( + !produced + .body + .transactions + .iter() + .any(|tx| crate::extract_bridge_deposit_id(tx) == Some(HashType(deposit_op_id))), + "the re-delivered mint must be skipped, not re-included in a block" + ); + + // A reconstructed withdraw's finalized L1 event was already re-delivered (and + // dropped) by cold-start backfill, so it will never be consumed again. + // Reconstruction must not count it, or the count stays phantom-inflated forever. + let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data"); + let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key"); + assert!( + !seq_b + .block_store() + .dbio() + .consume_unseen_withdraw_count(key) + .unwrap(), + "reconstruction must not leave a phantom unseen-withdraw count" + ); +} + +/// A reconstructed withdraw block must not touch the unseen-withdraw counter. +/// Its finalized L1 Withdraw event was already re-delivered (and dropped as a +/// no-op) by cold-start backfill, so counting it during reconstruction would +/// leave a permanent phantom that nothing ever consumes. +#[tokio::test] +async fn reconstructed_withdraw_leaves_no_phantom_unseen_count() { + let recipient = initial_public_user_accounts()[0].account_id; + let withdraw_amount = 100_u64; + let bedrock_account_pk = [0x33_u8; 32]; + + // Sequencer A produces a single withdraw block; treat its chain as the channel. + let config_a = bridge_funded_config(); + let (mut seq_a, mempool_a) = + SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let withdraw_tx = build_public_withdraw_tx( + recipient, + 0, + withdraw_amount, + bedrock_account_pk, + &create_signing_key_for_account1(), + ); + mempool_a + .push((TransactionOrigin::User, withdraw_tx.clone())) + .await + .unwrap(); + seq_a.produce_new_block().await.unwrap(); + + let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data"); + let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key"); + // Producing the withdraw counts it as unseen, awaiting its L1 event. + assert!( + seq_a + .block_store() + .dbio() + .consume_unseen_withdraw_count(key) + .unwrap(), + "producing a withdraw must count it as unseen" + ); + + let messages = channel_from_store(seq_a.block_store(), 10); + let tip_slot = messages.last().unwrap().1; + let channel_id = config_a.bedrock_config.channel_id; + + // Sequencer B reconstructs A's chain from a fresh store. + let config_b = bridge_funded_config(); + let (mut store_b, mut state_b) = + SequencerCore::::open_or_create_store(&config_b); + let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); + SequencerCore::::verify_and_reconstruct( + &mock_b, + &mut store_b, + &mut state_b, + true, + ) + .await + .expect("reconstruct"); + + assert!( + !store_b.dbio().consume_unseen_withdraw_count(key).unwrap(), + "reconstruction must not leave a phantom unseen-withdraw count" + ); +} + +/// A deposit whose L1 event was observed (an unfulfilled pending record +/// exists) and whose L2 mint is already contained in a finalized channel block. +/// Reconstruction must reconcile the pending record against that block โ€” marking +/// it submitted so the startup replay does not re-inject it โ€” and apply the mint +/// exactly once. +#[tokio::test] +async fn reconstruction_reconciles_already_finished_deposit() { + let recipient = initial_public_user_accounts()[0].account_id; + let deposit_amount = 400_u64; + let deposit_op_id = [0x1a_u8; 32]; + + // Sequencer A: a single block that fully processes the bridge deposit. + let config_a = bridge_funded_config(); + let (mut seq_a, mempool_a) = + SequencerCoreWithMockClients::start_from_config(config_a.clone()).await; + let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient); + let deposit_tx = + crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx"); + mempool_a + .push((TransactionOrigin::Sequencer, deposit_tx)) + .await + .unwrap(); + seq_a.produce_new_block().await.unwrap(); + let deposit_block_id = seq_a.block_store().latest_block_meta().unwrap().unwrap().id; + + let messages = channel_from_store(seq_a.block_store(), 10); + let tip_slot = messages.last().unwrap().1; + let channel_id = config_a.bedrock_config.channel_id; + + // Sequencer B: fresh store, but with the *unfulfilled* pending deposit event + // pre-seeded, as the cold-start backfill would when it re-observes this + // already-finalized deposit. + let config_b = bridge_funded_config(); + let (mut store_b, mut state_b) = + SequencerCore::::open_or_create_store(&config_b); + assert!( + store_b + .dbio() + .add_pending_deposit_event(deposit_record.clone()) + .unwrap() + ); + + let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages); + SequencerCore::::verify_and_reconstruct( + &mock_b, + &mut store_b, + &mut state_b, + true, + ) + .await + .expect("reconstruct"); + + // The mint was applied exactly once. + let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient); + assert_eq!( + state_b.get_account_by_id(vault_id).balance, + u128::from(deposit_amount), + "already-finished deposit must be applied exactly once" + ); + + // The pending event is now marked submitted in the reconstructed block, so the + // startup replay would not re-queue it โ€” no double mint on restart. + let record = store_b + .get_unfulfilled_deposit_events() + .unwrap() + .into_iter() + .find(|event| event.deposit_op_id == HashType(deposit_op_id)) + .expect("pending deposit event should still be recorded"); + assert_eq!( + record.submitted_in_block_id, + Some(deposit_block_id), + "reconstruction must reconcile the already-finished deposit against its channel block" + ); +} + +#[tokio::test] +async fn committed_local_against_missing_channel_fails_without_anchor() { + // A sequencer that has committed blocks โ€” a non-genesis tip plus a persisted + // checkpoint โ€” but only ever produced (so it never recorded a per-block + // anchor). Restarting it against a wiped/missing channel must still fail, + // driven by the committed-blocks invariant rather than an anchor probe. + let config = setup_sequencer_config(); + { + let (mut seq, _handle) = + SequencerCoreWithMockClients::start_from_config(config.clone()).await; + seq.produce_new_block().await.unwrap(); + seq.produce_new_block().await.unwrap(); + assert!(seq.block_store().latest_block_meta().unwrap().unwrap().id > 1); + } // drop releases the store so we can reopen it + + // Reopen: blocks beyond genesis, no anchor. `is_fresh_start = false` stands in + // for a checkpoint persisted by a prior sync (the mock never emits one). + let (mut store, mut state) = SequencerCore::::open_or_create_store(&config); + assert!(store.get_zone_anchor().unwrap().is_none()); + assert!(store.latest_block_meta().unwrap().unwrap().id > 1); + + // The channel is gone: no tip, no messages. + let mock = + MockBlockPublisher::with_canned_channel(config.bedrock_config.channel_id, None, vec![]); + let result = SequencerCore::::verify_and_reconstruct( + &mock, &mut store, &mut state, false, + ) + .await; + assert!( + result.is_err(), + "committed blocks against a missing channel must abort startup" + ); +} diff --git a/lez/storage/src/sequencer/mod.rs b/lez/storage/src/sequencer/mod.rs index 1a3a1886..47ebc41b 100644 --- a/lez/storage/src/sequencer/mod.rs +++ b/lez/storage/src/sequencer/mod.rs @@ -22,7 +22,7 @@ use crate::{ LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell, LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PendingDepositEventRecord, PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell, WithdrawalReconciliationKey, - ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, + ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned, ZoneSdkCheckpointCellRef, }, }; @@ -34,6 +34,10 @@ pub const DB_META_LAST_FINALIZED_BLOCK_ID: &str = "last_finalized_block_id"; pub const DB_META_LATEST_BLOCK_META_KEY: &str = "latest_block_meta"; /// Key base for storing the zone-sdk sequencer checkpoint (opaque bytes). pub const DB_META_ZONE_SDK_CHECKPOINT_KEY: &str = "zone_sdk_checkpoint"; +/// Key base for storing the last channel block read back and verified from +/// Bedrock (its L1 slot + `id`/`hash`) โ€” the anchor for the startup +/// consistency check and the resume point for reconstruction. +pub const DB_META_ZONE_CURSOR_KEY: &str = "zone_cursor"; /// Key base for storing queued deposit events that were not yet /// fulfilled on L2. pub const DB_META_PENDING_DEPOSIT_EVENTS_KEY: &str = "pending_deposit_events"; @@ -340,8 +344,9 @@ impl RocksDBIO { self.put_batch(&LatestBlockMetaCellRef(block_meta), (), batch) } - pub fn latest_block_meta(&self) -> DbResult { - self.get::(()).map(|val| val.0) + pub fn latest_block_meta(&self) -> DbResult> { + self.get_opt::(()) + .map(|val| val.map(|cell| cell.0)) } pub fn get_zone_sdk_checkpoint_bytes(&self) -> DbResult>> { @@ -359,6 +364,14 @@ impl RocksDBIO { self.del::(()) } + pub fn get_zone_anchor(&self) -> DbResult> { + Ok(self.get_opt::(())?.map(|cell| cell.0)) + } + + pub fn put_zone_anchor(&self, anchor: &ZoneAnchorRecord) -> DbResult<()> { + self.put(&ZoneAnchorCell(*anchor), ()) + } + pub fn get_pending_deposit_events(&self) -> DbResult> { Ok(self .get_opt::(())? @@ -434,6 +447,14 @@ impl RocksDBIO { Ok(removed) } + /// Whether a bridge deposit for `deposit_op_id` is already recorded as + /// included in a block (its pending record is marked submitted). + pub fn is_deposit_event_submitted(&self, deposit_op_id: HashType) -> DbResult { + Ok(self.get_pending_deposit_events()?.iter().any(|record| { + record.deposit_op_id == deposit_op_id && record.submitted_in_block_id.is_some() + })) + } + fn increment_unseen_withdraw_count( &self, withdrawal: WithdrawalReconciliationKey, diff --git a/lez/storage/src/sequencer/sequencer_cells.rs b/lez/storage/src/sequencer/sequencer_cells.rs index 7672e271..368fb648 100644 --- a/lez/storage/src/sequencer/sequencer_cells.rs +++ b/lez/storage/src/sequencer/sequencer_cells.rs @@ -9,7 +9,8 @@ use crate::{ sequencer::{ CF_LEE_STATE_NAME, DB_LEE_STATE_KEY, DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_DEPOSIT_EVENTS_KEY, - DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY, + DB_META_UNSEEN_WITHDRAW_COUNT_KEY, DB_META_ZONE_CURSOR_KEY, + DB_META_ZONE_SDK_CHECKPOINT_KEY, }, }; @@ -132,6 +133,39 @@ impl SimpleWritableCell for ZoneSdkCheckpointCellRef<'_> { } } +/// The last channel block read back and verified from Bedrock. +/// +/// Holds its L1 inscription `slot` plus the block's `id`/`hash`, and serves as +/// both the anchor for the startup consistency check and the resume point for +/// reconstruction. `slot` is stored as a raw `u64` because the zone-sdk `Slot` +/// does not derive borsh; the caller converts to/from `Slot`. +#[derive(Debug, Clone, Copy, BorshSerialize, BorshDeserialize)] +pub struct ZoneAnchorRecord { + pub slot: u64, + pub block_id: u64, + pub hash: HashType, +} + +#[derive(Debug, BorshSerialize, BorshDeserialize)] +pub struct ZoneAnchorCell(pub ZoneAnchorRecord); + +impl SimpleStorableCell for ZoneAnchorCell { + type KeyParams = (); + + const CELL_NAME: &'static str = DB_META_ZONE_CURSOR_KEY; + const CF_NAME: &'static str = CF_META_NAME; +} + +impl SimpleReadableCell for ZoneAnchorCell {} + +impl SimpleWritableCell for ZoneAnchorCell { + fn value_constructor(&self) -> DbResult> { + borsh::to_vec(&self).map_err(|err| { + DbError::borsh_cast_message(err, Some("Failed to serialize zone cursor".to_owned())) + }) + } +} + #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)] pub struct PendingDepositEventRecord { pub deposit_op_id: HashType, diff --git a/lez/wallet-ffi/Cargo.toml b/lez/wallet-ffi/Cargo.toml index b5fe7635..5440bee2 100644 --- a/lez/wallet-ffi/Cargo.toml +++ b/lez/wallet-ffi/Cargo.toml @@ -14,8 +14,6 @@ crate-type = ["rlib", "cdylib", "staticlib"] wallet.workspace = true lee.workspace = true lee_core.workspace = true -sequencer_service_rpc = { workspace = true, features = ["client"] } -common.workspace = true programs.workspace = true tokio.workspace = true diff --git a/lez/wallet-ffi/src/program_deployment.rs b/lez/wallet-ffi/src/program_deployment.rs index 2086fa5c..2e5550ae 100644 --- a/lez/wallet-ffi/src/program_deployment.rs +++ b/lez/wallet-ffi/src/program_deployment.rs @@ -1,9 +1,5 @@ use std::{ffi::CString, ptr, slice}; -use common::transaction::LeeTransaction; -use lee::ProgramDeploymentTransaction; -use sequencer_service_rpc::RpcClient as _; - use crate::{ block_on, error::{print_error, WalletFfiError}, @@ -60,14 +56,7 @@ pub unsafe extern "C" fn wallet_ffi_program_deployment( let elf = unsafe { slice::from_raw_parts(elf_data, elf_size) }.to_vec(); - let message = lee::program_deployment_transaction::Message::new(elf); - let transaction = ProgramDeploymentTransaction::new(message); - - match block_on( - wallet - .sequencer_client - .send_transaction(LeeTransaction::ProgramDeployment(transaction)), - ) { + match block_on(wallet.send_program_deployment_transaction(elf)) { Ok(tx_hash) => { let tx_hash = CString::new(tx_hash.to_string()) .map_or(ptr::null_mut(), std::ffi::CString::into_raw); diff --git a/lez/wallet-ffi/src/sync.rs b/lez/wallet-ffi/src/sync.rs index 5f7a4413..b65a0944 100644 --- a/lez/wallet-ffi/src/sync.rs +++ b/lez/wallet-ffi/src/sync.rs @@ -1,7 +1,5 @@ //! Block synchronization functions. -use sequencer_service_rpc::RpcClient as _; - use crate::{ block_on, error::{print_error, WalletFfiError}, @@ -136,7 +134,7 @@ pub unsafe extern "C" fn wallet_ffi_get_current_block_height( } }; - match block_on(wallet.sequencer_client.get_last_block_id()) { + match block_on(wallet.get_last_block_id()) { Ok(last_block_id) => { unsafe { *out_block_height = last_block_id; diff --git a/lez/wallet-ffi/src/wallet.rs b/lez/wallet-ffi/src/wallet.rs index b19aaae5..b2846912 100644 --- a/lez/wallet-ffi/src/wallet.rs +++ b/lez/wallet-ffi/src/wallet.rs @@ -86,6 +86,7 @@ fn c_str_to_path(ptr: *const c_char, name: &str) -> Result Result FfiCreateWalletOutput { let Ok(config_path) = c_str_to_path(config_path, "config_path") else { @@ -112,7 +114,17 @@ pub unsafe extern "C" fn wallet_ffi_create_new( return FfiCreateWalletOutput::default(); }; - match WalletCore::new_init_storage(config_path, storage_path, None, &password) { + let Ok(statistics_path) = c_str_to_path(statistics_path, "statistics_path") else { + return FfiCreateWalletOutput::default(); + }; + + match block_on(WalletCore::new_init_storage( + config_path, + storage_path, + statistics_path, + None, + &password, + )) { Ok((core, mnemonic)) => { let wrapper = Box::new(WalletWrapper { core: Mutex::new(core), @@ -143,7 +155,8 @@ pub unsafe extern "C" fn wallet_ffi_create_new( /// /// # Parameters /// - `config_path`: Path to the wallet configuration file (JSON) -/// - `storage_path`: Path where wallet data is stored +/// - `storage_path`: Path to the wallet storage (JSON) +/// - `statistics_path`: Path to the wallet statistics file (JSON) /// /// # Returns /// - Opaque wallet handle on success @@ -155,6 +168,7 @@ pub unsafe extern "C" fn wallet_ffi_create_new( pub unsafe extern "C" fn wallet_ffi_open( config_path: *const c_char, storage_path: *const c_char, + statistics_path: *const c_char, ) -> *mut WalletHandle { let Ok(config_path) = c_str_to_path(config_path, "config_path") else { return ptr::null_mut(); @@ -164,7 +178,16 @@ pub unsafe extern "C" fn wallet_ffi_open( return ptr::null_mut(); }; - match WalletCore::new_update_chain(config_path, storage_path, None) { + let Ok(statistics_path) = c_str_to_path(statistics_path, "statistics_path") else { + return ptr::null_mut(); + }; + + match block_on(WalletCore::new_update_chain( + config_path, + storage_path, + statistics_path, + None, + )) { Ok(core) => { let wrapper = Box::new(WalletWrapper { core: Mutex::new(core), @@ -216,7 +239,7 @@ pub unsafe extern "C" fn wallet_ffi_save(handle: *mut WalletHandle) -> WalletFfi Err(e) => return e, }; - let wallet = match wrapper.core.lock() { + let mut wallet = match wrapper.core.lock() { Ok(w) => w, Err(e) => { print_error(format!("Failed to lock wallet: {e}")); @@ -224,7 +247,10 @@ pub unsafe extern "C" fn wallet_ffi_save(handle: *mut WalletHandle) -> WalletFfi } }; - match wallet.store_persistent_data() { + match wallet + .store_persistent_data() + .and_then(|()| block_on(wallet.client_rotation())) + { Ok(()) => WalletFfiError::Success, Err(e) => { print_error(format!("Failed to save wallet: {e}")); @@ -334,7 +360,7 @@ pub unsafe extern "C" fn wallet_ffi_get_sequencer_addr(handle: *mut WalletHandle } }; - let addr = wallet.config().sequencer_addr.clone().to_string(); + let addr = wallet.leader_url().to_string(); match std::ffi::CString::new(addr) { Ok(s) => s.into_raw(), diff --git a/lez/wallet-ffi/wallet_ffi.h b/lez/wallet-ffi/wallet_ffi.h index 4104fb61..bbd7da1f 100644 --- a/lez/wallet-ffi/wallet_ffi.h +++ b/lez/wallet-ffi/wallet_ffi.h @@ -1612,6 +1612,7 @@ enum WalletFfiError wallet_ffi_vault_claim_private(struct WalletHandle *handle, * # Parameters * - `config_path`: Path to the wallet configuration file (JSON) * - `storage_path`: Path where wallet data will be stored + * - `statistics_path`: Path to the wallet statistics file (JSON) * - `password`: Password for encrypting the wallet seed * * # Returns @@ -1623,6 +1624,7 @@ enum WalletFfiError wallet_ffi_vault_claim_private(struct WalletHandle *handle, */ struct FfiCreateWalletOutput wallet_ffi_create_new(const char *config_path, const char *storage_path, + const char *statistics_path, const char *password); /** @@ -1632,7 +1634,8 @@ struct FfiCreateWalletOutput wallet_ffi_create_new(const char *config_path, * * # Parameters * - `config_path`: Path to the wallet configuration file (JSON) - * - `storage_path`: Path where wallet data is stored + * - `storage_path`: Path to the wallet storage (JSON) + * - `statistics_path`: Path to the wallet statistics file (JSON) * * # Returns * - Opaque wallet handle on success @@ -1641,7 +1644,9 @@ struct FfiCreateWalletOutput wallet_ffi_create_new(const char *config_path, * # Safety * All string parameters must be valid null-terminated UTF-8 strings. */ -struct WalletHandle *wallet_ffi_open(const char *config_path, const char *storage_path); +struct WalletHandle *wallet_ffi_open(const char *config_path, + const char *storage_path, + const char *statistics_path); /** * Destroy a wallet handle and free its resources. diff --git a/lez/wallet/configs/debug/wallet_config.json b/lez/wallet/configs/debug/wallet_config.json index 926ee298..c8bd872b 100644 --- a/lez/wallet/configs/debug/wallet_config.json +++ b/lez/wallet/configs/debug/wallet_config.json @@ -1,7 +1,10 @@ { - "sequencer_addr": "http://127.0.0.1:3040", + "sequencers": [{ + "sequencer_addr": "http://127.0.0.1:3040" + }], "seq_poll_timeout": "30s", "seq_tx_poll_max_blocks": 15, "seq_poll_max_retries": 10, - "seq_block_poll_max_amount": 100 + "seq_block_poll_max_amount": 100, + "calibration_limit": 100 } \ No newline at end of file diff --git a/lez/wallet/src/cli/account.rs b/lez/wallet/src/cli/account.rs index 275e13fc..c700d2e9 100644 --- a/lez/wallet/src/cli/account.rs +++ b/lez/wallet/src/cli/account.rs @@ -384,38 +384,61 @@ impl AccountSubcommand { Ok(SubcommandReturnValue::Empty) } - async fn handle_list(long: bool, wallet_core: &WalletCore) -> Result { - let key_chain = &wallet_core.storage.key_chain(); - let storage = wallet_core.storage(); + fn format_with_label( + wallet_core: &WalletCore, + id: AccountIdWithPrivacy, + chain_index: Option<&ChainIndex>, + ) -> String { + let id_str = chain_index.map_or_else(|| id.to_string(), |cci| format!("{cci} {id}")); - let format_with_label = |id: AccountIdWithPrivacy, chain_index: Option<&ChainIndex>| { - let id_str = chain_index.map_or_else(|| id.to_string(), |cci| format!("{cci} {id}")); - - let labels = storage.labels_for_account(id).format(", ").to_string(); - if labels.is_empty() { - id_str - } else { - format!("{id_str} [{labels}]") - } - }; - - if !long { - let accounts = key_chain - .account_ids() - .map(|(id, idx)| format_with_label(id, idx)) - .format("\n"); - println!("{accounts}"); - - return Ok(SubcommandReturnValue::Empty); + let labels = wallet_core + .storage() + .labels_for_account(id) + .format(", ") + .to_string(); + if labels.is_empty() { + id_str + } else { + format!("{id_str} [{labels}]") } + } + + async fn handle_list(long: bool, wallet_core: &WalletCore) -> Result { + let (public_account_ids, private_account_ids) = { + let key_chain = &wallet_core.storage.key_chain(); + + if !long { + let accounts = key_chain + .account_ids() + .map(|(id, idx)| Self::format_with_label(wallet_core, id, idx)) + .format("\n"); + println!("{accounts}"); + + return Ok(SubcommandReturnValue::Empty); + } + + let public_account_ids: Vec<_> = key_chain + .public_account_ids() + .map(|(id, chain_index)| (id, chain_index.cloned())) + .collect(); + let private_account_ids: Vec<_> = key_chain + .private_account_ids() + .map(|(id, chain_index)| (id, chain_index.cloned())) + .collect(); + + (public_account_ids, private_account_ids) + }; // Detailed listing with --long flag - // Public key tree accounts - for (id, chain_index) in key_chain.public_account_ids() { + for (id, chain_index) in public_account_ids { println!( "{}", - format_with_label(AccountIdWithPrivacy::Public(id), chain_index) + Self::format_with_label( + wallet_core, + AccountIdWithPrivacy::Public(id), + chain_index.as_ref() + ) ); match wallet_core.get_account_public(id).await { Ok(account) if account != Account::default() => { @@ -429,10 +452,14 @@ impl AccountSubcommand { } // Private key tree accounts - for (id, chain_index) in key_chain.private_account_ids() { + for (id, chain_index) in private_account_ids { println!( "{}", - format_with_label(AccountIdWithPrivacy::Private(id), chain_index) + Self::format_with_label( + wallet_core, + AccountIdWithPrivacy::Private(id), + chain_index.as_ref() + ) ); match wallet_core.get_account_private(id) { Some(account) if account != Account::default() => { diff --git a/lez/wallet/src/cli/chain.rs b/lez/wallet/src/cli/chain.rs index dfb22eba..b14a5aa3 100644 --- a/lez/wallet/src/cli/chain.rs +++ b/lez/wallet/src/cli/chain.rs @@ -1,7 +1,6 @@ use anyhow::Result; use clap::Subcommand; use common::HashType; -use sequencer_service_rpc::RpcClient as _; use crate::{ WalletCore, @@ -33,17 +32,17 @@ impl WalletSubcommand for ChainSubcommand { ) -> Result { match self { Self::CurrentBlockId => { - let latest_block_id = wallet_core.sequencer_client.get_last_block_id().await?; + let latest_block_id = wallet_core.get_last_block_id().await?; println!("Last block id is {latest_block_id}"); } Self::Block { id } => { - let block = wallet_core.sequencer_client.get_block(id).await?; + let block = wallet_core.get_block(id).await?; println!("Last block id is {block:#?}"); } Self::Transaction { hash } => { - let tx = wallet_core.sequencer_client.get_transaction(hash).await?; + let tx = wallet_core.get_transaction(hash).await?; println!("Transaction is {tx:#?}"); } diff --git a/lez/wallet/src/cli/config.rs b/lez/wallet/src/cli/config.rs index a78bd097..8c8c4b5c 100644 --- a/lez/wallet/src/cli/config.rs +++ b/lez/wallet/src/cli/config.rs @@ -36,8 +36,8 @@ impl ConfigSubcommand { println!("{config_str}"); } else if let Some(key) = key { match key.as_str() { - "sequencer_addr" => { - println!("{}", config.sequencer_addr); + "sequencers" => { + println!("{:?}", config.sequencers); } "seq_poll_timeout" => { println!("{:?}", config.seq_poll_timeout); @@ -51,13 +51,6 @@ impl ConfigSubcommand { "seq_block_poll_max_amount" => { println!("{}", config.seq_block_poll_max_amount); } - "basic_auth" => { - if let Some(basic_auth) = &config.basic_auth { - println!("{basic_auth}"); - } else { - println!("Not set"); - } - } _ => { println!("Unknown field"); } @@ -76,9 +69,6 @@ impl ConfigSubcommand { ) -> Result { let mut config = wallet_core.config().clone(); match key.as_str() { - "sequencer_addr" => { - config.sequencer_addr = value.parse()?; - } "seq_poll_timeout" => { config.seq_poll_timeout = humantime::parse_duration(&value) .map_err(|e| anyhow::anyhow!("Invalid duration: {e}"))?; @@ -92,9 +82,6 @@ impl ConfigSubcommand { "seq_block_poll_max_amount" => { config.seq_block_poll_max_amount = value.parse()?; } - "basic_auth" => { - config.basic_auth = Some(value.parse()?); - } "initial_accounts" => { anyhow::bail!("Setting this field from wallet is not supported"); } diff --git a/lez/wallet/src/cli/mod.rs b/lez/wallet/src/cli/mod.rs index 44b578a1..d436de52 100644 --- a/lez/wallet/src/cli/mod.rs +++ b/lez/wallet/src/cli/mod.rs @@ -3,10 +3,9 @@ use std::{io::Write as _, path::PathBuf, str::FromStr}; use anyhow::{Context as _, Result}; use bip39::Mnemonic; use clap::{Parser, Subcommand}; -use common::{HashType, transaction::LeeTransaction}; +use common::HashType; use derive_more::Display; use futures::TryFutureExt as _; -use lee::ProgramDeploymentTransaction; use lee_core::BlockId; use sequencer_service_rpc::RpcClient as _; @@ -108,9 +107,6 @@ pub struct Args { /// Continious run flag. #[arg(short, long)] pub continuous_run: bool, - /// Basic authentication in the format `user` or `user:password`. - #[arg(long)] - pub auth: Option, /// Wallet command. #[command(subcommand)] pub command: Option, @@ -220,7 +216,6 @@ pub async fn execute_subcommand( } Command::CheckHealth => { let remote_program_ids = wallet_core - .sequencer_client .get_program_ids() .await .expect("Error fetching program ids"); @@ -285,21 +280,25 @@ pub async fn execute_subcommand( "Failed to read program binary at {}", binary_filepath.display() ))?; - let message = lee::program_deployment_transaction::Message::new(bytecode); - let transaction = ProgramDeploymentTransaction::new(message); - let tx_hash = wallet_core - .sequencer_client - .send_transaction(LeeTransaction::ProgramDeployment(transaction)) + let response = wallet_core + .send_program_deployment_transaction(bytecode) .await .context("Transaction submission error")?; wallet_core - .poll_and_finalize_public_transaction(tx_hash) + .poll_and_finalize_public_transaction(response) .await .context("Transaction finalization error")? } }; + // Kind of a sledgehammer solution, but it is not clear if there is the case to not store + // statistics + wallet_core + .client_rotation() + .await + .context("Failed to rotate wallet")?; + Ok(subcommand_ret) } @@ -388,14 +387,13 @@ pub async fn execute_keys_restoration(wallet_core: &mut WalletCore, depth: u32) wallet_core.sync_to_latest_block().await?; + let leader_client = wallet_core.leader_owned(); + wallet_core .storage .key_chain_mut() .cleanup_trees_remove_uninit_layered(depth, |account_id| { - wallet_core - .sequencer_client - .get_account(account_id) - .map_err(Into::into) + leader_client.get_account(account_id).map_err(Into::into) }) .await?; diff --git a/lez/wallet/src/config.rs b/lez/wallet/src/config.rs index 77c2729f..ede71073 100644 --- a/lez/wallet/src/config.rs +++ b/lez/wallet/src/config.rs @@ -7,6 +7,17 @@ use log::warn; use serde::{Deserialize, Serialize}; use url::Url; +const DEFAULT_CALLIBRATION_LIMIT: usize = 100; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SequencerConnectionData { + /// Connection data of all known sequencers. + pub sequencer_addr: Url, + /// Basic authentication credentials. + #[serde(skip_serializing_if = "Option::is_none")] + pub basic_auth: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GasConfig { /// Gas spent per deploying one byte of data. @@ -28,8 +39,8 @@ pub struct GasConfig { #[optfield::optfield(pub WalletConfigOverrides, rewrap, attrs = (derive(Debug, Default, Clone)))] #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WalletConfig { - /// Sequencer URL. - pub sequencer_addr: Url, + /// Connection data of all known sequencers. + pub sequencers: Vec, /// Sequencer polling duration for new blocks. #[serde(with = "humantime_serde")] pub seq_poll_timeout: Duration, @@ -39,20 +50,23 @@ pub struct WalletConfig { pub seq_poll_max_retries: u64, /// Max amount of blocks to poll in one request. pub seq_block_poll_max_amount: u64, - /// Basic authentication credentials - #[serde(skip_serializing_if = "Option::is_none")] - pub basic_auth: Option, + /// Limit number of sequencer polls during calibration, should not be zero + #[serde(default = "default_calibration_limit")] + pub calibration_limit: usize, } impl Default for WalletConfig { fn default() -> Self { Self { - sequencer_addr: "http://127.0.0.1:3040".parse().unwrap(), + sequencers: vec![SequencerConnectionData { + sequencer_addr: "http://127.0.0.1:3040".parse().unwrap(), + basic_auth: None, + }], seq_poll_timeout: Duration::from_secs(12), seq_tx_poll_max_blocks: 5, seq_poll_max_retries: 5, seq_block_poll_max_amount: 100, - basic_auth: None, + calibration_limit: DEFAULT_CALLIBRATION_LIMIT, } } } @@ -97,26 +111,26 @@ impl WalletConfig { pub fn apply_overrides(&mut self, overrides: WalletConfigOverrides) { let Self { - sequencer_addr, + sequencers, seq_poll_timeout, seq_tx_poll_max_blocks, seq_poll_max_retries, seq_block_poll_max_amount, - basic_auth, + calibration_limit, } = self; let WalletConfigOverrides { - sequencer_addr: o_sequencer_addr, + sequencers: o_sequencers, seq_poll_timeout: o_seq_poll_timeout, seq_tx_poll_max_blocks: o_seq_tx_poll_max_blocks, seq_poll_max_retries: o_seq_poll_max_retries, seq_block_poll_max_amount: o_seq_block_poll_max_amount, - basic_auth: o_basic_auth, + calibration_limit: o_calibration_limit, } = overrides; - if let Some(v) = o_sequencer_addr { - warn!("Overriding wallet config 'sequencer_addr' to {v}"); - *sequencer_addr = v; + if let Some(v) = o_sequencers { + warn!("Overriding wallet config 'sequencers' to {v:?}"); + *sequencers = v; } if let Some(v) = o_seq_poll_timeout { warn!("Overriding wallet config 'seq_poll_timeout' to {v:?}"); @@ -134,9 +148,13 @@ impl WalletConfig { warn!("Overriding wallet config 'seq_block_poll_max_amount' to {v}"); *seq_block_poll_max_amount = v; } - if let Some(v) = o_basic_auth { - warn!("Overriding wallet config 'basic_auth' to {v:#?}"); - *basic_auth = v; + if let Some(v) = o_calibration_limit { + warn!("Overriding wallet config 'calibration_limit' to {v}"); + *calibration_limit = v; } } } + +const fn default_calibration_limit() -> usize { + DEFAULT_CALLIBRATION_LIMIT +} diff --git a/lez/wallet/src/helperfunctions.rs b/lez/wallet/src/helperfunctions.rs index 6fe78681..0af99c1f 100644 --- a/lez/wallet/src/helperfunctions.rs +++ b/lez/wallet/src/helperfunctions.rs @@ -66,6 +66,15 @@ pub fn fetch_persistent_storage_path() -> Result { Ok(accs_path) } +/// Fetch path to statistics storage from default home. +/// +/// File must be created through setup beforehand. +pub fn fetch_statistics_path() -> Result { + let home = get_home()?; + let statistics_path = home.join("statistics.json"); + Ok(statistics_path) +} + #[expect(dead_code, reason = "Maybe used later")] pub(crate) fn produce_random_nonces(size: usize) -> Vec { let mut result = vec![[0; 16]; size]; diff --git a/lez/wallet/src/lib.rs b/lez/wallet/src/lib.rs index fdd208af..4984f784 100644 --- a/lez/wallet/src/lib.rs +++ b/lez/wallet/src/lib.rs @@ -7,33 +7,38 @@ reason = "Most of the shadows come from args parsing which is ok" )] -use std::{collections::HashSet, path::PathBuf}; +use std::{ + collections::{BTreeMap, HashMap, HashSet}, + path::PathBuf, +}; pub use account_manager::AccountIdentity; use anyhow::{Context as _, Result}; use bip39::Mnemonic; -use common::{HashType, transaction::LeeTransaction}; +use common::{HashType, block::Block, transaction::LeeTransaction}; use config::WalletConfig; use key_protocol::key_management::key_tree::chain_index::ChainIndex; use lee::{ - Account, AccountId, PrivacyPreservingTransaction, ProgramId, + Account, AccountId, PrivacyPreservingTransaction, ProgramDeploymentTransaction, ProgramId, privacy_preserving_transaction::{ circuit::ProgramWithDependencies, message::{EncryptedAccountData, Message}, }, }; use lee_core::{ - Commitment, CommitmentSetDigest, MembershipProof, SharedSecretKey, account::Nonce, + BlockId, Commitment, CommitmentSetDigest, MembershipProof, SharedSecretKey, account::Nonce, program::InstructionData, }; use log::info; -use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use sequencer_service_rpc::{RpcClient as _, SequencerClient}; use storage::Storage; use tokio::io::AsyncWriteExt as _; +use url::Url; use crate::{ account::{AccountIdWithPrivacy, Label}, config::WalletConfigOverrides, + multi_client::{MultiSequencerClient, Statistics, extract_statistics_from_path}, poller::TxPoller, storage::key_chain::{NullifierIndex, SharedAccountEntry}, }; @@ -43,6 +48,7 @@ mod account_manager; pub mod cli; pub mod config; pub mod helperfunctions; +pub mod multi_client; pub mod poller; pub mod program_facades; pub mod signing; @@ -84,7 +90,6 @@ pub enum ExecutionFailureKind { KeycardError(#[from] pyo3::PyErr), } -#[expect(clippy::partial_pub_fields, reason = "TODO: make all fields private")] pub struct WalletCore { config_path: PathBuf, config_overrides: Option, @@ -93,45 +98,65 @@ pub struct WalletCore { storage: Storage, storage_path: PathBuf, - poller: TxPoller, - pub sequencer_client: SequencerClient, + statistics_path: PathBuf, + statistics: HashMap, + + multi_sequencer_client: MultiSequencerClient, } impl WalletCore { /// Construct wallet using [`HOME_DIR_ENV_VAR`] env var for paths or user home dir if not set. - pub fn from_env() -> Result { + pub async fn from_env() -> Result { let config_path = helperfunctions::fetch_config_path()?; let storage_path = helperfunctions::fetch_persistent_storage_path()?; + let statistics_path = helperfunctions::fetch_statistics_path()?; - Self::new_update_chain(config_path, storage_path, None) + Self::new_update_chain(config_path, storage_path, statistics_path, None).await } - pub fn new_update_chain( + pub async fn new_update_chain( config_path: PathBuf, storage_path: PathBuf, + statistics_path: PathBuf, config_overrides: Option, ) -> Result { let storage = Storage::from_path(&storage_path) .with_context(|| format!("Failed to load storage from {}", storage_path.display()))?; - Self::new(config_path, storage_path, config_overrides, storage) + Self::new( + config_path, + storage_path, + statistics_path, + config_overrides, + storage, + ) + .await } - pub fn new_init_storage( + pub async fn new_init_storage( config_path: PathBuf, storage_path: PathBuf, + statistics_path: PathBuf, config_overrides: Option, password: &str, ) -> Result<(Self, Mnemonic)> { let (storage, mnemonic) = Storage::new(password).context("Failed to create storage")?; - let wallet = Self::new(config_path, storage_path, config_overrides, storage)?; + let wallet = Self::new( + config_path, + storage_path, + statistics_path, + config_overrides, + storage, + ) + .await?; Ok((wallet, mnemonic)) } - fn new( + async fn new( config_path: PathBuf, storage_path: PathBuf, + statistics_path: PathBuf, config_overrides: Option, storage: Storage, ) -> Result { @@ -146,34 +171,24 @@ impl WalletCore { config.apply_overrides(config_overrides); } - let sequencer_client = { - let mut builder = SequencerClientBuilder::default(); - if let Some(basic_auth) = &config.basic_auth { - builder = builder.set_headers( - std::iter::once(( - "Authorization".parse().expect("Header name is valid"), - format!("Basic {basic_auth}") - .parse() - .context("Invalid basic auth format")?, - )) - .collect(), - ); - } - builder - .build(config.sequencer_addr.clone()) - .context("Failed to create sequencer client")? - }; + let mut statistics = extract_statistics_from_path(&statistics_path)?; - let tx_poller = TxPoller::new(&config, sequencer_client.clone()); + let multi_sequencer_client = MultiSequencerClient::new( + &config.sequencers, + &mut statistics, + config.calibration_limit, + ) + .await?; Ok(Self { config_path, config_overrides, config, - storage_path, storage, - poller: tx_poller, - sequencer_client, + storage_path, + statistics_path, + statistics, + multi_sequencer_client, }) } @@ -187,6 +202,21 @@ impl WalletCore { self.config = config; } + #[must_use] + pub fn optimal_poller(&self) -> TxPoller { + TxPoller::new(self.config(), self.leader_owned()) + } + + #[must_use] + pub fn leader_owned(&self) -> SequencerClient { + self.multi_sequencer_client.leader().clone() + } + + #[must_use] + pub fn leader_url(&self) -> Url { + self.multi_sequencer_client.leader_url().clone() + } + /// Get storage. #[must_use] pub const fn storage(&self) -> &Storage { @@ -223,6 +253,39 @@ impl WalletCore { Ok(()) } + /// Rotates multi-client and stores metrics. + pub async fn client_rotation(&mut self) -> Result<()> { + let leader_statistic = self + .statistics + .get_mut(self.multi_sequencer_client.leader_url()) + .ok_or_else(|| anyhow::anyhow!("Leader URL is not present in statistics"))?; + + self.multi_sequencer_client + .update_statistics(leader_statistic) + .await; + + self.multi_sequencer_client + .rotate( + &self.config.sequencers, + &mut self.statistics, + self.config.calibration_limit, + ) + .await?; + + let statistics_serialized = serde_json::to_vec_pretty(&self.statistics)?; + let mut file = tokio::fs::File::create(&self.statistics_path) + .await + .context("Failed to create file")?; + file.write_all(&statistics_serialized) + .await + .context("Failed to write to file")?; + file.sync_all().await.context("Failed to sync file")?; + + println!("Stored statistics at {}", self.statistics_path.display()); + + Ok(()) + } + /// Store persistent data at home. pub async fn store_config_changes(&self) -> Result<()> { let config = serde_json::to_vec_pretty(&self.config)?; @@ -414,14 +477,27 @@ impl WalletCore { }) } + #[must_use] + pub fn get_statistics(&self, sequencer_url: &Url) -> Option<&Statistics> { + self.statistics.get(sequencer_url) + } + /// Get account balance. pub async fn get_account_balance(&self, acc: AccountId) -> Result { - Ok(self.sequencer_client.get_account_balance(acc).await?) + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_account_balance(acc).await) + .await?) } /// Get accounts nonces. - pub async fn get_accounts_nonces(&self, accs: Vec) -> Result> { - Ok(self.sequencer_client.get_accounts_nonces(accs).await?) + pub async fn get_accounts_nonces(&self, accs: &[AccountId]) -> Result> { + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| { + client.get_accounts_nonces(accs.to_vec()).await + }) + .await?) } pub async fn get_account(&self, account_id: AccountIdWithPrivacy) -> Result { @@ -437,9 +513,36 @@ impl WalletCore { } } + pub async fn get_last_block_id(&self) -> Result { + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_last_block_id().await) + .await?) + } + + pub async fn get_block(&self, block_id: u64) -> Result> { + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_block(block_id).await) + .await?) + } + + pub async fn get_transaction( + &self, + hash: HashType, + ) -> Result> { + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_transaction(hash).await) + .await?) + } + /// Get public account. pub async fn get_account_public(&self, account_id: AccountId) -> Result { - Ok(self.sequencer_client.get_account(account_id).await?) + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_account(account_id).await) + .await?) } #[must_use] @@ -474,14 +577,31 @@ impl WalletCore { Some(Commitment::new(&account_id, account)) } + pub async fn get_program_ids(&self) -> Result> { + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| client.get_program_ids().await) + .await?) + } + + /// Poll transactions. + pub async fn poll_native_token_transfer( + &self, + hash: HashType, + ) -> Result<(LeeTransaction, BlockId)> { + self.optimal_poller().poll_tx(hash).await + } + pub async fn get_proofs_and_root( &self, commitments: Vec, ) -> Result<(Vec>, CommitmentSetDigest)> { - self.sequencer_client - .get_proofs_and_root(commitments) - .await - .map_err(Into::into) + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| { + client.get_proofs_and_root(commitments.clone()).await + }) + .await?) } pub fn decode_insert_privacy_preserving_transaction_results( @@ -499,7 +619,7 @@ impl WalletCore { match acc_decode_data { AccDecodeData::Decode(secret, acc_account_id) => { let acc_ead = tx.message.encrypted_private_post_states[output_index].clone(); - let acc_comm = tx.message.new_commitments[output_index].clone(); + let acc_comm = tx.message.new_commitments[output_index]; let (kind, res_acc) = lee_core::EncryptionScheme::decrypt( &acc_ead.ciphertext, @@ -530,7 +650,7 @@ impl WalletCore { tx_hash: HashType, ) -> Result { println!("Transaction hash is {tx_hash}"); - let (tx, block_id) = self.poller.poll_tx(tx_hash).await?; + let (tx, block_id) = self.optimal_poller().poll_tx(tx_hash).await?; println!("Transaction is included in block {block_id}"); println!("Transaction data is {tx:?}"); self.store_persistent_data()?; @@ -544,7 +664,7 @@ impl WalletCore { acc_decode_data: &[AccDecodeData], ) -> Result { println!("Transaction hash is {tx_hash}"); - let (tx, block_id) = self.poller.poll_tx(tx_hash).await?; + let (tx, block_id) = self.optimal_poller().poll_tx(tx_hash).await?; println!("Transaction is included in block {block_id}"); println!("Transaction data is {tx:?}"); if let common::transaction::LeeTransaction::PrivacyPreserving(private_tx) = tx { @@ -620,12 +740,16 @@ impl WalletCore { .map(|keys| keys.ssk) .collect(); - Ok(( - self.sequencer_client - .send_transaction(LeeTransaction::PrivacyPreserving(tx)) - .await?, - shared_secrets, - )) + let call_res = self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| { + client + .send_transaction(LeeTransaction::PrivacyPreserving(tx.clone())) + .await + }) + .await; + + Ok((call_res?, shared_secrets)) } pub async fn send_pub_tx( @@ -685,13 +809,31 @@ impl WalletCore { let tx = lee::public_transaction::PublicTransaction::new(message, witness_set); Ok(self - .sequencer_client - .send_transaction(LeeTransaction::Public(tx)) + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| { + client + .send_transaction(LeeTransaction::Public(tx.clone())) + .await + }) + .await?) + } + + pub async fn send_program_deployment_transaction(&self, bytecode: Vec) -> Result { + let message = lee::program_deployment_transaction::Message::new(bytecode); + let transaction = ProgramDeploymentTransaction::new(message); + + Ok(self + .multi_sequencer_client + .metered_call(async |client: &SequencerClient| { + client + .send_transaction(LeeTransaction::ProgramDeployment(transaction.clone())) + .await + }) .await?) } pub async fn sync_to_latest_block(&mut self) -> Result { - let latest_block_id = self.sequencer_client.get_last_block_id().await?; + let latest_block_id = self.get_last_block_id().await?; println!("Latest block is {latest_block_id}"); self.sync_to_block(latest_block_id).await?; Ok(latest_block_id) @@ -713,7 +855,7 @@ impl WalletCore { println!("Syncing to block {block_id}. Blocks to sync: {num_of_blocks}"); - let poller = self.poller.clone(); + let poller = self.optimal_poller(); let mut blocks = std::pin::pin!(poller.poll_block_range(last_synced_block.saturating_add(1)..=block_id)); diff --git a/lez/wallet/src/main.rs b/lez/wallet/src/main.rs index 838ee856..c9cf0820 100644 --- a/lez/wallet/src/main.rs +++ b/lez/wallet/src/main.rs @@ -8,8 +8,7 @@ use clap::{CommandFactory as _, Parser as _}; use wallet::{ WalletCore, cli::{Args, execute_continuous_run, execute_subcommand, read_password_from_stdin}, - config::WalletConfigOverrides, - helperfunctions::{fetch_config_path, fetch_persistent_storage_path}, + helperfunctions::{fetch_config_path, fetch_persistent_storage_path, fetch_statistics_path}, }; // TODO #169: We have sample configs for sequencer, but not for wallet @@ -21,7 +20,6 @@ use wallet::{ async fn main() -> Result<()> { let Args { continuous_run, - auth, command, } = Args::parse(); @@ -30,16 +28,11 @@ async fn main() -> Result<()> { let config_path = fetch_config_path().context("Could not fetch config path")?; let storage_path = fetch_persistent_storage_path().context("Could not fetch persistent storage path")?; - - // Override basic auth if provided via CLI - let config_overrides = WalletConfigOverrides { - basic_auth: auth.map(|auth| auth.parse()).transpose()?.map(Some), - ..Default::default() - }; + let statistics_path = fetch_statistics_path().context("Could not fetch statistics path")?; if let Some(command) = command { let mut wallet = if storage_path.exists() { - WalletCore::new_update_chain(config_path, storage_path, Some(config_overrides))? + WalletCore::new_update_chain(config_path, storage_path, statistics_path, None).await? } else { // TODO: Maybe move to `WalletCore::from_env()` or similar? @@ -49,9 +42,11 @@ async fn main() -> Result<()> { let (wallet, mnemonic) = WalletCore::new_init_storage( config_path, storage_path, - Some(config_overrides), + statistics_path, + None, &password, - )?; + ) + .await?; println!(); println!("IMPORTANT: Write down your recovery phrase and store it securely."); @@ -68,7 +63,7 @@ async fn main() -> Result<()> { Ok(()) } else if continuous_run { let mut wallet = - WalletCore::new_update_chain(config_path, storage_path, Some(config_overrides))?; + WalletCore::new_update_chain(config_path, storage_path, statistics_path, None).await?; execute_continuous_run(&mut wallet).await } else { let help = Args::command().render_long_help(); diff --git a/lez/wallet/src/multi_client.rs b/lez/wallet/src/multi_client.rs new file mode 100644 index 00000000..499cf7a1 --- /dev/null +++ b/lez/wallet/src/multi_client.rs @@ -0,0 +1,1038 @@ +#![expect( + clippy::float_arithmetic, + reason = "One should expect floating point arithmetic in statistic calculations" +)] +#![expect( + clippy::cast_precision_loss, + reason = "Operated numbers is not big enough to have precision loss" +)] + +use std::{collections::HashMap, path::Path, sync::Arc}; + +use anyhow::{Context as _, Result}; +use lee_core::BlockId; +use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuilder}; +use serde::{Deserialize, Serialize}; +use tokio::sync::RwLock; +use url::Url; + +use crate::config::SequencerConnectionData; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Statistics { + pub latency_avg: f32, + pub latency_var: f32, + pub sample_size: usize, + pub latest_block_id: BlockId, + pub errors: u64, +} + +#[derive(Debug, Clone)] +pub struct StatisticsUpdate { + pub latency: f32, + pub new_latest_block_id: Option, + pub is_failed: bool, +} + +impl Statistics { + pub fn apply_updates(&mut self, updates: &[StatisticsUpdate]) { + let CumulativeUpdates { + failure_count, + latest_block_id, + cumulative_latency, + cumulative_latency_squares, + additional_sample_size, + } = CumulativeUpdates::from_metric_updates(updates); + + self.errors = self.errors.saturating_add(failure_count); + if let Some(latest_block_id) = latest_block_id { + self.latest_block_id = latest_block_id; + } + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let orig_size_f = self.sample_size as f32; + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let mod_size_f = additional_sample_size as f32; + + let latency_avg_old = self.latency_avg; + let latency_avg_new = + cumulative_avg(latency_avg_old, cumulative_latency, orig_size_f, mod_size_f); + + let latency_var_new = cumulative_var( + latency_avg_old, + latency_avg_new, + self.latency_var, + cumulative_latency, + cumulative_latency_squares, + orig_size_f, + mod_size_f, + ); + + self.latency_avg = latency_avg_new; + self.latency_var = latency_var_new; + self.sample_size = self.sample_size.saturating_add(additional_sample_size); + } +} + +#[derive(Clone)] +pub struct MultiSequencerClient { + // For now we store only leader, it is possible, that + // in future for important sends(for example for transactions) + // we would want to distribute call between known sequencers + leader: SequencerClient, + leader_url: Url, + /// Wrapping statistic updates in Arc to not break interfaces too much. + /// + /// It is assumed, that wallet methods can be accesed via immutable reference. + statistic_updates: Arc>>, +} + +impl MultiSequencerClient { + async fn setup( + conn_data: &[SequencerConnectionData], + statistics: &mut HashMap, + calibration_limit: usize, + ) -> Result<(Url, SequencerClient)> { + let mut client_list = HashMap::new(); + + for SequencerConnectionData { + sequencer_addr, + basic_auth, + } in conn_data + { + let sequencer_client = { + let mut builder = SequencerClientBuilder::default(); + if let Some(basic_auth) = &basic_auth { + builder = builder.set_headers( + std::iter::once(( + "Authorization".parse().expect("Header name is valid"), + format!("Basic {basic_auth}") + .parse() + .context("Invalid basic auth format")?, + )) + .collect(), + ); + } + builder + .build(sequencer_addr) + .context("Failed to create sequencer client")? + }; + + // If there is statistics for client, actualize it + if let Some(metric_mut) = statistics.get_mut(sequencer_addr) { + let metric_updates = actualize_client(&sequencer_client).await; + + log::debug!( + "Metered call for {sequencer_addr:?}, metric updates is {metric_updates:?}" + ); + + metric_mut.apply_updates(&[metric_updates]); + // Otherwise calibrate client data + } else if let Some(client_statistics) = + calibrate_client(&sequencer_client, calibration_limit).await + { + statistics.insert(sequencer_addr.clone(), client_statistics); + } else { + log::warn!("Client {sequencer_addr:?} failed all {calibration_limit} calibration attempts, it may be unhealthy. + \n Consider bumping calibration_limit or remove this client altogether"); + } + + client_list.insert(sequencer_addr.clone(), sequencer_client); + } + + // Dropping client list, for reasons why, see comment in structure definition. + + let res = choose_leader(&client_list, statistics) + .ok_or_else(|| anyhow::anyhow!("Failed to find leader"))?; + + log::info!("Chosen leader is {:?}", res.0); + + Ok(res) + } + + pub async fn new( + conn_data: &[SequencerConnectionData], + statistics: &mut HashMap, + calibration_limit: usize, + ) -> Result { + let (leader_url, leader) = Self::setup(conn_data, statistics, calibration_limit).await?; + + Ok(Self { + leader, + leader_url, + statistic_updates: Arc::new(RwLock::new(vec![])), + }) + } + + /// Re-choose leader, `statistic_updates` must be empty. + pub async fn rotate( + &mut self, + conn_data: &[SequencerConnectionData], + statistics: &mut HashMap, + calibration_limit: usize, + ) -> Result<()> { + let (leader_url, leader) = Self::setup(conn_data, statistics, calibration_limit).await?; + + log::info!("Chosen leader is {leader_url:?}"); + + self.leader = leader; + self.leader_url = leader_url; + + Ok(()) + } + + #[must_use] + pub const fn leader(&self) -> &SequencerClient { + &self.leader + } + + #[must_use] + pub const fn leader_url(&self) -> &Url { + &self.leader_url + } + + // Keeping this call abstract, in case if we need to do more than one request + pub async fn metered_call Result>( + &self, + call: I, + ) -> Result { + let (resp, statistics_update) = + tokio::join!(call(self.leader()), actualize_client(self.leader())); + + log::debug!( + "Metered call for {:?}, metric updates is {:?}", + self.leader_url, + statistics_update + ); + + { + let mut statistic_updates_guard = self.statistic_updates.write().await; + statistic_updates_guard.push(statistics_update); + } + + resp + } + + /// Update statistics of a leader, clear statist updates log. + pub async fn update_statistics(&self, leader_statistic: &mut Statistics) { + let mut statistic_updates = self.statistic_updates.write().await; + leader_statistic.apply_updates(statistic_updates.as_ref()); + statistic_updates.clear(); + } +} + +struct CumulativeUpdates { + pub failure_count: u64, + pub latest_block_id: Option, + /// Necessary for cumulative average calculation. + pub cumulative_latency: f32, + /// Necessary for cumulative variance calculation. + pub cumulative_latency_squares: f32, + pub additional_sample_size: usize, +} + +impl CumulativeUpdates { + fn from_metric_updates(metric_updates: &[StatisticsUpdate]) -> Self { + let (failure_count, latest_block_id, cumulative_latency, cumulative_latency_squares) = + metric_updates + .iter() + .fold((0_u64, None, 0_f32, 0_f32), |acc, x| { + let StatisticsUpdate { + latency, + new_latest_block_id, + is_failed, + } = x; + ( + if *is_failed { + acc.0.saturating_add(1) + } else { + acc.0 + }, + match (acc.1, new_latest_block_id) { + (None, None) => None, + (None, Some(val)) | (Some(val), None) => Some(val), + (Some(val_old), Some(val_new)) => Some(std::cmp::max(val_old, val_new)), + }, + if *is_failed { acc.2 } else { acc.2 + latency }, + if *is_failed { + acc.3 + } else { + latency.mul_add(*latency, acc.3) + }, + ) + }); + + Self { + failure_count, + latest_block_id: latest_block_id.copied(), + cumulative_latency, + cumulative_latency_squares, + additional_sample_size: metric_updates.len().saturating_sub( + usize::try_from(failure_count).expect("Sample size should fit usize"), + ), + } + } +} + +pub fn extract_statistics_from_path( + path: &Path, +) -> Result, anyhow::Error> { + match std::fs::File::open(path) { + Ok(file) => { + let reader = std::io::BufReader::new(file); + Ok(serde_json::from_reader(reader)?) + } + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + println!("Statistics not found, choosing empty"); + Ok(HashMap::new()) + } + Err(err) => Err(err).context("IO error"), + } +} + +/// Measuring `get_last_block_id` as it should be the fastest request on sequencer. +async fn measure_request_duration(client: &SequencerClient) -> (u128, Option) { + let now = tokio::time::Instant::now(); + let block_id = client.get_last_block_id().await.ok(); + ( + tokio::time::Instant::now().duration_since(now).as_millis(), + block_id, + ) +} + +pub async fn calibrate_client( + client: &SequencerClient, + calibration_limit: usize, +) -> Option { + let mut latencies = vec![]; + let mut latest_block_id = 0; + let mut errors: u64 = 0; + + // ToDo: Add some DDoS adaptation + for _ in 0..calibration_limit { + let (latency, block_id) = measure_request_duration(client).await; + + let Some(block_id) = block_id else { + errors = errors.saturating_add(1); + continue; + }; + + latest_block_id = block_id; + latencies.push(latency); + } + + // There is no point in guard numbers, exclude client if it fails all requests. + if latencies.is_empty() { + return None; + } + + // Precision loss is fine there + let sample_size = latencies.len(); + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let latency_avg = (latencies.iter().sum::() as f32) / (sample_size as f32); + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let latency_var = latencies.iter().fold(0_f32, |acc, x| { + ((*x as f32) - latency_avg).mul_add((*x as f32) - latency_avg, acc) + }) / (sample_size as f32); + + Some(Statistics { + latency_avg, + latency_var, + sample_size, + latest_block_id, + errors, + }) +} + +pub async fn actualize_client(client: &SequencerClient) -> StatisticsUpdate { + let (latency, block_id) = measure_request_duration(client).await; + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let latency = latency as f32; + + StatisticsUpdate { + latency, + new_latest_block_id: block_id, + is_failed: block_id.is_none(), + } +} + +#[must_use] +pub fn choose_leader( + client_list: &HashMap, + statistics: &HashMap, +) -> Option<(Url, SequencerClient)> { + // Sort out all unmetered clients + let mut client_vec: Vec<_> = client_list + .keys() + .filter(|item| statistics.contains_key(*item)) + .collect(); + + if client_vec.is_empty() { + return None; + } + + // Considering the nature of our requests, the latest_block_id is the dominant characteristic + let max_block_id_addr = client_vec.iter().fold(client_vec[0], |acc, x| { + let old_latest_block_id = statistics.get(acc).unwrap().latest_block_id; + let new_latest_block_id = statistics.get(*x).unwrap().latest_block_id; + if new_latest_block_id > old_latest_block_id { + *x + } else { + acc + } + }); + + let max_block_id = statistics.get(max_block_id_addr).unwrap().latest_block_id; + + // Sort out all clients running late + client_vec = client_vec + .iter() + .filter_map(|x| { + let latest_block_id = statistics.get(*x).unwrap().latest_block_id; + + (latest_block_id == max_block_id).then_some(*x) + }) + .collect(); + + // Get the clients with lesser or equal to average error ratio + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let avg_err_ratio = client_vec.iter().fold(0_f32, |acc, x| { + acc + error_ratio( + statistics.get(*x).unwrap().errors, + statistics.get(*x).unwrap().sample_size, + ) + }) / (client_vec.len() as f32); + + client_vec.sort_by(|a, b| { + let err_ratio_a = error_ratio( + statistics.get(*a).unwrap().errors, + statistics.get(*a).unwrap().sample_size, + ); + let err_ratio_b = error_ratio( + statistics.get(*b).unwrap().errors, + statistics.get(*b).unwrap().sample_size, + ); + + err_ratio_a + .partial_cmp(&err_ratio_b) + .expect("Ratios must be a valid numbers") + }); + + let client_vec = client_vec[..(client_vec + .iter() + .position(|item| { + error_ratio( + statistics.get(*item).unwrap().errors, + statistics.get(*item).unwrap().sample_size, + ) > avg_err_ratio + }) + .unwrap_or(client_vec.len()))] + .to_vec(); + + // Choose clients with least latency and variance + let min_lat_var_addr = client_vec.iter().fold(client_vec[0], |acc, x| { + let old = statistics.get(acc).unwrap(); + let (old_lat, old_var) = (old.latency_avg, old.latency_var); + let new = statistics.get(*x).unwrap(); + let (new_lat, new_var) = (new.latency_avg, new.latency_var); + + let new_std = new_var.sqrt(); + let old_std = old_var.sqrt(); + + // Client is better if its average is better and variance does not make it worse + // So basically we want this: + // [-old_std............new_lat.......old_lat...............+new_std.........+old_std] + // + // However one can argue that this: + // + // [-old_std...................old_lat........new_lat.........+new_std.......+old_std] + // + // is still better, but it is up to discussion + if (new_lat <= old_lat) && ((new_lat + new_std) < (old_lat + old_std)) { + *x + } else { + acc + } + }); + + Some(( + min_lat_var_addr.clone(), + client_list.get(min_lat_var_addr).unwrap().clone(), + )) +} + +/// Helperfunction to calculate cumulative average. +/// +/// Cumulative average calculation is the following problem: +/// +/// We want to calculate avarage of a sample of size `N + N_1` +/// where average for `N` is known. +/// +/// To do so we need: +/// - old average value +/// - sum_{`i=1}^{N_1}{n_i`} +/// - `N` +/// - `N_1` +fn cumulative_avg( + latency_avg_old: f32, + cumulative_latency: f32, + orig_size_f: f32, + mod_size_f: f32, +) -> f32 { + latency_avg_old.mul_add(orig_size_f, cumulative_latency) / (orig_size_f + mod_size_f) +} + +/// Helperfunction to calculate cumulative variance. +/// +/// Cumulative variance calculation is the following problem: +/// +/// We want to calculate variance of a sample of size `N + N_1` +/// where average for `N` is known. +/// +/// To do so we need: +/// - old average value +/// - new average value +/// - old variance +/// - sum_{`i=1}^{N_1}{n_i`} +/// - sum_{`i=1}^{N_1}{n_i^2`} +/// - `N` +/// - `N_1` +fn cumulative_var( + latency_avg_old: f32, + latency_avg_new: f32, + latency_var: f32, + cumulative_latency: f32, + cumulative_latency_squares: f32, + orig_size_f: f32, + mod_size_f: f32, +) -> f32 { + // The formula was atrocious before. + // `mul_add` function have less precision loss with drawback of being absolutely unreadable + ((2_f32 * cumulative_latency).mul_add( + -latency_avg_new, + mod_size_f.mul_add( + latency_avg_new * latency_avg_new, + latency_var.mul_add( + orig_size_f, + (latency_avg_new - latency_avg_old) + * (latency_avg_new - latency_avg_old) + * orig_size_f, + ), + ), + ) + cumulative_latency_squares) + / (orig_size_f + mod_size_f) +} + +fn error_ratio(errors: u64, size: usize) -> f32 { + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let errors_f = errors as f32; + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let size_f = size as f32; + + errors_f / (errors_f + size_f) +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use sequencer_service_rpc::{SequencerClient, SequencerClientBuilder}; + use url::Url; + + use crate::multi_client::{ + CumulativeUpdates, Statistics, StatisticsUpdate, choose_leader, cumulative_avg, + cumulative_var, + }; + + fn update_statistics( + statistics: &mut HashMap, + leader_url: &Url, + metric_updates: &[StatisticsUpdate], + ) -> Result<(), anyhow::Error> { + let leader_metric = statistics + .get_mut(leader_url) + .ok_or_else(|| anyhow::anyhow!("Leader URL is not present in statistics"))?; + + leader_metric.apply_updates(metric_updates); + + Ok(()) + } + + fn client_from_url_unchecked(url: &Url) -> SequencerClient { + let builder = SequencerClientBuilder::default(); + builder.build(url).unwrap() + } + + #[test] + fn cumulative_updates_test() { + let statistics_updates_vec = vec![ + StatisticsUpdate { + latency: 100_f32, + new_latest_block_id: Some(15), + is_failed: false, + }, + StatisticsUpdate { + latency: 115_f32, + new_latest_block_id: Some(16), + is_failed: false, + }, + StatisticsUpdate { + latency: 50_f32, + new_latest_block_id: None, + is_failed: true, + }, + ]; + + let CumulativeUpdates { + failure_count, + latest_block_id, + cumulative_latency, + cumulative_latency_squares, + additional_sample_size, + } = CumulativeUpdates::from_metric_updates(&statistics_updates_vec); + + let epsilon = 0.01_f32; + + let sum_squared_manual = 100_f32.mul_add(100_f32, 115_f32 * 115_f32); + + assert_eq!(additional_sample_size, 2); + assert_eq!(failure_count, 1); + assert_eq!(latest_block_id, Some(16)); + assert!((cumulative_latency - 215_f32).abs() < epsilon); + assert!((cumulative_latency_squares - sum_squared_manual).abs() < epsilon); + } + + #[test] + fn cumulative_avg_test() { + let mut sample = vec![100_f32; 40]; + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let old_sample_size_f = sample.len() as f32; + + let old_avg = sample.iter().sum::() / old_sample_size_f; + + let new_samples = vec![ + 101_f32, 110_f32, 112_f32, 97_f32, 78_f32, 25_f32, 75_f32, 189_f32, 120_f32, 50_f32, + ]; + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let mod_sample_size_f = new_samples.len() as f32; + + let cumulative = new_samples.iter().sum(); + + sample.extend_from_slice(&new_samples); + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let new_sample_size_f = sample.len() as f32; + + let new_avg_1 = sample.iter().sum::() / new_sample_size_f; + + let new_avg_2 = cumulative_avg(old_avg, cumulative, old_sample_size_f, mod_sample_size_f); + + let epsilon = 0.01_f32; + + assert!((new_avg_1 - new_avg_2).abs() < epsilon); + } + + #[test] + fn cumulative_var_test() { + let mut sample = vec![100_f32; 40]; + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let old_sample_size_f = sample.len() as f32; + let old_avg = sample.iter().sum::() / old_sample_size_f; + + let old_var = sample + .iter() + .fold(0_f32, |acc, x| (x - old_avg).mul_add(x - old_avg, acc)) + / old_sample_size_f; + + let new_samples = vec![ + 101_f32, 110_f32, 112_f32, 97_f32, 78_f32, 25_f32, 75_f32, 189_f32, 120_f32, 50_f32, + ]; + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let mod_sample_size_f = new_samples.len() as f32; + + let cumulative = new_samples.iter().sum(); + let cumulative_squares = new_samples + .iter() + .fold(0_f32, |acc, x| (*x).mul_add(*x, acc)); + + let new_avg = cumulative_avg(old_avg, cumulative, old_sample_size_f, mod_sample_size_f); + + sample.extend_from_slice(&new_samples); + + #[expect(clippy::as_conversions, reason = "int to float conversion is safe")] + let new_var_1 = sample + .iter() + .fold(0_f32, |acc, x| (x - new_avg).mul_add(x - new_avg, acc)) + / (sample.len() as f32); + + let new_var_2 = cumulative_var( + old_avg, + new_avg, + old_var, + cumulative, + cumulative_squares, + old_sample_size_f, + mod_sample_size_f, + ); + + let epsilon = 0.01_f32; + + assert!((new_var_1 - new_var_2).abs() < epsilon); + } + + #[test] + fn metric_updates_correctness() { + let statistics_updates_vec = vec![ + StatisticsUpdate { + latency: 100_f32, + new_latest_block_id: Some(105), + is_failed: false, + }, + StatisticsUpdate { + latency: 115_f32, + new_latest_block_id: Some(106), + is_failed: false, + }, + StatisticsUpdate { + latency: 50_f32, + new_latest_block_id: None, + is_failed: true, + }, + ]; + + let addr_leader = Url::parse("https://127.0.0.1:3040").unwrap(); + + let leader_statistics = Statistics { + latency_avg: 100_f32, + latency_var: 25_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }; + + let cumulative_latency = 100_f32 + 115_f32; + let cumulative_latency_squares = 100_f32.mul_add(100_f32, 115_f32 * 115_f32); + + let avg_manual = cumulative_avg( + leader_statistics.latency_avg, + cumulative_latency, + 10_f32, + 2_f32, + ); + let var_manual = cumulative_var( + leader_statistics.latency_avg, + avg_manual, + leader_statistics.latency_var, + cumulative_latency, + cumulative_latency_squares, + 10_f32, + 2_f32, + ); + + let mut metric_map = HashMap::new(); + metric_map.insert(addr_leader.clone(), leader_statistics); + + update_statistics(&mut metric_map, &addr_leader, &statistics_updates_vec).unwrap(); + + let Statistics { + latency_avg, + latency_var, + sample_size, + latest_block_id, + errors, + } = metric_map[&addr_leader]; + + let epsilon = 0.01_f32; + + assert_eq!(errors, 6); + assert_eq!(latest_block_id, 106); + assert_eq!(sample_size, 12); + assert!((latency_avg - avg_manual).abs() < epsilon); + assert!((latency_var - var_manual).abs() < epsilon); + } + + #[test] + fn choose_leader_latest_block() { + let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap(); + let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap(); + let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap(); + let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap(); + + let leader = client_from_url_unchecked(&addr_leader); + let client_1 = client_from_url_unchecked(&addr_1); + let client_2 = client_from_url_unchecked(&addr_2); + let client_3 = client_from_url_unchecked(&addr_3); + + let mut client_list = HashMap::new(); + + client_list.insert(addr_leader.clone(), leader); + client_list.insert(addr_1.clone(), client_1); + client_list.insert(addr_2.clone(), client_2); + client_list.insert(addr_3.clone(), client_3); + + let mut statistics = HashMap::new(); + + statistics.insert( + addr_3, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 97, + errors: 5, + }, + ); + + statistics.insert( + addr_2, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 98, + errors: 5, + }, + ); + + statistics.insert( + addr_1, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 99, + errors: 5, + }, + ); + + statistics.insert( + addr_leader.clone(), + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap(); + + assert_eq!(leader_url, addr_leader); + } + + #[test] + fn choose_leader_least_errors() { + let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap(); + let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap(); + let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap(); + let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap(); + + let leader = client_from_url_unchecked(&addr_leader); + let client_1 = client_from_url_unchecked(&addr_1); + let client_2 = client_from_url_unchecked(&addr_2); + let client_3 = client_from_url_unchecked(&addr_3); + + let mut client_list = HashMap::new(); + + client_list.insert(addr_leader.clone(), leader); + client_list.insert(addr_1.clone(), client_1); + client_list.insert(addr_2.clone(), client_2); + client_list.insert(addr_3.clone(), client_3); + + let mut statistics = HashMap::new(); + + statistics.insert( + addr_3, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_2, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 4, + }, + ); + + statistics.insert( + addr_1, + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 3, + }, + ); + + statistics.insert( + addr_leader.clone(), + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 2, + }, + ); + + let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap(); + + assert_eq!(leader_url, addr_leader); + } + + #[test] + fn choose_leader_simple_latency_check() { + let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap(); + let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap(); + let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap(); + let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap(); + + let leader = client_from_url_unchecked(&addr_leader); + let client_1 = client_from_url_unchecked(&addr_1); + let client_2 = client_from_url_unchecked(&addr_2); + let client_3 = client_from_url_unchecked(&addr_3); + + let mut client_list = HashMap::new(); + + client_list.insert(addr_leader.clone(), leader); + client_list.insert(addr_1.clone(), client_1); + client_list.insert(addr_2.clone(), client_2); + client_list.insert(addr_3.clone(), client_3); + + let mut statistics = HashMap::new(); + + statistics.insert( + addr_3, + Statistics { + latency_avg: 103_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_2, + Statistics { + latency_avg: 102_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_1, + Statistics { + latency_avg: 101_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_leader.clone(), + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap(); + + assert_eq!(leader_url, addr_leader); + } + + #[test] + fn choose_leader_latency_var_check() { + let addr_leader = Url::parse("http://127.0.0.1:3040").unwrap(); + let addr_1 = Url::parse("http://127.0.0.1:3041").unwrap(); + let addr_2 = Url::parse("http://127.0.0.1:3042").unwrap(); + let addr_3 = Url::parse("http://127.0.0.1:3043").unwrap(); + + let leader = client_from_url_unchecked(&addr_leader); + let client_1 = client_from_url_unchecked(&addr_1); + let client_2 = client_from_url_unchecked(&addr_2); + let client_3 = client_from_url_unchecked(&addr_3); + + let mut client_list = HashMap::new(); + + client_list.insert(addr_leader.clone(), leader); + client_list.insert(addr_1.clone(), client_1); + client_list.insert(addr_2.clone(), client_2); + client_list.insert(addr_3.clone(), client_3); + + let mut statistics = HashMap::new(); + + statistics.insert( + addr_3, + Statistics { + latency_avg: 100_f32, + latency_var: 13_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_2, + Statistics { + latency_avg: 100_f32, + latency_var: 12_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_1, + Statistics { + latency_avg: 100_f32, + latency_var: 11_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + statistics.insert( + addr_leader.clone(), + Statistics { + latency_avg: 100_f32, + latency_var: 10_f32, + sample_size: 10, + latest_block_id: 100, + errors: 5, + }, + ); + + let (leader_url, _) = choose_leader(&client_list, &statistics).unwrap(); + + assert_eq!(leader_url, addr_leader); + } +} diff --git a/test_fixtures/fixtures/prebuilt_sequencer_db.dump b/test_fixtures/fixtures/prebuilt_sequencer_db.dump index 78995bc6..5a95e544 100644 Binary files a/test_fixtures/fixtures/prebuilt_sequencer_db.dump and b/test_fixtures/fixtures/prebuilt_sequencer_db.dump differ diff --git a/test_fixtures/src/bin/regenerate_test_fixture.rs b/test_fixtures/src/bin/regenerate_test_fixture.rs index 80da8fdc..038d4b0c 100644 --- a/test_fixtures/src/bin/regenerate_test_fixture.rs +++ b/test_fixtures/src/bin/regenerate_test_fixture.rs @@ -11,9 +11,9 @@ use sequencer_core::block_store::SequencerStore; use test_fixtures::{ config, setup::{ - prebuilt_sequencer_db_dump_path, setup_bedrock_node, + SequencerSetup, prebuilt_sequencer_db_dump_path, setup_bedrock_node, setup_private_accounts_with_initial_supply, setup_public_accounts_with_initial_supply, - setup_sequencer, setup_wallet, + setup_wallet, }, }; use wallet::config::WalletConfigOverrides; @@ -49,15 +49,12 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { let genesis = config::genesis_from_accounts(&initial_public_accounts, &initial_private_accounts); - let (sequencer_handle, temp_sequencer_dir) = setup_sequencer( - config::SequencerPartialConfig::default(), - bedrock_addr, - genesis, - config::bedrock_channel_id(), - None, - ) - .await - .context("Failed to setup Sequencer for fixture generation")?; + let (sequencer_handle, temp_sequencer_dir) = + SequencerSetup::new(config::SequencerPartialConfig::default(), bedrock_addr) + .with_genesis(genesis) + .setup() + .await + .context("Failed to setup Sequencer for fixture generation")?; let (mut wallet, _temp_wallet_dir, _wallet_password) = setup_wallet( sequencer_handle.addr(), @@ -65,6 +62,7 @@ async fn generate_prebuilt_fixture(dest: &Path) -> Result<()> { &initial_private_accounts, WalletConfigOverrides::default(), ) + .await .context("Failed to setup wallet for fixture generation")?; setup_public_accounts_with_initial_supply(&mut wallet, &initial_public_accounts) diff --git a/test_fixtures/src/config.rs b/test_fixtures/src/config.rs index 960f349b..f69c9c67 100644 --- a/test_fixtures/src/config.rs +++ b/test_fixtures/src/config.rs @@ -8,7 +8,7 @@ use lee::{AccountId, PrivateKey, PublicKey}; use lee_core::Identifier; use sequencer_core::config::{BedrockConfig, CrossZoneConfig, GenesisAction, SequencerConfig}; use url::Url; -use wallet::config::WalletConfig; +use wallet::config::{SequencerConnectionData, WalletConfig}; pub const INITIAL_PUBLIC_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; pub const INITIAL_PRIVATE_BALANCES_FOR_WALLET: [u128; 2] = [10_000, 20_000]; @@ -80,7 +80,26 @@ pub fn sequencer_config( home: PathBuf, bedrock_addr: SocketAddr, genesis_transactions: Vec, + cross_zone: Option, +) -> Result { + sequencer_config_with_channel( + partial, + home, + bedrock_addr, + bedrock_channel_id(), + genesis_transactions, + cross_zone, + ) +} + +/// Like [`sequencer_config`] but with an explicit Bedrock `channel_id`, so tests +/// can point a sequencer at a fresh/empty channel (e.g. to model a wiped Bedrock). +pub fn sequencer_config_with_channel( + partial: SequencerPartialConfig, + home: PathBuf, + bedrock_addr: SocketAddr, channel_id: ChannelId, + genesis_transactions: Vec, cross_zone: Option, ) -> Result { let SequencerPartialConfig { @@ -194,13 +213,16 @@ pub fn genesis_from_accounts( pub fn wallet_config(sequencer_addr: SocketAddr) -> Result { Ok(WalletConfig { - sequencer_addr: addr_to_url(UrlProtocol::Http, sequencer_addr) - .context("Failed to convert sequencer addr to URL")?, + sequencers: vec![SequencerConnectionData { + sequencer_addr: addr_to_url(UrlProtocol::Http, sequencer_addr) + .context("Failed to convert sequencer addr to URL")?, + basic_auth: None, + }], seq_poll_timeout: Duration::from_secs(30), seq_tx_poll_max_blocks: 15, seq_poll_max_retries: 10, seq_block_poll_max_amount: 100, - basic_auth: None, + calibration_limit: 5, }) } diff --git a/test_fixtures/src/lib.rs b/test_fixtures/src/lib.rs index 687da077..75eaf941 100644 --- a/test_fixtures/src/lib.rs +++ b/test_fixtures/src/lib.rs @@ -24,8 +24,8 @@ use wallet::{ use crate::{ indexer_client::IndexerClient, setup::{ - setup_bedrock_node, setup_indexer, setup_private_accounts_with_initial_supply, - setup_public_accounts_with_initial_supply, setup_sequencer, setup_sequencer_from_prebuilt, + SequencerSetup, setup_bedrock_node, setup_indexer, + setup_private_accounts_with_initial_supply, setup_public_accounts_with_initial_supply, setup_wallet, sync_wallet_from_prebuilt, }, }; @@ -359,11 +359,8 @@ impl TestContextBuilder { let partial_config = sequencer_partial_config.unwrap_or_default(); - let (sequencer_handle, temp_sequencer_dir) = if use_prebuilt { - setup_sequencer_from_prebuilt(partial_config, bedrock_addr) - .await - .context("Failed to setup Sequencer from prebuilt database")? - } else { + let mut sequencer_setup = SequencerSetup::new(partial_config, bedrock_addr); + if !use_prebuilt { // Wallet genesis must always be present so that // setup_public/private_accounts_with_initial_supply can claim from the vault PDAs. // When a test supplies custom genesis, merge rather than replace. @@ -376,16 +373,12 @@ impl TestContextBuilder { } None => wallet_genesis, }; - setup_sequencer( - partial_config, - bedrock_addr, - genesis, - config::bedrock_channel_id(), - None, - ) + sequencer_setup = sequencer_setup.with_genesis(genesis); + } + let (sequencer_handle, temp_sequencer_dir) = sequencer_setup + .setup() .await - .context("Failed to setup Sequencer")? - }; + .context("Failed to setup Sequencer")?; let (mut wallet, temp_wallet_dir, wallet_password) = setup_wallet( sequencer_handle.addr(), @@ -393,6 +386,7 @@ impl TestContextBuilder { &initial_private_accounts, wallet_config_overrides, ) + .await .context("Failed to setup wallet")?; if use_prebuilt { @@ -456,6 +450,10 @@ impl BlockingTestContext { self.ctx.as_ref().expect("TestContext is set") } + pub const fn ctx_mut(&mut self) -> &mut TestContext { + self.ctx.as_mut().expect("TestContext is set") + } + pub const fn runtime(&self) -> &tokio::runtime::Runtime { &self.runtime } diff --git a/test_fixtures/src/setup.rs b/test_fixtures/src/setup.rs index 325e1628..ce816919 100644 --- a/test_fixtures/src/setup.rs +++ b/test_fixtures/src/setup.rs @@ -1,4 +1,7 @@ -use std::{net::SocketAddr, path::PathBuf}; +use std::{ + net::SocketAddr, + path::{Path, PathBuf}, +}; use anyhow::{Context as _, Result, bail}; use indexer_service::{ChannelId, IndexerHandle}; @@ -20,12 +23,106 @@ use crate::{ private_mention, public_mention, }; -/// How to initialize the sequencer's database. -pub enum SequencerInit<'dump> { - /// Apply these genesis actions from scratch. - Genesis(Vec), - /// Restore an existing chain from a prebuilt dump (genesis actions are then irrelevant). - Prebuilt(&'dump DbDump), +pub struct SequencerSetup { + partial: config::SequencerPartialConfig, + bedrock_addr: SocketAddr, + channel_id: ChannelId, + genesis_transactions: Option>, + cross_zone: Option, +} + +impl SequencerSetup { + #[must_use] + pub fn new(partial: config::SequencerPartialConfig, bedrock_addr: SocketAddr) -> Self { + Self { + partial, + bedrock_addr, + channel_id: config::bedrock_channel_id(), + genesis_transactions: None, + cross_zone: None, + } + } + + /// Set the Bedrock channel ID to use for the sequencer. + /// If not set, the default channel ID from the Bedrock config will be used. + #[must_use] + pub const fn with_channel_id(mut self, channel_id: ChannelId) -> Self { + self.channel_id = channel_id; + self + } + + /// Set the cross-zone configuration to use for the sequencer. + /// If not set, the sequencer will be configured to run in single-zone mode. + #[must_use] + pub fn with_cross_zone(mut self, cross_zone: sequencer_core::config::CrossZoneConfig) -> Self { + self.cross_zone = Some(cross_zone); + self + } + + /// Set the genesis transactions to apply when initializing the sequencer. + /// If not set, the sequencer will be initialized from a prebuilt database dump. + #[must_use] + pub fn with_genesis(mut self, genesis_transactions: Vec) -> Self { + self.genesis_transactions = Some(genesis_transactions); + self + } + + /// Set up the sequencer in a fresh temporary home directory, returning the + /// owning [`TempDir`] alongside the handle. + pub async fn setup(self) -> Result<(SequencerHandle, TempDir)> { + let temp_sequencer_dir = + tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; + + let sequencer_handle = self.setup_at(temp_sequencer_dir.path()).await?; + + Ok((sequencer_handle, temp_sequencer_dir)) + } + + /// Set up the sequencer in an explicit `home` directory owned by the caller. + /// + /// Useful for tests that restart the sequencer against the same on-disk store. + pub async fn setup_at(self, home: &Path) -> Result { + let Self { + partial, + bedrock_addr, + channel_id, + genesis_transactions, + cross_zone, + } = self; + + debug!("Using sequencer home at {}", home.display()); + + let genesis_transactions = if let Some(genesis) = genesis_transactions { + genesis + } else { + let dump = load_prebuilt_dump()?; + // `SequencerCore::open_or_create_store` looks for `/rocksdb`. + let dst = home.join("rocksdb"); + let _store = SequencerStore::restore_db_from_dump( + &dst, + &dump, + lee::PrivateKey::try_new(config::SEQUENCER_SIGNING_KEY)?, + ) + .context("Failed to restore prebuilt sequencer database from dump")?; + // TODO: Technically not correct, we should reconstruct the genesis transactions + // from the dump, but this crutch doesn't affect anything for now + Vec::new() + }; + + let config = config::sequencer_config_with_channel( + partial, + home.to_owned(), + bedrock_addr, + channel_id, + genesis_transactions, + cross_zone, + ) + .context("Failed to create Sequencer config")?; + + sequencer_service::run(config, 0) + .await + .context("Failed to run Sequencer Service") + } } /// Committed single-file dump of the prebuilt sequencer database (`just regenerate-test-fixture`). @@ -139,82 +236,7 @@ pub async fn setup_indexer( .map(|handle| (handle, temp_indexer_dir)) } -pub async fn setup_sequencer( - partial: config::SequencerPartialConfig, - bedrock_addr: SocketAddr, - genesis_transactions: Vec, - channel_id: ChannelId, - cross_zone: Option, -) -> Result<(SequencerHandle, TempDir)> { - setup_sequencer_inner( - partial, - bedrock_addr, - SequencerInit::Genesis(genesis_transactions), - channel_id, - cross_zone, - ) - .await -} - -/// Like [`setup_sequencer`], but rebuilds the sequencer database from the committed prebuilt dump -/// so it starts from an existing chain instead of applying genesis from scratch. -pub async fn setup_sequencer_from_prebuilt( - partial: config::SequencerPartialConfig, - bedrock_addr: SocketAddr, -) -> Result<(SequencerHandle, TempDir)> { - let dump = load_prebuilt_dump()?; - setup_sequencer_inner( - partial, - bedrock_addr, - SequencerInit::Prebuilt(&dump), - config::bedrock_channel_id(), - None, - ) - .await -} - -async fn setup_sequencer_inner( - partial: config::SequencerPartialConfig, - bedrock_addr: SocketAddr, - init: SequencerInit<'_>, - channel_id: ChannelId, - cross_zone: Option, -) -> Result<(SequencerHandle, TempDir)> { - let temp_sequencer_dir = - tempfile::tempdir().context("Failed to create temp dir for sequencer home")?; - - debug!( - "Using temp sequencer home at {}", - temp_sequencer_dir.path().display() - ); - - let genesis_transactions = match init { - SequencerInit::Genesis(genesis) => genesis, - SequencerInit::Prebuilt(dump) => { - // `SequencerCore::open_or_create_store` looks for `/rocksdb`. - let dst = temp_sequencer_dir.path().join("rocksdb"); - SequencerStore::restore_db_from_dump(&dst, dump) - .context("Failed to restore prebuilt sequencer database from dump")?; - Vec::new() - } - }; - - let config = config::sequencer_config( - partial, - temp_sequencer_dir.path().to_owned(), - bedrock_addr, - genesis_transactions, - channel_id, - cross_zone, - ) - .context("Failed to create Sequencer config")?; - - let sequencer_handle = sequencer_service::run(config, 0).await?; - - Ok((sequencer_handle, temp_sequencer_dir)) -} - -pub fn setup_wallet( +pub async fn setup_wallet( sequencer_addr: SocketAddr, initial_public_accounts: &[(PrivateKey, u128)], initial_private_accounts: &[InitialPrivateAccountForWallet], @@ -232,14 +254,17 @@ pub fn setup_wallet( .context("Failed to write wallet config in temp dir")?; let storage_path = temp_wallet_dir.path().join("storage.json"); + let metrics_path = temp_wallet_dir.path().join("metrics.json"); let wallet_password = "test_pass".to_owned(); let (mut wallet, _mnemonic) = WalletCore::new_init_storage( config_path, storage_path, + metrics_path, Some(config_overrides), &wallet_password, ) + .await .context("Failed to init wallet")?; for (private_key, _balance) in initial_public_accounts { diff --git a/tools/cross_zone_chat/src/main.rs b/tools/cross_zone_chat/src/main.rs index 26ce7b3a..e9205d97 100644 --- a/tools/cross_zone_chat/src/main.rs +++ b/tools/cross_zone_chat/src/main.rs @@ -66,7 +66,7 @@ use sequencer_service_rpc::{RpcClient as _, SequencerClient, SequencerClientBuil use serde::{Deserialize, Serialize}; use test_fixtures::{ config::{self, SequencerPartialConfig, UrlProtocol, bedrock_channel_id, bedrock_channel_id_b}, - setup::{setup_bedrock_node, setup_sequencer}, + setup::{SequencerSetup, setup_bedrock_node}, }; const HTTP_PORT: u16 = 8088; @@ -289,14 +289,20 @@ async fn main() -> Result<()> { let cross_zone_b = watch_peer(zone_a, receiver_id); let partial = SequencerPartialConfig::default(); - let (seq_a, _home_a) = - setup_sequencer(partial, bedrock_addr, vec![], channel_a, Some(cross_zone_a)) - .await - .context("Failed to set up zone A sequencer")?; - let (seq_b, _home_b) = - setup_sequencer(partial, bedrock_addr, vec![], channel_b, Some(cross_zone_b)) - .await - .context("Failed to set up zone B sequencer")?; + let (seq_a, _home_a) = SequencerSetup::new(partial, bedrock_addr) + .with_genesis(vec![]) + .with_channel_id(channel_a) + .with_cross_zone(cross_zone_a) + .setup() + .await + .context("Failed to set up zone A sequencer")?; + let (seq_b, _home_b) = SequencerSetup::new(partial, bedrock_addr) + .with_genesis(vec![]) + .with_channel_id(channel_b) + .with_cross_zone(cross_zone_b) + .setup() + .await + .context("Failed to set up zone B sequencer")?; let state = Arc::new(AppState { zone_a: ZoneRuntime {