logos-blockchain/logos-execution-zone
1
0
Fork 0
mirror of https://github.com/logos-blockchain/logos-execution-zone.git synced 2026-08-05 04:23:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/dev' into artem/bundle-actions

Browse Source
This commit is contained in:
agureev 2026-08-04 14:21:00 +04:00
commit ccb426960f
101 changed files with 7882 additions and 1392 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
.github/workflows/ci.yml vendored
View File

@ -287,7 +287,7 @@ jobs:
needs: ci-image
if: github.event_name == 'push'
runs-on: ubuntu-latest
timeout-minutes: 90
timeout-minutes: 150
steps:
- uses: actions/checkout@v5
with:
@ -353,3 +353,34 @@ jobs:
exit 1
fi
echo "✅ Artifacts are up to date"
# The dashboard generator is plain Rust (no Docker), so this runs directly in
# the CI image container rather than through `run-in-ci-image`.
dashboards:
needs: ci-image
runs-on: ubuntu-latest
container:
image: ${{ needs.ci-image.outputs.image }}
credentials:
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
timeout-minutes: 30
name: dashboards
steps:
- uses: actions/checkout@v5
with:
ref: ${{ github.event.pull_request.head.sha || github.head_ref }}
- name: Regenerate dashboards
run: just regenerate-dashboards
- name: Check if dashboards match repository
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
if ! git diff --exit-code monitoring/grafana/dashboards/; then
echo "❌ Dashboards in the repository are out of date!"
echo "Please run 'just regenerate-dashboards' and commit the changes."
exit 1
fi
echo "✅ Dashboards are up to date"

1
.gitignore vendored
View File

@ -11,6 +11,7 @@ data/
rocksdb*
sequencer/service/data/
storage.json
statistics.json
result

6
CONTRIBUTING.md
View File

@ -6,6 +6,10 @@ This document describes the guidelines for contributing to the project. We will
If you have any questions, come say hi to our [Discord](https://discord.gg/tGJwgGrSPN)!
## Metrics
We have guidelines about metrics, for more information refer to [metrics](docs/metrics/metrics.md).
## Commit title format
We use [Conventional Commits](https://www.conventionalcommits.org/).
@ -59,7 +63,7 @@ Could be squashed to an empty commit if they belong to the same PR.
## Default branch
By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable.
By default all PRs must be directed into the `dev` branch. This helps us to keep releases stable.
## Branch workflow

669
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

25
Cargo.toml
View File

@ -18,9 +18,11 @@ members = [
"lez/system_accounts",
"lez/chain_state",
"lez/sequencer/core",
"lez/sequencer/core/metrics",
"lez/sequencer/service",
"lez/sequencer/service/protocol",
"lez/sequencer/service/rpc",
"lez/sequencer/service/metrics",
"lez/indexer/core",
"lez/indexer/service",
"lez/indexer/service/protocol",
@ -67,6 +69,7 @@ members = [
"tools/crypto_primitives_bench",
"tools/integration_bench",
"tools/cross_zone_chat",
"tools/dashboard_gen",
]
[workspace.dependencies]
@ -78,8 +81,10 @@ mempool = { path = "lez/mempool" }
storage = { path = "lez/storage" }
key_protocol = { path = "lee/key_protocol" }
sequencer_core = { path = "lez/sequencer/core" }
sequencer_core_metrics = { path = "lez/sequencer/core/metrics" }
sequencer_service_protocol = { path = "lez/sequencer/service/protocol" }
sequencer_service_rpc = { path = "lez/sequencer/service/rpc" }
sequencer_service_metrics = { path = "lez/sequencer/service/metrics" }
sequencer_service = { path = "lez/sequencer/service" }
indexer_core = { path = "lez/indexer/core" }
indexer_service = { path = "lez/indexer/service" }
@ -132,6 +137,7 @@ openssl = { version = "0.10", features = ["vendored"] }
openssl-probe = { version = "0.1.2" }
serde = { version = "1.0.60", default-features = false, features = ["derive"] }
serde_json = "1.0.81"
serde_yaml = "0.9.34"
serde_with = "3.16.1"
actix = "0.13.0"
actix-cors = "0.7.1"
@ -141,6 +147,8 @@ actix-rt = "*"
lazy_static = "1.5.0"
env_logger = "0.11"
log = "0.4.28"
metrics = "0.24.6"
metrics-exporter-prometheus = "0.18.3"
lru = "0.16.3"
thiserror = "2.0"
sha2 = "0.10.8"
@ -162,6 +170,7 @@ base64 = "0.22.1"
bip39 = "2.2.0"
hmac-sha512 = "1.1.7"
chrono = "0.4.41"
time = "0.3"
borsh = "1.5.7"
zstd = "0.13"
base58 = "0.2.0"
@ -171,14 +180,16 @@ url = { version = "2.5.4", features = ["serde"] }
tokio-retry = "0.3.0"
schemars = "1.2"
async-stream = "0.3.6"
strum = { version = "0.28.0", features = ["derive"] }
logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-core = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-chain-broadcast-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-chain-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-zone-sdk = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-http-api-common = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "d8711bbc3d43d3ef9755ef9b73af32fd0f703160" }
logos-blockchain-common-http-client = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-key-management-system-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-codec = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-core = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-chain-broadcast-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-chain-service = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-zone-sdk = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
logos-blockchain-http-api-common = { git = "https://github.com/logos-blockchain/logos-blockchain.git", rev = "0dc34e2c5a6e2ff772140378659489a602ee06bc" }
keycard-rs = { git = "https://github.com/keycard-tech/keycard-rs", rev = "9535a657ba04b1e6916de51777e22b4837c1a84d" }

48
Justfile
View File

@ -24,13 +24,15 @@ build-artifacts:
just regenerate-test-fixture; \
fi
RISC0_DOCKER_CONTAINER_TAG := "r0.1.91.1"
build-artifact methods_path features="":
@echo "Building artifacts for {{methods_path}}"
@rm -rf target/{{methods_path}}/riscv32im-risc0-zkvm-elf/docker/*.bin
@if [ "{{features}}" = "" ]; then \
CARGO_TARGET_DIR=target/{{methods_path}} cargo risczero build --manifest-path {{methods_path}}/Cargo.toml; \
RISC0_DOCKER_CONTAINER_TAG={{RISC0_DOCKER_CONTAINER_TAG}} CARGO_TARGET_DIR=target/{{methods_path}} cargo risczero build --manifest-path {{methods_path}}/Cargo.toml; \
else \
CARGO_TARGET_DIR=target/{{methods_path}} cargo risczero build --no-default-features --features {{features}} --manifest-path {{methods_path}}/Cargo.toml; \
RISC0_DOCKER_CONTAINER_TAG={{RISC0_DOCKER_CONTAINER_TAG}} CARGO_TARGET_DIR=target/{{methods_path}} cargo risczero build --no-default-features --features {{features}} --manifest-path {{methods_path}}/Cargo.toml; \
fi
@mkdir -p {{ARTIFACTS}}/{{methods_path}}
@cp target/{{methods_path}}/riscv32im-risc0-zkvm-elf/docker/*.bin {{ARTIFACTS}}/{{methods_path}}
@ -51,6 +53,13 @@ regenerate-test-fixture:
@echo "🧪 Regenerating test fixture"
RISC0_DEV_MODE=1 cargo run -p test_fixtures --bin regenerate_test_fixture
# Regenerate the committed Grafana dashboards from the Rust generator
# (tools/dashboard_gen) and commit the result. CI checks these are up to date.
regenerate-dashboards:
@echo "📊 Regenerating Grafana dashboards"
@cargo build -q -p dashboard_gen
@cargo run -q -p dashboard_gen -- sequencer > monitoring/grafana/dashboards/sequencer.json
# Run criterion benches: fast crypto primitives, then the slow PPE verify (real proving setup).
bench:
@echo "📊 Running criterion benches"
@ -63,19 +72,24 @@ run-bedrock:
@echo "⛓️ Running bedrock"
docker compose up
# Run Sequencer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
# Optional home/port let a second instance run off the same config, e.g.
# `just run-sequencer "" "$TMPDIR/lez-sequencer2" 3041` for the multi-sequencer demo.
# Run Prometheus + Grafana in docker. Grafana: http://localhost:3000 (anonymous
# admin), Prometheus: http://localhost:9090. Scrapes the sequencer's /metrics.
[working-directory: 'monitoring']
run-monitoring:
@echo "📊 Running Prometheus (http://localhost:9090) + Grafana (http://localhost:3000)"
docker compose up
# Run Sequencer. Extra args are forwarded to the binary. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
[working-directory: 'lez/sequencer/service']
run-sequencer standalone="" home="" port="3040":
run-sequencer *args:
@echo "🧠 Running sequencer"
@if [ "{{standalone}}" = "standalone" ]; then \
echo "🧪 Running in standalone mode"; \
RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \
else \
echo "🚀 Running in normal mode"; \
RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json --port {{port}} {{ if home != "" { "--home " + quote(home) } else { "" } }}; \
fi
RUST_LOG=info cargo run --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}}
# Run Sequencer with mocked Bedrock clients. Takes the same args as `run-sequencer`.
[working-directory: 'lez/sequencer/service']
run-sequencer-standalone *args:
@echo "🧪 Running sequencer in standalone mode"
RUST_LOG=info cargo run --features standalone --release -p sequencer_service -- configs/debug/sequencer_config.json {{args}}
# Run Indexer. Run with RISC0_DEV_MODE=1 to disable proof verification for faster iteration.
[working-directory: 'lez/indexer/service']
@ -101,6 +115,11 @@ run-wallet +args:
@echo "🔑 Running wallet"
LEE_WALLET_HOME_DIR=$(pwd)/configs/debug cargo run --release -p wallet -- {{args}}
# Query sequencer metrics in raw format. Useful for quick debugging. For a more detailed view, use `just run-monitoring`.
get-sequencer-metrics:
@echo "📊 Querying sequencer's metrics"
curl http://localhost:9000/metrics
# Import test accounts supplied in sequencer configuration.
wallet-import-test-accounts:
@echo "⚙️ Initializing accounts"
@ -141,4 +160,5 @@ clean:
rm -rf lez/wallet/configs/debug/storage.json
rm -rf lez/wallet/configs/debug/statistics.json
rm -rf rocksdb*
cd bedrock && docker compose down -v
cd bedrock && docker compose down -v && cd ..
cd monitoring && docker compose down -v && cd ..

BIN
artifacts/lee/privacy_preserving_circuit/privacy_preserving_circuit.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/amm.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/associated_token_account.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/authenticated_transfer.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/bridge.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/bridge_lock.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/clock.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/cross_zone_inbox.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/cross_zone_outbox.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/faucet.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/pinata.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/pinata_token.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/ping_receiver.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/ping_sender.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/token.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/vault.bin
View File

Binary file not shown.

BIN
artifacts/lez/programs/wrapped_token.bin
View File

Binary file not shown.

157
bedrock/deployment-settings.yaml
View File

@ -1,48 +1,47 @@
blend:
common:
num_blend_layers: 3
num_blend_layers: 1
minimum_network_size: 30
protocol_name: /blend/integration-tests
data_replication_factor: 0
protocol_name: /logos-blockchain-LEZ-DEV/blend/1.0.0
data_replication_factor: 1
core:
scheduler:
cover:
message_frequency_per_round: 1.0
delayer:
maximum_release_delay_in_rounds: 3
maximum_release_delay_in_rounds: 1
minimum_messages_coefficient: 1
normalization_constant: 1.03
activity_threshold_sensitivity: 1
network:
kademlia_protocol_name: /integration/logos-blockchain/kad/1.0.0
identify_protocol_name: /integration/logos-blockchain/identify/1.0.0
chain_sync_protocol_name: /integration/logos-blockchain/chainsync/1.0.0
kademlia_protocol_name: /logos-blockchain-LEZ-DEV/kad/1.0.0
identify_protocol_name: /logos-blockchain-LEZ-DEV/identify/1.0.0
chain_sync_protocol_name: /logos-blockchain-LEZ-DEV/chainsync/1.0.0
cryptarchia:
epoch_config:
epoch_stake_distribution_stabilization: 3
epoch_period_nonce_buffer: 3
epoch_period_nonce_stabilization: 4
security_param: 10
security_param: 5
slot_activation_coeff:
numerator: 1
denominator: 2
learning_rate: 0.1
learning_rate: 0.5
sdp_config:
service_params:
BN:
inactivity_period: 1
retention_period: 1
inactivity_period: 2
epoch: 0
min_stake:
threshold: 1
timestamp: 0
gossipsub_protocol: /integration/logos-blockchain/cryptarchia/proto/1.0.0
gossipsub_protocol: /logos-blockchain-LEZ-DEV/cryptarchia/1.0.0
genesis_block:
header:
version: Bedrock
parent_block: '0000000000000000000000000000000000000000000000000000000000000000'
slot: 0
block_root: b5f8787ac23674822414c70eea15d842da38f2e806ede1a73cf7b5cf0277da07
block_root: cb5951ac1ffa1aa5d0e585fb54e784bd9c025b28d752324e98b3837f34648692
proof_of_leadership:
proof: '0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
entropy_contribution: '0000000000000000000000000000000000000000000000000000000000000000'
@ -56,24 +55,134 @@ cryptarchia:
payload:
inputs: []
outputs:
- value: 1
pk: d204000000000000000000000000000000000000000000000000000000000000
- value: 100
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1
pk: ed266e6e887b9b97059dc1aa1b7b2e19b934291753c6336a163fe4ebaa28e717
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 1000000
pk: '2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26'
- value: 100000
pk: '6b2bcd3029fba573cff0c332dc4de7430faf5e261383d693d8dbb5b97665660a'
- value: 18446744073709551615
pk: c2a6a4a0981d5bdcf8ddeb8d7934fd8c5510efeb1053f613b45871670b6f7b19
- opcode: 17
payload:
channel_id: '0000000000000000000000000000000000000000000000000000000000000000'
# chain_id_len=12 (u64_le), chain_id=logos-devnet (utf-8),
# genesis_time=2026-01-10T07:47:56Z (u64_le), epoch_nonce=[0u8; 32]
inscription: '0c000000000000006c6f676f732d6465766e65742c046269000000000000000000000000000000000000000000000000000000000000000000000000'
inscription: '05302e322e3123766c6a2d2ddf918544bca603c5a291c7dd1b902d6769ff4b00021506780e075c06051a'
parent: '0000000000000000000000000000000000000000000000000000000000000000'
signer: '0000000000000000000000000000000000000000000000000000000000000000'
- opcode: 32
payload:
service_type: BN
locators:
- /ip4/65.109.51.37/udp/3400/quic-v1
provider_id: '59c662860b737f4e2515599adb3434856db8070b373a449ff66955ad3da6b473'
zk_id: '6b2bcd3029fba573cff0c332dc4de7430faf5e261383d693d8dbb5b97665660a'
locked_note_id: '7e449a14172fc90679f6fca7b49a2d58c305ebf7ac42ef20202e533c31115222'
ops_proofs:
- !ZkSig
pi_a: '0000000000000000000000000000000000000000000000000000000000000000'
pi_b: '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
pi_c: '0000000000000000000000000000000000000000000000000000000000000000'
- !Ed25519Sig '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
- !Ed25519Sig '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
- !ZkAndEd25519Sigs
zk_sig:
pi_a: '0000000000000000000000000000000000000000000000000000000000000000'
pi_b: '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
pi_c: '0000000000000000000000000000000000000000000000000000000000000000'
ed25519_sig: '00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000'
faucet_pk: c2a6a4a0981d5bdcf8ddeb8d7934fd8c5510efeb1053f613b45871670b6f7b19
time:
slot_duration: '1.0'
slot_duration: '1.000000000'
mempool:
pubsub_topic: mantle_e2e_tests
pubsub_topic: /logos-blockchain-LEZ-DEV/mempool/1.0.0

2
bedrock/docker-compose.yml
View File

@ -1,7 +1,7 @@
services:
logos-blockchain-node-0:
image: ghcr.io/logos-blockchain/logos-blockchain@sha256:91d6c5bf07e07fcfba5e7cf07d21ee686a6bc4b9f6210f2d28bffbcad9a3729f
image: ghcr.io/logos-blockchain/logos-blockchain:0.2.1-lssa
ports:
- "${PORT:-18080}:18080/tcp"
volumes:

10
bedrock/scripts/run_logos_blockchain_node.sh
View File

@ -7,14 +7,6 @@ export POL_PROOF_DEV_MODE=true
# Use static configs mounted from host. Both node-config.yaml and
# deployment-settings.yaml have matching validator keys so the node
# can produce blocks as a single-validator network.
# Copy deployment-settings to a writable path because sed -i can't
# rename on a bind-mounted file.
cp /etc/logos-blockchain/deployment-settings.yaml /deployment-settings.yaml
# Set chain_start_time to "now" so the chain starts immediately.
sed -i "s/PLACEHOLDER_CHAIN_START_TIME/$(date -u '+%Y-%m-%d %H:%M:%S.000000 +00:00:00')/" \
/deployment-settings.yaml
exec /usr/bin/logos-blockchain-node \
/etc/logos-blockchain/node-config.yaml \
--deployment /deployment-settings.yaml
--deployment /etc/logos-blockchain/deployment-settings.yaml

2
docker-compose.yml
View File

@ -11,3 +11,5 @@ include:
lez/indexer/service/docker-compose.yml
- path:
lez/explorer_service/docker-compose.yml
- path:
monitoring/docker-compose.yml

113
docs/metrics/metrics.md Normal file
View File

@ -0,0 +1,113 @@
# Metrics
Services expose Prometheus metrics; Grafana dashboards are generated from Rust so panel queries and metric names cannot drift apart.
## Metrics crates
Every crate that emits metrics gets a sibling `metrics` crate — `lez/sequencer/core/metrics` → `sequencer_core_metrics`. Each has two halves:
| Module | Gated by | Contents |
|---|---|---|
| `names` | always compiled | `pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";` — one const per metric |
| `record` | `record` feature | `record_*` / `increment_*` functions, plus `init()` |
The emitting crate depends on it with `features = ["record"]`; consumers that only need the names (i.e. `dashboard_gen`) take the default features and pull in nothing. Dashboards reference the same consts the recording code does, so **renaming a metric is a compile error rather than a silently empty panel**.
## Naming
The recorder runs with `with_recommended_naming(true)`, which enforces Prometheus convention:
| Kind | Suffix | Example |
|---|---|---|
| Counter | `_total` | `blocks_produced_total`, `submitted_transactions_total` |
| Histogram | unit | `block_creation_time_seconds` |
| Gauge | none | `mempool_size` |
**Spell the suffix in the const.** The exporter appends a missing unit suffix to the *rendered* name, but bucket matchers (below) run against the registered name — a duration metric named without `_seconds` renders correctly yet silently gets the wrong buckets.
## Metric types
| Type | Use for | Example |
|---|---|---|
| Counter | monotonically increasing event counts | `mempool_failed_transactions_total` |
| Gauge | a value that moves both ways | `mempool_size`, `chain_height` (a reorg lowers it) |
| Histogram | distributions — latencies, sizes, per-batch counts | `mempool_transaction_application_time_seconds` |
Each metric gets a private constructor plus a public recording wrapper, so its description, unit and labels are declared once:
```rust
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
```
Labels are passed as `"origin" => <&'static str>::from(origin)`; keep them low-cardinality (enums, never IDs or hashes).
## `init()`
Each `record` module exposes `init()`, called once at startup after the recorder is installed. It publishes every metric at zero.
This is not cosmetic. A metric only materialises when first touched, and `rate()`/`increase()` need a sample from *before* an increment to see it — a series that springs into existence at `1` reads as `0` until the second event, so the first one is lost forever. Zero-publishing also means an idle service exports `0` instead of nothing at all.
For histograms, creating the handle publishes zeroed buckets without recording an observation (recording a fake `0` would skew the distribution). Label combinations must each be registered, so `init()` iterates the label enums via `strum::EnumIter`.
## Metrics in libraries
**Yes, record metrics from library crates.** The `metrics` facade is a no-op until a recorder is installed, so a library that records costs nothing to a consumer that never installs one — including tests. Libraries record; only the binary installs the exporter.
## Exporter setup
`sequencer_service`'s `main.rs` installs the Prometheus recorder on the config's `metrics_address` (default `0.0.0.0:9000`) with **explicit histogram buckets**. This matters: without buckets, `metrics-exporter-prometheus` renders histograms as rolling-window summaries whose quantiles **reset to `0`** once the window (default 60 s) drains — an idle period reads as "took 0 s" rather than "no data". With buckets you get real `_bucket`/`_sum`/`_count` counters that never decay, are aggregatable, and honour the dashboard's time range.
Ladders are matched by name suffix, so a new timing metric is covered automatically:
```rust
.set_buckets(COUNT_BUCKETS) // fallback
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS)
```
## `dashboard_gen`
`tools/dashboard_gen` is a small Grafana dashboard builder plus the dashboard definitions. It prints JSON to stdout; the result is committed under `monitoring/grafana/dashboards/` and CI fails if it is stale.
```
src/lib.rs, schema.rs, styling.rs, unit.rs the builder library
src/dashboards/<name>.rs one dashboard per module
src/main.rs CLI: `dashboard_gen sequencer`
```
Panels are built fluently, and every query is composed from the `names` consts:
```rust
Panel::timeseries("Block production rate")
.width(18)
.target(rate_per_min(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL, "blocks/min"))
```
Query helpers: `rate_per_min` for counters, `avg` for histograms, and `selected_percentile` for percentile lines — the latter reads a `percentile` dashboard dropdown created by `percentile_variable`, so one panel serves p50/p90/p95/p99 instead of drawing all four. Rate windows use `$__rate_interval`, which tracks the panel's zoom.
**Extending it:**
| Goal | Change |
|---|---|
| New panel | Add a `Panel::…` to a row in the dashboard module |
| New dashboard | `src/dashboards/<name>.rs` with `pub fn dashboard()`, a `pub mod` line, a `DashboardKind` variant, and a `just regenerate-dashboards` line |
| Grafana option we don't model yet | Add the field to `schema.rs` and a setter on `Panel` (styling setters live in `styling.rs` and panic when handed a redundant default) |
The builder deliberately models only the subset of Grafana's schema we use.
## Justfile
| Recipe | Purpose |
|---|---|
| `just regenerate-dashboards` | Rebuild the committed dashboard JSON. Run after touching metric names or dashboard code — CI checks it is current. |
| `just run-monitoring` | Prometheus (`:9090`) + Grafana (`:3000`, anonymous admin) in docker, scraping the sequencer every 5 s |
| `just get-sequencer-metrics` | `curl` the raw `/metrics` endpoint — quickest way to confirm a metric name and value |

2
flake.nix
View File

@ -124,7 +124,7 @@
commonArgs = {
inherit src;
buildInputs = [ pkgs.openssl ];
buildInputs = [ pkgs.openssl pkgs.pcsclite ];
nativeBuildInputs = [
pkgs.pkg-config
pkgs.clang

6
integration_tests/Cargo.toml
View File

@ -38,17 +38,11 @@ programs.workspace = true
test_programs.workspace = true
testnet_initial_state.workspace = true
logos-blockchain-http-api-common.workspace = true
logos-blockchain-core.workspace = true
logos-blockchain-zone-sdk.workspace = true
logos-blockchain-key-management-system-service.workspace = true
anyhow.workspace = true
log.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
futures.workspace = true
hex.workspace = true
tempfile.workspace = true
bytesize.workspace = true
reqwest.workspace = true
borsh.workspace = true
num-bigint.workspace = true

677
integration_tests/tests/bridge.rs
View File

@ -1,43 +1,23 @@
#![expect(
clippy::tests_outside_test_module,
clippy::arithmetic_side_effects,
reason = "We don't care about these in tests"
)]
use std::{ops::Deref as _, time::Duration};
use std::time::Duration;
use anyhow::Context as _;
use borsh::BorshSerialize;
use common::transaction::LeeTransaction;
use futures::StreamExt as _;
use integration_tests::{
TIME_TO_WAIT_FOR_BLOCK_SECONDS, TestContext, account_balance, get_account,
wait_for_indexer_to_catch_up,
};
use lee::{
AccountId, execute_and_prove, privacy_preserving_transaction, program::Program,
public_transaction,
execute_and_prove, privacy_preserving_transaction, program::Program, public_transaction,
};
use lee_core::{InputAccountIdentity, account::AccountWithMetadata};
use log::info;
use logos_blockchain_core::mantle::{ledger::Inputs, ops::channel::deposit::DepositOp};
use logos_blockchain_http_api_common::bodies::{
channel::ChannelDepositRequestBody,
wallet::{
balance::WalletBalanceResponseBody,
transfer_funds::{WalletTransferFundsRequestBody, WalletTransferFundsResponseBody},
},
};
use logos_blockchain_zone_sdk::{
CommonHttpClient, ZoneMessage, adapter::NodeHttpClient, indexer::ZoneIndexer,
};
use num_bigint::BigUint;
use sequencer_service_rpc::RpcClient as _;
use test_fixtures::public_mention;
use tokio::test;
use wallet::cli::{Command, execute_subcommand, programs::bridge::BridgeSubcommand};
const TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK: Duration = Duration::from_mins(2);
// const TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK: Duration = Duration::from_mins(2);
#[test]
async fn public_bridge_deposit_invocation_is_dropped() -> anyhow::Result<()> {
@ -247,377 +227,386 @@ async fn private_bridge_deposit_invocation_is_dropped() -> anyhow::Result<()> {
Ok(())
}
async fn submit_bedrock_deposit(
bedrock_addr: std::net::SocketAddr,
bedrock_account_pk: &str,
recipient_id: AccountId,
amount: u64,
) -> anyhow::Result<()> {
#[derive(BorshSerialize)]
struct DepositMetadata {
recipient_id: AccountId,
}
// async fn submit_bedrock_deposit(
// bedrock_addr: std::net::SocketAddr,
// bedrock_account_pk: &str,
// recipient_id: AccountId,
// amount: u64,
// ) -> anyhow::Result<()> {
// #[derive(BorshSerialize)]
// struct DepositMetadata {
// recipient_id: AccountId,
// }
// Encode deposit metadata
let metadata = borsh::to_vec(&DepositMetadata { recipient_id })
.context("Failed to encode deposit metadata")?
.try_into()
.context("Encoded metadata is too big")?;
// // Encode deposit metadata
// let metadata = borsh::to_vec(&DepositMetadata { recipient_id })
// .context("Failed to encode deposit metadata")?
// .try_into()
// .context("Encoded metadata is too big")?;
let channel_id = integration_tests::config::bedrock_channel_id();
let client = reqwest::Client::new();
// let channel_id = integration_tests::config::bedrock_channel_id();
// let client = reqwest::Client::new();
let query_balance = || async {
let balance_response = client
.get(format!(
"http://{bedrock_addr}/wallet/{bedrock_account_pk}/balance"
))
.send()
.await
.context("Failed to query Bedrock wallet balance")?;
// let mut balance = bedrock_wallet_balance(bedrock_addr, bedrock_account_pk).await?;
let balance_response = check_response_success(balance_response).await?;
// info!(
// "Queried Bedrock balance for key {bedrock_account_pk}: {:?}",
// balance.balance
// );
balance_response
.json::<WalletBalanceResponseBody>()
.await
.context("Failed to decode Bedrock balance response")
};
// if balance.balance < amount {
// anyhow::bail!(
// "Bedrock wallet with key {bedrock_account_pk} has insufficient balance {:?} for
// deposit amount {:?}", balance.balance,
// amount
// );
// }
let mut balance = query_balance().await?;
// let mut selected_note_id = balance
// .notes
// .iter()
// .find_map(|(note_id, value)| (*value == amount).then_some(*note_id));
info!(
"Queried Bedrock balance for key {bedrock_account_pk}: {:?}",
balance.balance
);
// if selected_note_id.is_none() {
// let transfer_body = WalletTransferFundsRequestBody {
// tip: None,
// change_public_key: balance.address,
// funding_public_keys: vec![balance.address],
// recipient_public_key: balance.address,
// amount,
// };
if balance.balance < amount {
anyhow::bail!(
"Bedrock wallet with key {bedrock_account_pk} has insufficient balance {:?} for deposit amount {:?}",
balance.balance,
amount
);
}
// let transfer_response = client
// .post(format!(
// "http://{bedrock_addr}/wallet/transactions/transfer-funds"
// ))
// .json(&transfer_body)
// .send()
// .await
// .context("Failed to submit Bedrock transfer-funds request")?;
// let transfer_response = check_response_success(transfer_response).await?;
let mut selected_note_id = balance
.notes
.iter()
.find_map(|(note_id, value)| (*value == amount).then_some(*note_id));
// let transfer: WalletTransferFundsResponseBody = transfer_response
// .json()
// .await
// .context("Failed to decode Bedrock transfer-funds response")?;
if selected_note_id.is_none() {
let transfer_body = WalletTransferFundsRequestBody {
tip: None,
change_public_key: balance.address,
funding_public_keys: vec![balance.address],
recipient_public_key: balance.address,
amount,
};
// info!(
// "Submitted transfer-funds to create exact deposit note, tx hash {:?}",
// transfer.hash
// );
let transfer_response = client
.post(format!(
"http://{bedrock_addr}/wallet/transactions/transfer-funds"
))
.json(&transfer_body)
.send()
.await
.context("Failed to submit Bedrock transfer-funds request")?;
let transfer_response = check_response_success(transfer_response).await?;
// let mut found_note = None;
// for _ in 0..20 {
// tokio::time::sleep(Duration::from_millis(500)).await;
// balance = bedrock_wallet_balance(bedrock_addr, bedrock_account_pk).await?;
// found_note = balance
// .notes
// .iter()
// .find_map(|(note_id, value)| (*value == amount).then_some(*note_id));
// if found_note.is_some() {
// break;
// }
// }
let transfer: WalletTransferFundsResponseBody = transfer_response
.json()
.await
.context("Failed to decode Bedrock transfer-funds response")?;
// selected_note_id = found_note;
// }
info!(
"Submitted transfer-funds to create exact deposit note, tx hash {:?}",
transfer.hash
);
// let Some(selected_note_id) = selected_note_id else {
// anyhow::bail!(
// "Failed to locate exact-value note {amount:?} for Bedrock deposit; available notes:
// {:?}", balance.notes,
// );
// };
let mut found_note = None;
for _ in 0..20 {
tokio::time::sleep(Duration::from_millis(500)).await;
balance = query_balance().await?;
found_note = balance
.notes
.iter()
.find_map(|(note_id, value)| (*value == amount).then_some(*note_id));
if found_note.is_some() {
break;
}
}
// let body = ChannelDepositRequestBody {
// tip: None,
// deposit: DepositOp {
// channel_id,
// inputs: Inputs::new(selected_note_id),
// metadata,
// },
// change_public_key: balance.address,
// funding_public_keys: vec![balance.address],
// max_tx_fee: u64::MAX.into(),
// };
selected_note_id = found_note;
}
// let response = client
// .post(format!("http://{bedrock_addr}/channel/deposit"))
// .json(&body)
// .send()
// .await
// .context("Failed to submit Bedrock deposit request")?;
// let response = check_response_success(response).await?;
let Some(selected_note_id) = selected_note_id else {
anyhow::bail!(
"Failed to locate exact-value note {amount:?} for Bedrock deposit; available notes: {:?}",
balance.notes,
);
};
// let body_text = response
// .text()
// .await
// .unwrap_or_else(|_| "<failed to decode>".to_owned());
// info!(
// "Successfully submitted Bedrock deposit request for recipient {recipient_id} and amount
// {amount}, response body: {body_text}", );
let body = ChannelDepositRequestBody {
tip: None,
deposit: DepositOp {
channel_id,
inputs: Inputs::new(selected_note_id),
metadata,
},
change_public_key: balance.address,
funding_public_keys: vec![balance.address],
max_tx_fee: 1_000_u64.into(),
};
// Ok(())
// }
let response = client
.post(format!("http://{bedrock_addr}/channel/deposit"))
.json(&body)
.send()
.await
.context("Failed to submit Bedrock deposit request")?;
let response = check_response_success(response).await?;
// /// The Bedrock wallet state of `bedrock_account_pk`: its total balance and the
// /// notes it owns, keyed by note id.
// async fn bedrock_wallet_balance(
// bedrock_addr: std::net::SocketAddr,
// bedrock_account_pk: &str,
// ) -> anyhow::Result<WalletBalanceResponseBody> {
// let response = reqwest::Client::new()
// .get(format!(
// "http://{bedrock_addr}/wallet/{bedrock_account_pk}/balance"
// ))
// .send()
// .await
// .context("Failed to query Bedrock wallet balance")?;
let body_text = response
.text()
.await
.unwrap_or_else(|_| "<failed to decode>".to_owned());
info!(
"Successfully submitted Bedrock deposit request for recipient {recipient_id} and amount {amount}, response body: {body_text}",
);
// check_response_success(response)
// .await?
// .json::<WalletBalanceResponseBody>()
// .await
// .context("Failed to decode Bedrock balance response")
// }
Ok(())
}
// async fn check_response_success(response: reqwest::Response) -> anyhow::Result<reqwest::Response>
// { if response.status().is_success() {
// Ok(response)
// } else {
// let status = response.status();
// let body_text = response.text().await.unwrap_or_default();
// anyhow::bail!("Request failed with status {status} and body {body_text}");
// }
// }
async fn check_response_success(response: reqwest::Response) -> anyhow::Result<reqwest::Response> {
if response.status().is_success() {
Ok(response)
} else {
let status = response.status();
let body_text = response.text().await.unwrap_or_default();
anyhow::bail!("Request failed with status {status} and body {body_text}");
}
}
// async fn wait_for_vault_balance(
// ctx: &TestContext,
// vault_id: AccountId,
// expected_balance: u128,
// ) -> anyhow::Result<()> {
// let timeout = TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK
// + Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS);
// tokio::time::timeout(timeout, async {
// loop {
// let balance = account_balance(ctx, vault_id).await?;
// if balance == expected_balance {
// return Ok(());
// }
// tokio::time::sleep(Duration::from_millis(500)).await;
// }
// })
// .await
// .with_context(|| {
// format!("Timed out waiting for vault {vault_id} balance to reach {expected_balance}")
// })?
// }
async fn wait_for_vault_balance(
ctx: &TestContext,
vault_id: AccountId,
expected_balance: u128,
) -> anyhow::Result<()> {
let timeout = TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK
+ Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS);
tokio::time::timeout(timeout, async {
loop {
let balance = account_balance(ctx, vault_id).await?;
if balance == expected_balance {
return Ok(());
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
})
.await
.with_context(|| {
format!("Timed out waiting for vault {vault_id} balance to reach {expected_balance}")
})?
}
// /// Test deposit and withdraw round trip.
// ///
// /// Implemented as one test instead of two separate tests for deposit and withdraw, because the
// /// withdraw test depends on the deposit to set up the necessary state (funds in vault) for
// testing /// withdraw functionality.
// #[test]
// async fn bedrock_deposit_claim_and_withdraw_round_trip_succeeds() -> anyhow::Result<()> {
// let mut ctx = TestContext::new().await?;
/// Test deposit and withdraw round trip.
///
/// Implemented as one test instead of two separate tests for deposit and withdraw, because the
/// withdraw test depends on the deposit to set up the necessary state (funds in vault) for testing
/// withdraw functionality.
#[test]
async fn bedrock_deposit_claim_and_withdraw_round_trip_succeeds() -> anyhow::Result<()> {
let mut ctx = TestContext::new().await?;
// let bedrock_account_pk = "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26";
// let recipient_id = ctx.existing_public_accounts()[0];
// let amount = 1_u64;
// let vault_program_id = programs::vault().id();
// let recipient_vault_id = vault_core::compute_vault_account_id(vault_program_id,
// recipient_id);
let bedrock_account_pk = "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26";
let recipient_id = ctx.existing_public_accounts()[0];
let amount = 1_u64;
let vault_program_id = programs::vault().id();
let recipient_vault_id = vault_core::compute_vault_account_id(vault_program_id, recipient_id);
// let vault_balance_before = account_balance(&ctx, recipient_vault_id).await?;
// let recipient_balance_before = account_balance(&ctx, recipient_id).await?;
let vault_balance_before = account_balance(&ctx, recipient_vault_id).await?;
let recipient_balance_before = account_balance(&ctx, recipient_id).await?;
// // Submit deposit to Bedrock
// submit_bedrock_deposit(ctx.bedrock_addr(), bedrock_account_pk, recipient_id, amount)
// .await
// .context("Failed to submit Bedrock deposit for round-trip setup")?;
// Submit deposit to Bedrock
submit_bedrock_deposit(ctx.bedrock_addr(), bedrock_account_pk, recipient_id, amount)
.await
.context("Failed to submit Bedrock deposit for round-trip setup")?;
// // Wait for vault to receive the deposit (minted from bridge to vault)
// wait_for_vault_balance(
// &ctx,
// recipient_vault_id,
// vault_balance_before + u128::from(amount),
// )
// .await?;
// Wait for vault to receive the deposit (minted from bridge to vault)
wait_for_vault_balance(
&ctx,
recipient_vault_id,
vault_balance_before + u128::from(amount),
)
.await?;
// // Now claim funds from vault back to recipient
// let nonces = ctx
// .wallet()
// .get_accounts_nonces(&[recipient_id])
// .await
// .context("Failed to get nonce for vault claim")?;
// Now claim funds from vault back to recipient
let nonces = ctx
.wallet()
.get_accounts_nonces(&[recipient_id])
.await
.context("Failed to get nonce for vault claim")?;
// let signing_key = ctx
// .wallet()
// .storage()
// .key_chain()
// .pub_account_signing_key(recipient_id)
// .with_context(|| format!("Missing signing key for account {recipient_id}"))?;
let signing_key = ctx
.wallet()
.storage()
.key_chain()
.pub_account_signing_key(recipient_id)
.with_context(|| format!("Missing signing key for account {recipient_id}"))?;
// let claim_message = public_transaction::Message::try_new(
// vault_program_id,
// vec![recipient_id, recipient_vault_id],
// nonces,
// vault_core::Instruction::Claim {
// amount: u128::from(amount),
// },
// )
// .context("Failed to build vault claim message")?;
let claim_message = public_transaction::Message::try_new(
vault_program_id,
vec![recipient_id, recipient_vault_id],
nonces,
vault_core::Instruction::Claim {
amount: u128::from(amount),
},
)
.context("Failed to build vault claim message")?;
// let claim_witness_set =
// public_transaction::WitnessSet::for_message(&claim_message, &[signing_key]);
// let claim_tx = LeeTransaction::Public(lee::PublicTransaction::new(
// claim_message,
// claim_witness_set,
// ));
let claim_witness_set =
public_transaction::WitnessSet::for_message(&claim_message, &[signing_key]);
let claim_tx = LeeTransaction::Public(lee::PublicTransaction::new(
claim_message,
claim_witness_set,
));
// let claim_hash = ctx.sequencer_client().send_transaction(claim_tx).await?;
let claim_hash = ctx.sequencer_client().send_transaction(claim_tx).await?;
// tokio::time::sleep(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS)).await;
tokio::time::sleep(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS)).await;
// let claim_on_chain = ctx.sequencer_client().get_transaction(claim_hash).await?;
// let vault_balance_after_claim = account_balance(&ctx, recipient_vault_id).await?;
// let recipient_balance_after_claim = account_balance(&ctx, recipient_id).await?;
let claim_on_chain = ctx.sequencer_client().get_transaction(claim_hash).await?;
let vault_balance_after_claim = account_balance(&ctx, recipient_vault_id).await?;
let recipient_balance_after_claim = account_balance(&ctx, recipient_id).await?;
// assert!(
// claim_on_chain.is_some(),
// "Vault claim transaction must be included on-chain"
// );
// assert_eq!(
// vault_balance_after_claim, vault_balance_before,
// "Vault balance should return to initial state after claim"
// );
// assert_eq!(
// recipient_balance_after_claim,
// recipient_balance_before + u128::from(amount),
// "Recipient balance should increase by claimed amount"
// );
assert!(
claim_on_chain.is_some(),
"Vault claim transaction must be included on-chain"
);
assert_eq!(
vault_balance_after_claim, vault_balance_before,
"Vault balance should return to initial state after claim"
);
assert_eq!(
recipient_balance_after_claim,
recipient_balance_before + u128::from(amount),
"Recipient balance should increase by claimed amount"
);
// // The indexer must replay the deposit and claim blocks and reach the same
// // state as the sequencer — including the bridge system account the deposit
// // modifies, which is the case the hot fix unblocks.
// wait_for_indexer_to_catch_up(&ctx).await?;
// let bridge_account_id = system_accounts::bridge_account_id();
// for account_id in [recipient_id, recipient_vault_id, bridge_account_id] {
// let indexer_account = indexer_service_rpc::RpcClient::get_account(
// // `deref` is needed for correct trait resolution
// // of the async `get_account` method on `RpcClient`
// ctx.indexer_client().deref(),
// account_id.into(),
// )
// .await?;
// let sequencer_account = get_account(&ctx, account_id).await?;
// assert_eq!(
// indexer_account,
// sequencer_account.into(),
// "Indexer and sequencer diverged for account {account_id} after deposit"
// );
// }
// The indexer must replay the deposit and claim blocks and reach the same
// state as the sequencer — including the bridge system account the deposit
// modifies, which is the case the hot fix unblocks.
wait_for_indexer_to_catch_up(&ctx).await?;
let bridge_account_id = system_accounts::bridge_account_id();
for account_id in [recipient_id, recipient_vault_id, bridge_account_id] {
let indexer_account = indexer_service_rpc::RpcClient::get_account(
// `deref` is needed for correct trait resolution
// of the async `get_account` method on `RpcClient`
ctx.indexer_client().deref(),
account_id.into(),
)
.await?;
let sequencer_account = get_account(&ctx, account_id).await?;
assert_eq!(
indexer_account,
sequencer_account.into(),
"Indexer and sequencer diverged for account {account_id} after deposit"
);
}
// // Withdraw back to Bedrock and wait for finalized withdraw event.
// let sender_id = recipient_id;
// Withdraw back to Bedrock and wait for finalized withdraw event.
let sender_id = recipient_id;
// let observer = create_zone_indexer_observer(ctx.bedrock_addr())?;
// let observe_fut =
// wait_for_finalized_withdraw_op(&observer, ctx.bedrock_addr(), amount,
// bedrock_account_pk);
let observer = create_zone_indexer_observer(ctx.bedrock_addr())?;
let observe_fut = wait_for_finalized_withdraw_op(&observer, amount, bedrock_account_pk);
// let withdraw_fut = execute_subcommand(
// ctx.wallet_mut(),
// Command::Bridge(BridgeSubcommand::Withdraw {
// from: public_mention(sender_id),
// amount,
// bedrock_account_pk: bedrock_account_pk.to_owned(),
// }),
// );
let withdraw_fut = execute_subcommand(
ctx.wallet_mut(),
Command::Bridge(BridgeSubcommand::Withdraw {
from: public_mention(sender_id),
amount,
bedrock_account_pk: bedrock_account_pk.to_owned(),
}),
);
// let (observe_result, withdraw_result) = tokio::join!(observe_fut, withdraw_fut);
let (observe_result, withdraw_result) = tokio::join!(observe_fut, withdraw_fut);
// withdraw_result.context("Failed to execute wallet bridge withdraw command")?;
withdraw_result.context("Failed to execute wallet bridge withdraw command")?;
// observe_result
// .context("Failed while waiting for finalized withdraw event from zone indexer")?;
observe_result
.context("Failed while waiting for finalized withdraw event from zone indexer")?;
// // Sleep to observe sequencer log about validated withdraw event
// tokio::time::sleep(Duration::from_secs(1)).await;
// Sleep to observe sequencer log about validated withdraw event
tokio::time::sleep(Duration::from_secs(1)).await;
// Ok(())
// }
Ok(())
}
// fn create_zone_indexer_observer(
// bedrock_addr: std::net::SocketAddr,
// ) -> anyhow::Result<ZoneIndexer<NodeHttpClient>> {
// let bedrock_url = integration_tests::config::addr_to_url(
// integration_tests::config::UrlProtocol::Http,
// bedrock_addr,
// )
// .context("Failed to convert Bedrock addr to URL for zone indexer observer")?;
fn create_zone_indexer_observer(
bedrock_addr: std::net::SocketAddr,
) -> anyhow::Result<ZoneIndexer<NodeHttpClient>> {
let bedrock_url = integration_tests::config::addr_to_url(
integration_tests::config::UrlProtocol::Http,
bedrock_addr,
)
.context("Failed to convert Bedrock addr to URL for zone indexer observer")?;
// let node = NodeHttpClient::new(CommonHttpClient::new(None), bedrock_url);
let node = NodeHttpClient::new(CommonHttpClient::new(None), bedrock_url);
// Ok(ZoneIndexer::new(
// integration_tests::config::bedrock_channel_id(),
// node,
// ))
// }
Ok(ZoneIndexer::new(
integration_tests::config::bedrock_channel_id(),
node,
))
}
// /// Waits for a finalized withdraw that pays `expected_amount` to `receiver_pk`.
// ///
// /// A withdraw op releases channel-owned notes and carries nothing but their
// /// ids — the value and recipient live in the note itself. A released note keeps
// /// its id, value and public key, so the pairing is checked on the receiver's
// /// Bedrock wallet: one of the released notes must land there with the expected
// /// value.
// async fn wait_for_finalized_withdraw_op(
// observer: &ZoneIndexer<NodeHttpClient>,
// bedrock_addr: std::net::SocketAddr,
// expected_amount: u64,
// receiver_pk: &str,
// ) -> anyhow::Result<()> {
// let timeout = TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK
// + Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS);
async fn wait_for_finalized_withdraw_op(
observer: &ZoneIndexer<NodeHttpClient>,
expected_amount: u64,
receiver_pk: &str,
) -> anyhow::Result<()> {
let timeout = TIME_TO_FINALIZE_DEPOSIT_EVENT_ON_BEDROCK
+ Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS);
// tokio::time::timeout(timeout, async {
// // The wallet can trail the channel event, so released notes accumulate
// // across polls instead of being checked once when first observed.
// let mut released_notes = HashSet::new();
let bedrock_account_pk_bytes = hex::decode(receiver_pk)
.context("Failed to decode expected receiver public key from hex")?;
let expected_receiver_pk =
logos_blockchain_key_management_system_service::keys::ZkPublicKey::from(
BigUint::from_bytes_le(&bedrock_account_pk_bytes),
);
// loop {
// let stream = observer
// .follow()
// .await
// .context("Failed to read zone indexer message batch")?;
// let mut stream = std::pin::pin!(stream);
tokio::time::timeout(timeout, async {
loop {
let stream = observer
.follow()
.await
.context("Failed to read zone indexer message batch")?;
let mut stream = std::pin::pin!(stream);
// while let Some(message) = stream.next().await {
// info!("Observed zone message {message:?}");
while let Some(message) = stream.next().await {
info!("Observed zone message {message:?}");
// if let ZoneMessage::Withdraw(withdraw) = message {
// released_notes.extend(withdraw.inputs.iter().copied());
// }
// }
let ZoneMessage::Withdraw(withdraw) = message else {
continue;
};
// if !released_notes.is_empty() {
// let balance = bedrock_wallet_balance(bedrock_addr, receiver_pk).await?;
// if released_notes
// .iter()
// .any(|note_id| balance.notes.get(note_id) == Some(&expected_amount))
// {
// return Ok(());
// }
// }
let mut iter = withdraw.outputs.iter();
let Some(note) = iter.next() else {
continue;
};
if iter.next().is_some() {
// Withdraw op should only have one output
continue;
}
if note.value == expected_amount && note.pk == expected_receiver_pk {
return Ok(());
}
}
tokio::time::sleep(Duration::from_millis(500)).await;
}
})
.await
.with_context(|| {
format!("Timed out waiting for finalized withdraw message with amount {expected_amount}")
})?
}
// tokio::time::sleep(Duration::from_millis(500)).await;
// }
// })
// .await
// .with_context(|| {
// format!("Timed out waiting for finalized withdraw message with amount {expected_amount}")
// })?
// }

7
integration_tests/tests/cross_zone_bridge.rs
View File

@ -30,7 +30,7 @@ use lee::{
AccountId, PrivateKey, PublicKey, PublicTransaction,
public_transaction::{Message, WitnessSet},
};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, GenesisAction};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute, GenesisAction};
use sequencer_service_rpc::RpcClient as _;
use tokio::test;
@ -58,7 +58,10 @@ async fn lock_on_zone_a_mints_wrapped_token_on_zone_b() -> Result<()> {
let cross_zone = CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: *channel_a.as_ref(),
allowed_targets: vec![wrapped_token_id],
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::bridge_lock().id(),
target_program_id: wrapped_token_id,
}],
expected_block_signing_pubkey: None,
}],
};

7
integration_tests/tests/cross_zone_ping.rs
View File

@ -23,7 +23,7 @@ use integration_tests::{
use lee::{AccountId, PublicTransaction, public_transaction::Message};
use lee_core::program::ProgramId;
use ping_core::{ReceiverInstruction, SenderInstruction, ping_record_pda};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute};
use sequencer_service_rpc::{RpcClient as _, SequencerClient};
use tokio::test;
@ -49,7 +49,10 @@ async fn ping_crosses_from_zone_a_to_zone_b() -> Result<()> {
let cross_zone = CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: zone_a,
allowed_targets: vec![receiver_id],
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::ping_sender().id(),
target_program_id: receiver_id,
}],
expected_block_signing_pubkey: None,
}],
};

154
integration_tests/tests/cross_zone_state_machine.rs
View File

@ -13,7 +13,7 @@
use std::collections::BTreeMap;
use cross_zone_inbox_core::{
CrossZoneMessage, InboxConfig, Instruction as InboxInstruction, SeenShard,
CrossZoneMessage, CrossZoneRoute, InboxConfig, Instruction as InboxInstruction, SeenShard,
inbox_config_account_id, inbox_seen_shard_account_id, message_key,
};
use cross_zone_outbox_core::{OutboxRecord, outbox_pda};
@ -44,15 +44,21 @@ fn seed_inbox_config(
state: &mut V03State,
self_zone: [u8; 32],
src_zone: [u8; 32],
src_program_id: lee_core::program::ProgramId,
target: lee_core::program::ProgramId,
) {
let inbox_id = programs::cross_zone_inbox().id();
let mut allowed_targets = BTreeMap::new();
allowed_targets.insert(src_zone, vec![target]);
let mut allowed_routes = BTreeMap::new();
allowed_routes.insert(
src_zone,
vec![CrossZoneRoute {
src_program_id,
target_program_id: target,
}],
);
let config = InboxConfig {
self_zone,
allowed_peers: BTreeMap::new(),
allowed_targets,
allowed_routes,
};
*state = std::mem::replace(state, V03State::new()).with_public_accounts([(
inbox_config_account_id(inbox_id),
@ -109,7 +115,7 @@ fn inbox_dispatch_delivers_payload_to_ping_receiver() {
let src_block_id = 5;
let mut state = base_state();
seed_inbox_config(&mut state, self_zone, src_zone, receiver_id);
seed_inbox_config(&mut state, self_zone, src_zone, [9_u32; 8], receiver_id);
// The payload is the ping_receiver instruction, serialized as risc0 words in
// little-endian bytes (the contract the inbox reverses when forwarding).
@ -243,7 +249,13 @@ fn inbox_dispatch_mints_wrapped_token() {
let src_block_id = 5;
let mut state = base_state();
seed_inbox_config(&mut state, self_zone, src_zone, wrapped_token_id);
seed_inbox_config(
&mut state,
self_zone,
src_zone,
[9_u32; 8],
wrapped_token_id,
);
seed_wrapped_config(&mut state);
let msg = CrossZoneMessage {
@ -285,6 +297,126 @@ fn inbox_dispatch_mints_wrapped_token() {
);
}
/// A zone that bridges must allow `wrapped_token` as a target. When that
/// allowance was per peer rather than per source program, it was enough for any
/// emitter on the peer to reach it, and `ping_sender` lets its caller choose the
/// target and payload freely. Any user on the peer could therefore mint wrapped
/// tokens with no lock and no escrow behind them, by routing a `Mint` payload
/// through the ping emitter. The route is the pair, so this must not execute.
#[test]
fn a_mint_from_an_unrouted_emitter_is_rejected() {
let inbox_id = programs::cross_zone_inbox().id();
let wrapped_token_id = programs::wrapped_token().id();
let self_zone = [1_u8; 32];
let src_zone = [2_u8; 32];
let src_block_id = 5;
let mut state = base_state();
// The config a bridging zone writes: the lock program may mint, nothing else.
seed_inbox_config(
&mut state,
self_zone,
src_zone,
programs::bridge_lock().id(),
wrapped_token_id,
);
seed_wrapped_config(&mut state);
let msg = CrossZoneMessage {
src_zone,
src_block_id,
src_tx_index: 0,
// The emitter a user can drive directly, aimed at the bridge's target.
src_program_id: programs::ping_sender().id(),
target_program_id: wrapped_token_id,
payload: mint_payload(),
l1_inclusion_witness: None,
};
let seen_id = inbox_seen_shard_account_id(inbox_id, &src_zone, src_block_id);
let wrapped_config_id = wrapped_token_core::config_account_id(wrapped_token_id);
let holding_id = wrapped_token_core::holding_account_id(wrapped_token_id, &RECIPIENT);
let message = Message::try_new(
inbox_id,
vec![
inbox_config_account_id(inbox_id),
seen_id,
wrapped_config_id,
holding_id,
],
vec![],
InboxInstruction::Dispatch(msg),
)
.expect("build dispatch message");
let tx = PublicTransaction::new(message, WitnessSet::from_raw_parts(vec![]));
assert!(
ValidatedStateDiff::from_public_transaction(&tx, &state, 1, 0).is_err(),
"a delivery from an emitter with no route to wrapped_token must not mint"
);
}
/// The same target reached by the emitter the route names still works. Without
/// this, the test above would pass equally against an inbox that rejected every
/// delivery.
#[test]
fn a_mint_from_the_routed_emitter_is_accepted() {
let inbox_id = programs::cross_zone_inbox().id();
let wrapped_token_id = programs::wrapped_token().id();
let bridge_lock_id = programs::bridge_lock().id();
let self_zone = [1_u8; 32];
let src_zone = [2_u8; 32];
let src_block_id = 5;
let mut state = base_state();
seed_inbox_config(
&mut state,
self_zone,
src_zone,
bridge_lock_id,
wrapped_token_id,
);
seed_wrapped_config(&mut state);
let msg = CrossZoneMessage {
src_zone,
src_block_id,
src_tx_index: 0,
src_program_id: bridge_lock_id,
target_program_id: wrapped_token_id,
payload: mint_payload(),
l1_inclusion_witness: None,
};
let seen_id = inbox_seen_shard_account_id(inbox_id, &src_zone, src_block_id);
let wrapped_config_id = wrapped_token_core::config_account_id(wrapped_token_id);
let holding_id = wrapped_token_core::holding_account_id(wrapped_token_id, &RECIPIENT);
let message = Message::try_new(
inbox_id,
vec![
inbox_config_account_id(inbox_id),
seen_id,
wrapped_config_id,
holding_id,
],
vec![],
InboxInstruction::Dispatch(msg),
)
.expect("build dispatch message");
let tx = PublicTransaction::new(message, WitnessSet::from_raw_parts(vec![]));
let diff = ValidatedStateDiff::from_public_transaction(&tx, &state, 1, 0)
.expect("the routed emitter must still deliver");
let minted = wrapped_token_core::read_balance(
&diff.public_diff()[&holding_id].data.clone().into_inner(),
);
assert_eq!(minted, LOCK_AMOUNT);
}
/// A dispatch whose message key is already in the seen-shard is an idempotent
/// no-op: the inbox makes no chained call, so the wrapped token is not minted a
/// second time. This is the bridge's replay defense.
@ -299,7 +431,13 @@ fn mint_replay_rejected() {
let src_tx_index = 0;
let mut state = base_state();
seed_inbox_config(&mut state, self_zone, src_zone, wrapped_token_id);
seed_inbox_config(
&mut state,
self_zone,
src_zone,
[9_u32; 8],
wrapped_token_id,
);
seed_wrapped_config(&mut state);
// Seed the seen-shard as already containing this message's key, so the inbox

7
integration_tests/tests/cross_zone_verified.rs
View File

@ -22,7 +22,7 @@ use integration_tests::{
use lee::{AccountId, PublicTransaction, public_transaction::Message};
use lee_core::program::ProgramId;
use ping_core::{ReceiverInstruction, SenderInstruction, ping_record_pda};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute};
use sequencer_service_rpc::RpcClient as _;
use tokio::test;
@ -46,7 +46,10 @@ async fn indexer_verifies_and_delivers_cross_zone_ping() -> Result<()> {
let cross_zone = CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: zone_a,
allowed_targets: vec![receiver_id],
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::ping_sender().id(),
target_program_id: receiver_id,
}],
expected_block_signing_pubkey: None,
}],
};

230
integration_tests/tests/cross_zone_watcher_restart.rs Normal file
View File

@ -0,0 +1,230 @@
#![expect(
clippy::tests_outside_test_module,
reason = "top-level test functions are conventional for integration tests"
)]
//! A sequencer restart must resume its cross-zone watcher from the persisted
//! per-peer delivery floor instead of re-reading the peer channel from genesis.
//!
//! Re-reading is safe (the dispatch key is content-addressed and the inbox
//! no-ops a replay) so on-chain state cannot tell the two apart. What does tell
//! them apart is the transactions: a watcher that lost its cursor re-injects
//! every already-delivered dispatch, which shows up as inbox transactions in
//! blocks produced after the restart. This is the only test that covers the
//! wiring from `spawn_watchers` through the store, so a silent regression here
//! would leave every other test green while the feature does nothing.
use std::time::Duration;
use anyhow::{Context as _, Result};
use common::transaction::LeeTransaction;
use cross_zone_outbox_core::outbox_pda;
use integration_tests::{
config::{self, SequencerPartialConfig},
setup::{SequencerSetup, sequencer_client, setup_bedrock_node},
};
use lee::{AccountId, PublicTransaction, public_transaction::Message};
use lee_core::program::ProgramId;
use ping_core::{ReceiverInstruction, SenderInstruction, ping_record_pda};
use sequencer_core::config::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute};
use sequencer_service_rpc::{RpcClient as _, SequencerClient};
use tokio::test;
const DELIVERY_TIMEOUT: Duration = Duration::from_secs(480);
/// Blocks zone B must produce after the restart before we judge the watcher.
/// A watcher that lost its cursor re-reads the peer channel on its first pass,
/// so a handful of blocks is ample room for the replay to appear.
const BLOCKS_AFTER_RESTART: u64 = 5;
const RESTART_TIMEOUT: Duration = Duration::from_secs(240);
const PING_PAYLOAD: &[u8] = b"hello-cross-zone";
#[test]
async fn restarted_watcher_resumes_instead_of_replaying_the_peer_channel() -> Result<()> {
// Declared first so it outlives both zones (drops run in reverse order).
let (_bedrock, bedrock_addr) = setup_bedrock_node()
.await
.context("Failed to set up shared Bedrock node")?;
let partial = SequencerPartialConfig::default();
let channel_a = config::bedrock_channel_id();
let channel_b = config::bedrock_channel_id_b();
let zone_a: [u8; 32] = *channel_a.as_ref();
let zone_b: [u8; 32] = *channel_b.as_ref();
let receiver_id = programs::ping_receiver().id();
let cross_zone = CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: zone_a,
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::ping_sender().id(),
target_program_id: receiver_id,
}],
expected_block_signing_pubkey: None,
}],
};
let (seq_a, _seq_a_home) = SequencerSetup::new(partial, bedrock_addr)
.with_channel_id(channel_a)
.with_genesis(vec![])
.setup()
.await
.context("Failed to set up zone A sequencer")?;
// Zone B keeps an explicit home so it can be restarted on the same store.
let home_b = tempfile::tempdir().context("Failed to create zone B home")?;
let mut seq_b = SequencerSetup::new(partial, bedrock_addr)
.with_channel_id(channel_b)
.with_genesis(vec![])
.with_cross_zone(cross_zone.clone())
.setup_at(home_b.path())
.await
.context("Failed to set up zone B sequencer")?;
// Deliver one ping, so the peer channel holds a dispatch worth replaying.
sequencer_client(seq_a.addr())?
.send_transaction(build_ping_tx(zone_b, receiver_id))
.await
.context("Failed to submit ping on zone A")?;
let record_id = ping_record_pda(receiver_id);
let delivered = wait_for_delivery(sequencer_client(seq_b.addr())?, record_id).await?;
assert_eq!(
delivered, PING_PAYLOAD,
"Zone B must record the payload before the restart"
);
let tip_before = sequencer_client(seq_b.addr())?.get_last_block_id().await?;
// Restart zone B on the same home. Zone A stays quiet from here, so any
// inbox transaction after the restart is a replay, not a new delivery.
//
// `shutdown` rather than `drop`: dropping aborts the main loop without
// awaiting it and leaves the watchers and the publisher's drive task holding
// the store, so the reopen below would race the `RocksDB` lock.
seq_b.shutdown().await;
seq_b = SequencerSetup::new(partial, bedrock_addr)
.with_channel_id(channel_b)
.with_genesis(vec![])
.with_cross_zone(cross_zone)
.setup_at(home_b.path())
.await
.context("Failed to restart zone B sequencer")?;
let client_b = sequencer_client(seq_b.addr())?;
let tip_after = wait_for_block_id(
&client_b,
tip_before.saturating_add(BLOCKS_AFTER_RESTART),
RESTART_TIMEOUT,
)
.await?;
let replayed = count_inbox_transactions(&client_b, tip_before.saturating_add(1), tip_after)
.await
.context("Failed to scan zone B blocks after the restart")?;
assert_eq!(
replayed,
0,
"a restarted watcher must resume from its persisted delivery floor; found {replayed} inbox transaction(s) in blocks {}..={tip_after}, which means it re-read the peer channel from genesis",
tip_before.saturating_add(1)
);
// The delivery itself must survive the restart untouched.
let account = client_b.get_account(record_id).await?;
assert_eq!(
account.data.into_inner(),
PING_PAYLOAD,
"the delivered payload must survive the restart"
);
Ok(())
}
/// Counts inbox transactions across `from..=to`, the signature of a re-injected
/// dispatch.
async fn count_inbox_transactions(client: &SequencerClient, from: u64, to: u64) -> Result<usize> {
let inbox_id = programs::cross_zone_inbox().id();
let mut count = 0_usize;
for block_id in from..=to {
let Some(block) = client.get_block(block_id).await? else {
continue;
};
for tx in &block.body.transactions {
if let LeeTransaction::Public(public_tx) = tx
&& public_tx.message().program_id == inbox_id
{
count = count.saturating_add(1);
}
}
}
Ok(count)
}
/// Waits until the sequencer's tip reaches `target`, returning the tip.
async fn wait_for_block_id(
client: &SequencerClient,
target: u64,
timeout: Duration,
) -> Result<u64> {
let wait = async {
loop {
let tip = client.get_last_block_id().await?;
if tip >= target {
return Ok::<u64, anyhow::Error>(tip);
}
tokio::time::sleep(Duration::from_secs(2)).await;
}
};
tokio::time::timeout(timeout, wait)
.await
.context("Zone B did not produce enough blocks after the restart")?
}
/// Builds a top-level `ping_sender` transaction that chains into the outbox to emit
/// a message carrying a `ping_receiver::Record` instruction for the target zone.
fn build_ping_tx(target_zone: [u8; 32], receiver_id: ProgramId) -> LeeTransaction {
let outbox_id = programs::cross_zone_outbox().id();
let ordinal = 0;
let words = risc0_zkvm::serde::to_vec(&ReceiverInstruction::Record {
payload: PING_PAYLOAD.to_vec(),
})
.expect("serialize ping instruction");
let payload: Vec<u8> = words.iter().flat_map(|word| word.to_le_bytes()).collect();
let send = SenderInstruction::Send {
outbox_program_id: outbox_id,
target_zone,
target_program_id: receiver_id,
target_accounts: vec![ping_record_pda(receiver_id).into_value()],
payload,
ordinal,
};
let outbox_account = outbox_pda(outbox_id, &target_zone, ordinal);
let message = Message::try_new(
programs::ping_sender().id(),
vec![outbox_account],
vec![],
send,
)
.expect("build ping message");
LeeTransaction::Public(PublicTransaction::new(
message,
lee::public_transaction::WitnessSet::from_raw_parts(vec![]),
))
}
/// Polls zone B's sequencer until the ping record PDA holds a payload.
async fn wait_for_delivery(client: SequencerClient, record_id: AccountId) -> Result<Vec<u8>> {
let wait = async {
loop {
let account = client.get_account(record_id).await?;
let data = account.data.into_inner();
if !data.is_empty() {
return Ok::<Vec<u8>, anyhow::Error>(data);
}
tokio::time::sleep(Duration::from_secs(3)).await;
}
};
tokio::time::timeout(DELIVERY_TIMEOUT, wait)
.await
.context("Zone B did not record the cross-zone payload in time")?
}

1
integration_tests/tests/multi_sequencer.rs
View File

@ -70,6 +70,7 @@ async fn multi_sequencer_committee_converges() -> Result<()> {
&BedrockConfig {
channel_id: config::bedrock_channel_id(),
node_url: config::addr_to_url(config::UrlProtocol::Http, bedrock_addr)?,
funding_key: config::bedrock_funding_key(),
auth: None,
},
&Ed25519Key::from_bytes(&key_a),

124
integration_tests/tests/wallet_ffi.rs
View File

@ -170,13 +170,13 @@ unsafe extern "C" {
fn wallet_ffi_free_transfer_result(result: *mut FfiTransferResult);
fn wallet_ffi_bridge_withdraw(
handle: *mut WalletHandle,
from: *const FfiBytes32,
amount: u64,
bedrock_account_pk: *const FfiBytes32,
out_result: *mut FfiTransferResult,
) -> error::WalletFfiError;
// fn wallet_ffi_bridge_withdraw(
// handle: *mut WalletHandle,
// from: *const FfiBytes32,
// amount: u64,
// bedrock_account_pk: *const FfiBytes32,
// out_result: *mut FfiTransferResult,
// ) -> error::WalletFfiError;
fn wallet_ffi_get_vault_balance(
handle: *mut WalletHandle,
@ -1521,68 +1521,68 @@ fn restore_keys_from_seed_ffi() -> Result<()> {
Ok(())
}
#[test]
fn test_wallet_ffi_bridge_withdraw() -> Result<()> {
let ctx = BlockingTestContext::new()?;
let home = tempfile::tempdir()?;
let FfiCreateWalletOutput {
wallet: wallet_ffi_handle,
mnemonic: _,
} = new_wallet_ffi_with_test_context_config(&ctx, home.path())?;
let from: FfiBytes32 = ctx.ctx().existing_public_accounts()[0].into();
let bridge_account: FfiBytes32 = system_accounts::bridge_account_id().into();
let bedrock_account_pk = FfiBytes32::from_bytes([0x42; 32]);
let amount = 100_u64;
// #[test]
// fn test_wallet_ffi_bridge_withdraw() -> Result<()> {
// let ctx = BlockingTestContext::new()?;
// let home = tempfile::tempdir()?;
// let FfiCreateWalletOutput {
// wallet: wallet_ffi_handle,
// mnemonic: _,
// } = new_wallet_ffi_with_test_context_config(&ctx, home.path())?;
// let from: FfiBytes32 = ctx.ctx().existing_public_accounts()[0].into();
// let bridge_account: FfiBytes32 = system_accounts::bridge_account_id().into();
// let bedrock_account_pk = FfiBytes32::from_bytes([0x42; 32]);
// let amount = 100_u64;
let mut transfer_result = FfiTransferResult::default();
unsafe {
wallet_ffi_bridge_withdraw(
wallet_ffi_handle,
&raw const from,
amount,
&raw const bedrock_account_pk,
&raw mut transfer_result,
)
.unwrap();
}
// let mut transfer_result = FfiTransferResult::default();
// unsafe {
// wallet_ffi_bridge_withdraw(
// wallet_ffi_handle,
// &raw const from,
// amount,
// &raw const bedrock_account_pk,
// &raw mut transfer_result,
// )
// .unwrap();
// }
info!("Waiting for next block creation");
std::thread::sleep(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS));
// info!("Waiting for next block creation");
// std::thread::sleep(Duration::from_secs(TIME_TO_WAIT_FOR_BLOCK_SECONDS));
let from_balance = unsafe {
let mut out_balance: [u8; 16] = [0; 16];
wallet_ffi_get_balance(
wallet_ffi_handle,
&raw const from,
true,
&raw mut out_balance,
)
.unwrap();
u128::from_le_bytes(out_balance)
};
// let from_balance = unsafe {
// let mut out_balance: [u8; 16] = [0; 16];
// wallet_ffi_get_balance(
// wallet_ffi_handle,
// &raw const from,
// true,
// &raw mut out_balance,
// )
// .unwrap();
// u128::from_le_bytes(out_balance)
// };
let bridge_balance = unsafe {
let mut out_balance: [u8; 16] = [0; 16];
wallet_ffi_get_balance(
wallet_ffi_handle,
&raw const bridge_account,
true,
&raw mut out_balance,
)
.unwrap();
u128::from_le_bytes(out_balance)
};
// let bridge_balance = unsafe {
// let mut out_balance: [u8; 16] = [0; 16];
// wallet_ffi_get_balance(
// wallet_ffi_handle,
// &raw const bridge_account,
// true,
// &raw mut out_balance,
// )
// .unwrap();
// u128::from_le_bytes(out_balance)
// };
assert_eq!(from_balance, 9900);
assert_eq!(bridge_balance, 1_000_100);
// assert_eq!(from_balance, 9900);
// assert_eq!(bridge_balance, 1_000_100);
unsafe {
wallet_ffi_free_transfer_result(&raw mut transfer_result);
wallet_ffi_destroy(wallet_ffi_handle);
}
// unsafe {
// wallet_ffi_free_transfer_result(&raw mut transfer_result);
// wallet_ffi_destroy(wallet_ffi_handle);
// }
Ok(())
}
// Ok(())
// }
#[test]
fn test_wallet_ffi_transfer_generic_public() -> Result<()> {

2
lee/state_machine/src/lib.rs
View File

@ -30,6 +30,8 @@ pub mod program_deployment_transaction;
pub mod public_transaction;
mod signature;
mod state;
#[cfg(feature = "test-utils")]
pub mod test_utils;
mod validated_state_diff;
mod privacy_preserving_circuit {

28
lee/state_machine/src/test_utils.rs Normal file
View File

@ -0,0 +1,28 @@
//! Test-only constructors for otherwise-opaque state types.
//!
//! A [`ValidatedStateDiff`] can normally only be produced by the transaction validation
//! functions, which guarantees it has been checked before any state mutation. These
//! helpers let downstream crates unit-test *post-execution* validation logic — e.g. the
//! system-account and bridge guards in `common` — against a hand-built diff, without
//! running a program in the zkVM.
use std::collections::HashMap;
use crate::{
Account, AccountId,
validated_state_diff::{StateDiff, ValidatedStateDiff},
};
/// Builds a [`ValidatedStateDiff`] carrying only the given public-account changes.
#[must_use]
pub const fn validated_state_diff_from_public_diff(
public_diff: HashMap<AccountId, Account>,
) -> ValidatedStateDiff {
ValidatedStateDiff::new_unchecked(StateDiff {
signer_account_ids: Vec::new(),
public_diff,
new_commitments: Vec::new(),
new_nullifiers: Vec::new(),
program: None,
})
}

18
lee/state_machine/src/validated_state_diff/mod.rs
View File

@ -35,10 +35,24 @@ pub struct StateDiff {
/// The validated output of executing or verifying a transaction, ready to be applied to the state.
///
/// Can only be constructed by the transaction validation functions inside this crate, ensuring the
/// diff has been checked before any state mutation occurs.
/// It can only be constructed by the transaction validation functions inside this crate, ensuring
/// the diff has been checked before any state mutation occurs. Under the `test-utils` feature the
/// [`crate::test_utils`] module additionally exposes a hand-rolled constructor for unit-testing
/// downstream validation logic; that feature must never be enabled in a production build.
pub struct ValidatedStateDiff(StateDiff);
#[cfg(feature = "test-utils")]
impl ValidatedStateDiff {
/// Test-only constructor that wraps an already-built [`StateDiff`] **without validating it**.
///
/// Kept in this module so the wrapped field can stay private: in a normal build (feature off)
/// the only ways to obtain a `ValidatedStateDiff` remain the `from_*_transaction` validators.
#[must_use]
pub const fn new_unchecked(state_diff: StateDiff) -> Self {
Self(state_diff)
}
}
impl ValidatedStateDiff {
pub fn from_public_transaction(
tx: &PublicTransaction,

3
lez/common/Cargo.toml
View File

@ -25,3 +25,6 @@ log.workspace = true
hex.workspace = true
borsh.workspace = true
logos-blockchain-common-http-client.workspace = true
[dev-dependencies]
lee = { workspace = true, features = ["test-utils"] }

35
lez/common/src/test_utils.rs
View File

@ -1,4 +1,12 @@
// Backs the hand-built state/diff helpers below, which are compiled only for `common`'s own
// unit tests. They rely on `lee::test_utils`, gated behind `lee`'s `test-utils` feature and
// enabled here via dev-dependencies, so it never reaches a production build.
#[cfg(test)]
use std::collections::HashMap;
use lee::AccountId;
#[cfg(test)]
use lee::{Account, PrivateKey, PublicKey, V03State, ValidatedStateDiff};
use crate::{
HashType,
@ -13,6 +21,33 @@ pub fn sequencer_sign_key_for_testing() -> lee::PrivateKey {
lee::PrivateKey::try_new([37; 32]).unwrap()
}
/// A syntactically valid `Public` transaction. Its contents are irrelevant to the
/// bridge guard, which only branches on the transaction *variant* and the diff.
#[cfg(test)]
#[must_use]
pub fn any_public_transaction() -> LeeTransaction {
let sender_key = PrivateKey::try_new([9_u8; 32]).expect("valid key");
let sender_id = AccountId::from(&PublicKey::new_from_private_key(&sender_key));
let recipient_key = PrivateKey::try_new([8_u8; 32]).expect("valid key");
let recipient_id = AccountId::from(&PublicKey::new_from_private_key(&recipient_key));
create_transaction_native_token_transfer(sender_id, 0, recipient_id, 1, &sender_key)
}
/// Builds a state whose only entry is `account_id` (set to `pre`) and a single-entry diff
/// that maps `account_id` to `post`, so the validation guards can be exercised in isolation.
#[cfg(test)]
#[must_use]
pub fn state_and_diff(
account_id: AccountId,
pre: Account,
post: Account,
) -> (V03State, ValidatedStateDiff) {
let state = V03State::new().with_public_accounts([(account_id, pre)]);
let diff =
lee::test_utils::validated_state_diff_from_public_diff(HashMap::from([(account_id, post)]));
(state, diff)
}
// Dummy producers
/// Produce dummy block with provided transactions + clock transaction an the end.

116
lez/common/src/transaction.rs
View File

@ -35,6 +35,15 @@ impl LeeTransaction {
})
}
#[must_use]
pub const fn kind(&self) -> TxKind {
match self {
Self::Public(_) => TxKind::Public,
Self::PrivacyPreserving(_) => TxKind::PrivacyPreserving,
Self::ProgramDeployment(_) => TxKind::ProgramDeployment,
}
}
#[must_use]
pub fn affected_public_account_ids(&self) -> Vec<AccountId> {
match self {
@ -255,9 +264,112 @@ fn validate_doesnt_modify_account(
#[cfg(test)]
mod tests {
use lee::{AccountId, PrivateKey, PublicKey, V03State};
use lee::{Account, AccountId, PrivateKey, PublicKey, V03State};
use lee_core::account::Nonce;
use crate::test_utils::create_transaction_native_token_transfer;
use super::validate_doesnt_modify_account;
use crate::test_utils::{
any_public_transaction, create_transaction_native_token_transfer, state_and_diff,
};
#[test]
fn bridge_guard_allows_balance_only_increase() {
// A diff that *only* increases the bridge balance (the legitimate deposit shape)
// must be accepted.
let bridge_id = system_accounts::bridge_account_id();
let pre = Account {
balance: 500,
nonce: Nonce(7),
..Account::default()
};
let post = Account {
balance: 600,
..pre.clone()
};
let (state, diff) = state_and_diff(bridge_id, pre, post);
let tx = any_public_transaction();
assert!(
tx.validate_bridge_account_modification(&state, &diff)
.is_ok(),
"a balance-only increase of the bridge account must be allowed",
);
}
#[test]
fn bridge_guard_rejects_data_modification_even_when_balance_increases() {
// A diff that changes the bridge account's data (here: the nonce) while *also*
// increasing its balance must be rejected.
let bridge_id = system_accounts::bridge_account_id();
let pre = Account {
balance: 500,
nonce: Nonce(7),
..Account::default()
};
let post = Account {
balance: 600,
nonce: Nonce(8),
..pre.clone()
};
let (state, diff) = state_and_diff(bridge_id, pre, post);
let tx = any_public_transaction();
assert!(
tx.validate_bridge_account_modification(&state, &diff)
.is_err(),
"modifying bridge account data must be rejected even if the balance increases",
);
}
#[test]
fn bridge_guard_rejects_zero_value_deposit() {
// A diff that touches the bridge account without *strictly* increasing its balance
// must be rejected — a zero-value deposit is not a real credit.
let bridge_id = system_accounts::bridge_account_id();
let pre = Account {
balance: 500,
nonce: Nonce(7),
..Account::default()
};
let post = pre.clone();
let (state, diff) = state_and_diff(bridge_id, pre, post);
let tx = any_public_transaction();
assert!(
tx.validate_bridge_account_modification(&state, &diff)
.is_err(),
"a bridge diff that does not strictly increase the balance must be rejected",
);
}
#[test]
fn validate_doesnt_modify_account_flags_a_changed_account() {
// Directly exercise the system-account guard with a diff that genuinely changes a
// clock account, then with one that leaves it untouched. The inverted comparison would
// treat a changed account as unchanged and wave it through (and would flag an *unchanged*
// account instead).
let clock_id = system_accounts::clock_account_ids()[0];
let pre = Account {
balance: 1_000,
..Account::default()
};
let changed = Account {
balance: 2_000,
..Account::default()
};
let (state, diff) = state_and_diff(clock_id, pre.clone(), changed);
assert!(
validate_doesnt_modify_account(&state, &diff, clock_id).is_err(),
"a diff that changes a system account must be rejected",
);
let (unchanged_state, unchanged_diff) = state_and_diff(clock_id, pre.clone(), pre);
assert!(
validate_doesnt_modify_account(&unchanged_state, &unchanged_diff, clock_id).is_ok(),
"a diff that leaves a system account unchanged must be accepted",
);
}
#[test]
fn system_account_ids_are_distinct_and_non_default() {

3
lez/configs/docker-all-in-one/sequencer_config.json
View File

@ -11,7 +11,8 @@
"max_retries": 5
},
"channel_id": "0101010101010101010101010101010101010101010101010101010101010101",
"node_url": "http://logos-blockchain-node-0:18080"
"node_url": "http://logos-blockchain-node-0:18080",
"funding_key": "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26"
},
"genesis": [
{

11
lez/cross_zone/src/lib.rs
View File

@ -143,17 +143,16 @@ pub fn build_dispatch_from_emission(
build_inbox_dispatch_tx(programs::cross_zone_inbox().id(), &msg, target_ids)
}
/// The inbox config a zone derives from its cross-zone config: the per-peer target
/// allowlists plus its own zone id.
/// The inbox config a zone derives from its cross-zone config: the per-peer
/// delivery routes plus its own zone id.
fn inbox_config(self_zone: ZoneId, cross_zone: &CrossZoneConfig) -> InboxConfig {
let mut allowed_targets = BTreeMap::new();
let mut allowed_routes = BTreeMap::new();
for peer in &cross_zone.peers {
allowed_targets.insert(peer.channel_id, peer.allowed_targets.clone());
allowed_routes.insert(peer.channel_id, peer.allowed_routes.clone());
}
InboxConfig {
self_zone,
allowed_peers: BTreeMap::new(),
allowed_targets,
allowed_routes,
}
}

1
lez/mempool/Cargo.toml
View File

@ -12,3 +12,4 @@ tokio = { workspace = true, features = ["sync"] }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
futures.workspace = true

33
lez/mempool/src/lib.rs
View File

@ -18,6 +18,19 @@ impl<T> MemPool<T> {
(mem_pool, sender)
}
/// Returns the total number of items in the mempool, including both the front buffer and the
/// channel.
#[must_use]
pub fn len(&self) -> usize {
self.front_buffer.len().saturating_add(self.receiver.len())
}
/// Returns true if the mempool is empty, false otherwise.
#[must_use]
pub fn is_empty(&self) -> bool {
self.front_buffer.is_empty() && self.receiver.is_empty()
}
/// Pop an item from the mempool first checking the front buffer (LIFO) then the channel (FIFO).
pub fn pop(&mut self) -> Option<T> {
use tokio::sync::mpsc::error::TryRecvError;
@ -74,6 +87,7 @@ impl<T> MemPoolHandle<T> {
#[cfg(test)]
mod tests {
use futures::FutureExt as _;
use tokio::test;
use super::*;
@ -82,6 +96,7 @@ mod tests {
async fn mempool_new() {
let (mut pool, _handle): (MemPool<u64>, _) = MemPool::new(10);
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
@ -89,10 +104,12 @@ mod tests {
let (mut pool, handle) = MemPool::new(10);
handle.push(1).await.unwrap();
assert_eq!(pool.len(), 1);
let item = pool.pop();
assert_eq!(item, Some(1));
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
@ -103,29 +120,23 @@ mod tests {
handle.push(2).await.unwrap();
handle.push(3).await.unwrap();
assert_eq!(pool.len(), 3);
assert_eq!(pool.pop(), Some(1));
assert_eq!(pool.pop(), Some(2));
assert_eq!(pool.pop(), Some(3));
assert_eq!(pool.pop(), None);
}
#[test]
async fn pop_empty() {
let (mut pool, _handle): (MemPool<u64>, _) = MemPool::new(10);
assert_eq!(pool.pop(), None);
assert_eq!(pool.len(), 0);
}
#[test]
async fn max_size() {
let (mut pool, handle) = MemPool::new(2);
let (_pool, handle) = MemPool::new(2);
handle.push(1).await.unwrap();
handle.push(2).await.unwrap();
// This should block if buffer is full, but we'll use try_send in a real scenario
// For now, just verify we can pop items
assert_eq!(pool.pop(), Some(1));
assert_eq!(pool.pop(), Some(2));
// This should block if buffer is full
assert_eq!(handle.push(3).now_or_never(), None);
}
#[test]

1
lez/programs/bridge/Cargo.toml
View File

@ -7,5 +7,4 @@ license = { workspace = true }
[dependencies]
bridge_core.workspace = true
vault_core.workspace = true
authenticated_transfer_core.workspace = true
lee_core.workspace = true

46
lez/programs/bridge/src/main.rs
View File

@ -104,33 +104,35 @@ fn main() {
}
}
Instruction::Withdraw {
amount,
amount: _,
bedrock_account_pk: _,
} => {
let [sender, bridge] = pre_states
.try_into()
.expect("Withdraw requires exactly 2 accounts");
panic!("Withdraws are disabled in the current version of LEZ");
assert_eq!(
bridge.account_id,
bridge_core::compute_bridge_account_id(self_program_id),
"Second account must be bridge PDA"
);
// let [sender, bridge] = pre_states
// .try_into()
// .expect("Withdraw requires exactly 2 accounts");
let auth_transfer_program_id = bridge.account.program_owner;
assert_eq!(
sender.account.program_owner, auth_transfer_program_id,
"Sender account must be owned by the authenticated transfer program"
);
// assert_eq!(
// bridge.account_id,
// bridge_core::compute_bridge_account_id(self_program_id),
// "Second account must be bridge PDA"
// );
let chained_calls = vec![ChainedCall::new(
auth_transfer_program_id,
vec![sender, bridge],
&authenticated_transfer_core::Instruction::Transfer {
amount: u128::from(amount),
},
)];
(unchanged_post_states(&pre_states_clone), chained_calls)
// let auth_transfer_program_id = bridge.account.program_owner;
// assert_eq!(
// sender.account.program_owner, auth_transfer_program_id,
// "Sender account must be owned by the authenticated transfer program"
// );
// let chained_calls = vec![ChainedCall::new(
// auth_transfer_program_id,
// vec![sender, bridge],
// &authenticated_transfer_core::Instruction::Transfer {
// amount: u128::from(amount),
// },
// )];
// (unchanged_post_states(&pre_states_clone), chained_calls)
}
};

118
lez/programs/cross_zone_inbox/core/src/lib.rs
View File

@ -23,13 +23,30 @@ pub type ExpectedPubkey = [u8; 32];
/// Content-addressed replay key for a delivered message.
pub type MessageKey = [u8; 32];
/// One delivery a peer is allowed to make: a program on the peer that may emit,
/// paired with the program here it may reach.
///
/// The pair is the unit rather than two independent lists. A bridging peer needs
/// `wrapped_token` reachable, and any emitter that lets its caller choose the
/// target (`ping_sender` does) would otherwise reach it too, minting tokens with
/// no lock behind them. Naming the pair is what stops two separately reasonable
/// entries composing into a route nobody wrote down.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)]
pub struct CrossZoneRoute {
/// The program on the peer zone that emitted the message.
pub src_program_id: ProgramId,
/// The program on this zone it may be delivered to.
pub target_program_id: ProgramId,
}
/// A peer zone whose outbox a zone watches for inbound cross-zone messages.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CrossZonePeer {
/// The peer's Bedrock channel; its 32 bytes double as the peer's zone id.
pub channel_id: ZoneId,
/// Programs on the local zone a message from this peer is allowed to target.
pub allowed_targets: Vec<ProgramId>,
/// The deliveries this peer may make: which of its programs may emit, and
/// what each of them may reach here.
pub allowed_routes: Vec<CrossZoneRoute>,
/// The peer's block-signing public key, pinned to reject blocks inscribed by
/// anyone other than that zone's sequencer. `None` skips the check (the
/// channel signer is still authenticated by the zone-sdk).
@ -60,17 +77,32 @@ pub struct CrossZoneMessage {
pub l1_inclusion_witness: Option<Vec<u8>>,
}
/// Peer and per-peer target allowlists, plus this inbox's own zone id.
/// Per-peer delivery routes, plus this inbox's own zone id.
#[derive(
Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize,
)]
pub struct InboxConfig {
pub self_zone: ZoneId,
pub allowed_peers: BTreeMap<ZoneId, ExpectedPubkey>,
pub allowed_targets: BTreeMap<ZoneId, Vec<ProgramId>>,
/// Which deliveries each peer may make. A peer absent from this map may
/// deliver nothing.
pub allowed_routes: BTreeMap<ZoneId, Vec<CrossZoneRoute>>,
}
impl InboxConfig {
/// Whether `src_zone` may deliver from `src_program_id` to
/// `target_program_id`. A peer with no routes may deliver nothing.
#[must_use]
pub fn permits(
&self,
src_zone: &ZoneId,
src_program_id: ProgramId,
target_program_id: ProgramId,
) -> bool {
self.allowed_routes
.get(src_zone)
.is_some_and(|routes| routes_permit(routes, src_program_id, target_program_id))
}
/// Borsh-encoded form stored in the inbox config account.
#[must_use]
pub fn to_bytes(&self) -> Vec<u8> {
@ -122,6 +154,25 @@ pub enum Instruction {
InitConfig(InboxConfig),
}
/// Whether `routes` authorize a delivery from `src_program_id` to
/// `target_program_id`.
///
/// The one place the rule lives. The inbox guest decides with it and the
/// sequencer's watcher drops unroutable messages with it, and those two must
/// agree: a watcher stricter than the guest loses messages silently, and one
/// looser records deliveries the guest will refuse, which production then feeds
/// in and gives up on.
#[must_use]
pub fn routes_permit(
routes: &[CrossZoneRoute],
src_program_id: ProgramId,
target_program_id: ProgramId,
) -> bool {
routes.iter().any(|route| {
route.src_program_id == src_program_id && route.target_program_id == target_program_id
})
}
/// Content-addressed replay key for a delivered message.
///
/// Hashes `(src_zone, src_block_id, src_tx_index)` under a domain separator.
@ -191,6 +242,63 @@ mod tests {
[b; 32]
}
fn program(n: u32) -> ProgramId {
[n; 8]
}
/// The route is the pair. Two entries that are each reasonable on their own,
/// a lock program that may mint and a ping emitter that may reach a
/// receiver, must not compose into the lock program's target being
/// reachable from the ping emitter: that emitter lets its caller choose the
/// target, so it would mint with nothing locked behind it.
#[test]
fn a_route_authorizes_one_pair_and_does_not_compose() {
let lock = program(1);
let wrapped_token = program(2);
let ping_sender = program(3);
let ping_receiver = program(4);
let mut allowed_routes = BTreeMap::new();
allowed_routes.insert(
zone(9),
vec![
CrossZoneRoute {
src_program_id: lock,
target_program_id: wrapped_token,
},
CrossZoneRoute {
src_program_id: ping_sender,
target_program_id: ping_receiver,
},
],
);
let config = InboxConfig {
self_zone: zone(1),
allowed_routes,
};
assert!(config.permits(&zone(9), lock, wrapped_token));
assert!(config.permits(&zone(9), ping_sender, ping_receiver));
assert!(
!config.permits(&zone(9), ping_sender, wrapped_token),
"an emitter whose caller picks the target must not reach the bridge's target"
);
assert!(
!config.permits(&zone(9), lock, ping_receiver),
"a route grants its own target, not every target the peer has"
);
}
#[test]
fn a_peer_with_no_routes_may_deliver_nothing() {
let config = InboxConfig {
self_zone: zone(1),
allowed_routes: BTreeMap::new(),
};
assert!(!config.permits(&zone(9), program(1), program(2)));
}
#[test]
fn message_key_is_stable_and_content_addressed() {
assert_eq!(message_key(&zone(1), 7, 3), message_key(&zone(1), 7, 3));

12
lez/programs/cross_zone_inbox/src/main.rs
View File

@ -85,13 +85,13 @@ fn dispatch(
msg.src_zone != cfg.self_zone,
"Source zone must not be this zone"
);
let allowed_targets = cfg
.allowed_targets
.get(&msg.src_zone)
.expect("Source zone is not an allowed peer");
// Checked as a pair. The emitting program is as much a part of the
// authorization as the target: an emitter whose caller chooses the target
// reaches everything the peer may reach, so a target allowlist on its own
// lets any such emitter stand in for every other one.
assert!(
allowed_targets.contains(&msg.target_program_id),
"Target program is not allowed for this peer"
cfg.permits(&msg.src_zone, msg.src_program_id, msg.target_program_id),
"No route from this source program to this target program for this peer"
);
let key = message_key(&msg.src_zone, msg.src_block_id, msg.src_tx_index);

3
lez/sequencer/core/Cargo.toml
View File

@ -12,6 +12,7 @@ lee.workspace = true
lee_core.workspace = true
chain_state.workspace = true
common.workspace = true
sequencer_core_metrics = { workspace = true, features = ["record"] }
storage.workspace = true
mempool.workspace = true
logos-blockchain-zone-sdk.workspace = true
@ -26,6 +27,7 @@ cross_zone_inbox_core.workspace = true
logos-blockchain-key-management-system-service.workspace = true
logos-blockchain-core.workspace = true
logos-blockchain-http-api-common.workspace = true
anyhow.workspace = true
serde.workspace = true
serde_json.workspace = true
@ -57,3 +59,4 @@ test_programs.workspace = true
lee = { workspace = true, features = ["test-utils"] }
key_protocol.workspace = true
token_core.workspace = true
ping_core.workspace = true

19
lez/sequencer/core/metrics/Cargo.toml Normal file
View File

@ -0,0 +1,19 @@
[package]
name = "sequencer_core_metrics"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[features]
default = []
# Enable metrics record
record = ["dep:common", "dep:metrics", "dep:strum"]
[dependencies]
common = { workspace = true, optional = true }
metrics = { workspace = true, optional = true }
strum = { workspace = true, optional = true }

9
lez/sequencer/core/metrics/src/lib.rs Normal file
View File

@ -0,0 +1,9 @@
//! This crate provides all metrics exposed by the sequencer core crate.
#[cfg(feature = "record")]
pub use record::*;
pub mod names;
#[cfg(feature = "record")]
pub mod record;

9
lez/sequencer/core/metrics/src/names.rs Normal file
View File

@ -0,0 +1,9 @@
pub const BLOCK_CREATION_TIME: &str = "block_creation_time_seconds";
pub const CHAIN_HEIGHT: &str = "chain_height";
pub const BLOCKS_PRODUCED_TOTAL: &str = "blocks_produced_total";
pub const MEMPOOL_SIZE: &str = "mempool_size";
pub const MEMPOOL_MAX_SIZE: &str = "mempool_max_size";
pub const MEMPOOL_TRANSACTION_APPLICATION_TIME: &str =
"mempool_transaction_application_time_seconds";
pub const TRANSACTIONS_PER_BLOCK: &str = "transactions_per_block";
pub const MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str = "mempool_failed_transactions_total";

167
lez/sequencer/core/metrics/src/record.rs Normal file
View File

@ -0,0 +1,167 @@
#![expect(
clippy::cast_precision_loss,
clippy::as_conversions,
reason = "It's okay for metrics"
)]
use std::time::Duration;
use metrics::{Counter, Histogram, Unit, counter, gauge, histogram};
use strum::IntoEnumIterator as _;
use crate::names;
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum TransactionOrigin {
User,
Sequencer,
}
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum TxKind {
Public,
PrivacyPreserving,
ProgramDeployment,
}
/// Whether applying a transaction to the block's working state succeeded.
#[derive(Debug, Clone, Copy, strum::IntoStaticStr, strum::EnumIter)]
#[strum(serialize_all = "snake_case")]
pub enum ApplyStatus {
Applied,
Failed,
}
impl From<common::transaction::TxKind> for TxKind {
fn from(kind: common::transaction::TxKind) -> Self {
match kind {
common::transaction::TxKind::Public => Self::Public,
common::transaction::TxKind::PrivacyPreserving => Self::PrivacyPreserving,
common::transaction::TxKind::ProgramDeployment => Self::ProgramDeployment,
}
}
}
/// Initialize metrics.
pub fn init() {
blocks_produced_total_counter().increment(0);
mempool_failed_transactions_total_counter().increment(0);
record_mempool_size(0);
record_chain_height(0);
drop(block_creation_time_histogram());
drop(transactions_per_block_histogram());
for origin in TransactionOrigin::iter() {
for kind in TxKind::iter() {
for status in ApplyStatus::iter() {
drop(mempool_transaction_application_time_histogram(
origin, kind, status,
));
}
}
}
}
fn block_creation_time_histogram() -> Histogram {
histogram!(
description: "Time taken to create a block",
unit: Unit::Seconds,
names::BLOCK_CREATION_TIME
)
}
pub fn record_block_creation_time(duration: Duration) {
block_creation_time_histogram().record(duration.as_secs_f64());
}
/// Height of the chain head, which moves backwards on a reorg, hence a gauge.
pub fn record_chain_height(height: u64) {
gauge!(
description: "Height of the chain head",
unit: Unit::Count,
names::CHAIN_HEIGHT
)
.set(height as f64);
}
fn blocks_produced_total_counter() -> Counter {
counter!(
description: "Number of blocks produced by this sequencer and applied to the head",
unit: Unit::Count,
names::BLOCKS_PRODUCED_TOTAL
)
}
pub fn increment_blocks_produced_total() {
blocks_produced_total_counter().increment(1);
}
pub fn record_mempool_size(size: usize) {
gauge!(
description: "Size of the mempool",
unit: Unit::Count,
names::MEMPOOL_SIZE
)
.set(u64::try_from(size).expect("Mempool size should fit into u64") as f64);
}
pub fn record_mempool_max_size(size: usize) {
gauge!(
description: "Configured maximum size of the mempool",
unit: Unit::Count,
names::MEMPOOL_MAX_SIZE
)
.set(u64::try_from(size).expect("Mempool max size should fit into u64") as f64);
}
fn mempool_transaction_application_time_histogram(
origin: TransactionOrigin,
kind: TxKind,
status: ApplyStatus,
) -> Histogram {
histogram!(
description: "Time taken to apply a mempool transaction",
unit: Unit::Seconds,
names::MEMPOOL_TRANSACTION_APPLICATION_TIME,
"origin" => <&'static str>::from(origin),
"kind" => <&'static str>::from(kind),
"status" => <&'static str>::from(status),
)
}
pub fn record_mempool_transaction_application_time(
origin: TransactionOrigin,
kind: TxKind,
status: ApplyStatus,
duration: Duration,
) {
mempool_transaction_application_time_histogram(origin, kind, status)
.record(duration.as_secs_f64());
}
fn transactions_per_block_histogram() -> Histogram {
histogram!(
description: "Number of transactions from mempool included in block",
unit: Unit::Count,
names::TRANSACTIONS_PER_BLOCK
)
}
pub fn record_transactions_per_block(count: usize) {
transactions_per_block_histogram()
.record(u64::try_from(count).expect("Block transaction count should fit into u64") as f64);
}
fn mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions from mempool that failed to be included in blocks",
unit: Unit::Count,
names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_mempool_failed_transactions_total() {
mempool_failed_transactions_total_counter().increment(1);
}

185
lez/sequencer/core/src/block_publisher.rs
View File

@ -1,14 +1,18 @@
use std::{sync::Arc, time::Duration};
use std::time::Duration;
use anyhow::{Context as _, Result, anyhow, ensure};
use common::block::Block;
use futures::Stream;
use log::{info, warn};
pub use logos_blockchain_core::mantle::ops::channel::{Ed25519PublicKey, MsgId};
pub use logos_blockchain_core::mantle::{
ledger::NoteId,
ops::channel::{Ed25519PublicKey, MsgId},
};
use logos_blockchain_core::{
mantle::{
MantleTx, SignedMantleTx, Transaction as _,
SignedMantleTx,
channel::{SlotTimeframe, SlotTimeout},
gas::GasCost,
ops::{
Op, OpProof,
channel::{
@ -17,9 +21,12 @@ use logos_blockchain_core::{
inscribe::Inscription,
},
},
traits::Hashable as _,
transactions::{MantleTxBuilder, OpsProofs},
},
proofs::channel_multi_sig_proof::{ChannelMultiSigProof, IndexedSignature},
};
use logos_blockchain_http_api_common::bodies::wallet::fund::WalletFundRequestBody;
pub use logos_blockchain_key_management_system_service::keys::{
ED25519_SECRET_KEY_SIZE, Ed25519Key, ZkKey,
};
@ -29,18 +36,15 @@ use logos_blockchain_zone_sdk::{
adapter::{Node as _, NodeHttpClient},
indexer::ZoneIndexer,
sequencer::{
DepositInfo, Event, FinalizedOp, InscriptionInfo, OrphanedTx,
SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg, WithdrawInfo,
ZoneSequencer,
ChannelUpdateTx, DepositInfo, Event, FinalizedOp, FundingConfig, InscriptionInfo,
PendingTx, SequencerConfig as ZoneSdkSequencerConfig, TurnNotification, WithdrawArg,
WithdrawInfo, ZoneSequencer,
},
};
use tokio::{
sync::{mpsc, oneshot, watch},
task::JoinHandle,
};
use tokio::sync::{mpsc, oneshot, watch};
use tokio_util::sync::CancellationToken;
use crate::config::BedrockConfig;
use crate::{config::BedrockConfig, task_group::TaskGroup};
/// Channel capacity for the publish inbox. One publish per produced block, drained
/// in microseconds by the drive task — 32 is huge headroom and just provides
@ -75,14 +79,29 @@ pub struct FollowUpdate {
/// persist the whole event in one write.
pub type OnFollowSink = Box<dyn Fn(FollowUpdate) + Send + 'static>;
/// What one publish produced.
pub struct PublishOutcome {
/// The `MsgId` zone-sdk assigned the published inscription.
pub this_msg: MsgId,
/// The checkpoint that now holds the inscription as pending.
pub checkpoint: SequencerCheckpoint,
/// Channel notes the bundled withdrawals release, empty for a plain
/// publish.
/// A [`ChannelWithdrawOp`](logos_blockchain_core::mantle::ops::channel::withdraw::ChannelWithdrawOp)
/// carries nothing but the note ids it releases, so these are the only
/// handle the local withdraw intent shares with the Bedrock Withdraw event
/// that later reports it.
pub released_notes: Vec<NoteId>,
}
/// Commands the drive task executes with `&mut sequencer`.
enum Command {
/// Publish an inscription (+ atomic withdrawals); responds with the assigned
/// `MsgId` and the checkpoint that now includes it as pending.
/// Publish an inscription (+ atomic withdrawals); responds with the
/// [`PublishOutcome`].
Publish {
inscription: Inscription,
withdrawals: Vec<WithdrawArg>,
resp: oneshot::Sender<Result<(MsgId, SequencerCheckpoint)>>,
resp: oneshot::Sender<Result<PublishOutcome>>,
},
}
@ -98,9 +117,8 @@ pub trait BlockPublisherTrait: Sized {
on_follow: OnFollowSink,
) -> Result<Self>;
/// Publish a block and return the `MsgId` zone-sdk assigned its inscription
/// together with the checkpoint that now holds it as pending. Zone-sdk
/// drives the actual submission and retries internally.
/// Publish a block and return what zone-sdk made of it. Zone-sdk drives the
/// actual submission and retries internally.
///
/// The checkpoint must be persisted with the block — restoring an older one
/// drops the inscription from the pending set, and it is never resubmitted.
@ -108,7 +126,7 @@ pub trait BlockPublisherTrait: Sized {
&self,
block: &Block,
withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)>;
) -> Result<PublishOutcome>;
fn channel_id(&self) -> ChannelId;
@ -120,6 +138,14 @@ pub trait BlockPublisherTrait: Sized {
/// are processed past that point, so the node must halt.
fn driver_cancellation(&self) -> CancellationToken;
/// The publisher's background tasks, for a caller that needs to know when
/// they have actually stopped. Its sinks capture a store handle, so the
/// `RocksDB` lock outlives the sequencer until the drive task is gone.
/// Empty by default, for publishers that run no tasks.
fn background_tasks(&self) -> TaskGroup {
TaskGroup::default()
}
/// Current channel frontier slot on the connected chain, or `None` if the
/// channel does not exist there. Drives the startup frontier check.
async fn channel_tip_slot(&self) -> Result<Option<Slot>>;
@ -143,19 +169,12 @@ pub struct ZoneSdkPublisher {
turn_rx: watch::Receiver<TurnNotification>,
// Cancelled when the drive task ends for any reason, including a panic.
driver_cancellation: CancellationToken,
// Aborts the drive task when the last clone is dropped.
_drive_task: Arc<DriveTaskGuard>,
// Stops the drive task when the last clone is dropped, and lets a shutdown
// path wait until it has actually stopped.
drive_task: TaskGroup,
indexer: ZoneIndexer<NodeHttpClient>,
}
struct DriveTaskGuard(JoinHandle<()>);
impl Drop for DriveTaskGuard {
fn drop(&mut self) {
self.0.abort();
}
}
impl BlockPublisherTrait for ZoneSdkPublisher {
async fn new(
config: &BedrockConfig,
@ -169,6 +188,11 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
let zone_sdk_config = ZoneSdkSequencerConfig {
resubmit_interval,
funding: Some(FundingConfig {
funding_pk: config.funding_key,
max_tx_fee: GasCost::new(logos_blockchain_core::mantle::Value::MAX),
priority_fee: FundingConfig::DEFAULT_PRIORITY_FEE,
}),
..ZoneSdkSequencerConfig::default()
};
@ -211,15 +235,20 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
let published = if withdrawals.is_empty() {
sequencer.handle()
.publish(data_bounded)
.await
.context("Failed to publish block")
} else {
sequencer.handle()
.publish_atomic_withdraw(data_bounded, withdrawals)
.await
.context("Failed to publish block with withdrawals")
};
let msg_result = published
.map(|(result, checkpoint)| (result.tx.inscription().this_msg, checkpoint));
let msg_result = published.map(|(result, checkpoint)| PublishOutcome {
this_msg: result.tx.inscription().this_msg,
checkpoint,
released_notes: released_notes(&result.tx),
});
match &msg_result {
Ok(_) if withdraw_count == 0 => {
info!("Published block with the size of {data_byte_size} bytes");
@ -235,9 +264,6 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
}
},
event = sequencer.next_event() => {
let Some(event) = event else {
continue;
};
match event {
Event::BlocksProcessed {
checkpoint,
@ -247,12 +273,13 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
let adopted = channel_update
.adopted
.iter()
.filter_map(channel_update_inscription)
.filter_map(block_from_inscription)
.collect();
let orphaned = channel_update
.orphaned
.iter()
.map(orphan_inscription)
.filter_map(channel_update_inscription)
.filter_map(block_from_inscription)
.collect();
@ -297,6 +324,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
notification.ends_at_slot
);
}
Event::MempoolPending(_tx_hash) => {}
}
}
}
@ -318,7 +346,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
command_tx,
turn_rx,
driver_cancellation,
_drive_task: Arc::new(DriveTaskGuard(drive_task)),
drive_task: TaskGroup::new(vec![drive_task]),
})
}
@ -326,7 +354,7 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
&self,
block: &Block,
withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)> {
) -> Result<PublishOutcome> {
let data = borsh::to_vec(block).context("Failed to serialize block")?;
let data_bounded: Inscription = data
.try_into()
@ -359,6 +387,10 @@ impl BlockPublisherTrait for ZoneSdkPublisher {
self.driver_cancellation.clone()
}
fn background_tasks(&self) -> TaskGroup {
self.drive_task.clone()
}
async fn channel_tip_slot(&self) -> Result<Option<Slot>> {
Ok(self
.node
@ -392,16 +424,31 @@ fn block_from_inscription(inscription: &InscriptionInfo) -> Option<(MsgId, Block
.map(|block| (inscription.this_msg, block))
}
/// Channel notes the withdraws bundled with a published tx release; empty for a
/// plain inscription. See [`PublishOutcome::released_notes`].
fn released_notes(tx: &PendingTx) -> Vec<NoteId> {
match tx {
PendingTx::Inscription(_) => Vec::new(),
PendingTx::AtomicWithdraw(bundle) => bundle
.withdraws
.iter()
.flat_map(|withdraw| withdraw.op.inputs.iter().copied())
.collect(),
}
}
/// The inscription carried by an orphaned tx (plain or atomic-withdraw bundle).
const fn orphan_inscription(orphan: &OrphanedTx) -> &InscriptionInfo {
const fn channel_update_inscription(orphan: &ChannelUpdateTx) -> Option<&InscriptionInfo> {
match orphan {
OrphanedTx::Inscription(info) => info,
OrphanedTx::AtomicWithdraw(bundle) => &bundle.inscription,
ChannelUpdateTx::Inscription(info) => Some(info),
ChannelUpdateTx::AtomicWithdraw(bundle) => Some(&bundle.inscription),
ChannelUpdateTx::Custom(_signed_mantle_tx) => None,
}
}
/// Signs a `ChannelConfig` op (accredited keys + rotation params) with
/// `signing_key` and posts it straight to the bedrock node.
/// `signing_key`, funds it from `config.funding_key` via the node's wallet,
/// and posts it straight to the bedrock node.
///
/// A standalone one-shot — no running sequencer involved, so authorization is
/// holding the admin key: the L1 rejects non-admin signers. `Ok(())` means the
@ -414,12 +461,12 @@ pub async fn post_channel_config(
posting_timeframe: u32,
posting_timeout: u32,
configuration_threshold: u16,
withdraw_threshold: u16,
transfer_threshold: u16,
) -> Result<()> {
ensure!(!keys.is_empty(), "Channel key list must not be empty");
for (name, threshold) in [
("configuration_threshold", configuration_threshold),
("withdraw_threshold", withdraw_threshold),
("transfer_threshold", transfer_threshold),
] {
ensure!(
threshold >= 1 && usize::from(threshold) <= keys.len(),
@ -440,27 +487,51 @@ pub async fn post_channel_config(
posting_timeframe: SlotTimeframe::from(posting_timeframe),
posting_timeout: SlotTimeout::from(posting_timeout),
configuration_threshold,
withdraw_threshold,
};
let mantle_tx = MantleTx([Op::ChannelConfig(config_op)].into());
let tx_hash = mantle_tx.hash();
// The admin key is `keys[0]`, hence signature index 0.
let signature = IndexedSignature::new(
0,
signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref()),
);
let proof = ChannelMultiSigProof::new(vec![signature])
.map_err(|err| anyhow!("Failed to assemble channel multi-sig proof: {err:?}"))?;
let signed_tx = SignedMantleTx {
ops_proofs: vec![OpProof::ChannelMultiSigProof(proof)],
mantle_tx,
transfer_threshold,
};
let node = NodeHttpClient::new(
CommonHttpClient::new(config.auth.clone().map(Into::into)),
config.node_url.clone(),
);
// Fund the op from the node's wallet: the node appends a fee transfer
// (paid from `funding_key`, change back to it) and returns its proof.
let tx_builder = MantleTxBuilder::new()
.extend_ops([Op::ChannelConfig(config_op)])
.map_err(|err| anyhow!("Too many ops in channel config transaction: {err:?}"))?;
let funded = node
.fund_tx(WalletFundRequestBody {
tip: None,
tx_builder,
change_public_key: config.funding_key,
funding_public_keys: vec![config.funding_key],
max_tx_fee: GasCost::new(logos_blockchain_core::mantle::Value::MAX),
priority_fee: FundingConfig::DEFAULT_PRIORITY_FEE,
})
.await
.context("Failed to fund channel config transaction")?;
let mantle_tx = funded.funded_tx;
// Sign the funded tx: the appended fee transfer changes the hash.
let tx_hash = mantle_tx.hash();
// The admin key is `keys[0]`, hence signature index 0.
let signature = IndexedSignature::new(
0,
signing_key.sign_payload(tx_hash.as_signing_bytes().as_ref()),
);
let proof = ChannelMultiSigProof::try_new(signature.into())
.map_err(|err| anyhow!("Failed to assemble channel multi-sig proof: {err:?}"))?;
// Proofs follow op order; funding appends the transfer as the last op.
let mut ops_proofs: OpsProofs = OpProof::ChannelMultiSigProof(proof).into();
if let Some(transfer_proof) = funded.transfer_proof {
ops_proofs
.try_push(transfer_proof)
.map_err(|err| anyhow!("Too many operation proofs: {err:?}"))?;
}
let signed_tx = SignedMantleTx::new(mantle_tx, ops_proofs);
node.post_transaction(signed_tx)
.await
.context("Failed to post channel config transaction")

36
lez/sequencer/core/src/block_store.rs
View File

@ -9,10 +9,12 @@ use common::{
use lee::V03State;
use lee_core::BlockId;
use log::info;
use logos_blockchain_zone_sdk::sequencer::SequencerCheckpoint;
use logos_blockchain_zone_sdk::{Slot, sequencer::SequencerCheckpoint};
use storage::sequencer::{
RocksDBIO,
sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord},
sequencer_cells::{
PeerZoneKey, PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord,
},
};
pub use storage::{DbResult, sequencer::DbDump};
@ -233,6 +235,36 @@ pub(crate) fn block_to_transactions_map(block: &Block) -> HashMap<HashType, u64>
.collect()
}
/// A cross-zone watcher's delivery floor on `peer_zone`'s channel.
///
/// The highest slot every message of which was delivered, or `None` before it
/// has delivered anything from that peer. Stored as a little-endian `u64`.
///
/// Free functions rather than only [`SequencerStore`] methods because each
/// watcher runs as its own spawned task and holds an `Arc<RocksDBIO>`;
/// `SequencerStore` is not `Clone`.
pub fn get_cross_zone_peer_floor(dbio: &RocksDBIO, peer_zone: PeerZoneKey) -> Result<Option<Slot>> {
let Some(bytes) = dbio.get_cross_zone_peer_floor_bytes(peer_zone)? else {
return Ok(None);
};
let bytes: [u8; 8] = bytes.as_slice().try_into().with_context(|| {
format!(
"Stored cross-zone peer floor is {} bytes, expected 8",
bytes.len()
)
})?;
Ok(Some(Slot::new(u64::from_le_bytes(bytes))))
}
pub fn set_cross_zone_peer_floor(
dbio: &RocksDBIO,
peer_zone: PeerZoneKey,
floor: Slot,
) -> Result<()> {
dbio.put_cross_zone_peer_floor_bytes(peer_zone, &floor.to_le_bytes())?;
Ok(())
}
#[cfg(test)]
mod tests {
use common::{block::HashableBlockData, test_utils::sequencer_sign_key_for_testing};

17
lez/sequencer/core/src/config.rs
View File

@ -1,6 +1,7 @@
use std::{
fs::File,
io::BufReader,
net::{IpAddr, Ipv4Addr, SocketAddr},
path::{Path, PathBuf},
time::Duration,
};
@ -8,10 +9,11 @@ use std::{
use anyhow::Result;
use bytesize::ByteSize;
use common::config::BasicAuth;
pub use cross_zone_inbox_core::{CrossZoneConfig, CrossZonePeer};
pub use cross_zone_inbox_core::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute};
use humantime_serde;
use lee::{AccountId, Balance};
use logos_blockchain_core::mantle::ops::channel::ChannelId;
use logos_blockchain_key_management_system_service::keys::ZkPublicKey;
use serde::{Deserialize, Serialize};
use url::Url;
@ -62,6 +64,9 @@ pub struct SequencerConfig {
/// Cross-zone messaging configuration. `None` disables the watcher.
#[serde(default)]
pub cross_zone: Option<CrossZoneConfig>,
/// Address the Prometheus metrics exporter binds to.
#[serde(default = "default_metrics_address")]
pub metrics_address: Option<SocketAddr>,
}
#[derive(Clone, Serialize, Deserialize)]
@ -72,9 +77,14 @@ pub struct BedrockConfig {
pub node_url: Url,
/// Bedrock auth.
pub auth: Option<BasicAuth>,
pub funding_key: ZkPublicKey,
}
impl SequencerConfig {
/// Address [`Self::metrics_address`] falls back to when the config omits it.
pub const DEFAULT_METRICS_ADDRESS: SocketAddr =
SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 9000);
pub fn from_path(config_home: &Path) -> Result<Self> {
let file = File::open(config_home)?;
let reader = BufReader::new(file);
@ -86,3 +96,8 @@ impl SequencerConfig {
const fn default_max_block_size() -> ByteSize {
ByteSize::mib(1)
}
#[expect(clippy::unnecessary_wraps, reason = "Required by serde")]
const fn default_metrics_address() -> Option<SocketAddr> {
Some(SequencerConfig::DEFAULT_METRICS_ADDRESS)
}

1095
lez/sequencer/core/src/cross_zone_watcher.rs
View File

File diff suppressed because it is too large Load Diff

553
lez/sequencer/core/src/lib.rs
View File

@ -16,6 +16,7 @@ use common::{
transaction::{LeeTransaction, clock_invocation},
};
use config::{GenesisAction, SequencerConfig};
use cross_zone_inbox_core::CrossZoneMessage;
use futures::StreamExt as _;
use itertools::Itertools as _;
use lee::{AccountId, PublicTransaction, public_transaction::Message};
@ -33,12 +34,16 @@ use num_bigint::BigUint;
pub use storage::error::DbError;
use storage::sequencer::{
RocksDBIO, StoreUpdate,
sequencer_cells::{PendingDepositEventRecord, WithdrawalReconciliationKey, ZoneAnchorRecord},
sequencer_cells::{
PendingCrossZoneDispatchRecord, PendingDepositEventRecord, WithdrawalReconciliationKey,
ZoneAnchorRecord,
},
};
use crate::{
block_publisher::{BlockPublisherTrait, MsgId, ZoneSdkPublisher},
block_publisher::{BlockPublisherTrait, MsgId, NoteId, ZoneSdkPublisher},
block_store::SequencerStore,
task_group::{StoreRelease, TaskGroup},
};
pub mod block_publisher;
@ -48,6 +53,23 @@ pub mod cross_zone_watcher;
#[cfg(feature = "mock")]
pub mod mock;
pub mod task_group;
/// Failed production attempts before a cross-zone dispatch is given up on.
///
/// One attempt per block, so this is tens of seconds of retrying. Enough for a
/// failure that is not the message's fault to clear, short enough that a message
/// which will never execute stops being retried.
const RETIRE_DISPATCH_AFTER_FAILURES: u32 = 3;
/// Cross-zone deliveries one block may carry.
///
/// Each one costs a guest execution whether it succeeds or fails, and what
/// queues them up is chosen by peer zones. Without a bound, a backlog decides
/// how long a block takes to build and leaves no room for user transactions,
/// since store-drained work is taken before the mempool. The rest wait one
/// block; nothing is dropped.
const MAX_DISPATCHES_PER_BLOCK: usize = 16;
/// The origin of a transaction.
#[derive(Clone, Copy)]
@ -58,6 +80,15 @@ pub enum TransactionOrigin {
Sequencer,
}
impl From<TransactionOrigin> for sequencer_core_metrics::TransactionOrigin {
fn from(origin: TransactionOrigin) -> Self {
match origin {
TransactionOrigin::User => Self::User,
TransactionOrigin::Sequencer => Self::Sequencer,
}
}
}
#[derive(Clone, Debug, BorshDeserialize)]
struct DepositMetadata {
recipient_id: lee::AccountId,
@ -71,6 +102,10 @@ pub struct SequencerCore<BP: BlockPublisherTrait = ZoneSdkPublisher> {
mempool: MemPool<(TransactionOrigin, LeeTransaction)>,
sequencer_config: SequencerConfig,
block_publisher: BP,
/// Cross-zone watchers, stopped when this sequencer is dropped. They hold a
/// store handle, so leaving them running would keep the `RocksDB` lock held
/// and make the home directory unopenable by a restarting sequencer.
watchers: TaskGroup,
}
impl<BP: BlockPublisherTrait> SequencerCore<BP> {
@ -118,6 +153,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
)
.expect("Failed to create database with genesis block");
// Incrementing count for genesis.
sequencer_core_metrics::increment_blocks_produced_total();
(store, genesis_state)
}
}
@ -169,6 +207,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
pub async fn start_from_config(
config: SequencerConfig,
) -> (Self, MemPoolHandle<(TransactionOrigin, LeeTransaction)>) {
sequencer_core_metrics::init();
let bedrock_signing_key =
load_or_create_signing_key(&config.home.join("bedrock_signing_key"))
.expect("Failed to load or create bedrock signing key");
@ -189,6 +229,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let is_fresh_start = initial_checkpoint.is_none();
let (mempool, mempool_handle) = MemPool::new(config.mempool_max_size);
sequencer_core_metrics::record_mempool_max_size(config.mempool_max_size);
let block_publisher = BP::new(
&config.bedrock_config,
@ -202,14 +243,17 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// Cross-zone messaging: start a watcher per configured peer. The inbox
// config account is seeded into genesis state in `build_genesis_state`.
if let Some(cross_zone) = &config.cross_zone {
cross_zone_watcher::spawn_watchers(
&config.bedrock_config,
cross_zone,
config.block_create_timeout,
&mempool_handle,
);
}
let watchers = config
.cross_zone
.as_ref()
.map_or_else(TaskGroup::default, |cross_zone| {
cross_zone_watcher::spawn_watchers(
&config.bedrock_config,
cross_zone,
config.block_create_timeout,
&store.dbio(),
)
});
// Before producing, verify our local state still belongs to the chain
// the channel serves and replay any channel blocks we are missing
// (e.g. from other sequencers).
@ -239,7 +283,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let mut last_checkpoint = None;
for block in &pending_blocks {
let (_msg, checkpoint) = block_publisher
let outcome = block_publisher
.publish_block(block, vec![])
.await
.unwrap_or_else(|err| {
@ -248,7 +292,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
block.header.block_id
)
});
last_checkpoint = Some(checkpoint);
last_checkpoint = Some(outcome.checkpoint);
}
// These blocks are already stored, so only the sdk's pending set
@ -267,8 +311,11 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
mempool,
sequencer_config: config,
block_publisher,
watchers,
};
sequencer_core_metrics::record_chain_height(sequencer_core.chain_height());
(sequencer_core, mempool_handle)
}
@ -423,6 +470,12 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.context("Failed to read stored block")?
{
Some(stored) if stored.header.hash == block_hash => {
// Already applied, but the channel serving it is what makes
// it irreversible, so its deliveries are settled and their
// records are owed nothing. Without this a restart leaves a
// record for every delivery it already published, and
// nothing downstream would ever remove them.
settle_reconstructed_deliveries(store, &stored);
store
.set_zone_anchor(&record)
.context("Failed to persist zone anchor")?;
@ -464,6 +517,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.iter()
.filter_map(extract_bridge_deposit_id)
.collect();
// The same for the deliveries it carries: the inbox has seen them, so
// the drain would skip them anyway, and the records are owed nothing.
let finalized_dispatch_keys = settled_dispatch_keys(&store.dbio(), block);
// The tip meta stays pinned to the head tip even when the reconstructed
// block lands below it, and the anchor only advances if the block
@ -477,6 +533,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
head_tip: head_tip.as_ref(),
final_snapshot: final_meta.as_ref().map(|meta| (chain.final_state(), meta)),
remove_deposit_records: &finalized_deposit_ids,
remove_dispatch_records: &finalized_dispatch_keys,
zone_anchor: Some(&record),
..StoreUpdate::new(chain.head_state())
})
@ -502,18 +559,21 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.build_block_from_mempool()
.context("Failed to build block from mempool transactions")?;
let withdrawal_reconciliation_keys: Vec<_> = withdrawals
.iter()
.map(|withdraw| withdraw_event_reconciliation_key(&withdraw.outputs))
.collect::<Result<Vec<_>>>()
.context("Failed to build reconciliation keys for block withdrawals")?;
let (this_msg, checkpoint) = self
let block_publisher::PublishOutcome {
this_msg,
checkpoint,
released_notes,
} = self
.block_publisher
.publish_block(&block, withdrawals)
.await
.context("Failed to publish block to Bedrock")?;
let withdrawal_reconciliation_keys: Vec<_> = released_notes
.iter()
.map(withdrawal_reconciliation_key)
.collect();
self.record_produced_block(
this_msg,
&block,
@ -553,6 +613,9 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
chain.head_state(),
Some(&checkpoint_bytes),
)?;
sequencer_core_metrics::increment_blocks_produced_total();
sequencer_core_metrics::record_chain_height(block.header.block_id);
}
// Neither branch persists anything, checkpoint included: the
// inscription it holds as pending belongs to a block that is not
@ -638,8 +701,42 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
fn build_block_from_mempool(&mut self) -> Result<BlockWithMeta> {
let now = Instant::now();
// Build on the head: its tip is the parent, its state the validation base.
let (prev_block_hash, new_block_height, mut working_state) = {
// Decoded outside the chain lock, and read before it is taken: the usual
// case is no delivery records at all, and decoding is the expensive part.
// One that does not decode is dropped rather than kept, since nothing
// will ever turn those bytes into a block transaction.
let mut settled = Vec::new();
let recorded_dispatches: Vec<_> = self
.store
.dbio()
.get_pending_cross_zone_dispatches()
.context("Failed to load pending cross-zone dispatches")?
.into_iter()
.filter_map(
|record| match borsh::from_slice::<LeeTransaction>(&record.transaction) {
Ok(tx) => {
let message = extract_cross_zone_dispatch(&tx);
Some((record.message_key, message, tx))
}
Err(err) => {
warn!(
"Dropping pending cross-zone dispatch {} that does not decode: {err:#}",
hex::encode(record.message_key)
);
settled.push(record.message_key);
None
}
},
)
.collect();
// Build on the head: its tip is the parent, its state the validation
// base.
//
// The delivery records are classified in here rather than after, so the
// final state can be read by reference. Cloning it cost a full state
// copy on every block of every zone, cross-zone or not.
let (prev_block_hash, new_block_height, mut working_state, pending_dispatches) = {
let chain = self.chain.lock().expect("chain state mutex poisoned");
let tip = chain.head_tip();
let height = tip.as_ref().map_or(GENESIS_BLOCK_ID, |head| {
@ -648,9 +745,43 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.expect("block id should not overflow")
});
let prev = tip.map_or(HashType([0; 32]), |head| head.hash);
(prev, height, chain.head_state().clone())
// Three outcomes per record. Already in the final state means the
// delivery is irreversible, so the record is dropped; that is the
// only thing that removes a record the watcher re-added after its
// delivery had already settled, which it does whenever it re-reads a
// slot it has consumed. Already in the head state but not the final
// one means the delivery is on this chain but could still orphan, so
// the record is skipped and kept. Otherwise it goes in this block.
let mut pending: VecDeque<LeeTransaction> = VecDeque::new();
for (key, message, tx) in recorded_dispatches {
match message {
Some(message) if dispatch_already_delivered(chain.final_state(), &message) => {
settled.push(key);
}
Some(message) if dispatch_already_delivered(chain.head_state(), &message) => {}
_ if pending.len() >= MAX_DISPATCHES_PER_BLOCK => {}
_ => pending.push_back(tx),
}
}
(prev, height, chain.head_state().clone(), pending)
};
if !settled.is_empty()
&& let Err(err) = self
.store
.dbio()
.drop_settled_cross_zone_dispatches(&settled)
{
// Only bookkeeping: the deliveries themselves are irreversible, and
// the next turn tries again.
warn!(
"Failed to drop {} settled delivery record(s): {err:#}",
settled.len()
);
}
let mut valid_transactions = Vec::new();
let mut withdrawals = Vec::new();
@ -664,7 +795,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
// build on — it was minted by us or by a peer whose block we adopted.
// An orphan reverts the receipt with the block, so the next turn
// re-mints without any bookkeeping of our own.
let mut pending_deposits: VecDeque<LeeTransaction> = self
let pending_deposits: VecDeque<LeeTransaction> = self
.store
.get_pending_deposit_events()
.context("Failed to load pending deposit events")?
@ -691,10 +822,14 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
let clock_tx = clock_invocation(new_block_timestamp);
let clock_lee_tx = LeeTransaction::Public(clock_tx.clone());
// Pending deposit mints first, then user work. `from_store` is not the
// same as a `Sequencer` origin — the cross-zone watcher pushes those
// into the mempool too.
while let Some((origin, tx, from_store)) = pending_deposits
sequencer_core_metrics::record_mempool_size(self.mempool.len());
// Everything drained from the store first, then user work. `from_store`
// is not the same as a `Sequencer` origin: it says the transaction has a
// record behind it and so needs no requeue, where the origin only says
// it was not submitted by a user.
let mut pending_from_store = pending_deposits;
pending_from_store.extend(pending_dispatches);
while let Some((origin, tx, from_store)) = pending_from_store
.pop_front()
.map(|tx| (TransactionOrigin::Sequencer, tx, true))
.or_else(|| self.mempool.pop().map(|(origin, tx)| (origin, tx, false)))
@ -719,27 +854,74 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.len();
if block_size > max_block_size {
// Would a block carrying nothing but this still be too big? Then
// it does not fit in any block and deferring it defers it for
// ever. A store-drained transaction is at the head of the queue
// every turn, so breaking here would stop production reaching
// anything behind it, including the whole mempool, permanently.
// Count it against the delivery instead so it is given up on.
//
// Measured on its own rather than from `block_size`, which also
// counts whatever this block already holds: a transaction that
// merely does not fit *today* is the ordinary deferral below.
if from_store
&& !self.fits_in_an_empty_block(
&tx,
&clock_lee_tx,
new_block_height,
prev_block_hash,
new_block_timestamp,
)?
{
error!(
"Sequencer-drained transaction {tx_hash} cannot fit in any block under the \
{max_block_size} byte limit; giving up on it rather than stalling production",
);
self.count_dispatch_failure(&tx);
continue;
}
warn!(
"Transaction with hash {tx_hash} deferred to next block: \
block size {block_size} bytes would exceed limit of {max_block_size} bytes",
);
// A deposit mint needs no requeue: its record stays unfulfilled
// in the store and is drained again on the next turn.
// Anything drained from the store needs no requeue: its record
// stays there and is drained again on the next turn.
if !from_store {
self.mempool.push_front((origin, tx));
}
break;
}
if Self::apply_mempool_transaction(
let before_tx_apply = Instant::now();
let applied = Self::apply_mempool_transaction(
&mut working_state,
origin,
&tx,
new_block_height,
new_block_timestamp,
&mut withdrawals,
) {
);
if applied {
sequencer_core_metrics::record_mempool_transaction_application_time(
origin.into(),
tx.kind().into(),
sequencer_core_metrics::ApplyStatus::Applied,
before_tx_apply.elapsed(),
);
valid_transactions.push(tx);
} else {
sequencer_core_metrics::increment_mempool_failed_transactions_total();
sequencer_core_metrics::record_mempool_transaction_application_time(
origin.into(),
tx.kind().into(),
sequencer_core_metrics::ApplyStatus::Failed,
before_tx_apply.elapsed(),
);
// A failed transaction is simply left out of the block, except a
// dispatch: that one is re-fed from the store every turn, so one
// that can never execute would fail on every block for ever.
self.count_dispatch_failure(&tx);
}
if valid_transactions.len() >= self.sequencer_config.max_num_tx_in_block {
@ -751,6 +933,7 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
.transition_from_public_transaction(&clock_tx, new_block_height, new_block_timestamp)
.context("Clock transaction failed. Aborting block production.")?;
valid_transactions.push(clock_lee_tx);
sequencer_core_metrics::record_transactions_per_block(valid_transactions.len());
let hashable_data = HashableBlockData {
block_id: new_block_height,
@ -769,6 +952,8 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
now.elapsed().as_secs()
);
sequencer_core_metrics::record_block_creation_time(now.elapsed());
Ok(BlockWithMeta { block, withdrawals })
}
@ -828,6 +1013,93 @@ impl<BP: BlockPublisherTrait> SequencerCore<BP> {
&self.block_publisher
}
/// Whether a block carrying nothing but `tx` and the clock would be within
/// the size limit.
///
/// Distinguishes "does not fit in this block" from "does not fit in any
/// block". The first is an ordinary deferral; the second, for a transaction
/// the store re-feeds every turn, is a permanent stall unless it is given up
/// on.
fn fits_in_an_empty_block(
&self,
tx: &LeeTransaction,
clock_tx: &LeeTransaction,
block_id: u64,
prev_block_hash: HashType,
timestamp: u64,
) -> Result<bool> {
let alone = HashableBlockData {
block_id,
transactions: vec![tx.clone(), clock_tx.clone()],
prev_block_hash,
timestamp,
};
let size = borsh::to_vec(&alone)
.context("Failed to serialize block for size check")?
.len();
let max = usize::try_from(self.sequencer_config.max_block_size.as_u64())
.expect("`max_block_size` should fit into usize");
Ok(size <= max)
}
/// Counts one failed production attempt against `tx` if it is a cross-zone
/// delivery, giving up on it once too many accumulate.
///
/// A delivery's payload and target accounts are chosen on the peer zone and
/// validated by nobody in between, so one can fail for good; but a failure
/// can equally be a property of the moment, so give up only after several.
/// Giving up drops the record, which is also what keeps a peer from growing
/// the pending list with deliveries that can never execute.
fn count_dispatch_failure(&self, tx: &LeeTransaction) {
let Some(message) = extract_cross_zone_dispatch(tx) else {
return;
};
let key = cross_zone_inbox_core::message_key(
&message.src_zone,
message.src_block_id,
message.src_tx_index,
);
match self
.store
.dbio()
.record_dispatch_failure(key, RETIRE_DISPATCH_AFTER_FAILURES)
{
Ok(true) => error!(
"Giving up on cross-zone delivery {} after {RETIRE_DISPATCH_AFTER_FAILURES} failed attempts; it will not be retried",
hex::encode(key)
),
Ok(false) => warn!(
"Cross-zone delivery {} failed to execute, will retry next block",
hex::encode(key)
),
Err(err) => error!(
"Failed to count the failed attempt for cross-zone delivery {}: {err:#}",
hex::encode(key)
),
}
}
/// A weak reference to this sequencer's store, for a shutdown path that
/// needs to observe the database actually closing rather than infer it.
#[must_use]
pub fn store_release(&self) -> StoreRelease {
StoreRelease::new(&self.store.dbio())
}
/// Every background task that holds this sequencer's store handle.
///
/// Taken before the core is shared, so a shutdown path can wait for them
/// without owning the core. Until all of them have stopped the `RocksDB`
/// lock is still held and the home directory cannot be reopened, which is
/// what a restart does.
#[must_use]
pub fn background_tasks(&self) -> Vec<TaskGroup> {
vec![
self.watchers.clone(),
self.block_publisher.background_tasks(),
]
}
/// Whether this sequencer is currently authorized to write to the channel.
#[must_use]
pub fn is_our_turn(&self) -> bool {
@ -857,6 +1129,29 @@ fn deposit_already_minted(state: &lee::V03State, deposit_op_id: HashType) -> boo
.is_some_and(|receipt| *receipt != lee::Account::default())
}
/// Whether a cross-zone delivery is already on the chain we are building on.
///
/// The inbox records every delivered message key in a seen shard and no-ops a
/// replay, so that shard is the same kind of answer the deposit receipt gives:
/// state, not bookkeeping. An orphan reverts the entry with the block, so the
/// next turn re-delivers with nothing of ours to unwind.
fn dispatch_already_delivered(state: &lee::V03State, message: &CrossZoneMessage) -> bool {
let shard_id = cross_zone_inbox_core::inbox_seen_shard_account_id(
programs::cross_zone_inbox().id(),
&message.src_zone,
message.src_block_id,
);
state.get_account_by_id_ref(shard_id).is_some_and(|shard| {
cross_zone_inbox_core::SeenShard::from_bytes(shard.data.as_ref()).is_ok_and(|seen| {
seen.contains(&cross_zone_inbox_core::message_key(
&message.src_zone,
message.src_block_id,
message.src_tx_index,
))
})
})
}
/// Feed one channel delta into the follow state and mirror it to the store:
/// revert orphaned, then apply and persist adopted and finalized blocks.
/// Production builds on this same head. Wired to the publisher via
@ -893,33 +1188,19 @@ fn apply_follow_update(
let deposit_records: Vec<PendingDepositEventRecord> =
deposits.iter().map(pending_deposit_event_record).collect();
// A withdraw whose outputs we cannot read has no counter to reconcile
// against; log and drop it rather than fail the whole update.
// One reconciliation unit per released note, matching how the intents were
// recorded at publish time.
let consumed_withdrawals: Vec<WithdrawalReconciliationKey> = withdrawals
.iter()
.filter_map(|withdraw| {
withdraw_event_reconciliation_key(&withdraw.op.outputs)
.inspect_err(|err| {
error!(
"Failed to build reconciliation key for Bedrock Withdraw event with tx_hash {}: {err:#}",
hex::encode(withdraw.tx_hash.as_ref())
);
})
.ok()
})
.flat_map(|withdraw| withdraw.op.inputs.iter())
.map(withdrawal_reconciliation_key)
.collect();
// The lock is held across the persist below so disk writes land in apply
// order — the produce path persists under this same lock.
let (resubmit_txs, outcome) = {
let (resubmit_txs, outcome, head_height) = {
let mut chain = chain.lock().expect("chain state mutex poisoned");
// User txs of orphaned blocks, returned to the mempool below.
let resubmit_txs: Vec<LeeTransaction> = orphaned
.iter()
.flat_map(|(_, block)| resubmittable_txs(block))
.collect();
// Outcomes align with `adopted`.
let outcomes = chain.apply_channel_update(&orphaned, &adopted);
let mut to_persist: Vec<(&Block, bool)> = adopted
@ -953,6 +1234,22 @@ fn apply_follow_update(
AcceptOutcome::Parked(_) | AcceptOutcome::RetryableFailure(_) => {}
}
}
// User txs of orphaned blocks, returned to the mempool below.
//
// Computed after the finalized tier has advanced, and only for blocks
// above it: the zone-sdk reports a block as orphaned once LIB pruning
// drops its inscription from the channel lineage, so every block of
// ours is orphaned a poll or two after it finalizes. Those transactions
// are irreversibly included, and returning them to the mempool puts
// them back in every block we produce from then on.
let final_height = chain.final_tip().map(|tip| tip.block_id);
let resubmit_txs: Vec<LeeTransaction> = orphaned
.iter()
.filter(|(_, block)| final_height.is_none_or(|id| block.header.block_id > id))
.flat_map(|(_, block)| resubmittable_txs(block))
.collect();
// Snapshot the advanced final tier so a restart re-anchors on it.
let final_meta = final_advanced.then(|| {
let tip = chain.final_tip().expect("advanced final tier has a tip");
@ -975,6 +1272,14 @@ fn apply_follow_update(
.filter_map(extract_bridge_deposit_id)
.collect();
// The same for cross-zone deliveries, keyed by message key: a record
// goes once its own delivery is irreversible, never because another
// block finalized at its height.
let finalized_dispatch_keys: Vec<[u8; 32]> = irreversible
.iter()
.flat_map(|block| settled_dispatch_keys(dbio, block))
.collect();
// A persist failure is fatal: the in-memory chain has already advanced,
// and continuing would leave a permanent gap in the store. The `panic!`
// ends the drive task, whose cancellation halts the node.
@ -987,14 +1292,17 @@ fn apply_follow_update(
finalized_up_to: last_finalized,
new_deposit_events: &deposit_records,
remove_deposit_records: &finalized_deposit_ids,
remove_dispatch_records: &finalized_dispatch_keys,
consumed_withdrawals: &consumed_withdrawals,
..StoreUpdate::new(chain.head_state())
})
.unwrap_or_else(|err| panic!("Failed to persist follow update: {err:#}"));
(resubmit_txs, outcome)
(resubmit_txs, outcome, head_tip.map_or(0, |tip| tip.id))
};
sequencer_core_metrics::record_chain_height(head_height);
if outcome.accepted_deposits > 0 {
info!(
"Recorded {} Bedrock Deposit event(s); their mints are drained from the store on our next turn",
@ -1003,9 +1311,8 @@ fn apply_follow_update(
}
for withdrawal in &outcome.unmatched_withdrawals {
warn!(
"Unexpected Bedrock Withdraw event of {} to {}: no matching unseen withdraw found",
withdrawal.amount,
hex::encode(withdrawal.bedrock_account_pk)
"Unexpected Bedrock Withdraw event releasing channel note {}: no matching unseen withdraw found",
hex::encode(withdrawal.released_note_id)
);
}
@ -1226,6 +1533,104 @@ fn is_sequencer_only_tx(tx: &LeeTransaction) -> bool {
if is_sequencer_only_program(tx.message().program_id))
}
/// The cross-zone message an inbox dispatch delivers, or `None` if `tx` is not
/// a dispatch.
#[must_use]
fn extract_cross_zone_dispatch(tx: &LeeTransaction) -> Option<CrossZoneMessage> {
let LeeTransaction::Public(tx) = tx else {
return None;
};
let message = tx.message();
if message.program_id != programs::cross_zone_inbox().id() {
return None;
}
match risc0_zkvm::serde::from_slice::<cross_zone_inbox_core::Instruction, u32>(
&message.instruction_data,
) {
Ok(cross_zone_inbox_core::Instruction::Dispatch(msg)) => Some(msg),
Ok(cross_zone_inbox_core::Instruction::InitConfig(_)) | Err(_) => None,
}
}
/// The content-addressed key of the message an inbox dispatch delivers.
///
/// A delivery in an irreversible block settles its pending record, so the record
/// is dropped by identity rather than by the height it happened to land at.
#[must_use]
fn extract_cross_zone_dispatch_key(tx: &LeeTransaction) -> Option<[u8; 32]> {
extract_cross_zone_dispatch(tx).map(|msg| {
cross_zone_inbox_core::message_key(&msg.src_zone, msg.src_block_id, msg.src_tx_index)
})
}
/// The keys of the deliveries `block` carries, reporting any whose transaction
/// is not the one we recorded for that key.
///
/// The key covers `(src_zone, src_block_id, src_tx_index)` and nothing about the
/// payload, and so does the inbox's own replay check, so a sequencer that
/// publishes a dispatch with the right key and a forged payload settles our
/// correct record along with it. The forgery is caught downstream by the
/// indexer, which re-derives every delivery and halts, but the local record is
/// the last copy of what we believed and it is about to be dropped either way.
/// Saying so in the log is what makes the halt diagnosable.
fn settled_dispatch_keys(dbio: &RocksDBIO, block: &Block) -> Vec<[u8; 32]> {
let recorded = dbio.get_pending_cross_zone_dispatches().unwrap_or_default();
let (keys, forged) = classify_settled_deliveries(&recorded, block);
for key in forged {
error!(
"Cross-zone delivery {} settled with a transaction that is not the one this node recorded for that key. The message key does not cover the payload, so a peer's sequencer can publish a different delivery under it.",
hex::encode(key)
);
}
keys
}
/// Splits the deliveries `block` carries into every settled key, and the subset
/// whose transaction is not the one `recorded` holds for that key.
///
/// Separated from the logging so the detection is testable: a forged delivery
/// leaves no trace in state that differs from an honest one, precisely because
/// the key does not cover the payload.
fn classify_settled_deliveries(
recorded: &[PendingCrossZoneDispatchRecord],
block: &Block,
) -> (Vec<[u8; 32]>, Vec<[u8; 32]>) {
let mut keys = Vec::new();
let mut forged = Vec::new();
for tx in &block.body.transactions {
let Some(key) = extract_cross_zone_dispatch_key(tx) else {
continue;
};
let mismatched = recorded
.iter()
.find(|record| record.message_key == key)
.is_some_and(|record| {
borsh::to_vec(tx).is_ok_and(|encoded| encoded != record.transaction)
});
if mismatched {
forged.push(key);
}
keys.push(key);
}
(keys, forged)
}
/// Drops the records of deliveries carried by a reconstructed block.
///
/// A persist failure is only logged: the deliveries are already irreversible, so
/// the worst case is a record the next drain drops instead.
fn settle_reconstructed_deliveries(store: &SequencerStore, block: &Block) {
let keys = settled_dispatch_keys(&store.dbio(), block);
if keys.is_empty() {
return;
}
if let Err(err) = store.dbio().drop_settled_cross_zone_dispatches(&keys) {
warn!("Failed to settle reconstructed delivery records: {err:#}");
}
}
#[must_use]
fn extract_bridge_deposit_id(tx: &LeeTransaction) -> Option<HashType> {
let LeeTransaction::Public(tx) = tx else {
@ -1283,35 +1688,21 @@ fn extract_bridge_withdraw_data(tx: &LeeTransaction) -> Option<WithdrawArg> {
})
}
fn withdraw_event_reconciliation_key(
outputs: &logos_blockchain_core::mantle::ledger::Outputs,
) -> Result<WithdrawalReconciliationKey> {
let [note] = outputs.as_ref().as_slice() else {
return Err(anyhow!(
"Unsupported withdraw output count for reconciliation: {}",
outputs.len()
));
};
// `extract_bridge_withdraw_data` maps [u8;32] LE -> BigUint -> ZkPublicKey.
// Reconcile by reversing that direction here.
let mut bedrock_account_pk = BigUint::from(note.pk.into_inner()).to_bytes_le();
if bedrock_account_pk.len() > 32 {
return Err(anyhow!(
"Withdraw recipient public key is too large: {} bytes",
bedrock_account_pk.len()
));
}
bedrock_account_pk.resize(32, 0);
let bedrock_account_pk: [u8; 32] = bedrock_account_pk
/// The reconciliation identity of one released channel note.
///
/// A `ChannelWithdrawOp` releases notes the channel already owns and carries
/// only their ids — the recipient key and value live in the note itself, which
/// neither the op nor the Bedrock Withdraw event reports. The note id is
/// therefore the one handle both sides share, and it is unique: a note is spent
/// once.
fn withdrawal_reconciliation_key(note_id: &NoteId) -> WithdrawalReconciliationKey {
let released_note_id: [u8; 32] = note_id
.as_bytes()
.as_ref()
.try_into()
.expect("Public key bytes were padded/truncated to 32 bytes");
.expect("`NoteId` is a 32-byte field element");
Ok(WithdrawalReconciliationKey {
amount: note.value,
bedrock_account_pk,
})
WithdrawalReconciliationKey { released_note_id }
}
/// Load signing key from file or generate a new one if it doesn't exist.

31
lez/sequencer/core/src/mock.rs
View File

@ -5,14 +5,17 @@ use common::block::Block;
use futures::Stream;
use logos_blockchain_core::{
header::HeaderId,
mantle::ops::channel::{ChannelId, MsgId},
mantle::{
ledger::{NoteId, Utxo},
ops::channel::{ChannelId, MsgId},
},
};
use logos_blockchain_key_management_system_service::keys::Ed25519Key;
use logos_blockchain_zone_sdk::{Slot, ZoneMessage, sequencer::WithdrawArg};
use tokio_util::sync::CancellationToken;
use crate::{
block_publisher::{BlockPublisherTrait, OnFollowSink, SequencerCheckpoint},
block_publisher::{BlockPublisherTrait, OnFollowSink, PublishOutcome, SequencerCheckpoint},
config::BedrockConfig,
};
@ -70,12 +73,16 @@ impl BlockPublisherTrait for MockBlockPublisher {
async fn publish_block(
&self,
block: &Block,
_bridge_withdrawals: Vec<WithdrawArg>,
) -> Result<(MsgId, SequencerCheckpoint)> {
withdrawals: Vec<WithdrawArg>,
) -> Result<PublishOutcome> {
// Deterministic per-block id so head dedup behaves in tests.
//
// TODO: should we allow more "mockability" here?
Ok((MsgId::from(block.header.hash.0), mock_checkpoint()))
Ok(PublishOutcome {
this_msg: MsgId::from(block.header.hash.0),
checkpoint: mock_checkpoint(),
released_notes: mock_released_notes(&withdrawals),
})
}
fn channel_id(&self) -> ChannelId {
@ -108,6 +115,20 @@ impl BlockPublisherTrait for MockBlockPublisher {
}
}
/// The notes the mock reports as released by `withdrawals`.
///
/// Zone-sdk picks the actual channel notes to release, so a mock has to invent
/// them: one note id per requested output, derived from the output itself so
/// tests can recompute the reconciliation keys of a block they produced.
#[must_use]
pub(crate) fn mock_released_notes(withdrawals: &[WithdrawArg]) -> Vec<NoteId> {
withdrawals
.iter()
.flat_map(|withdraw| withdraw.outputs.into_iter().enumerate())
.map(|(output_index, note)| Utxo::new([0; 32], output_index, *note).id())
.collect()
}
/// A zeroed checkpoint, for [`MockBlockPublisher::publish_block`] and for tests
/// building a [`crate::block_publisher::FollowUpdate`]. Tests only assert *that*
/// a checkpoint was persisted alongside its effects, never what is in it.

137
lez/sequencer/core/src/task_group.rs Normal file
View File

@ -0,0 +1,137 @@
//! A set of background tasks that can be stopped and waited on.
use std::sync::{Arc, Mutex, MutexGuard, PoisonError, Weak};
use log::warn;
use storage::sequencer::RocksDBIO;
use tokio::task::JoinHandle;
/// Background tasks owned by one component, stoppable on demand and stopped
/// anyway when the last handle goes away.
///
/// `JoinHandle::abort` only *requests* cancellation, and dropping a handle
/// detaches rather than cancels, so neither on its own says when a task has
/// actually stopped. That matters because these tasks hold a store handle:
/// until they are gone the `RocksDB` lock is still held and a restarting
/// sequencer cannot reopen its home directory. [`TaskGroup::shutdown`] is the
/// answer to "have they stopped yet"; the `Drop` below stays as the best-effort
/// path for panics and tests that never call it.
///
/// Cloneable so the owner can keep it (tying task lifetime to its own) while a
/// shutdown path elsewhere holds a clone.
#[derive(Clone, Default)]
pub struct TaskGroup(Arc<TaskGroupInner>);
#[derive(Default)]
struct TaskGroupInner(Mutex<Vec<JoinHandle<()>>>);
/// A weak handle to the store, for observing when it is finally closed.
///
/// Every strong reference lives inside a task or a server that shutdown stops,
/// but the last drop runs on whichever thread owned it, not on the one awaiting
/// shutdown. Watching the count is the difference between knowing the database
/// file is closed and assuming it from another crate's drop order.
pub struct StoreRelease(Weak<RocksDBIO>);
impl StoreRelease {
#[must_use]
pub fn new(store: &Arc<RocksDBIO>) -> Self {
Self(Arc::downgrade(store))
}
/// How many holders are left. Zero means the store is closed.
#[must_use]
pub fn holders(&self) -> usize {
self.0.strong_count()
}
}
impl Drop for TaskGroupInner {
fn drop(&mut self) {
for task in Self::take(&self.0) {
task.abort();
}
}
}
impl TaskGroupInner {
/// Empties the handle list, so a second shutdown (or a drop after one) is a
/// no-op rather than a second abort.
fn handles(handles: &Mutex<Vec<JoinHandle<()>>>) -> MutexGuard<'_, Vec<JoinHandle<()>>> {
handles.lock().unwrap_or_else(PoisonError::into_inner)
}
fn take(handles: &Mutex<Vec<JoinHandle<()>>>) -> Vec<JoinHandle<()>> {
std::mem::take(&mut *Self::handles(handles))
}
}
impl TaskGroup {
/// Takes ownership of already-spawned tasks.
#[must_use]
pub fn new(handles: Vec<JoinHandle<()>>) -> Self {
Self(Arc::new(TaskGroupInner(Mutex::new(handles))))
}
/// Whether any task has ended on its own.
///
/// These tasks run for the lifetime of the sequencer, so a finished one is a
/// task that panicked, and whatever it was doing is not happening any more.
#[must_use]
pub fn any_finished(&self) -> bool {
TaskGroupInner::handles(&self.0.0)
.iter()
.any(JoinHandle::is_finished)
}
/// Stops every task and waits for it to finish.
///
/// Returns only once the runtime has dropped each task's future, so whatever
/// they held (a store handle, a network client) is released by the time this
/// returns. Cancellation is the expected outcome, so it is not reported; a
/// panic is, since it means the task died on its own terms earlier.
pub async fn shutdown(&self) {
let handles = TaskGroupInner::take(&self.0.0);
for handle in handles {
handle.abort();
if let Err(err) = handle.await
&& err.is_panic()
{
warn!("Background task panicked before shutdown: {err}");
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn a_task_that_ends_on_its_own_is_visible() {
let group = TaskGroup::new(vec![tokio::spawn(async {})]);
// A watcher only ends by panicking, so "finished" is the signal that a
// peer's deliveries have stopped happening.
tokio::task::yield_now().await;
assert!(group.any_finished());
let running = TaskGroup::new(vec![tokio::spawn(std::future::pending())]);
assert!(!running.any_finished());
}
#[tokio::test]
async fn shutdown_ends_a_task_that_would_never_end_on_its_own() {
let group = TaskGroup::new(vec![tokio::spawn(std::future::pending())]);
// The watchers and the drive task are infinite loops, so awaiting one
// without cancelling it first hangs here for ever.
tokio::time::timeout(std::time::Duration::from_secs(5), group.shutdown())
.await
.expect("shutdown must not hang on a task that never finishes by itself");
// Shutting down twice is a no-op rather than a second abort.
tokio::time::timeout(std::time::Duration::from_secs(5), group.shutdown())
.await
.expect("a second shutdown must return immediately");
}
}

800
lez/sequencer/core/src/tests.rs
View File

@ -4,48 +4,54 @@ use std::{pin::pin, time::Duration};
use common::{
HashType,
block::{BedrockStatus, HashableBlockData},
block::{BedrockStatus, Block, HashableBlockData},
test_utils::sequencer_sign_key_for_testing,
transaction::{LeeTransaction, clock_invocation},
};
use key_protocol::key_management::KeyChain;
use lee::{
Account, AccountId, Data, PrivacyPreservingTransaction, PrivateKey, PublicKey,
PublicTransaction, V03State,
error::LeeError,
execute_and_prove,
privacy_preserving_transaction::{Message, circuit::ProgramWithDependencies},
program::Program,
Account, AccountId, Data, PrivateKey, PublicKey, PublicTransaction, V03State, program::Program,
};
use lee_core::{
Commitment, InputAccountIdentity, Nullifier,
account::{AccountWithMetadata, Nonce},
program::PdaSeed,
};
use logos_blockchain_core::mantle::{
ledger::Inputs,
ops::channel::{ChannelId, MsgId, deposit::Metadata},
tx::TxHash,
use lee_core::{account::Nonce, program::PdaSeed};
use logos_blockchain_core::{
events::DepositRecreatedNotes,
mantle::{
TxHash,
ledger::Inputs,
ops::channel::{ChannelId, MsgId, deposit::Metadata},
},
};
use logos_blockchain_key_management_system_service::keys::ZkPublicKey;
use logos_blockchain_zone_sdk::sequencer::DepositInfo;
use mempool::MemPoolHandle;
use storage::sequencer::sequencer_cells::PendingDepositEventRecord;
use ping_core::{ReceiverInstruction, ping_record_pda};
use storage::sequencer::sequencer_cells::{
PendingCrossZoneDispatchRecord, PendingDepositEventRecord,
};
use tempfile::tempdir;
use testnet_initial_state::{initial_pub_accounts_private_keys, initial_public_user_accounts};
use crate::{
TransactionOrigin, apply_follow_update,
MAX_DISPATCHES_PER_BLOCK, RETIRE_DISPATCH_AFTER_FAILURES, TransactionOrigin,
apply_follow_update,
block_publisher::FollowUpdate,
block_store::SequencerStore,
build_bridge_deposit_tx_from_event, build_genesis_state,
config::{BedrockConfig, GenesisAction, SequencerConfig},
deposit_already_minted, is_sequencer_only_program,
build_bridge_deposit_tx_from_event, build_genesis_state, classify_settled_deliveries,
config::{
BedrockConfig, CrossZoneConfig, CrossZonePeer, CrossZoneRoute, GenesisAction,
SequencerConfig,
},
deposit_already_minted, dispatch_already_delivered, extract_cross_zone_dispatch,
extract_cross_zone_dispatch_key, is_sequencer_only_program,
mock::{SequencerCoreWithMockClients, mock_checkpoint},
resubmittable_txs,
};
mod reconstruction;
/// The peer zone a cross-zone test receives from. Distinct from the test
/// channel id (`[0; 32]`), which the inbox guest rejects as a source.
const PEER_ZONE: [u8; 32] = [0xbe_u8; 32];
#[derive(borsh::BorshSerialize)]
struct DepositMetadataForEncoding {
recipient_id: lee::AccountId,
@ -79,10 +85,12 @@ fn setup_sequencer_config() -> SequencerConfig {
channel_id: ChannelId::from([0; 32]),
node_url: "http://not-used-in-unit-tests".parse().unwrap(),
auth: None,
funding_key: ZkPublicKey::zero(),
},
retry_pending_blocks_timeout: Duration::from_mins(4),
genesis: vec![],
cross_zone: None,
metrics_address: None,
}
}
@ -162,6 +170,83 @@ fn tx_is_bridge_deposit(
)
}
/// A config that receives `ping_receiver` messages from [`PEER_ZONE`], so
/// `build_genesis_state` seeds the inbox config PDA and a delivery has an
/// allowlist to pass.
fn cross_zone_test_config() -> SequencerConfig {
SequencerConfig {
cross_zone: Some(CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: PEER_ZONE,
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::ping_sender().id(),
target_program_id: programs::ping_receiver().id(),
}],
expected_block_signing_pubkey: None,
}],
}),
..setup_sequencer_config()
}
}
/// A `ping_receiver::Record` instruction as risc0 words, little-endian: the wire
/// form an emitter on the peer zone puts in the message payload.
fn ping_payload(payload: &[u8]) -> Vec<u8> {
risc0_zkvm::serde::to_vec(&ReceiverInstruction::Record {
payload: payload.to_vec(),
})
.expect("ping instruction serializes")
.iter()
.flat_map(|word| word.to_le_bytes())
.collect()
}
/// The dispatch transaction for a message at index 0 of [`PEER_ZONE`] block
/// `src_block_id`. Built through the same builder the watcher uses, so a change
/// to the encoding shows up here rather than passing silently.
fn dispatch_tx(src_block_id: u64, payload: Vec<u8>) -> LeeTransaction {
let receiver_id = programs::ping_receiver().id();
LeeTransaction::Public(cross_zone::build_dispatch_from_emission(
PEER_ZONE,
src_block_id,
0,
programs::ping_sender().id(),
receiver_id,
&[ping_record_pda(receiver_id).into_value()],
payload,
))
}
/// The pending record the watcher would leave behind for that dispatch.
fn dispatch_record(src_block_id: u64, payload: Vec<u8>) -> PendingCrossZoneDispatchRecord {
let tx = dispatch_tx(src_block_id, payload);
PendingCrossZoneDispatchRecord::recorded(
cross_zone_inbox_core::message_key(&PEER_ZONE, src_block_id, 0),
borsh::to_vec(&tx).expect("dispatch encodes"),
)
}
/// The message keys of the deliveries a block carries.
fn dispatches_in(block: &Block) -> Vec<[u8; 32]> {
block
.body
.transactions
.iter()
.filter_map(extract_cross_zone_dispatch_key)
.collect()
}
/// The pending dispatch records a sequencer still holds.
fn pending_dispatches(
sequencer: &SequencerCoreWithMockClients,
) -> Vec<PendingCrossZoneDispatchRecord> {
sequencer
.store
.dbio()
.get_pending_cross_zone_dispatches()
.expect("pending dispatches readable")
}
#[tokio::test]
async fn start_from_config() {
let config = setup_sequencer_config();
@ -477,6 +562,363 @@ async fn a_replayed_deposit_mint_no_ops_in_the_guest() {
);
}
#[tokio::test]
async fn recorded_dispatches_are_drained_from_the_store_on_production() {
let payload = b"hello-cross-zone".to_vec();
let record = dispatch_record(7, ping_payload(&payload));
let key = record.message_key;
let (mut sequencer, _mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
assert_eq!(
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap(),
1
);
// The delivery never goes through the mempool: the record is the queue, and
// production drains it. That is what makes the window between the watcher's
// durable read cursor and a block carrying the dispatch survivable.
assert!(
sequencer.mempool.pop().is_none(),
"deliveries are drained from the store, never queued in the mempool"
);
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer
.store
.get_block_at_id(block_id)
.unwrap()
.expect("produced block is stored");
assert_eq!(
dispatches_in(&block),
vec![key],
"the drained delivery should be included in the produced block"
);
let record_id = ping_record_pda(programs::ping_receiver().id());
assert_eq!(
sequencer.with_state(|state| state.get_account_by_id(record_id).data.into_inner()),
payload,
"the dispatch must reach its target program, not just sit in the block"
);
// The record stays until the delivery finalizes; re-delivery is prevented by
// the inbox seen-set now in head state, not by any marker on the record.
assert_eq!(
pending_dispatches(&sequencer)
.iter()
.map(|record| record.message_key)
.collect::<Vec<_>>(),
vec![key],
"the record remains until the delivery becomes irreversible"
);
}
#[tokio::test]
async fn a_delivered_dispatch_is_skipped_on_the_next_turn() {
// The seen-set is what replaces the submitted mark: the drain asks the state
// it is building on whether the inbox has already taken this message, so a
// record that outlives its delivery costs one skipped drain, not a replay.
let record = dispatch_record(11, ping_payload(b"once"));
let key = record.message_key;
let (mut sequencer, _mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
let first = sequencer.produce_new_block().await.unwrap();
let second = sequencer.produce_new_block().await.unwrap();
let delivered_in = |block_id: u64| {
dispatches_in(
&sequencer
.store
.get_block_at_id(block_id)
.unwrap()
.expect("produced block is stored"),
)
};
assert_eq!(delivered_in(first), vec![key]);
assert!(
delivered_in(second).is_empty(),
"the inbox seen-set must keep the drain from re-delivering"
);
let message = extract_cross_zone_dispatch(&dispatch_tx(11, ping_payload(b"once")))
.expect("the dispatch carries a cross-zone message");
assert!(
sequencer.with_state(|state| dispatch_already_delivered(state, &message)),
"the seen shard in head state is what the skip reads"
);
}
#[tokio::test]
async fn a_dispatch_that_never_executes_is_given_up_on_after_repeated_failures() {
// A payload that is not `u32`-aligned: the inbox guest rejects it outright,
// so this is a delivery that can never execute however often it is retried.
// Its content is chosen on the peer zone and validated by nobody in between,
// so without a give-up policy it would fail on every block for ever.
let record = dispatch_record(13, b"odd".to_vec());
let (mut sequencer, _mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
for attempt in 1..RETIRE_DISPATCH_AFTER_FAILURES {
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer
.store
.get_block_at_id(block_id)
.unwrap()
.expect("produced block is stored");
assert!(
dispatches_in(&block).is_empty(),
"a dispatch that fails to execute must not reach the block"
);
let records = pending_dispatches(&sequencer);
assert_eq!(records.len(), 1);
assert_eq!(
records[0].failed_attempts, attempt,
"the counter advances once per block, not once per process start"
);
}
// The attempt at the limit gives up on it, and giving up drops the record.
// Anything else leaves an entry no later block can ever remove, which is how
// a peer that can make deliveries fail would grow this list without bound.
sequencer.produce_new_block().await.unwrap();
assert!(
pending_dispatches(&sequencer).is_empty(),
"giving up on a delivery must drop its record, not flag it"
);
// And nothing re-feeds it, so it stops costing a guest execution per block.
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert!(dispatches_in(&block).is_empty());
assert!(pending_dispatches(&sequencer).is_empty());
}
#[tokio::test]
async fn a_redelivered_record_is_dropped_once_its_delivery_is_irreversible() {
// The watcher persists its floor only at slot boundaries, so a crash inside
// a slot makes the next run re-read it and re-record deliveries that have
// already settled. Their keys are in the inbox seen-set for good, so no
// future block will ever carry them and the settlement path cannot reach
// them. The drain dropping them is the only thing that does.
let record = dispatch_record(29, ping_payload(b"again"));
let key = record.message_key;
let (mut sequencer, mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record.clone()])
.unwrap();
let block_id = sequencer.produce_new_block().await.unwrap();
let delivery_block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert_eq!(dispatches_in(&delivery_block), vec![key]);
apply_follow_update(
&sequencer.store.dbio(),
&sequencer.chain(),
&mempool_handle,
FollowUpdate {
finalized: vec![(MsgId::from(delivery_block.header.hash.0), delivery_block)],
..empty_follow_update()
},
);
assert!(pending_dispatches(&sequencer).is_empty());
// The watcher re-reads the slot and records it again.
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
assert_eq!(pending_dispatches(&sequencer).len(), 1);
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert!(
dispatches_in(&block).is_empty(),
"the delivery is already on the chain, so it must not be delivered again"
);
assert!(
pending_dispatches(&sequencer).is_empty(),
"a record whose delivery is already irreversible must be dropped, not kept for ever"
);
}
#[tokio::test]
async fn a_delivery_still_reversible_keeps_its_record() {
// The counterpart to the test above, and the reason the drain checks two
// states rather than one. In head but not yet final means the delivery can
// still orphan, so skipping it is right but dropping its record would lose
// the delivery when it does.
let record = dispatch_record(31, ping_payload(b"pending"));
let key = record.message_key;
let (mut sequencer, _mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
sequencer.produce_new_block().await.unwrap();
sequencer.produce_new_block().await.unwrap();
assert_eq!(
pending_dispatches(&sequencer)
.iter()
.map(|record| record.message_key)
.collect::<Vec<_>>(),
vec![key],
"nothing has finalized, so the record must survive in case the block orphans"
);
}
#[test]
fn a_settled_delivery_that_is_not_the_one_we_recorded_is_reported() {
// The message key covers (src_zone, src_block_id, src_tx_index) and nothing
// about the payload, and so does the inbox's own replay check. So a peer's
// sequencer can publish a delivery under a key we hold with a payload we
// never saw, and it settles our correct record along with it. The indexer
// catches the forgery and halts; this record is the last local copy of what
// we believed, so the mismatch has to be reported before it is dropped.
let honest = dispatch_record(53, ping_payload(b"honest"));
let key = honest.message_key;
let forged = dispatch_tx(53, ping_payload(b"forged"));
assert_eq!(
extract_cross_zone_dispatch_key(&forged),
Some(key),
"the forged delivery must share the key, or it proves nothing"
);
let block = common::test_utils::produce_dummy_block(2, None, vec![forged]);
let (keys, mismatched) = classify_settled_deliveries(std::slice::from_ref(&honest), &block);
assert_eq!(keys, vec![key], "the record is settled either way");
assert_eq!(
mismatched,
vec![key],
"a delivery that differs from the one recorded under that key must be reported"
);
// The honest case must stay quiet, or the report is noise.
let honest_block = common::test_utils::produce_dummy_block(
2,
None,
vec![dispatch_tx(53, ping_payload(b"honest"))],
);
let (keys, mismatched) = classify_settled_deliveries(&[honest], &honest_block);
assert_eq!(keys, vec![key]);
assert!(mismatched.is_empty());
}
#[tokio::test]
async fn a_delivery_too_large_for_any_block_does_not_stall_production() {
// A store-drained transaction is at the head of the queue every turn, so one
// that cannot fit in any block would defer itself for ever and, because the
// deferral breaks the loop, stop production ever reaching the mempool behind
// it. The peer chooses the payload, so this is theirs to trigger.
let record = dispatch_record(41, ping_payload(&[7_u8; 8192]));
let mut config = cross_zone_test_config();
config.max_block_size = bytesize::ByteSize::kib(4);
let (mut sequencer, mempool_handle) =
SequencerCoreWithMockClients::start_from_config(config).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
let user_tx = common::test_utils::create_transaction_native_token_transfer(
initial_public_user_accounts()[0].account_id,
0,
initial_public_user_accounts()[1].account_id,
10,
&create_signing_key_for_account1(),
);
mempool_handle
.push((TransactionOrigin::User, user_tx.clone()))
.await
.unwrap();
// Production must get past it to the mempool in the very first block.
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert!(
block.body.transactions.contains(&user_tx),
"an oversized drained delivery must not stop production reaching the mempool"
);
assert!(dispatches_in(&block).is_empty());
// And it is given up on rather than retried for ever.
for _ in 1..RETIRE_DISPATCH_AFTER_FAILURES {
sequencer.produce_new_block().await.unwrap();
}
assert!(
pending_dispatches(&sequencer).is_empty(),
"a delivery that fits in no block must be given up on"
);
}
#[tokio::test]
async fn a_delivery_backlog_is_spread_across_blocks() {
// Each delivery costs a guest execution and peers decide how many queue up,
// so an unbounded drain would let a backlog decide how long a block takes to
// build and leave no room for user work, since store-drained transactions
// are taken before the mempool.
let backlog = MAX_DISPATCHES_PER_BLOCK + 3;
let records: Vec<_> = (0..backlog)
.map(|index| {
let src_block_id = 100 + u64::try_from(index).expect("test index fits");
dispatch_record(src_block_id, ping_payload(b"backlog"))
})
.collect();
let mut config = cross_zone_test_config();
config.max_num_tx_in_block = backlog + 10;
let (mut sequencer, _mempool_handle) =
SequencerCoreWithMockClients::start_from_config(config).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(records)
.unwrap();
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert_eq!(
dispatches_in(&block).len(),
MAX_DISPATCHES_PER_BLOCK,
"one block must not carry an unbounded number of deliveries"
);
// Deferred, not dropped: the rest go in the next block.
let block_id = sequencer.produce_new_block().await.unwrap();
let block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert_eq!(dispatches_in(&block).len(), 3);
}
#[test]
fn transaction_pre_check_pass() {
let tx = common::test_utils::produce_dummy_empty_transaction();
@ -1034,93 +1476,94 @@ async fn block_production_aborts_when_clock_account_data_is_corrupted() {
);
}
#[test]
fn private_bridge_withdraw_invocation_is_dropped() {
let sender_keys = KeyChain::new_os_random();
let sender_account_id = AccountId::for_regular_private_account(
&sender_keys.nullifier_public_key,
&sender_keys.viewing_public_key,
0,
);
let sender_private_account = Account {
program_owner: programs::authenticated_transfer().id(),
balance: 100,
nonce: Nonce(0xdead_beef),
data: Data::default(),
};
let bridge_account_id = system_accounts::bridge_account_id();
// #[test]
// fn private_bridge_withdraw_invocation_is_dropped() {
// let sender_keys = KeyChain::new_os_random();
// let sender_account_id = AccountId::for_regular_private_account(
// &sender_keys.nullifier_public_key,
// &sender_keys.viewing_public_key,
// 0,
// );
// let sender_private_account = Account {
// program_owner: programs::authenticated_transfer().id(),
// balance: 100,
// nonce: Nonce(0xdead_beef),
// data: Data::default(),
// };
// let bridge_account_id = system_accounts::bridge_account_id();
let mut state = V03State::new()
.with_public_accounts([(bridge_account_id, system_accounts::bridge_account())])
.with_private_accounts([(
Commitment::new(&sender_account_id, &sender_private_account),
Nullifier::for_account_initialization(&sender_account_id),
)]);
// let mut state = V03State::new()
// .with_public_accounts([(bridge_account_id, system_accounts::bridge_account())])
// .with_private_accounts([(
// Commitment::new(&sender_account_id, &sender_private_account),
// Nullifier::for_account_initialization(&sender_account_id),
// )]);
let sender_commitment = Commitment::new(&sender_account_id, &sender_private_account);
// let sender_commitment = Commitment::new(&sender_account_id, &sender_private_account);
let sender_pre = AccountWithMetadata::new(
sender_private_account,
true,
(
&sender_keys.nullifier_public_key,
&sender_keys.viewing_public_key,
0,
),
);
let bridge_pre = AccountWithMetadata::new(
state.get_account_by_id(bridge_account_id),
false,
bridge_account_id,
);
// let sender_pre = AccountWithMetadata::new(
// sender_private_account,
// true,
// (
// &sender_keys.nullifier_public_key,
// &sender_keys.viewing_public_key,
// 0,
// ),
// );
// let bridge_pre = AccountWithMetadata::new(
// state.get_account_by_id(bridge_account_id),
// false,
// bridge_account_id,
// );
let instruction = Program::serialize_instruction(bridge_core::Instruction::Withdraw {
amount: 1,
bedrock_account_pk: [0; 32],
})
.unwrap();
// let instruction = Program::serialize_instruction(bridge_core::Instruction::Withdraw {
// amount: 1,
// bedrock_account_pk: [0; 32],
// })
// .unwrap();
let program_with_deps = ProgramWithDependencies::new(
programs::bridge(),
[(
programs::authenticated_transfer().id(),
programs::authenticated_transfer(),
)]
.into(),
);
// let program_with_deps = ProgramWithDependencies::new(
// programs::bridge(),
// [(
// programs::authenticated_transfer().id(),
// programs::authenticated_transfer(),
// )]
// .into(),
// );
let (output, proof) = execute_and_prove(
vec![sender_pre, bridge_pre],
instruction,
vec![
InputAccountIdentity::PrivateAuthorizedUpdate {
vpk: sender_keys.viewing_public_key.clone(),
random_seed: [0; 32],
view_tag: 0,
nsk: sender_keys.private_key_holder.nullifier_secret_key,
membership_proof: state
.get_proof_for_commitment(&sender_commitment)
.expect("sender commitment must be in state"),
identifier: 0,
},
InputAccountIdentity::Public,
],
&program_with_deps,
)
.expect("Execution should succeed");
// let (output, proof) = execute_and_prove(
// vec![sender_pre, bridge_pre],
// instruction,
// vec![
// InputAccountIdentity::PrivateAuthorizedUpdate {
// vpk: sender_keys.viewing_public_key.clone(),
// random_seed: [0; 32],
// view_tag: 0,
// nsk: sender_keys.private_key_holder.nullifier_secret_key,
// membership_proof: state
// .get_proof_for_commitment(&sender_commitment)
// .expect("sender commitment must be in state"),
// identifier: 0,
// },
// InputAccountIdentity::Public,
// ],
// &program_with_deps,
// )
// .expect("Execution should succeed");
let message = Message::from_circuit_output(vec![], output);
let witness_set =
lee::privacy_preserving_transaction::WitnessSet::for_message(&message, proof, &[]);
let tx =
LeeTransaction::PrivacyPreserving(PrivacyPreservingTransaction::new(message, witness_set));
let res = tx.execute_check_on_state(&mut state, 1, 0);
// let message = Message::try_from_circuit_output(vec![bridge_account_id], vec![], output)
// .expect("Message construction should succeed");
// let witness_set =
// lee::privacy_preserving_transaction::WitnessSet::for_message(&message, proof, &[]);
// let tx =
// LeeTransaction::PrivacyPreserving(PrivacyPreservingTransaction::new(message,
// witness_set)); let res = tx.execute_check_on_state(&mut state, 1, 0);
assert!(
matches!(res, Err(LeeError::InvalidInput(_))),
"Bridge withdraw invocation should be rejected in private execution"
);
}
// assert!(
// matches!(res, Err(LeeError::InvalidInput(_))),
// "Bridge withdraw invocation should be rejected in private execution"
// );
// }
/// Builds a [`V03State`] with the clock program and `program` registered, the three clock
/// accounts initialized, and the clock advanced to `clock_timestamp` so that reads of the
@ -1581,6 +2024,7 @@ async fn follow_update_records_deposits_for_the_production_drain() {
inputs: Inputs::empty(),
amount: 5,
metadata: Metadata::try_from(metadata).expect("deposit metadata fits"),
notes: DepositRecreatedNotes::default(),
};
apply_follow_update(
@ -1735,6 +2179,68 @@ async fn follow_orphan_reverts_head_and_requeues_user_txs() {
);
}
#[tokio::test]
async fn follow_orphan_of_a_finalized_block_requeues_nothing() {
// The zone-sdk reports a block as orphaned once LIB pruning drops its
// inscription from the channel lineage, which happens a poll or two after
// every block of ours finalizes. Its transactions are irreversibly
// included, so requeueing them would put them back in every block we
// produce from then on.
let config = setup_sequencer_config();
let (mut sequencer, mempool_handle) =
SequencerCoreWithMockClients::start_from_config(config).await;
let acc1 = initial_public_user_accounts()[0].account_id;
let acc2 = initial_public_user_accounts()[1].account_id;
let tx = common::test_utils::create_transaction_native_token_transfer(
acc1,
0,
acc2,
10,
&create_signing_key_for_account1(),
);
mempool_handle
.push((TransactionOrigin::User, tx))
.await
.unwrap();
sequencer.produce_new_block().await.unwrap();
let block2 = sequencer.store.get_block_at_id(2).unwrap().unwrap();
apply_follow_update(
&sequencer.store.dbio(),
&sequencer.chain(),
&mempool_handle,
FollowUpdate {
finalized: vec![(MsgId::from(block2.header.hash.0), block2.clone())],
..empty_follow_update()
},
);
apply_follow_update(
&sequencer.store.dbio(),
&sequencer.chain(),
&mempool_handle,
FollowUpdate {
orphaned: vec![(MsgId::from(block2.header.hash.0), block2)],
..empty_follow_update()
},
);
assert_eq!(
sequencer.chain_height(),
2,
"an irreversible block cannot be reverted"
);
assert_eq!(
sequencer.with_state(|s| s.get_account_by_id(acc2).balance),
20010,
"the finalized transfer stands"
);
assert!(
sequencer.mempool.pop().is_none(),
"a transaction that is already irreversible must not be requeued"
);
}
#[tokio::test]
async fn follow_finalized_own_block_moves_final_tier_and_marks_store() {
let config = setup_sequencer_config();
@ -1773,6 +2279,98 @@ async fn follow_finalized_own_block_moves_final_tier_and_marks_store() {
assert!(matches!(stored.bedrock_status, BedrockStatus::Finalized));
}
#[tokio::test]
async fn follow_finalized_delivery_drops_its_pending_record() {
// The record exists to bridge the gap between the watcher's durable read
// cursor and a block that carries the delivery. Once that block is
// irreversible the delivery cannot be lost any more, so the record is owed
// nothing and goes with the same update that made the block irreversible.
let record = dispatch_record(17, ping_payload(b"settled"));
let key = record.message_key;
let (mut sequencer, mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
let block_id = sequencer.produce_new_block().await.unwrap();
let delivery_block = sequencer.store.get_block_at_id(block_id).unwrap().unwrap();
assert_eq!(dispatches_in(&delivery_block), vec![key]);
assert_eq!(
pending_dispatches(&sequencer).len(),
1,
"including the delivery is not enough to settle its record"
);
apply_follow_update(
&sequencer.store.dbio(),
&sequencer.chain(),
&mempool_handle,
FollowUpdate {
finalized: vec![(MsgId::from(delivery_block.header.hash.0), delivery_block)],
..empty_follow_update()
},
);
assert!(
pending_dispatches(&sequencer).is_empty(),
"a delivery in an irreversible block settles its record"
);
}
#[tokio::test]
async fn a_parked_finalized_block_does_not_drop_a_dispatch_record() {
// Keyed by message key, not by height: a finalized block the final tier
// parks never became irreversible, so nothing it happens to sit above may
// settle a record. Dropping one here would lose the delivery for good, since
// the watcher's floor has already moved past the peer block it came from.
let record = dispatch_record(19, ping_payload(b"parked"));
let key = record.message_key;
let delivery = dispatch_tx(19, ping_payload(b"parked"));
let (mut sequencer, mempool_handle) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
sequencer
.store
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
let tx = common::test_utils::produce_dummy_empty_transaction();
mempool_handle
.push((TransactionOrigin::User, tx))
.await
.unwrap();
sequencer.produce_new_block().await.unwrap();
// A skip-ahead block carrying the same delivery: not in head and linking to
// nothing we hold, so the final tier parks it instead of applying it.
let parked =
common::test_utils::produce_dummy_block(9, Some(HashType([44; 32])), vec![delivery]);
apply_follow_update(
&sequencer.store.dbio(),
&sequencer.chain(),
&mempool_handle,
FollowUpdate {
finalized: vec![(MsgId::from([9; 32]), parked)],
..empty_follow_update()
},
);
assert_eq!(
pending_dispatches(&sequencer)
.iter()
.map(|record| record.message_key)
.collect::<Vec<_>>(),
vec![key],
"a parked finalized block must not drop its delivery's record"
);
}
#[tokio::test]
async fn follow_finalized_backfill_block_is_applied_and_marked_finalized() {
let config = setup_sequencer_config();

537
lez/sequencer/core/src/tests/reconstruction.rs
View File

@ -298,219 +298,230 @@ fn deposit_event_record(
}
}
/// Builds a signed public bridge `Withdraw` transaction (the normal user path).
fn build_public_withdraw_tx(
sender: lee::AccountId,
nonce: u128,
amount: u64,
bedrock_account_pk: [u8; 32],
signing_key: &lee::PrivateKey,
) -> LeeTransaction {
let message = lee::public_transaction::Message::try_new(
programs::bridge().id(),
vec![sender, system_accounts::bridge_account_id()],
vec![nonce.into()],
bridge_core::Instruction::Withdraw {
amount,
bedrock_account_pk,
},
)
.unwrap();
let witness_set = lee::public_transaction::WitnessSet::for_message(&message, &[signing_key]);
LeeTransaction::Public(lee::PublicTransaction::new(message, witness_set))
}
// /// Builds a signed public bridge `Withdraw` transaction (the normal user path).
// fn build_public_withdraw_tx(
// sender: lee::AccountId,
// nonce: u128,
// amount: u64,
// bedrock_account_pk: [u8; 32],
// signing_key: &lee::PrivateKey,
// ) -> LeeTransaction {
// let message = lee::public_transaction::Message::try_new(
// programs::bridge().id(),
// vec![sender, system_accounts::bridge_account_id()],
// vec![nonce.into()],
// bridge_core::Instruction::Withdraw {
// amount,
// bedrock_account_pk,
// },
// )
// .unwrap();
// let witness_set = lee::public_transaction::WitnessSet::for_message(&message, &[signing_key]);
// LeeTransaction::Public(lee::PublicTransaction::new(message, witness_set))
// }
/// Cold-start backfill re-records an already-finalized deposit event as a
/// pending record before reconstruction replays the same deposit block.
/// Reconstruction must drop that record — its mint is permanently reflected in
/// the reconstructed state (the receipt PDA) — so the next production neither
/// re-mints the vault nor emits a stray deposit tx.
#[tokio::test]
async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() {
let recipient = initial_public_user_accounts()[0].account_id;
let deposit_amount = 500_u64;
let withdraw_amount = 100_u64;
let bedrock_account_pk = [0x22_u8; 32];
let deposit_op_id = [0x0d_u8; 32];
// /// The reconciliation key a produced block carries for `withdraw_tx`, keyed on
// /// the note [`MockBlockPublisher`] reports as released for it.
// fn produced_withdraw_key(withdraw_tx: &LeeTransaction) -> WithdrawalReconciliationKey {
// let withdraw_arg = crate::extract_bridge_withdraw_data(withdraw_tx).expect("withdraw data");
// let [note_id] = crate::mock::mock_released_notes(std::slice::from_ref(&withdraw_arg))[..]
// else {
// panic!("A bridge withdraw releases exactly one note");
// };
// Sequencer A produces a deposit block then a withdraw block.
let config_a = bridge_funded_config();
let (mut seq_a, mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
// crate::withdrawal_reconciliation_key(&note_id)
// }
let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient);
let deposit_tx =
crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx");
mempool_a
.push((TransactionOrigin::Sequencer, deposit_tx))
.await
.unwrap();
seq_a.produce_new_block().await.unwrap();
// /// Cold-start backfill re-records an already-finalized deposit event as a
// /// pending record before reconstruction replays the same deposit block.
// /// Reconstruction must drop that record — its mint is permanently reflected in
// /// the reconstructed state (the receipt PDA) — so the next production neither
// /// re-mints the vault nor emits a stray deposit tx.
// #[tokio::test]
// async fn reconstructed_deposit_is_not_reminted_after_backfill_redelivery() {
// let recipient = initial_public_user_accounts()[0].account_id;
// let deposit_amount = 500_u64;
// let withdraw_amount = 100_u64;
// let bedrock_account_pk = [0x22_u8; 32];
// let deposit_op_id = [0x0d_u8; 32];
let withdraw_tx = build_public_withdraw_tx(
recipient,
0,
withdraw_amount,
bedrock_account_pk,
&create_signing_key_for_account1(),
);
mempool_a
.push((TransactionOrigin::User, withdraw_tx.clone()))
.await
.unwrap();
seq_a.produce_new_block().await.unwrap();
// // Sequencer A produces a deposit block then a withdraw block.
// let config_a = bridge_funded_config();
// let (mut seq_a, mempool_a) =
// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
// let deposit_record = deposit_event_record(deposit_op_id, deposit_amount, recipient);
// let deposit_tx =
// crate::build_bridge_deposit_tx_from_event(&deposit_record).expect("build deposit tx");
// mempool_a
// .push((TransactionOrigin::Sequencer, deposit_tx))
// .await
// .unwrap();
// seq_a.produce_new_block().await.unwrap();
let config_b = bridge_funded_config();
let (mut seq_b, _mempool_b) = SequencerCoreWithMockClients::start_from_config(config_b).await;
// let withdraw_tx = build_public_withdraw_tx(
// recipient,
// 0,
// withdraw_amount,
// bedrock_account_pk,
// &create_signing_key_for_account1(),
// );
// mempool_a
// .push((TransactionOrigin::User, withdraw_tx.clone()))
// .await
// .unwrap();
// seq_a.produce_new_block().await.unwrap();
// Backfill re-delivery: the deposit event is re-recorded as a pending record
// before reconstruction runs. The mint no longer flows through the mempool
// (that sink was removed); the store drain is the only source.
assert!(
seq_b
.block_store()
.dbio()
.add_pending_deposit_event(deposit_record.clone())
.unwrap()
);
// let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
// let messages = channel_from_store(seq_a.block_store(), 10);
// let tip_slot = messages.last().unwrap().1;
// let channel_id = config_a.bedrock_config.channel_id;
let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(
&mock_b,
&seq_b.store,
&seq_b.chain,
true,
)
.await
.expect("reconstruct");
// let config_b = bridge_funded_config();
// let (mut seq_b, _mempool_b) =
// SequencerCoreWithMockClients::start_from_config(config_b).await;
let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
assert_eq!(tip_b.id, tip_a.id);
assert_eq!(tip_b.hash, tip_a.hash);
// // Backfill re-delivery: the deposit event is re-recorded as a pending record
// // before reconstruction runs. The mint no longer flows through the mempool
// // (that sink was removed); the store drain is the only source.
// assert!(
// seq_b
// .block_store()
// .dbio()
// .add_pending_deposit_event(deposit_record.clone())
// .unwrap()
// );
// Reconstruction replays the finalized deposit block, minting the receipt
// into state and dropping the re-recorded pending event — so the drain has
// nothing left to re-mint. This is the mechanism that protects against the
// re-delivery, in place of the removed mempool sink.
assert!(
seq_b
.block_store()
.dbio()
.get_pending_deposit_events()
.unwrap()
.is_empty(),
"reconstruction must drop the re-delivered pending deposit record"
);
// let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
// SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(
// &mock_b,
// &seq_b.store,
// &seq_b.chain,
// true,
// )
// .await
// .expect("reconstruct");
seq_b.produce_new_block().await.unwrap();
// let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
// assert_eq!(tip_b.id, tip_a.id);
// assert_eq!(tip_b.hash, tip_a.hash);
let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient);
let bridge_id = system_accounts::bridge_account_id();
let state_b = seq_b.chain().lock().unwrap().head_state().clone();
let state_a = seq_a.chain().lock().unwrap().head_state().clone();
for account in [vault_id, bridge_id, recipient] {
assert_eq!(
state_b.get_account_by_id(account).balance,
state_a.get_account_by_id(account).balance,
"reconstructed balance mismatch for {account:?}",
);
}
assert_eq!(
state_b.get_account_by_id(vault_id).balance,
u128::from(deposit_amount),
"deposit must mint into the recipient vault exactly once, not twice"
);
// // Reconstruction replays the finalized deposit block, minting the receipt
// // into state and dropping the re-recorded pending event — so the drain has
// // nothing left to re-mint. This is the mechanism that protects against the
// // re-delivery, in place of the removed mempool sink.
// assert!(
// seq_b
// .block_store()
// .dbio()
// .get_pending_deposit_events()
// .unwrap()
// .is_empty(),
// "reconstruction must drop the re-delivered pending deposit record"
// );
let produced = seq_b
.block_store()
.get_block_at_id(tip_b.id + 1)
.unwrap()
.expect("produced block present");
assert!(
!produced
.body
.transactions
.iter()
.any(|tx| crate::extract_bridge_deposit_id(tx) == Some(HashType(deposit_op_id))),
"the re-delivered mint must be skipped, not re-included in a block"
);
// seq_b.produce_new_block().await.unwrap();
// A reconstructed withdraw's finalized L1 event was already re-delivered (and
// dropped) by cold-start backfill, so it will never be consumed again.
// Reconstruction must not count it, or the count stays phantom-inflated forever.
let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data");
let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key");
assert!(
!seq_b
.block_store()
.dbio()
.consume_unseen_withdraw_count(key)
.unwrap(),
"reconstruction must not leave a phantom unseen-withdraw count"
);
}
// let vault_id = vault_core::compute_vault_account_id(programs::vault().id(), recipient);
// let bridge_id = system_accounts::bridge_account_id();
// let state_b = seq_b.chain().lock().unwrap().head_state().clone();
// let state_a = seq_a.chain().lock().unwrap().head_state().clone();
// for account in [vault_id, bridge_id, recipient] {
// assert_eq!(
// state_b.get_account_by_id(account).balance,
// state_a.get_account_by_id(account).balance,
// "reconstructed balance mismatch for {account:?}",
// );
// }
// assert_eq!(
// state_b.get_account_by_id(vault_id).balance,
// u128::from(deposit_amount),
// "deposit must mint into the recipient vault exactly once, not twice"
// );
/// A reconstructed withdraw block must not touch the unseen-withdraw counter.
/// Its finalized L1 Withdraw event was already re-delivered (and dropped as a
/// no-op) by cold-start backfill, so counting it during reconstruction would
/// leave a permanent phantom that nothing ever consumes.
#[tokio::test]
async fn reconstructed_withdraw_leaves_no_phantom_unseen_count() {
let recipient = initial_public_user_accounts()[0].account_id;
let withdraw_amount = 100_u64;
let bedrock_account_pk = [0x33_u8; 32];
// let produced = seq_b
// .block_store()
// .get_block_at_id(tip_b.id + 1)
// .unwrap()
// .expect("produced block present");
// assert!(
// !produced
// .body
// .transactions
// .iter()
// .any(|tx| crate::extract_bridge_deposit_id(tx) == Some(HashType(deposit_op_id))),
// "the re-delivered mint must be skipped, not re-included in a block"
// );
// Sequencer A produces a single withdraw block; treat its chain as the channel.
let config_a = bridge_funded_config();
let (mut seq_a, mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
let withdraw_tx = build_public_withdraw_tx(
recipient,
0,
withdraw_amount,
bedrock_account_pk,
&create_signing_key_for_account1(),
);
mempool_a
.push((TransactionOrigin::User, withdraw_tx.clone()))
.await
.unwrap();
seq_a.produce_new_block().await.unwrap();
// // A reconstructed withdraw's finalized L1 event was already re-delivered (and
// // dropped) by cold-start backfill, so it will never be consumed again.
// // Reconstruction must not count it, or the count stays phantom-inflated forever.
// let key = produced_withdraw_key(&withdraw_tx);
// assert!(
// !seq_b
// .block_store()
// .dbio()
// .consume_unseen_withdraw_count(key)
// .unwrap(),
// "reconstruction must not leave a phantom unseen-withdraw count"
// );
// }
let withdraw_arg = crate::extract_bridge_withdraw_data(&withdraw_tx).expect("withdraw data");
let key = crate::withdraw_event_reconciliation_key(&withdraw_arg.outputs).expect("recon key");
// Producing the withdraw counts it as unseen, awaiting its L1 event.
assert!(
seq_a
.block_store()
.dbio()
.consume_unseen_withdraw_count(key)
.unwrap(),
"producing a withdraw must count it as unseen"
);
// /// A reconstructed withdraw block must not touch the unseen-withdraw counter.
// /// Its finalized L1 Withdraw event was already re-delivered (and dropped as a
// /// no-op) by cold-start backfill, so counting it during reconstruction would
// /// leave a permanent phantom that nothing ever consumes.
// #[tokio::test]
// async fn reconstructed_withdraw_leaves_no_phantom_unseen_count() {
// let recipient = initial_public_user_accounts()[0].account_id;
// let withdraw_amount = 100_u64;
// let bedrock_account_pk = [0x33_u8; 32];
let messages = channel_from_store(seq_a.block_store(), 10);
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
// // Sequencer A produces a single withdraw block; treat its chain as the channel.
// let config_a = bridge_funded_config();
// let (mut seq_a, mempool_a) =
// SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
// let withdraw_tx = build_public_withdraw_tx(
// recipient,
// 0,
// withdraw_amount,
// bedrock_account_pk,
// &create_signing_key_for_account1(),
// );
// mempool_a
// .push((TransactionOrigin::User, withdraw_tx.clone()))
// .await
// .unwrap();
// seq_a.produce_new_block().await.unwrap();
// Sequencer B reconstructs A's chain from a fresh store.
let config_b = bridge_funded_config();
let (store_b, chain_b) = fresh_store_and_chain(&config_b);
let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(&mock_b, &store_b, &chain_b, true)
.await
.expect("reconstruct");
// let key = produced_withdraw_key(&withdraw_tx);
// // Producing the withdraw counts it as unseen, awaiting its L1 event.
// assert!(
// seq_a
// .block_store()
// .dbio()
// .consume_unseen_withdraw_count(key)
// .unwrap(),
// "producing a withdraw must count it as unseen"
// );
assert!(
!store_b.dbio().consume_unseen_withdraw_count(key).unwrap(),
"reconstruction must not leave a phantom unseen-withdraw count"
);
}
// let messages = channel_from_store(seq_a.block_store(), 10);
// let tip_slot = messages.last().unwrap().1;
// let channel_id = config_a.bedrock_config.channel_id;
// // Sequencer B reconstructs A's chain from a fresh store.
// let config_b = bridge_funded_config();
// let (store_b, chain_b) = fresh_store_and_chain(&config_b);
// let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
// SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(&mock_b, &store_b, &chain_b,
// true) .await
// .expect("reconstruct");
// assert!(
// !store_b.dbio().consume_unseen_withdraw_count(key).unwrap(),
// "reconstruction must not leave a phantom unseen-withdraw count"
// );
// }
/// A deposit whose L1 event was observed (an unfulfilled pending record
/// exists) and whose L2 mint is already contained in a finalized channel block.
@ -586,6 +597,146 @@ async fn reconstruction_reconciles_already_finished_deposit() {
);
}
/// A cross-zone delivery whose record is still pending locally, but whose block
/// arrives already finalized on the channel. Reconstruction must settle the
/// record on the way through: the delivery is permanently reflected in the
/// reconstructed state (the inbox seen shard), so the next production neither
/// re-delivers it nor leaves a record nothing will ever drop.
#[tokio::test]
async fn reconstructed_delivery_settles_its_pending_record() {
let payload = b"reconstructed".to_vec();
let record = dispatch_record(23, ping_payload(&payload));
let key = record.message_key;
// Sequencer A produces the block that carries the delivery.
let config_a = cross_zone_test_config();
let (mut seq_a, _mempool_a) =
SequencerCoreWithMockClients::start_from_config(config_a.clone()).await;
seq_a
.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record.clone()])
.unwrap();
seq_a.produce_new_block().await.unwrap();
let tip_a = seq_a.block_store().latest_block_meta().unwrap().unwrap();
let messages = channel_from_store(seq_a.block_store(), 10);
let tip_slot = messages.last().unwrap().1;
let channel_id = config_a.bedrock_config.channel_id;
// Sequencer B holds the same record, as its own watcher would after reading
// the peer block, and reconstructs A's chain from a fresh store.
let (mut seq_b, _mempool_b) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
assert_eq!(
seq_b
.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap(),
1
);
let mock_b = MockBlockPublisher::with_canned_channel(channel_id, Some(tip_slot), messages);
SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(
&mock_b,
&seq_b.store,
&seq_b.chain,
true,
)
.await
.expect("reconstruct");
let tip_b = seq_b.block_store().latest_block_meta().unwrap().unwrap();
assert_eq!(tip_b.id, tip_a.id);
assert_eq!(tip_b.hash, tip_a.hash);
assert!(
seq_b
.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
.unwrap()
.is_empty(),
"reconstruction must settle the record of a delivery it replayed"
);
// The delivery landed exactly once, and the next turn does not re-emit it.
let record_id = ping_record_pda(programs::ping_receiver().id());
assert_eq!(
seq_b.with_state(|state| state.get_account_by_id(record_id).data.into_inner()),
payload,
"the reconstructed delivery must reach its target program"
);
seq_b.produce_new_block().await.unwrap();
let produced = seq_b
.block_store()
.get_block_at_id(tip_b.id + 1)
.unwrap()
.expect("produced block present");
assert!(
!dispatches_in(&produced).contains(&key),
"the reconstructed delivery must not be re-emitted"
);
}
/// A delivery this node published itself, served back by the channel at or below
/// its own tip. That path verifies the block matches and returns early, so it is
/// reached on every restart. It must still settle the delivery's record: the
/// channel serving the block is what makes it irreversible, and nothing later
/// will ever put that key in a block again.
#[tokio::test]
async fn a_verified_own_block_settles_its_delivery_records() {
let record = dispatch_record(37, ping_payload(b"verified"));
let key = record.message_key;
let (mut seq, _mempool) =
SequencerCoreWithMockClients::start_from_config(cross_zone_test_config()).await;
seq.block_store()
.dbio()
.add_pending_cross_zone_dispatches(vec![record])
.unwrap();
let block_id = seq.produce_new_block().await.unwrap();
let block = seq
.block_store()
.get_block_at_id(block_id)
.unwrap()
.unwrap();
assert_eq!(dispatches_in(&block), vec![key]);
assert_eq!(
seq.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
.unwrap()
.len(),
1,
"producing the block is not what settles the record"
);
// The channel serves our own chain back, tip included.
let messages = channel_from_store(seq.block_store(), 10);
let tip_slot = messages.last().unwrap().1;
let mock = MockBlockPublisher::with_canned_channel(
seq.sequencer_config.bedrock_config.channel_id,
Some(tip_slot),
messages,
);
SequencerCore::<MockBlockPublisher>::verify_and_reconstruct(
&mock, &seq.store, &seq.chain, true,
)
.await
.expect("reconstruct");
assert!(
seq.block_store()
.dbio()
.get_pending_cross_zone_dispatches()
.unwrap()
.is_empty(),
"a delivery the channel confirms must not leave a record nothing can remove"
);
}
#[tokio::test]
async fn committed_local_against_missing_channel_fails_without_anchor() {
// A sequencer that has committed blocks — a non-genesis tip plus a persisted

2
lez/sequencer/service/Cargo.toml
View File

@ -15,6 +15,7 @@ mempool.workspace = true
sequencer_core = { workspace = true, features = ["testnet"] }
sequencer_service_protocol.workspace = true
sequencer_service_rpc = { workspace = true, features = ["server"] }
sequencer_service_metrics = { workspace = true, features = ["record"] }
programs.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
@ -22,6 +23,7 @@ anyhow.workspace = true
env_logger.workspace = true
hex.workspace = true
log.workspace = true
metrics-exporter-prometheus.workspace = true
tokio.workspace = true
tokio-util.workspace = true
jsonrpsee.workspace = true

3
lez/sequencer/service/Dockerfile
View File

@ -63,8 +63,9 @@ COPY --from=builder --chown=sequencer_service_user:sequencer_service_user /usr/l
VOLUME /var/lib/sequencer_service
# Expose default port
# Expose default ports
EXPOSE 3040
EXPOSE 9000
# Health check (TODO #244: Replace when a real health endpoint is available)
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \

3
lez/sequencer/service/configs/debug/sequencer_config.json
View File

@ -11,7 +11,8 @@
"max_retries": 5
},
"channel_id": "0101010101010101010101010101010101010101010101010101010101010101",
"node_url": "http://localhost:18080"
"node_url": "http://localhost:18080",
"funding_key": "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26"
},
"genesis": [
{

3
lez/sequencer/service/configs/docker/sequencer_config.json
View File

@ -11,7 +11,8 @@
"max_retries": 5
},
"channel_id": "0101010101010101010101010101010101010101010101010101010101010101",
"node_url": "http://host.docker.internal:18080"
"node_url": "http://host.docker.internal:18080",
"funding_key": "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26"
},
"genesis": [
{

1
lez/sequencer/service/docker-compose.yml
View File

@ -18,6 +18,7 @@ services:
container_name: sequencer_service
ports:
- "3040:3040"
- "9000:9000"
volumes:
# Mount configuration file
- ./configs/docker/sequencer_config.json:/etc/sequencer_service/sequencer_config.json

16
lez/sequencer/service/metrics/Cargo.toml Normal file
View File

@ -0,0 +1,16 @@
[package]
name = "sequencer_service_metrics"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[features]
default = []
# Enable metrics record
record = ["dep:metrics"]
[dependencies]
metrics = { workspace = true, optional = true }

9
lez/sequencer/service/metrics/src/lib.rs Normal file
View File

@ -0,0 +1,9 @@
//! This crate provides all metrics exposed by the sequencer service crate.
#[cfg(feature = "record")]
pub use record::*;
pub mod names;
#[cfg(feature = "record")]
pub mod record;

3
lez/sequencer/service/metrics/src/names.rs Normal file
View File

@ -0,0 +1,3 @@
pub const SUBMITTED_TRANSACTIONS_TOTAL: &str = "submitted_transactions_total";
pub const BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL: &str =
"before_mempool_failed_transactions_total";

32
lez/sequencer/service/metrics/src/record.rs Normal file
View File

@ -0,0 +1,32 @@
use metrics::{Counter, Unit, counter};
use crate::names;
pub fn init() {
submitted_transactions_total_counter().increment(0);
before_mempool_failed_transactions_total_counter().increment(0);
}
fn submitted_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions submitted",
unit: Unit::Count,
names::SUBMITTED_TRANSACTIONS_TOTAL
)
}
pub fn increment_submitted_transactions_total() {
submitted_transactions_total_counter().increment(1);
}
fn before_mempool_failed_transactions_total_counter() -> Counter {
counter!(
description: "Number of transactions that failed before reaching the mempool",
unit: Unit::Count,
names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL
)
}
pub fn increment_before_mempool_failed_transactions_total() {
before_mempool_failed_transactions_total_counter().increment(1);
}

6
lez/sequencer/service/src/bin/configure_channel.rs
View File

@ -31,9 +31,9 @@ struct Args {
/// Signatures required for future config changes.
#[clap(long, default_value_t = 1)]
configuration_threshold: u16,
/// Signatures required for channel withdrawals.
/// Signatures required for channel transfers.
#[clap(long, default_value_t = 1)]
withdraw_threshold: u16,
transfer_threshold: u16,
}
#[tokio::main]
@ -58,7 +58,7 @@ async fn main() -> Result<()> {
args.posting_timeframe,
args.posting_timeout,
args.configuration_threshold,
args.withdraw_threshold,
args.transfer_threshold,
)
.await
}

123
lez/sequencer/service/src/lib.rs
View File

@ -12,7 +12,11 @@ use sequencer_core::SequencerCore;
#[cfg(feature = "standalone")]
use sequencer_core::SequencerCoreWithMockClients as SequencerCore;
pub use sequencer_core::config::*;
use sequencer_core::{TransactionOrigin, block_publisher::BlockPublisherTrait as _};
use sequencer_core::{
TransactionOrigin,
block_publisher::BlockPublisherTrait as _,
task_group::{StoreRelease, TaskGroup},
};
use sequencer_service_rpc::RpcServer as _;
use tokio::{sync::Mutex, task::JoinHandle};
use tokio_util::sync::CancellationToken;
@ -26,12 +30,19 @@ const REQUEST_BODY_MAX_SIZE: ByteSize = ByteSize::mib(10);
/// Implements `Drop` to ensure all tasks are aborted and the RPC server is stopped when dropped.
pub struct SequencerHandle {
addr: SocketAddr,
/// Option because of `Drop` which forbids to simply move out of `self` in `stopped()`.
server_handle: Option<ServerHandle>,
server_handle: ServerHandle,
main_loop_handle: JoinHandle<Result<Never>>,
/// Cancelled when the publisher's drive task terminates (e.g. a panicked
/// persist sink); no channel events are processed past that point.
driver_cancellation: CancellationToken,
/// The core's background tasks, taken before the core was shared. This
/// handle owns no reference to the core itself, so without these there is
/// nothing to wait on: aborting the main loop only starts the teardown.
background_tasks: Vec<TaskGroup>,
/// The store, weakly. Every strong reference lives inside something this
/// handle stops, so watching the count go to zero is how shutdown knows the
/// database file is actually closed rather than assuming it from drop order.
store: StoreRelease,
}
impl SequencerHandle {
@ -40,29 +51,73 @@ impl SequencerHandle {
server_handle: ServerHandle,
main_loop_handle: JoinHandle<Result<Never>>,
driver_cancellation: CancellationToken,
background_tasks: Vec<TaskGroup>,
store: StoreRelease,
) -> Self {
Self {
addr,
server_handle: Some(server_handle),
server_handle,
main_loop_handle,
driver_cancellation,
background_tasks,
store,
}
}
/// Stops the sequencer and waits for every part of it to be gone.
///
/// `Drop` alone cannot do this: it aborts the main loop without awaiting it,
/// and the core lives behind `Arc`s held by that task and the RPC server, so
/// after a plain drop the store is still open for an unbounded stretch. That
/// is why restarting a sequencer on the same home directory used to need a
/// sleep, and why an in-process restart could fail outright with a `RocksDB`
/// lock error.
///
/// Order matters: the main loop stops first so nothing new is produced while
/// the publisher is torn down, then the background tasks that hold the store,
/// then the server. Consuming `self` drops the last references, so the store
/// is closed by the time this returns.
pub async fn shutdown(mut self) {
self.main_loop_handle.abort();
if let Err(err) = (&mut self.main_loop_handle).await
&& err.is_panic()
{
error!("Sequencer main loop panicked before shutdown: {err}");
}
for tasks in &self.background_tasks {
tasks.shutdown().await;
}
if let Err(err) = self.server_handle.stop() {
error!("An error occurred while stopping Sequencer RPC server: {err}");
}
self.server_handle.clone().stopped().await;
// Nothing this handle owns holds the store, so waiting here rather than
// after the drop is the same thing, and it keeps the guarantee inside
// the call the caller awaits.
wait_for_store_release(&self.store).await;
}
/// Wait for any of the sequencer tasks to fail and return the error.
#[expect(
clippy::integer_division_remainder_used,
reason = "Generated by select! macro, can't be easily rewritten to avoid this lint"
)]
pub async fn failed(mut self) -> Result<Never> {
pub async fn failed(&mut self) -> Result<Never> {
let Self {
addr: _,
server_handle,
main_loop_handle,
driver_cancellation,
} = &mut self;
background_tasks: _,
store: _,
} = self;
let server_handle = server_handle.take().expect("Server handle is set");
// Cloned rather than taken: `stopped()` consumes a handle, and taking
// this one would leave `shutdown` with no way to stop the server.
let server_handle = server_handle.clone();
tokio::select! {
() = server_handle.stopped() => {
Err(anyhow!("RPC Server stopped"))
@ -89,11 +144,16 @@ impl SequencerHandle {
server_handle,
main_loop_handle,
driver_cancellation,
background_tasks,
store: _,
} = self;
let stopped = server_handle.as_ref().is_none_or(ServerHandle::is_stopped)
let stopped = server_handle.is_stopped()
|| main_loop_handle.is_finished()
|| driver_cancellation.is_cancelled();
|| driver_cancellation.is_cancelled()
// A watcher only ends by panicking, and a peer whose deliveries have
// silently stopped is exactly what this predicate exists to catch.
|| background_tasks.iter().any(TaskGroup::any_finished);
!stopped
}
@ -110,21 +170,49 @@ impl Drop for SequencerHandle {
server_handle,
main_loop_handle,
driver_cancellation: _,
background_tasks: _,
store: _,
} = self;
main_loop_handle.abort();
let Some(handle) = server_handle else {
return;
};
if let Err(err) = handle.stop() {
if let Err(err) = server_handle.stop() {
error!("An error occurred while stopping Sequencer RPC server: {err}");
}
}
}
/// Waits until nothing holds the store any more.
///
/// Everything that holds one lives inside a task or a server this handle has
/// already stopped, but the last drop happens on whichever thread ran them, not
/// on this one. Without this the caller can reopen the database a moment too
/// early and hit a `RocksDB` lock error, which is the kind of failure that shows
/// up as an occasional flake rather than a bug.
async fn wait_for_store_release(store: &StoreRelease) {
/// Long enough for a drop that is already in flight, short enough that a
/// leak is reported rather than hung on.
const RELEASE_TIMEOUT: Duration = Duration::from_secs(10);
const POLL: Duration = Duration::from_millis(10);
let released = tokio::time::timeout(RELEASE_TIMEOUT, async {
while store.holders() > 0 {
tokio::time::sleep(POLL).await;
}
})
.await;
if released.is_err() {
error!(
"Sequencer store still held by {} reference(s) after shutdown; something outlived the tasks it should have died with",
store.holders()
);
}
}
pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<SequencerHandle> {
sequencer_service_metrics::init();
let block_timeout = config.block_create_timeout;
let max_block_size = config.max_block_size;
@ -134,6 +222,11 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<Seq
info!("Sequencer core set up");
let driver_cancellation = sequencer_core.block_publisher().driver_cancellation();
// Taken while the core is still owned here: once it is behind the `Arc`
// below, the only owners are the RPC server and the main loop task, and
// neither hands it back.
let background_tasks = sequencer_core.background_tasks();
let store = sequencer_core.store_release();
let seq_core_wrapped = Arc::new(Mutex::new(sequencer_core));
let mempool_handle_for_server = mempool_handle.clone();
@ -156,6 +249,8 @@ pub async fn run(config: SequencerConfig, listen_addr: SocketAddr) -> Result<Seq
server_handle,
main_loop_handle,
driver_cancellation,
background_tasks,
store,
))
}

105
lez/sequencer/service/src/main.rs
View File

@ -3,9 +3,11 @@ use std::{
path::PathBuf,
};
use anyhow::Result;
use anyhow::{Context as _, Result};
use clap::Parser;
use log::{error, info};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
use tokio::signal::unix::{SignalKind, signal};
use tokio_util::sync::CancellationToken;
#[derive(Debug, Parser)]
@ -23,6 +25,10 @@ struct Args {
/// so multiple instances can share one config file.
#[clap(long)]
home: Option<PathBuf>,
/// Override the config's `metrics_address`, so multiple instances can share
/// one config file without fighting over the exporter port.
#[clap(long)]
metrics_address: Option<SocketAddr>,
}
#[tokio::main]
@ -33,23 +39,18 @@ struct Args {
async fn main() -> Result<()> {
env_logger::init();
let Args {
config_path,
port,
listen_address,
home,
} = Args::parse();
let args = Args::parse();
// TODO: handle this cancellation token more gracefully within Sequencer service
// similar to how we do in Indexer
let cancellation_token = listen_for_shutdown_signal();
let mut config = sequencer_service::SequencerConfig::from_path(&config_path)?;
if let Some(home) = home {
config.home = home;
let mut config = sequencer_service::SequencerConfig::from_path(&args.config_path)?;
apply_config_overrides(&args, &mut config);
if let Some(metrics_address) = config.metrics_address {
install_prometheus_recorder(metrics_address)?;
}
let sequencer_handle =
sequencer_service::run(config, SocketAddr::new(listen_address, port)).await?;
let mut sequencer_handle =
sequencer_service::run(config, SocketAddr::new(args.listen_address, args.port)).await?;
tokio::select! {
() = cancellation_token.cancelled() => {
@ -60,21 +61,89 @@ async fn main() -> Result<()> {
}
}
// Stop the watchers, the publisher's drive task, the block loop and the RPC
// server, and wait for each. Dropping the handle only asks; the store stays
// open for an unbounded stretch after that, so a restart can find its own
// home directory locked, and a watcher can be killed between recording a
// delivery and handing it over.
sequencer_handle.shutdown().await;
info!("Sequencer shutdown complete");
Ok(())
}
fn apply_config_overrides(args: &Args, config: &mut sequencer_service::SequencerConfig) {
let Args {
home,
metrics_address,
config_path: _,
port: _,
listen_address: _,
} = args;
if let Some(home) = home {
config.home.clone_from(home);
}
if let Some(metrics_address) = metrics_address {
config.metrics_address = Some(*metrics_address);
}
}
/// Installs the recorder on `metrics_address`.
fn install_prometheus_recorder(metrics_address: SocketAddr) -> Result<()> {
/// Ladder for `*_seconds` histograms, densest across the 1–100 ms band where
/// block production and transaction application actually land.
const LATENCY_BUCKETS: &[f64] = &[
0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
/// Fallback ladder for histograms that count things rather than measure time.
const COUNT_BUCKETS: &[f64] = &[1.0, 2.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0];
PrometheusBuilder::new()
.with_http_listener(metrics_address)
.with_recommended_naming(true)
.set_buckets(COUNT_BUCKETS)
.context("Failed to set default histogram buckets")?
.set_buckets_for_metric(Matcher::Suffix("_seconds".to_owned()), LATENCY_BUCKETS)
.context("Failed to set latency histogram buckets")?
.install()
.context("Failed to install Prometheus recorder")
}
/// Cancelled on Ctrl-C or `SIGTERM`.
///
/// `SIGTERM` is what a container runtime sends first, so without it every
/// orchestrated stop is the ungraceful path.
#[expect(
clippy::integer_division_remainder_used,
reason = "Generated by select! macro, can't be easily rewritten to avoid this lint"
)]
fn listen_for_shutdown_signal() -> CancellationToken {
let cancellation_token = CancellationToken::new();
let cancellation_token_clone = cancellation_token.clone();
tokio::spawn(async move {
if let Err(err) = tokio::signal::ctrl_c().await {
error!("Failed to listen for Ctrl-C signal: {err}");
return;
let mut terminate = match signal(SignalKind::terminate()) {
Ok(terminate) => terminate,
Err(err) => {
error!("Failed to listen for SIGTERM: {err}");
return;
}
};
tokio::select! {
result = tokio::signal::ctrl_c() => match result {
Ok(()) => info!("Received Ctrl-C signal"),
Err(err) => {
error!("Failed to listen for Ctrl-C signal: {err}");
return;
}
},
_ = terminate.recv() => info!("Received SIGTERM"),
}
info!("Received Ctrl-C signal");
cancellation_token_clone.cancel();
});

83
lez/sequencer/service/src/service.rs
View File

@ -6,7 +6,7 @@ use jsonrpsee::{
types::{ErrorCode, ErrorObjectOwned},
};
use lee;
use log::warn;
use log::{error, warn};
use mempool::MemPoolHandle;
use sequencer_core::{
DbError, SequencerCore, TransactionOrigin, block_publisher::BlockPublisherTrait,
@ -44,49 +44,62 @@ impl<BC: BlockPublisherTrait + Send + Sync + 'static> sequencer_service_rpc::Rpc
for SequencerService<BC>
{
async fn send_transaction(&self, tx: LeeTransaction) -> Result<HashType, ErrorObjectOwned> {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
sequencer_service_metrics::increment_submitted_transactions_total();
let tx_hash = tx.hash();
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size = u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
let res = async move {
// Reserve ~200 bytes for block header overhead
const BLOCK_HEADER_OVERHEAD: u64 = 200;
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let encoded_tx =
borsh::to_vec(&tx).expect("Transaction borsh serialization should not fail");
let tx_size =
u64::try_from(encoded_tx.len()).expect("Transaction size should fit in u64");
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
));
}
let max_tx_size = self.max_block_size.saturating_sub(BLOCK_HEADER_OVERHEAD);
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
if tx_size > max_tx_size {
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
format!("Transaction too large: size {tx_size}, max {max_tx_size}"),
None::<()>,
)
})?;
));
}
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction".to_owned(),
None::<()>,
));
}
let authenticated_tx = tx
.transaction_stateless_check()
.inspect_err(|err| warn!("Error at pre_check {err:#?}"))
.map_err(|err| {
ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
format!("{err:?}"),
None::<()>,
)
})?;
// Sequencer-only programs (the cross-zone inbox) are injected by the
// watcher; a user must not invoke them top-level, or anyone could forge
// an inbound cross-zone delivery. Chained user calls are already rejected
// by the inbox guest's caller-is-none assertion.
if let LeeTransaction::Public(public_tx) = &authenticated_tx
&& sequencer_core::is_sequencer_only_program(public_tx.message().program_id)
{
return Err(ErrorObjectOwned::owned(
ErrorCode::InvalidParams.code(),
"Program is sequencer-only and cannot be invoked by a user transaction"
.to_owned(),
None::<()>,
));
}
Ok(authenticated_tx)
};
let authenticated_tx = res.await.inspect_err(|err| {
sequencer_service_metrics::increment_before_mempool_failed_transactions_total();
error!("Transaction failed before reaching mempool: {err:#?}");
})?;
self.mempool_handle
.push((TransactionOrigin::User, authenticated_tx))

249
lez/storage/src/sequencer/mod.rs
View File

@ -1,4 +1,8 @@
use std::{collections::BTreeMap, path::Path, sync::Arc};
use std::{
collections::BTreeMap,
path::Path,
sync::{Arc, Mutex, MutexGuard, PoisonError},
};
use borsh::{BorshDeserialize, BorshSerialize};
use common::{
@ -18,7 +22,9 @@ use crate::{
sequencer::sequencer_cells::{
FinalBlockMetaCellOwned, FinalBlockMetaCellRef, FinalLeeStateCellOwned,
FinalLeeStateCellRef, LEEStateCellOwned, LEEStateCellRef, LastFinalizedBlockIdCell,
LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PendingDepositEventRecord,
LatestBlockMetaCellOwned, LatestBlockMetaCellRef, PeerFloorCellOwned, PeerFloorCellRef,
PeerZoneKey, PendingCrossZoneDispatchRecord, PendingCrossZoneDispatchesCellOwned,
PendingCrossZoneDispatchesCellRef, PendingDepositEventRecord,
PendingDepositEventsCellOwned, PendingDepositEventsCellRef, UnseenWithdrawCountCell,
WithdrawalReconciliationKey, ZoneAnchorCell, ZoneAnchorRecord, ZoneSdkCheckpointCellOwned,
ZoneSdkCheckpointCellRef,
@ -40,9 +46,25 @@ pub const DB_META_ZONE_CURSOR_KEY: &str = "zone_cursor";
/// Key base for storing queued deposit events that were not yet
/// fulfilled on L2.
pub const DB_META_PENDING_DEPOSIT_EVENTS_KEY: &str = "pending_deposit_events";
/// Key base for storing a cross-zone watcher's delivery floor on one peer
/// channel (opaque bytes). Keyed per peer zone.
pub const DB_META_CROSS_ZONE_PEER_FLOOR_KEY: &str = "cross_zone_peer_floor";
/// Key base for storing cross-zone deliveries the watcher has recorded but
/// which are not yet known to be irreversibly delivered.
pub const DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY: &str = "pending_cross_zone_dispatches";
/// Key base for counting unseen L2 withdraw intents.
pub const DB_META_UNSEEN_WITHDRAW_COUNT_KEY: &str = "unseen_withdraw_count";
/// How many cross-zone deliveries may be pending at once.
///
/// The whole list is a single value, read on every block and rewritten on every
/// change, and what fills it is chosen by peer zones rather than by us. Refusing
/// to record past this bound turns "a peer decides how large our store gets"
/// into "a peer's messages wait", since a watcher that cannot record holds its
/// delivery floor and reads the slot again later.
pub const MAX_PENDING_CROSS_ZONE_DISPATCHES: usize = 4096;
/// Key base for storing the LEE state.
pub const DB_LEE_STATE_KEY: &str = "lee_state";
/// Key base for storing the LEE state at the last L1-finalized block.
@ -123,6 +145,8 @@ pub struct StoreUpdate<'update> {
pub new_deposit_events: &'update [PendingDepositEventRecord],
/// Deposit op ids whose mint finalized: their pending records are dropped.
pub remove_deposit_records: &'update [HashType],
/// Message keys whose delivery finalized: their pending records are dropped.
pub remove_dispatch_records: &'update [[u8; 32]],
/// L1 withdraw events to reconcile against the local unseen counters.
pub consumed_withdrawals: &'update [WithdrawalReconciliationKey],
/// L2 withdraw intents this update raises, awaiting their L1 event.
@ -146,6 +170,7 @@ impl<'update> StoreUpdate<'update> {
finalized_up_to: None,
new_deposit_events: &[],
remove_deposit_records: &[],
remove_dispatch_records: &[],
consumed_withdrawals: &[],
new_withdraw_intents: &[],
zone_anchor: None,
@ -165,8 +190,21 @@ pub struct StoreUpdateOutcome {
pub unmatched_withdrawals: Vec<WithdrawalReconciliationKey>,
}
#[expect(
clippy::partial_pub_fields,
reason = "the pending-record lock is an implementation detail and must stay private"
)]
pub struct RocksDBIO {
pub db: DBWithThreadMode<MultiThreaded>,
/// Serializes the read-modify-write cycles over the pending cross-zone
/// dispatch list.
///
/// The list is a single value holding the whole `Vec`, and three tasks
/// rewrite it: the watcher recording a delivery, the production loop
/// counting a failed attempt, and the publisher's drive task settling
/// finalized deliveries. Rocksdb makes the write atomic, not the cycle, so
/// without this the writer that read first silently drops the others.
pending_records: Mutex<()>,
}
impl DBIO for RocksDBIO {
@ -176,6 +214,18 @@ impl DBIO for RocksDBIO {
}
impl RocksDBIO {
/// Held across a pending-record read-modify-write. See
/// [`RocksDBIO::pending_records`].
///
/// A poisoned lock is recovered rather than propagated: the records behind
/// it are a plain `Vec` that a panicking writer cannot leave half-written,
/// since the write is a single rocksdb put.
fn lock_pending_records(&self) -> MutexGuard<'_, ()> {
self.pending_records
.lock()
.unwrap_or_else(PoisonError::into_inner)
}
pub fn open(path: &Path) -> DbResult<Self> {
let db_opts = Options::default();
Self::open_inner(path, &db_opts)
@ -289,7 +339,10 @@ impl RocksDBIO {
additional_info: Some("Failed to open or create DB".to_owned()),
})?;
let dbio = Self { db };
let dbio = Self {
db,
pending_records: Mutex::new(()),
};
Ok(dbio)
}
@ -539,12 +592,191 @@ impl RocksDBIO {
Ok(accepted)
}
/// One cross-zone watcher's delivery floor on `peer_zone`'s channel, or
/// `None` before it has delivered anything from that peer.
pub fn get_cross_zone_peer_floor_bytes(
&self,
peer_zone: PeerZoneKey,
) -> DbResult<Option<Vec<u8>>> {
Ok(self
.get_opt::<PeerFloorCellOwned>(peer_zone)?
.map(|cell| cell.0))
}
pub fn put_cross_zone_peer_floor_bytes(
&self,
peer_zone: PeerZoneKey,
bytes: &[u8],
) -> DbResult<()> {
self.put(&PeerFloorCellRef(bytes), peer_zone)
}
pub fn get_pending_cross_zone_dispatches(
&self,
) -> DbResult<Vec<PendingCrossZoneDispatchRecord>> {
Ok(self
.get_opt::<PendingCrossZoneDispatchesCellOwned>(())?
.map_or_else(Vec::new, |cell| cell.0))
}
fn put_pending_cross_zone_dispatches(
&self,
records: &[PendingCrossZoneDispatchRecord],
) -> DbResult<()> {
self.put(&PendingCrossZoneDispatchesCellRef(records), ())
}
fn put_pending_cross_zone_dispatches_batch(
&self,
records: &[PendingCrossZoneDispatchRecord],
batch: &mut WriteBatch,
) -> DbResult<()> {
self.put_batch(&PendingCrossZoneDispatchesCellRef(records), (), batch)
}
/// Records every delivery one peer block carries, in a single write.
///
/// Returns how many were new. Ones already recorded are skipped, so a slot
/// the watcher re-reads is not double-tracked.
///
/// Batched rather than one call per delivery because the whole list is one
/// value: recording a block's messages one at a time rewrites the list once
/// per message, which is quadratic in a block that carries many.
///
/// Fails without writing anything if the list would exceed
/// [`MAX_PENDING_CROSS_ZONE_DISPATCHES`]. The caller's floor then stays put
/// and the slot is read again later, which is the difference between
/// backpressure and an unbounded list a peer controls the size of.
pub fn add_pending_cross_zone_dispatches(
&self,
dispatches: Vec<PendingCrossZoneDispatchRecord>,
) -> DbResult<usize> {
if dispatches.is_empty() {
return Ok(0);
}
let _pending = self.lock_pending_records();
let mut records = self.get_pending_cross_zone_dispatches()?;
let before = records.len();
for dispatch in dispatches {
if records
.iter()
.any(|record| record.message_key == dispatch.message_key)
{
continue;
}
records.push(dispatch);
}
let accepted = records.len().saturating_sub(before);
if accepted == 0 {
return Ok(0);
}
if records.len() > MAX_PENDING_CROSS_ZONE_DISPATCHES {
return Err(DbError::db_interaction_error(format!(
"Refusing to hold more than {MAX_PENDING_CROSS_ZONE_DISPATCHES} pending cross-zone deliveries; {before} already pending"
)));
}
self.put_pending_cross_zone_dispatches(&records)?;
Ok(accepted)
}
/// Counts a failed production attempt against a delivery, dropping its
/// record once it reaches `retire_at`. Returns whether it was dropped.
///
/// Dropped rather than flagged: a retired record is one the drain will never
/// turn into a block transaction again, so nothing would ever remove it, and
/// a peer that can make deliveries fail could grow the list without bound.
/// The delivery is given up on either way; this way the cost is a log line
/// rather than a permanent entry.
///
/// A delivery with no record is already retired as far as this is concerned:
/// there is nothing left to count against.
pub fn record_dispatch_failure(&self, message_key: [u8; 32], retire_at: u32) -> DbResult<bool> {
let _pending = self.lock_pending_records();
let mut records = self.get_pending_cross_zone_dispatches()?;
let Some(position) = records
.iter()
.position(|record| record.message_key == message_key)
else {
return Ok(true);
};
let attempts = {
let record = &mut records[position];
record.failed_attempts = record.failed_attempts.saturating_add(1);
record.failed_attempts
};
let retired = attempts >= retire_at;
if retired {
records.remove(position);
}
self.put_pending_cross_zone_dispatches(&records)?;
Ok(retired)
}
/// Drops the records of deliveries that are settled for good, outside any
/// store update.
///
/// The settlement path in [`Self::store_update`] catches a delivery as its
/// block becomes irreversible. This catches the ones that path cannot: a
/// record re-added after its delivery had already settled, which the watcher
/// does whenever it re-reads a slot it has already consumed. Nothing would
/// ever put such a key in a block again, so without this it stays for ever.
pub fn drop_settled_cross_zone_dispatches(&self, message_keys: &[[u8; 32]]) -> DbResult<usize> {
if message_keys.is_empty() {
return Ok(0);
}
let _pending = self.lock_pending_records();
let to_remove: std::collections::HashSet<&[u8; 32]> = message_keys.iter().collect();
let mut records = self.get_pending_cross_zone_dispatches()?;
let before = records.len();
records.retain(|record| !to_remove.contains(&record.message_key));
let removed = before.saturating_sub(records.len());
if removed > 0 {
self.put_pending_cross_zone_dispatches(&records)?;
}
Ok(removed)
}
/// Drops the pending records of deliveries that just became irreversible,
/// staged into `batch` so they go with the update that made them so.
///
/// Removal only, unlike [`Self::stage_pending_deposit_events`]: a delivery is
/// recorded by the watcher through
/// [`Self::add_pending_cross_zone_dispatch`], on its own task and outside
/// any store update, so nothing ever adds one here.
fn stage_removed_dispatches(
&self,
remove_keys: &[[u8; 32]],
batch: &mut WriteBatch,
) -> DbResult<usize> {
if remove_keys.is_empty() {
return Ok(0);
}
let to_remove: std::collections::HashSet<&[u8; 32]> = remove_keys.iter().collect();
let mut records = self.get_pending_cross_zone_dispatches()?;
let before = records.len();
records.retain(|record| !to_remove.contains(&record.message_key));
let removed = before.saturating_sub(records.len());
if removed > 0 {
self.put_pending_cross_zone_dispatches_batch(&records, batch)?;
}
Ok(removed)
}
/// Stages the unseen-withdraw decrements for one update into `batch`,
/// returning one entry per occurrence that matched no local counter.
///
/// Occurrences are folded per key for the same reason as the deposit
/// records: two withdrawals in one update can share a reconciliation key,
/// and a per-occurrence disk read would miss the staged decrement.
/// records: should two withdrawals in one update share a reconciliation
/// key, a per-occurrence disk read would miss the staged decrement.
fn stage_consumed_withdrawals(
&self,
withdrawals: &[WithdrawalReconciliationKey],
@ -621,8 +853,8 @@ impl RocksDBIO {
/// Stages the unseen-withdraw increments for one update into `batch`.
///
/// Occurrences are folded per key for the same reason as
/// [`Self::stage_consumed_withdrawals`]: two intents in one update can share
/// a reconciliation key, and a per-occurrence disk read would miss the
/// [`Self::stage_consumed_withdrawals`]: should two intents in one update
/// share a reconciliation key, a per-occurrence disk read would miss the
/// staged increment and count the pair once.
fn stage_new_withdraw_intents(
&self,
@ -875,6 +1107,7 @@ impl RocksDBIO {
/// one, most carry nothing else) must not drag a full state serialization
/// with it.
pub fn store_update(&self, update: &StoreUpdate<'_>) -> DbResult<StoreUpdateOutcome> {
let _pending = self.lock_pending_records();
let StoreUpdate {
checkpoint,
blocks,
@ -884,6 +1117,7 @@ impl RocksDBIO {
finalized_up_to,
new_deposit_events,
remove_deposit_records,
remove_dispatch_records,
consumed_withdrawals,
new_withdraw_intents,
zone_anchor,
@ -940,6 +1174,7 @@ impl RocksDBIO {
remove_deposit_records,
&mut batch,
)?;
self.stage_removed_dispatches(remove_dispatch_records, &mut batch)?;
let unmatched_withdrawals =
self.stage_consumed_withdrawals(consumed_withdrawals, &mut batch)?;
self.stage_new_withdraw_intents(new_withdraw_intents, &mut batch)?;

158
lez/storage/src/sequencer/sequencer_cells.rs
View File

@ -8,7 +8,8 @@ use crate::{
error::DbError,
sequencer::{
CF_LEE_STATE_NAME, DB_FINAL_BLOCK_META_KEY, DB_FINAL_LEE_STATE_KEY, DB_LEE_STATE_KEY,
DB_META_LAST_FINALIZED_BLOCK_ID, DB_META_LATEST_BLOCK_META_KEY,
DB_META_CROSS_ZONE_PEER_FLOOR_KEY, DB_META_LAST_FINALIZED_BLOCK_ID,
DB_META_LATEST_BLOCK_META_KEY, DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY,
DB_META_PENDING_DEPOSIT_EVENTS_KEY, DB_META_UNSEEN_WITHDRAW_COUNT_KEY,
DB_META_ZONE_CURSOR_KEY, DB_META_ZONE_SDK_CHECKPOINT_KEY,
},
@ -245,6 +246,82 @@ pub struct PendingDepositEventRecord {
pub metadata: Vec<u8>,
}
/// A cross-zone delivery the watcher has read off a peer block but which is not
/// yet known to be irreversibly delivered.
///
/// The watcher's delivery floor is durable, so once it advances past a peer
/// block that block is never re-read. This record is what stands in its place:
/// block production drains it every turn, and it survives a restart. Mirrors
/// [`PendingDepositEventRecord`], which solves the same problem for deposits,
/// and like it carries no "submitted" mark: the record is dropped when the
/// delivery itself finalizes, and re-including one meanwhile is harmless
/// because the inbox no-ops a replay on chain.
#[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize)]
pub struct PendingCrossZoneDispatchRecord {
/// Content-addressed replay key of the delivered message, and this record's
/// identity.
pub message_key: [u8; 32],
/// The borsh-encoded dispatch transaction, so production can re-feed it
/// without re-reading the peer channel.
pub transaction: Vec<u8>,
/// Production attempts that ended in an execution failure.
///
/// A dispatch's payload and target accounts are chosen on the peer zone and
/// validated by nobody in between, so one can fail for good. A failure can
/// equally be a property of the moment, so a single one is not enough to
/// give up on a delivery. Once too many accumulate the record is dropped
/// rather than flagged, since a delivery nothing will retry is also a
/// delivery nothing would ever remove.
pub failed_attempts: u32,
}
impl PendingCrossZoneDispatchRecord {
/// A delivery the watcher has just read: never attempted.
#[must_use]
pub const fn recorded(message_key: [u8; 32], transaction: Vec<u8>) -> Self {
Self {
message_key,
transaction,
failed_attempts: 0,
}
}
}
#[derive(BorshDeserialize)]
pub struct PendingCrossZoneDispatchesCellOwned(pub Vec<PendingCrossZoneDispatchRecord>);
impl SimpleStorableCell for PendingCrossZoneDispatchesCellOwned {
type KeyParams = ();
const CELL_NAME: &'static str = DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY;
const CF_NAME: &'static str = CF_META_NAME;
}
impl SimpleReadableCell for PendingCrossZoneDispatchesCellOwned {}
#[derive(BorshSerialize)]
pub struct PendingCrossZoneDispatchesCellRef<'records>(
pub &'records [PendingCrossZoneDispatchRecord],
);
impl SimpleStorableCell for PendingCrossZoneDispatchesCellRef<'_> {
type KeyParams = ();
const CELL_NAME: &'static str = DB_META_PENDING_CROSS_ZONE_DISPATCHES_KEY;
const CF_NAME: &'static str = CF_META_NAME;
}
impl SimpleWritableCell for PendingCrossZoneDispatchesCellRef<'_> {
fn value_constructor(&self) -> DbResult<Vec<u8>> {
borsh::to_vec(&self).map_err(|err| {
DbError::borsh_cast_message(
err,
Some("Failed to serialize pending cross-zone dispatches cell".to_owned()),
)
})
}
}
#[derive(BorshDeserialize)]
pub struct PendingDepositEventsCellOwned(pub Vec<PendingDepositEventRecord>);
@ -278,10 +355,78 @@ impl SimpleWritableCell for PendingDepositEventsCellRef<'_> {
}
}
/// Identifies which peer channel a cross-zone watcher cursor belongs to. The
/// 32-byte peer channel id doubles as the peer's zone id.
pub type PeerZoneKey = [u8; 32];
/// Opaque bytes for one peer's cross-zone read cursor. As with the zone-sdk
/// checkpoint, the caller owns the encoding, since the cursor type derives serde
/// rather than borsh.
#[derive(BorshDeserialize)]
pub struct PeerFloorCellOwned(pub Vec<u8>);
impl SimpleStorableCell for PeerFloorCellOwned {
type KeyParams = PeerZoneKey;
const CELL_NAME: &'static str = DB_META_CROSS_ZONE_PEER_FLOOR_KEY;
const CF_NAME: &'static str = CF_META_NAME;
/// Folds the peer zone into the key so each peer keeps its own cursor.
fn key_constructor(peer_zone: Self::KeyParams) -> DbResult<Vec<u8>> {
borsh::to_vec(&(Self::CELL_NAME, peer_zone)).map_err(|err| {
DbError::borsh_cast_message(
err,
Some(format!(
"Failed to serialize {:?} key params",
Self::CELL_NAME
)),
)
})
}
}
impl SimpleReadableCell for PeerFloorCellOwned {}
#[derive(BorshSerialize)]
pub struct PeerFloorCellRef<'bytes>(pub &'bytes [u8]);
impl SimpleStorableCell for PeerFloorCellRef<'_> {
type KeyParams = PeerZoneKey;
const CELL_NAME: &'static str = DB_META_CROSS_ZONE_PEER_FLOOR_KEY;
const CF_NAME: &'static str = CF_META_NAME;
/// Folds the peer zone into the key so each peer keeps its own cursor.
fn key_constructor(peer_zone: Self::KeyParams) -> DbResult<Vec<u8>> {
borsh::to_vec(&(Self::CELL_NAME, peer_zone)).map_err(|err| {
DbError::borsh_cast_message(
err,
Some(format!(
"Failed to serialize {:?} key params",
Self::CELL_NAME
)),
)
})
}
}
impl SimpleWritableCell for PeerFloorCellRef<'_> {
fn value_constructor(&self) -> DbResult<Vec<u8>> {
borsh::to_vec(&self).map_err(|err| {
DbError::borsh_cast_message(
err,
Some("Failed to serialize cross-zone peer floor cell".to_owned()),
)
})
}
}
/// Identity of one withdrawal, shared by the intent recorded when the
/// sequencer publishes it and the Bedrock Withdraw event that later reports
/// it: the id of the channel note the withdrawal releases.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct WithdrawalReconciliationKey {
pub amount: u64,
pub bedrock_account_pk: [u8; 32],
pub released_note_id: [u8; 32],
}
#[derive(Debug, BorshSerialize, BorshDeserialize)]
@ -294,12 +439,9 @@ impl SimpleStorableCell for UnseenWithdrawCountCell {
const CF_NAME: &'static str = CF_META_NAME;
fn key_constructor(key_params: Self::KeyParams) -> DbResult<Vec<u8>> {
let WithdrawalReconciliationKey {
amount,
bedrock_account_pk,
} = key_params;
let WithdrawalReconciliationKey { released_note_id } = key_params;
borsh::to_vec(&(Self::CELL_NAME, amount, bedrock_account_pk)).map_err(|err| {
borsh::to_vec(&(Self::CELL_NAME, released_note_id)).map_err(|err| {
DbError::borsh_cast_message(
err,
Some(format!(

187
lez/storage/src/sequencer/tests.rs
View File

@ -37,6 +37,17 @@ fn deposit_record(seed: u8) -> PendingDepositEventRecord {
}
}
fn dispatch_record(seed: u8) -> PendingCrossZoneDispatchRecord {
PendingCrossZoneDispatchRecord::recorded([seed; 32], vec![seed; 4])
}
/// A distinct message key per index, for filling the pending list.
fn key_from_index(index: usize) -> [u8; 32] {
let mut key = [0_u8; 32];
key[..8].copy_from_slice(&u64::try_from(index).expect("test index fits").to_le_bytes());
key
}
fn stored_balance(dbio: &RocksDBIO) -> u128 {
dbio.get_lee_state()
.unwrap()
@ -399,19 +410,182 @@ fn finalized_deposit_records_are_removed_by_op_id() {
assert_eq!(stored, vec![second]);
}
#[test]
fn dispatch_records_round_trip_and_dedupe_by_message_key() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let record = dispatch_record(1);
assert_eq!(
dbio.add_pending_cross_zone_dispatches(vec![record.clone()])
.unwrap(),
1
);
// The watcher re-reads a slot it stalled on, so the same delivery arrives
// again; recording it twice would double-count its failed attempts.
assert_eq!(
dbio.add_pending_cross_zone_dispatches(vec![record.clone(), dispatch_record(2)])
.unwrap(),
1,
"only the delivery not already held is newly recorded"
);
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap(),
vec![record, dispatch_record(2)]
);
}
#[test]
fn recording_past_the_cap_writes_nothing() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
// What fills this list is chosen by peer zones, so the bound is what stops a
// peer deciding how large our store gets. Refusing the whole write leaves
// the watcher's floor where it is, so the slot is read again later and
// nothing is lost.
let full: Vec<_> = (0..MAX_PENDING_CROSS_ZONE_DISPATCHES)
.map(|seed| PendingCrossZoneDispatchRecord::recorded(key_from_index(seed), vec![0_u8; 4]))
.collect();
assert_eq!(
dbio.add_pending_cross_zone_dispatches(full).unwrap(),
MAX_PENDING_CROSS_ZONE_DISPATCHES
);
let over = PendingCrossZoneDispatchRecord::recorded(
key_from_index(MAX_PENDING_CROSS_ZONE_DISPATCHES),
vec![0_u8; 4],
);
assert!(
dbio.add_pending_cross_zone_dispatches(vec![over]).is_err(),
"recording past the cap must fail so the caller holds its floor"
);
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap().len(),
MAX_PENDING_CROSS_ZONE_DISPATCHES,
"a refused write must leave the list untouched"
);
// Re-offering only what is already held is not growth, so it still succeeds.
assert_eq!(
dbio.add_pending_cross_zone_dispatches(vec![PendingCrossZoneDispatchRecord::recorded(
key_from_index(0),
vec![0_u8; 4]
)])
.unwrap(),
0
);
}
#[test]
fn settled_dispatch_records_are_dropped_outside_an_update() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
// The watcher re-reads a slot it already consumed and re-records a delivery
// that settled long ago. Its key will never appear in a future block, so the
// store-update path cannot reach it and this is the only thing that does.
let first = dispatch_record(1);
let second = dispatch_record(2);
dbio.add_pending_cross_zone_dispatches(vec![first.clone(), second.clone()])
.unwrap();
assert_eq!(
dbio.drop_settled_cross_zone_dispatches(&[first.message_key])
.unwrap(),
1
);
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap(),
vec![second]
);
// Dropping one that is already gone is a no-op, not an error.
assert_eq!(
dbio.drop_settled_cross_zone_dispatches(&[first.message_key])
.unwrap(),
0
);
}
#[test]
fn finalized_dispatch_records_are_removed_by_message_key() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let first = dispatch_record(1);
let second = dispatch_record(2);
dbio.add_pending_cross_zone_dispatches(vec![first.clone(), second.clone()])
.unwrap();
// Only the finalized delivery's key is dropped. Two deliveries can sit in
// the same block, so a record must go by its own identity rather than by
// anything about the height its delivery landed at.
dbio.store_update(&StoreUpdate {
remove_dispatch_records: &[first.message_key],
..StoreUpdate::new(&state_with_balance(100))
})
.unwrap();
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap(),
vec![second]
);
}
#[test]
fn record_dispatch_failure_drops_the_record_at_the_limit() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let record = dispatch_record(1);
let key = record.message_key;
let survivor = dispatch_record(2);
dbio.add_pending_cross_zone_dispatches(vec![record, survivor.clone()])
.unwrap();
assert!(!dbio.record_dispatch_failure(key, 3).unwrap());
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap()[0].failed_attempts,
1,
"a failure short of the limit is counted, not given up on"
);
assert!(!dbio.record_dispatch_failure(key, 3).unwrap());
assert!(
dbio.record_dispatch_failure(key, 3).unwrap(),
"the third failure is the one it is given up on"
);
// Dropped rather than flagged: a delivery the drain will never feed into a
// block again is one nothing would ever remove, so flagging it would let a
// peer that can make deliveries fail grow the list without bound.
assert_eq!(
dbio.get_pending_cross_zone_dispatches().unwrap(),
vec![survivor],
"giving up on a delivery drops its record and leaves the others alone"
);
// A key with no record reads as given up on: there is nothing left to count
// against, and nothing will feed it into a block.
assert!(
dbio.record_dispatch_failure(key, 3).unwrap(),
"a failure against a dropped delivery must not re-create its record"
);
assert_eq!(dbio.get_pending_cross_zone_dispatches().unwrap().len(), 1);
}
#[test]
fn repeated_withdrawal_key_in_one_update_folds_once_per_occurrence() {
let temp_dir = tempdir().unwrap();
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let key = WithdrawalReconciliationKey {
amount: 7,
bedrock_account_pk: [3; 32],
released_note_id: [3; 32],
};
// Two local intents for the same key in one update — two withdrawals of the
// same amount to the same L1 key. A per-occurrence disk read would miss the
// staged increment and record the pair as one.
// Two local intents for the same key in one update. A per-occurrence disk
// read would miss the staged increment and record the pair as one.
dbio.store_update(&StoreUpdate {
new_withdraw_intents: &[key, key],
..StoreUpdate::new(&state_with_balance(100))
@ -450,8 +624,7 @@ fn unmatched_withdrawal_is_reported_and_writes_nothing() {
let (dbio, _genesis) = dbio_with_genesis(temp_dir.path());
let key = WithdrawalReconciliationKey {
amount: 5,
bedrock_account_pk: [4; 32],
released_note_id: [4; 32],
};
let outcome = dbio
.store_update(&StoreUpdate {

36
monitoring/docker-compose.yml Normal file
View File

@ -0,0 +1,36 @@
# Prometheus + Grafana monitoring stack.
services:
prometheus:
image: prom/prometheus:v3.13.1
container_name: prometheus
command:
- --config.file=/etc/prometheus/prometheus.yml
ports:
- "9090:9090"
volumes:
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
- prometheus_data:/prometheus
# Lets Prometheus reach services running natively on the host.
extra_hosts:
- "host.docker.internal:host-gateway"
grafana:
image: grafana/grafana:13.1.1
container_name: grafana
ports:
- "3000:3000"
environment:
# Dev-only: open Grafana with no login, full access.
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_AUTH_DISABLE_LOGIN_FORM=true
volumes:
- ./grafana/provisioning:/etc/grafana/provisioning:ro
- ./grafana/dashboards:/var/lib/grafana/dashboards:ro
- grafana_data:/var/lib/grafana
depends_on:
- prometheus
volumes:
prometheus_data:
grafana_data:

377
monitoring/grafana/dashboards/sequencer.json Normal file
View File

@ -0,0 +1,377 @@
{
"annotations": { "list": [ ] },
"editable": true,
"graphTooltip": 1,
"panels": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "color": { "mode": "fixed", "fixedColor": "blue" }, "unit": "short", "decimals": 0 },
"overrides": [ ]
},
"gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 },
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "chain_height",
"legendFormat": "height",
"refId": "A"
}
],
"title": "Chain height",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "color": { "mode": "fixed", "fixedColor": "green" }, "unit": "short", "decimals": 0 },
"overrides": [ ]
},
"gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 },
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "area",
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "blocks_produced_total",
"legendFormat": "produced",
"refId": "A"
}
],
"title": "Blocks produced by this sequencer since startup",
"type": "stat"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"overrides": [
{
"matcher": { "id": "byName", "options": "produced · blocks/min" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
}
]
},
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 },
"id": 3,
"options": {
"legend": { "displayMode": "list", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "single" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(blocks_produced_total[$__rate_interval]) * 60",
"legendFormat": "produced · blocks/min",
"refId": "A"
}
],
"title": "Block production rate",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" },
"overrides": [
{
"matcher": { "id": "byName", "options": "avg" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
]
}
]
},
"gridPos": { "h": 9, "w": 24, "x": 0, "y": 7 },
"id": 4,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(block_creation_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(block_creation_time_seconds_sum[$__rate_interval]) / rate(block_creation_time_seconds_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "B"
}
],
"title": "Block creation time",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "s" },
"overrides": [ ]
},
"gridPos": { "h": 9, "w": 12, "x": 0, "y": 16 },
"id": 5,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le, kind, origin, status) (rate(mempool_transaction_application_time_seconds_bucket[$__rate_interval])))",
"legendFormat": "{{kind}} · {{origin}} · {{status}}",
"refId": "A"
}
],
"title": "Transaction application time",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": { "custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10 }, "unit": "short" },
"overrides": [
{
"matcher": { "id": "byName", "options": "avg" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "text" } }
]
}
]
},
"gridPos": { "h": 9, "w": 12, "x": 12, "y": 16 },
"id": 6,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "histogram_quantile(${percentile}, sum by (le) (rate(transactions_per_block_bucket[$__rate_interval])))",
"legendFormat": "${percentile:text}",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(transactions_per_block_sum[$__rate_interval]) / rate(transactions_per_block_count[$__rate_interval])",
"legendFormat": "avg",
"refId": "B"
}
],
"title": "Transactions per block",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 1,
"min": 0.0,
"max": 100.0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 70.0 },
{ "color": "red", "value": 90.0 }
]
}
},
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 25 },
"id": 7,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "100 * mempool_size / mempool_max_size",
"legendFormat": "utilization",
"refId": "A"
}
],
"title": "Mempool utilization",
"type": "gauge"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 10, "spanNulls": true },
"unit": "short",
"min": 0.0
},
"overrides": [
{
"matcher": { "id": "byName", "options": "capacity" },
"properties": [
{ "id": "custom.lineStyle", "value": { "dash": [ 8, 4 ], "fill": "dash" } },
{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }
]
},
{
"matcher": { "id": "byName", "options": "queued" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } } ]
}
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 25 },
"id": 8,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_size",
"legendFormat": "queued",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "mempool_max_size",
"legendFormat": "capacity",
"refId": "B"
}
],
"title": "Mempool size vs capacity",
"type": "timeseries"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"unit": "percent",
"decimals": 2,
"min": 0.0,
"max": 100.0,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "orange", "value": 1.0 },
{ "color": "red", "value": 5.0 }
]
}
},
"overrides": [ ]
},
"gridPos": { "h": 8, "w": 6, "x": 0, "y": 33 },
"id": 9,
"options": {
"reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false },
"showThresholdLabels": false,
"showThresholdMarkers": true
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "100 * (increase(before_mempool_failed_transactions_total[$__range]) + increase(mempool_failed_transactions_total[$__range])) / clamp_min(increase(submitted_transactions_total[$__range]), 1)",
"legendFormat": "failed",
"refId": "A"
}
],
"title": "Failed transactions share",
"type": "gauge"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"fieldConfig": {
"defaults": {
"custom": { "drawStyle": "line", "lineWidth": 1, "fillOpacity": 35, "gradientMode": "opacity" },
"unit": "short",
"min": 0.0
},
"overrides": [
{
"matcher": { "id": "byName", "options": "submitted" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · before mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "orange" } } ]
},
{
"matcher": { "id": "byName", "options": "failed · in mempool" },
"properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ]
}
]
},
"gridPos": { "h": 8, "w": 18, "x": 6, "y": 33 },
"id": 10,
"options": {
"legend": { "displayMode": "table", "placement": "bottom", "calcs": [ "last", "max" ] },
"tooltip": { "mode": "multi", "sort": "desc" }
},
"targets": [
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(submitted_transactions_total[$__rate_interval]) * 60",
"legendFormat": "submitted",
"refId": "A"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(before_mempool_failed_transactions_total[$__rate_interval]) * 60",
"legendFormat": "failed · before mempool",
"refId": "B"
},
{
"datasource": { "type": "prometheus", "uid": "prometheus" },
"expr": "rate(mempool_failed_transactions_total[$__rate_interval]) * 60",
"legendFormat": "failed · in mempool",
"refId": "C"
}
],
"title": "Submitted vs failed transactions (per minute)",
"type": "timeseries"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [ "sequencer" ],
"templating": {
"list": [
{
"current": { "selected": true, "text": "p95", "value": "0.95" },
"includeAll": false,
"label": "Percentile",
"multi": false,
"name": "percentile",
"options": [
{ "selected": false, "text": "p50", "value": "0.5" },
{ "selected": false, "text": "p90", "value": "0.9" },
{ "selected": true, "text": "p95", "value": "0.95" },
{ "selected": false, "text": "p99", "value": "0.99" }
],
"query": "p50 : 0.5, p90 : 0.9, p95 : 0.95, p99 : 0.99",
"type": "custom"
}
]
},
"time": { "from": "now-15m", "to": "now" },
"timezone": "",
"title": "Sequencer",
"uid": "sequencer"
}

9
monitoring/grafana/provisioning/dashboards/dashboards.yml Normal file
View File

@ -0,0 +1,9 @@
apiVersion: 1
providers:
- name: sequencer
type: file
allowUiUpdates: true
options:
path: /var/lib/grafana/dashboards
foldersFromFilesStructure: false

13
monitoring/grafana/provisioning/datasources/prometheus.yml Normal file
View File

@ -0,0 +1,13 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: true
jsonData:
# Must match prometheus.yml's `scrape_interval`
timeInterval: 5s

10
monitoring/prometheus/prometheus.yml Normal file
View File

@ -0,0 +1,10 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: sequencer
metrics_path: /metrics
static_configs:
- targets:
- host.docker.internal:9000

5
test_fixtures/Cargo.toml
View File

@ -25,8 +25,10 @@ bip39.workspace = true
bytesize.workspace = true
env_logger.workspace = true
futures.workspace = true
hex.workspace = true
jsonrpsee = { workspace = true, features = ["ws-client"] }
log.workspace = true
num-bigint.workspace = true
serde.workspace = true
serde_json.workspace = true
tempfile.workspace = true
@ -34,3 +36,6 @@ testcontainers = { version = "0.27.3", features = ["docker-compose"] }
tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
tokio-util.workspace = true
url.workspace = true
time.workspace = true
logos-blockchain-key-management-system-service.workspace = true

BIN
test_fixtures/fixtures/prebuilt_sequencer_db.dump
View File

Binary file not shown.

33
test_fixtures/src/config.rs
View File

@ -6,6 +6,8 @@ use indexer_service::{ChannelId, ClientConfig, IndexerConfig};
use key_protocol::key_management::{KeyChain, secret_holders::SeedHolder};
use lee::{AccountId, PrivateKey, PublicKey};
use lee_core::Identifier;
use logos_blockchain_key_management_system_service::keys::ZkPublicKey;
use num_bigint::BigUint;
use sequencer_core::config::{BedrockConfig, CrossZoneConfig, GenesisAction, SequencerConfig};
use url::Url;
use wallet::config::{MultiSequencerClientConfig, SequencerConnectionData, WalletConfig};
@ -76,29 +78,11 @@ impl std::fmt::Display for UrlProtocol {
}
pub fn sequencer_config(
partial: SequencerPartialConfig,
home: PathBuf,
bedrock_addr: SocketAddr,
genesis_transactions: Vec<GenesisAction>,
cross_zone: Option<CrossZoneConfig>,
) -> Result<SequencerConfig> {
sequencer_config_with_channel(
partial,
home,
bedrock_addr,
bedrock_channel_id(),
genesis_transactions,
cross_zone,
)
}
/// Like [`sequencer_config`] but with an explicit Bedrock `channel_id`, so tests
/// can point a sequencer at a fresh/empty channel (e.g. to model a wiped Bedrock).
pub fn sequencer_config_with_channel(
partial: SequencerPartialConfig,
home: PathBuf,
bedrock_addr: SocketAddr,
channel_id: ChannelId,
funding_key: ZkPublicKey,
genesis_transactions: Vec<GenesisAction>,
cross_zone: Option<CrossZoneConfig>,
) -> Result<SequencerConfig> {
@ -122,9 +106,11 @@ pub fn sequencer_config_with_channel(
channel_id,
node_url: addr_to_url(UrlProtocol::Http, bedrock_addr)
.context("Failed to convert bedrock addr to URL")?,
funding_key,
auth: None,
},
cross_zone,
metrics_address: Some(SequencerConfig::DEFAULT_METRICS_ADDRESS),
})
}
@ -280,3 +266,12 @@ pub fn bedrock_channel_id_b() -> ChannelId {
.unwrap_or_else(|_| unreachable!());
ChannelId::from(channel_id)
}
/// Funding key of the Bedrock test node, matching `funding_pk` in `bedrock/node-config.yaml`.
#[must_use]
pub fn bedrock_funding_key() -> ZkPublicKey {
const PUBLIC_KEY_HEX: &str = "2e03b2eff5a45478e7e79668d2a146cf2c5c7925bce927f2b1c67f2ab4fc0d26";
let bytes = hex::decode(PUBLIC_KEY_HEX).expect("Fixed funding key must be valid hex");
ZkPublicKey::from(BigUint::from_bytes_le(&bytes))
}

2
test_fixtures/src/lib.rs
View File

@ -217,7 +217,7 @@ impl Drop for TestContext {
temp_wallet_dir: _,
} = self;
let sequencer_handle = sequencer_handle
let mut sequencer_handle = sequencer_handle
.take()
.expect("Sequencer handle should be present in TestContext drop");
if !sequencer_handle.is_healthy() {

3
test_fixtures/src/setup.rs
View File

@ -131,11 +131,12 @@ impl SequencerSetup {
Vec::new()
};
let config = config::sequencer_config_with_channel(
let config = config::sequencer_config(
partial,
home.to_owned(),
bedrock_addr,
channel_id,
config::bedrock_funding_key(),
genesis_transactions,
cross_zone,
)

7
tools/cross_zone_chat/src/main.rs
View File

@ -54,7 +54,7 @@ use axum::{
routing::{get, post},
};
use common::{block::BedrockStatus, transaction::LeeTransaction};
use cross_zone_inbox_core::{CrossZoneConfig, CrossZonePeer, Instruction, ZoneId};
use cross_zone_inbox_core::{CrossZoneConfig, CrossZonePeer, CrossZoneRoute, Instruction, ZoneId};
use cross_zone_outbox_core::outbox_pda;
use lee::{
ProgramId, PublicTransaction,
@ -348,7 +348,10 @@ fn watch_peer(peer: ZoneId, receiver_id: ProgramId) -> CrossZoneConfig {
CrossZoneConfig {
peers: vec![CrossZonePeer {
channel_id: peer,
allowed_targets: vec![receiver_id],
allowed_routes: vec![CrossZoneRoute {
src_program_id: programs::ping_sender().id(),
target_program_id: receiver_id,
}],
expected_block_signing_pubkey: None,
}],
}

17
tools/dashboard_gen/Cargo.toml Normal file
View File

@ -0,0 +1,17 @@
[package]
name = "dashboard_gen"
version = "0.1.0"
edition = "2024"
license = { workspace = true }
[lints]
workspace = true
[dependencies]
sequencer_core_metrics.workspace = true
sequencer_service_metrics.workspace = true
clap = { workspace = true, features = ["derive"] }
serde = { workspace = true, features = ["derive", "alloc"] }
serde_json.workspace = true
json-pretty-compact = "0.1.2"

3
tools/dashboard_gen/src/dashboards.rs Normal file
View File

@ -0,0 +1,3 @@
//! One module per dashboard, each exposing a `dashboard()` builder.
pub mod sequencer;

195
tools/dashboard_gen/src/dashboards/sequencer.rs Normal file
View File

@ -0,0 +1,195 @@
//! The sequencer dashboard: chain progress, block timings, mempool and
//! transaction outcomes.
#![expect(
clippy::non_ascii_literal,
reason = "legend separators use `·` intentionally, matching the rendered Grafana labels"
)]
use dashboard_gen::{
Color, Dashboard, FieldOverride, GradientMode, Panel, Target, Thresholds, Unit, avg,
percentile_legend, percentile_variable, rate_per_min, selected_percentile,
};
const PERCENTILES: &[u32] = &[50, 90, 95, 99];
const DEFAULT_PERCENTILE: u32 = 95;
pub fn dashboard() -> Dashboard {
Dashboard::new("Sequencer", "sequencer")
.tag("sequencer")
.variable(percentile_variable(PERCENTILES, DEFAULT_PERCENTILE))
.row(
7,
[
Panel::stat("Chain height")
.width(6)
.unit(Unit::Short)
.decimals(0)
.color(Color::fixed("blue"))
.target(
Target::new(sequencer_core_metrics::names::CHAIN_HEIGHT).legend("height"),
),
Panel::stat("Blocks produced by this sequencer since startup")
.width(6)
.unit(Unit::Short)
.decimals(0)
.color(Color::fixed("green"))
.target(
Target::new(sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL)
.legend("produced"),
),
Panel::timeseries("Block production rate")
.width(12)
.unit(Unit::Short)
.target(rate_per_min(
sequencer_core_metrics::names::BLOCKS_PRODUCED_TOTAL,
"produced · blocks/min",
))
.with_override(
FieldOverride::by_name("produced · blocks/min").color(Color::fixed("green")),
),
],
)
.row(
9,
[Panel::timeseries("Block creation time")
.width(24)
.unit(Unit::Seconds)
.target(selected_percentile(
sequencer_core_metrics::names::BLOCK_CREATION_TIME,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::BLOCK_CREATION_TIME))
.with_override(
FieldOverride::by_name("avg")
.dashed_line()
.color(Color::fixed("text")),
)],
)
.row(
9,
[
Panel::timeseries("Transaction application time")
.width(12)
.unit(Unit::Seconds)
.target(selected_percentile(
sequencer_core_metrics::names::MEMPOOL_TRANSACTION_APPLICATION_TIME,
&["kind", "origin", "status"],
"{{kind}} · {{origin}} · {{status}}",
)),
Panel::timeseries("Transactions per block")
.width(12)
.unit(Unit::Short)
.target(selected_percentile(
sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK,
&[],
&percentile_legend(),
))
.target(avg(sequencer_core_metrics::names::TRANSACTIONS_PER_BLOCK))
.with_override(
FieldOverride::by_name("avg")
.dashed_line()
.color(Color::fixed("text")),
),
],
)
.row(
8,
[
Panel::gauge("Mempool utilization")
.width(6)
.unit(Unit::Percent)
.decimals(1)
.min(0.0)
.max(100.0)
.thresholds(
Thresholds::base("green")
.step(70.0, "orange")
.step(90.0, "red"),
)
.target(
Target::new(format!(
"100 * {size} / {max_size}",
size = sequencer_core_metrics::names::MEMPOOL_SIZE,
max_size = sequencer_core_metrics::names::MEMPOOL_MAX_SIZE,
))
.legend("utilization"),
),
Panel::timeseries("Mempool size vs capacity")
.width(18)
.unit(Unit::Short)
.span_nulls()
.min(0.0)
.target(
Target::new(sequencer_core_metrics::names::MEMPOOL_SIZE).legend("queued"),
)
.target(
Target::new(sequencer_core_metrics::names::MEMPOOL_MAX_SIZE)
.legend("capacity"),
)
.with_override(
FieldOverride::by_name("capacity")
.dashed_line()
.color(Color::fixed("red")),
)
.with_override(FieldOverride::by_name("queued").color(Color::fixed("blue"))),
],
)
.row(
8,
[
Panel::gauge("Failed transactions share")
.width(6)
.unit(Unit::Percent)
.decimals(2)
.min(0.0)
.max(100.0)
.thresholds(
Thresholds::base("green")
.step(1.0, "orange")
.step(5.0, "red"),
)
.target(
Target::new(format!(
// Both failure stages against the same submission base;
// `clamp_min` keeps an idle window (nothing submitted)
// reading as 0% instead of a division by zero.
"100 * (increase({before_mempool}[$__range]) + increase({in_mempool}[$__range])) / clamp_min(increase({submitted}[$__range]), 1)",
before_mempool = sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
in_mempool = sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
submitted = sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
))
.legend("failed"),
),
Panel::timeseries("Submitted vs failed transactions (per minute)")
.width(18)
.unit(Unit::Short)
.min(0.0)
.fill_opacity(35)
.gradient_mode(GradientMode::Opacity)
.target(rate_per_min(
sequencer_service_metrics::names::SUBMITTED_TRANSACTIONS_TOTAL,
"submitted",
))
.target(rate_per_min(
sequencer_service_metrics::names::BEFORE_MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · before mempool",
))
.target(rate_per_min(
sequencer_core_metrics::names::MEMPOOL_FAILED_TRANSACTIONS_TOTAL,
"failed · in mempool",
))
.with_override(
FieldOverride::by_name("submitted").color(Color::fixed("green")),
)
.with_override(
FieldOverride::by_name("failed · before mempool")
.color(Color::fixed("orange")),
)
.with_override(
FieldOverride::by_name("failed · in mempool").color(Color::fixed("red")),
),
],
)
}

603
tools/dashboard_gen/src/lib.rs Normal file
View File

@ -0,0 +1,603 @@
//! A tiny, hand-rolled Grafana dashboard builder.
//!
//! This is a deliberately small subset of what the (Rust-less) Grafana
//! Foundation SDK does: model only the panel types and options we actually use,
//! expose a fluent builder, and render to the same dashboard JSON Grafana
//! provisions. The payoff over hand-written JSON:
//!
//! * metric names live in Rust `const`s, so a rename is a compile error here;
//! * repetitive structure (percentile targets, grid layout, legend/tooltip defaults) collapses into
//! a single call instead of copy-pasted JSON.
//!
//! Build a [`Dashboard`] and serialize it directly.
// Styling vocabularies passed to the optional `styling` setters are part of the
// public API (and are used by this module's `Panel` fields and `finalize`).
pub use schema::{
AxisPlacement, Color, GradientMode, LineInterpolation, ShowPoints, StackingMode, Thresholds,
Variable,
};
use schema::{
Calc, Custom, Datasource, Defaults, DrawStyle, EmptyList, FieldConfig, Fill, GaugeOptions,
GraphMode, GridPos, Legend, LegendDisplay, LineStyle, Matcher, MatcherKind, Options,
OverrideProperty, PanelModel, PanelType, Placement, PropertyId, PropertyValue, ReduceOptions,
SortOrder, Stacking, StatColorMode, StatOptions, Templating, TimeRange, TimeSeriesOptions,
Tooltip, TooltipMode, VariableKind, VariableOption,
};
use serde::Serialize;
pub use unit::Unit;
mod schema;
mod styling;
mod unit;
/// Datasource uid every panel/target points at. Dashboards stay portable across
/// environments because they reference the datasource by this stable uid rather
/// than by a per-environment URL.
pub const DATASOURCE_UID: &str = "prometheus";
/// Window every rate query — counter and histogram alike — rates over.
/// `$__rate_interval` tracks the panel's zoom, so the window always covers the
/// step Grafana samples at: a fixed one shorter than the step leaves gaps the
/// query never looks at, dropping events from the graph entirely.
const RATE_WINDOW: &str = "$__rate_interval";
/// Dashboard variable holding the quantile every percentile query reads.
const PERCENTILE_VAR: &str = "percentile";
/// Area fill under a timeseries line, as a percentage. Enough to read a series'
/// shape at a glance without drowning the ones stacked behind it.
pub(crate) const DEFAULT_FILL_OPACITY: u32 = 10;
/// A single Prometheus query within a panel.
#[derive(Clone, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Target {
datasource: Datasource,
expr: String,
#[serde(rename = "legendFormat")]
legend: String,
ref_id: String,
}
impl Target {
pub fn new(expr: impl Into<String>) -> Self {
Self {
datasource: Datasource::prometheus(),
expr: expr.into(),
legend: String::new(),
ref_id: ref_letter(0),
}
}
#[must_use]
pub fn legend(mut self, legend: impl Into<String>) -> Self {
self.legend = legend.into();
self
}
}
/// A per-series style override, matched by series name.
#[derive(Serialize)]
pub struct FieldOverride {
matcher: Matcher,
properties: Vec<OverrideProperty>,
}
impl FieldOverride {
pub fn by_name(name: impl Into<String>) -> Self {
Self {
matcher: Matcher {
id: MatcherKind::ByName,
options: name.into(),
},
properties: Vec::new(),
}
}
#[must_use]
pub fn color(mut self, color: Color) -> Self {
self.properties.push(OverrideProperty {
id: PropertyId::Color,
value: PropertyValue::Color(color),
});
self
}
#[must_use]
pub fn dashed_line(mut self) -> Self {
self.properties.push(OverrideProperty {
id: PropertyId::LineStyle,
value: PropertyValue::LineStyle(LineStyle {
dash: [8, 4],
fill: Fill::Dash,
}),
});
self
}
}
#[derive(Clone, Copy)]
enum Kind {
Stat,
TimeSeries,
Gauge,
}
/// A dashboard panel builder. Grid position and panel id are assigned by
/// [`Dashboard::row`]; everything else is set here.
pub struct Panel {
title: String,
kind: Kind,
targets: Vec<Target>,
width: u32,
unit: Option<Unit>,
decimals: Option<u32>,
color: Option<Color>,
min: Option<f64>,
max: Option<f64>,
thresholds: Option<Thresholds>,
span_nulls: bool,
overrides: Vec<FieldOverride>,
// Optional timeseries styling, set via the `styling` setters.
fill_opacity: Option<u32>,
line_interpolation: Option<LineInterpolation>,
show_points: Option<ShowPoints>,
gradient_mode: Option<GradientMode>,
stacking: Option<StackingMode>,
axis_placement: Option<AxisPlacement>,
axis_label: Option<String>,
}
impl Panel {
fn new(title: impl Into<String>, kind: Kind) -> Self {
Self {
title: title.into(),
kind,
targets: Vec::new(),
width: 0,
unit: None,
decimals: None,
color: None,
min: None,
max: None,
thresholds: None,
span_nulls: false,
overrides: Vec::new(),
fill_opacity: None,
line_interpolation: None,
show_points: None,
gradient_mode: None,
stacking: None,
axis_placement: None,
axis_label: None,
}
}
/// A single big-number panel.
pub fn stat(title: impl Into<String>) -> Self {
Self::new(title, Kind::Stat)
}
/// A time-series line panel.
pub fn timeseries(title: impl Into<String>) -> Self {
Self::new(title, Kind::TimeSeries)
}
/// A radial gauge panel. Pair it with [`Panel::min`]/[`Panel::max`] — the
/// dial needs a range to fill — and [`Panel::thresholds`] for its coloring.
pub fn gauge(title: impl Into<String>) -> Self {
Self::new(title, Kind::Gauge)
}
/// Grid width in Grafana's 24-column units. Unset panels split the row's
/// remaining width evenly.
#[must_use]
pub const fn width(mut self, width: u32) -> Self {
self.width = width;
self
}
#[must_use]
pub fn unit(mut self, unit: Unit) -> Self {
self.unit = Some(unit);
self
}
#[must_use]
pub const fn decimals(mut self, decimals: u32) -> Self {
self.decimals = Some(decimals);
self
}
#[must_use]
pub fn color(mut self, color: Color) -> Self {
self.color = Some(color);
self
}
/// Lower bound of the value scale.
#[must_use]
pub const fn min(mut self, min: f64) -> Self {
self.min = Some(min);
self
}
/// Upper bound of the value scale.
#[must_use]
pub const fn max(mut self, max: f64) -> Self {
self.max = Some(max);
self
}
#[must_use]
pub fn thresholds(mut self, thresholds: Thresholds) -> Self {
self.thresholds = Some(thresholds);
self
}
#[must_use]
pub const fn span_nulls(mut self) -> Self {
self.span_nulls = true;
self
}
#[must_use]
pub fn target(mut self, target: Target) -> Self {
self.targets.push(target);
self
}
#[must_use]
pub fn targets(mut self, targets: impl IntoIterator<Item = Target>) -> Self {
self.targets.extend(targets);
self
}
#[must_use]
pub fn with_override(mut self, over: FieldOverride) -> Self {
self.overrides.push(over);
self
}
fn finalize(self, id: u32, grid_pos: GridPos) -> PanelModel {
let targets: Vec<Target> = self
.targets
.into_iter()
.enumerate()
.map(|(i, mut t)| {
t.ref_id = ref_letter(i);
t
})
.collect();
let unit = self.unit.unwrap_or_else(default_unit);
let (defaults, options, panel_type) = match self.kind {
Kind::Stat => {
let defaults = Defaults {
color: self.color,
custom: None,
unit,
decimals: self.decimals,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
let options = Options::Stat(StatOptions {
color_mode: StatColorMode::Value,
graph_mode: GraphMode::Area,
reduce_options: ReduceOptions {
calcs: vec![Calc::LastNotNull],
fields: String::new(),
values: false,
},
});
(defaults, options, PanelType::Stat)
}
Kind::Gauge => {
let defaults = Defaults {
color: self.color,
custom: None,
unit,
decimals: self.decimals,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
let options = Options::Gauge(GaugeOptions {
reduce_options: ReduceOptions {
calcs: vec![Calc::LastNotNull],
fields: String::new(),
values: false,
},
show_threshold_labels: false,
show_threshold_markers: true,
});
(defaults, options, PanelType::Gauge)
}
Kind::TimeSeries => {
let defaults = Defaults {
color: None,
custom: Some(Custom {
draw_style: DrawStyle::Line,
line_width: 1,
fill_opacity: self.fill_opacity.unwrap_or(DEFAULT_FILL_OPACITY),
span_nulls: self.span_nulls.then_some(true),
line_interpolation: self.line_interpolation,
show_points: self.show_points,
gradient_mode: self.gradient_mode,
stacking: self.stacking.map(|mode| Stacking {
mode,
group: "A".to_owned(),
}),
axis_placement: self.axis_placement,
axis_label: self.axis_label,
}),
unit,
decimals: None,
min: self.min,
max: self.max,
thresholds: self.thresholds,
};
// Panels with several series read better as a sortable table with
// a multi-series tooltip; single-series panels stay compact. A
// `{{label}}` legend fans one target out into a series per label
// value, so it counts as several too.
let multi =
targets.len() > 1 || targets.iter().any(|target| target.legend.contains("{{"));
let options = Options::TimeSeries(TimeSeriesOptions {
legend: Legend {
display_mode: if multi {
LegendDisplay::Table
} else {
LegendDisplay::List
},
placement: Placement::Bottom,
calcs: vec![Calc::Last, Calc::Max],
},
tooltip: if multi {
Tooltip {
mode: TooltipMode::Multi,
sort: Some(SortOrder::Desc),
}
} else {
Tooltip {
mode: TooltipMode::Single,
sort: None,
}
},
});
(defaults, options, PanelType::Timeseries)
}
};
PanelModel {
datasource: Datasource::prometheus(),
field_config: FieldConfig {
defaults,
overrides: self.overrides,
},
grid_pos,
id,
options,
targets,
title: self.title,
panel_type,
}
}
}
/// A dashboard, built row by row. Serialize it directly to get the JSON.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Dashboard {
annotations: EmptyList,
editable: bool,
graph_tooltip: u32,
panels: Vec<PanelModel>,
refresh: String,
schema_version: u32,
tags: Vec<String>,
templating: Templating,
time: TimeRange,
timezone: String,
title: String,
uid: String,
// Layout cursor — not part of the dashboard schema.
#[serde(skip)]
next_id: u32,
#[serde(skip)]
cursor_y: u32,
}
impl Dashboard {
pub fn new(title: impl Into<String>, uid: impl Into<String>) -> Self {
Self {
annotations: EmptyList::default(),
editable: true,
graph_tooltip: 1,
panels: Vec::new(),
refresh: "5s".to_owned(),
schema_version: 39,
tags: Vec::new(),
templating: Templating::default(),
time: TimeRange {
from: "now-15m".to_owned(),
to: "now".to_owned(),
},
timezone: String::new(),
title: title.into(),
uid: uid.into(),
next_id: 1,
cursor_y: 0,
}
}
#[must_use]
pub fn tag(mut self, tag: impl Into<String>) -> Self {
self.tags.push(tag.into());
self
}
#[must_use]
pub fn refresh(mut self, refresh: impl Into<String>) -> Self {
self.refresh = refresh.into();
self
}
/// Add a dropdown to the dashboard's top bar, e.g. [`percentile_variable`].
#[must_use]
pub fn variable(mut self, variable: Variable) -> Self {
self.templating.list.push(variable);
self
}
/// Place a horizontal row of panels at the current vertical cursor. Panel
/// ids, x offsets and y are assigned here; unset widths split the remaining
/// 24 columns evenly.
#[must_use]
pub fn row(mut self, height: u32, panels: impl IntoIterator<Item = Panel>) -> Self {
let panels: Vec<Panel> = panels.into_iter().collect();
let specified: u32 = panels.iter().map(|p| p.width).sum();
let auto_count = u32::try_from(panels.iter().filter(|p| p.width == 0).count()).unwrap_or(0);
// `checked_div` yields `None` when there are no auto-width panels; the
// fallback width is unused in that case.
let auto_width = 24_u32
.saturating_sub(specified)
.checked_div(auto_count)
.unwrap_or(0);
let mut x = 0;
for panel in panels {
let w = if panel.width == 0 {
auto_width
} else {
panel.width
};
let grid_pos = GridPos {
h: height,
w,
x,
y: self.cursor_y,
};
let id = self.next_id;
self.next_id = self.next_id.saturating_add(1);
x = x.saturating_add(w);
self.panels.push(panel.finalize(id, grid_pos));
}
self.cursor_y = self.cursor_y.saturating_add(height);
self
}
}
/// The dropdown driving every [`selected_percentile`] query, offering
/// `percentiles` (e.g. `[50, 90, 95, 99]`) with `default` pre-selected.
///
/// Panics if `default` is not one of `percentiles`, or if any of them falls
/// outside `p1..=p99`.
#[must_use]
pub fn percentile_variable(percentiles: &[u32], default: u32) -> Variable {
assert!(
percentiles.contains(&default),
"default p{default} is not one of the offered percentiles {percentiles:?}",
);
let options: Vec<VariableOption> = percentiles
.iter()
.map(|&p| VariableOption {
selected: p == default,
text: format!("p{p}"),
value: quantile(p),
})
.collect();
let current = options
.iter()
.find(|option| option.selected)
.cloned()
.expect("`default` is one of `percentiles`, asserted above");
let query = options
.iter()
.map(|option| format!("{} : {}", option.text, option.value))
.collect::<Vec<_>>()
.join(", ");
Variable {
current,
include_all: false,
label: "Percentile".to_owned(),
// A quantile is a scalar argument to `histogram_quantile`, so exactly
// one may be selected.
multi: false,
name: PERCENTILE_VAR.to_owned(),
options,
query,
kind: VariableKind::Custom,
}
}
/// The legend fragment that renders the dropdown's current choice, e.g. `p95`.
#[must_use]
pub fn percentile_legend() -> String {
format!("${{{PERCENTILE_VAR}:text}}")
}
/// A [`histogram_quantile`] line over `metric`'s buckets, at whatever quantile
/// [`percentile_variable`] currently holds. `labels` stay split out into their
/// own series; every other label is summed away.
///
/// [`histogram_quantile`]: https://prometheus.io/docs/prometheus/latest/querying/functions/#histogram_quantile
#[must_use]
pub fn selected_percentile(metric: &str, labels: &[&str], legend: &str) -> Target {
// `le` carries the bucket boundary, so it must survive the aggregation.
let grouping = std::iter::once("le")
.chain(labels.iter().copied())
.collect::<Vec<_>>()
.join(", ");
Target::new(format!(
"histogram_quantile(${{{PERCENTILE_VAR}}}, sum by ({grouping}) (rate({metric}_bucket[{RATE_WINDOW}])))"
))
.legend(legend)
}
/// A percentile as its `histogram_quantile` argument, derived without float
/// math: zero-pad to two digits then drop trailing zeros (50 → `0.5`).
///
/// Panics outside `1..=99`, the only range two digits render: p100 would come
/// out as `0.1` and p0 as `0.`, neither of them loudly.
fn quantile(percentile: u32) -> String {
assert!(
(1..=99).contains(&percentile),
"p{percentile} is outside the supported range p1..=p99",
);
let quantile = format!("0.{percentile:02}");
quantile.trim_end_matches('0').to_owned()
}
/// An `avg` target for a histogram metric: `rate(sum) / rate(count)`.
#[must_use]
pub fn avg(metric: &str) -> Target {
Target::new(format!(
"rate({metric}_sum[{RATE_WINDOW}]) / rate({metric}_count[{RATE_WINDOW}])"
))
.legend("avg")
}
/// A per-minute rate target for a counter metric.
#[must_use]
pub fn rate_per_min(metric: &str, legend: &str) -> Target {
//`rate` is per-second whatever the window, so `* 60` is per-minute regardless of the panel's
//`rate` zoom.
Target::new(format!("rate({metric}[{RATE_WINDOW}]) * 60")).legend(legend)
}
const fn default_unit() -> Unit {
Unit::Short
}
fn ref_letter(index: usize) -> String {
let offset = u8::try_from(index).unwrap_or(0);
char::from(b'A'.saturating_add(offset)).to_string()
}

45
tools/dashboard_gen/src/main.rs Normal file
View File

@ -0,0 +1,45 @@
//! Builds one of the Grafana dashboards and prints its JSON to stdout.
#![expect(
clippy::print_stdout,
reason = "CLI tool: emitting the dashboard JSON on stdout is the deliverable"
)]
use clap::{Parser, ValueEnum};
use dashboard_gen::Dashboard;
use json_pretty_compact::PrettyCompactFormatter;
use serde::Serialize as _;
mod dashboards;
#[derive(Debug, Parser)]
#[clap(version)]
struct Args {
/// Which dashboard to build.
dashboard: DashboardKind,
}
#[derive(Debug, Clone, Copy, ValueEnum)]
enum DashboardKind {
Sequencer,
}
impl DashboardKind {
fn build(self) -> Dashboard {
match self {
Self::Sequencer => dashboards::sequencer::dashboard(),
}
}
}
fn main() {
let Args { dashboard } = Args::parse();
let formatter = PrettyCompactFormatter::new();
let mut output = Vec::new();
let mut ser = serde_json::Serializer::with_formatter(&mut output, formatter);
dashboard.build().serialize(&mut ser).unwrap();
let json = String::from_utf8(output).unwrap();
println!("{json}");
}

434
tools/dashboard_gen/src/schema.rs Normal file
View File

@ -0,0 +1,434 @@
//! The serializable Grafana dashboard schema — the internal data model the
//! public builders assemble into.
//!
//! Every type here is `Serialize`-only: the model is sized for what we emit.
use serde::Serialize;
use crate::{DATASOURCE_UID, FieldOverride, Target, unit::Unit};
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DatasourceKind {
Prometheus,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Fill {
Dash,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum DrawStyle {
Line,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum MatcherKind {
ByName,
}
#[derive(Clone, Copy, Serialize)]
pub enum PropertyId {
#[serde(rename = "color")]
Color,
#[serde(rename = "custom.lineStyle")]
LineStyle,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum Calc {
LastNotNull,
Last,
Max,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum StatColorMode {
Value,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GraphMode {
Area,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum LegendDisplay {
Table,
List,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum Placement {
Bottom,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum TooltipMode {
Single,
Multi,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum SortOrder {
Desc,
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum PanelType {
Stat,
Timeseries,
Gauge,
}
#[derive(Clone, Copy, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ThresholdMode {
Absolute,
/// Steps expressed as a percentage of the min–max range. The builder never
/// emits this; it exists so Grafana exports using it still parse.
Percentage,
}
/// One threshold step: the color values at or above `value` take. The base step
/// carries `value: null` — Grafana's "everything below the first threshold".
#[derive(Clone, Serialize)]
pub struct ThresholdStep {
pub color: String,
pub value: Option<f64>,
}
/// A threshold ladder, driving gauge/stat coloring.
#[derive(Clone, Serialize)]
pub struct Thresholds {
pub mode: ThresholdMode,
pub steps: Vec<ThresholdStep>,
}
impl Thresholds {
/// Start a ladder with the color used below every threshold.
pub fn base(color: impl Into<String>) -> Self {
Self {
mode: ThresholdMode::Absolute,
steps: vec![ThresholdStep {
color: color.into(),
value: None,
}],
}
}
/// Add a step: values at or above `value` render in `color`.
#[must_use]
pub fn step(mut self, value: f64, color: impl Into<String>) -> Self {
self.steps.push(ThresholdStep {
color: color.into(),
value: Some(value),
});
self
}
}
// Optional timeseries styling vocabularies. Each derives `PartialEq` so the
// public setters can panic when handed the Grafana default (see `styling`).
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub enum LineInterpolation {
Linear,
Smooth,
StepBefore,
StepAfter,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum ShowPoints {
Auto,
Never,
Always,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum GradientMode {
None,
Opacity,
Hue,
Scheme,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum StackingMode {
None,
Normal,
Percent,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum AxisPlacement {
Auto,
Left,
Right,
Hidden,
}
#[derive(Serialize)]
pub struct Stacking {
pub mode: StackingMode,
pub group: String,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Custom {
pub draw_style: DrawStyle,
pub line_width: u32,
pub fill_opacity: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub span_nulls: Option<bool>,
// Optional styling — omitted (left at Grafana's default) unless a setter
// fills it in. See `styling`.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub line_interpolation: Option<LineInterpolation>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub show_points: Option<ShowPoints>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gradient_mode: Option<GradientMode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub stacking: Option<Stacking>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub axis_placement: Option<AxisPlacement>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub axis_label: Option<String>,
}
#[derive(Clone, Serialize)]
pub struct Datasource {
#[serde(rename = "type")]
pub kind: DatasourceKind,
pub uid: String,
}
impl Datasource {
pub fn prometheus() -> Self {
Self {
kind: DatasourceKind::Prometheus,
uid: DATASOURCE_UID.to_owned(),
}
}
}
#[derive(Clone, Serialize)]
#[serde(rename_all = "kebab-case")]
#[serde(tag = "mode")]
pub enum Color {
Fixed {
#[serde(rename = "fixedColor")]
fixed_color: String,
},
PaletteClassic,
}
impl Color {
pub fn fixed(color: impl Into<String>) -> Self {
Self::Fixed {
fixed_color: color.into(),
}
}
#[must_use]
pub const fn palette_classic() -> Self {
Self::PaletteClassic
}
}
#[derive(Serialize)]
pub struct LineStyle {
pub dash: [u32; 2],
pub fill: Fill,
}
#[derive(Serialize)]
#[serde(untagged)]
pub enum PropertyValue {
Color(Color),
LineStyle(LineStyle),
}
#[derive(Serialize)]
pub struct OverrideProperty {
pub id: PropertyId,
pub value: PropertyValue,
}
#[derive(Serialize)]
pub struct Matcher {
pub id: MatcherKind,
pub options: String,
}
#[derive(Serialize)]
pub struct Defaults {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub color: Option<Color>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub custom: Option<Custom>,
pub unit: Unit,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub decimals: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub min: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thresholds: Option<Thresholds>,
}
#[derive(Serialize)]
pub struct FieldConfig {
pub defaults: Defaults,
pub overrides: Vec<FieldOverride>,
}
#[derive(Serialize)]
pub struct ReduceOptions {
pub calcs: Vec<Calc>,
// Empty string means "all fields"; genuinely free-form, not a vocabulary.
pub fields: String,
pub values: bool,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct StatOptions {
pub color_mode: StatColorMode,
pub graph_mode: GraphMode,
pub reduce_options: ReduceOptions,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Legend {
pub display_mode: LegendDisplay,
pub placement: Placement,
pub calcs: Vec<Calc>,
}
#[derive(Serialize)]
pub struct Tooltip {
pub mode: TooltipMode,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub sort: Option<SortOrder>,
}
#[derive(Serialize)]
pub struct TimeSeriesOptions {
pub legend: Legend,
pub tooltip: Tooltip,
}
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct GaugeOptions {
pub reduce_options: ReduceOptions,
pub show_threshold_labels: bool,
pub show_threshold_markers: bool,
}
#[derive(Serialize)]
#[serde(untagged)]
pub enum Options {
Stat(StatOptions),
TimeSeries(TimeSeriesOptions),
Gauge(GaugeOptions),
}
#[derive(Clone, Copy, Serialize)]
#[serde(rename_all = "lowercase")]
pub enum VariableKind {
/// A fixed list of choices, spelled out in the dashboard itself.
Custom,
}
/// One choice in a [`Variable`] dropdown: `text` is displayed, `value` is what
/// `$name` interpolates to in a query.
#[derive(Clone, Serialize)]
pub struct VariableOption {
pub selected: bool,
pub text: String,
pub value: String,
}
/// A dashboard-level dropdown, rendered in the top bar.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct Variable {
pub current: VariableOption,
pub include_all: bool,
pub label: String,
pub multi: bool,
pub name: String,
pub options: Vec<VariableOption>,
/// Grafana's own encoding of `options`, as `text : value` pairs.
pub query: String,
#[serde(rename = "type")]
pub kind: VariableKind,
}
#[derive(Serialize, Default)]
pub struct Templating {
pub list: Vec<Variable>,
}
#[derive(Clone, Copy, Serialize)]
pub struct GridPos {
pub h: u32,
pub w: u32,
pub x: u32,
pub y: u32,
}
/// A fully positioned panel, ready to serialize.
#[derive(Serialize)]
#[serde(rename_all = "camelCase")]
pub struct PanelModel {
pub datasource: Datasource,
pub field_config: FieldConfig,
pub grid_pos: GridPos,
pub id: u32,
pub options: Options,
pub targets: Vec<Target>,
pub title: String,
#[serde(rename = "type")]
pub panel_type: PanelType,
}
#[expect(
clippy::trailing_empty_array,
reason = "Grafana expects `list: []` for the blocks we don't populate"
)]
#[derive(Serialize, Default)]
pub struct EmptyList {
pub list: [u8; 0],
}
#[derive(Serialize)]
pub struct TimeRange {
// Free-form Grafana time expressions, not a closed vocabulary.
pub from: String,
pub to: String,
}

124
tools/dashboard_gen/src/styling.rs Normal file
View File

@ -0,0 +1,124 @@
//! Optional timeseries styling setters.
//!
//! These are real setters — each fills in a field of the panel's `custom` block.
//! Every one documents Grafana's default and **panics if handed that default**:
//! passing the default is always redundant (Grafana emits it anyway), and the
//! generator only serializes fields that differ from the default. So if a call
//! wouldn't change the rendered panel, it's a mistake worth catching loudly at
//! generation time rather than shipping a no-op.
//!
//! These affect timeseries panels only; on a stat panel the `custom` block is
//! not emitted, so the value is silently dropped.
use crate::{
DEFAULT_FILL_OPACITY, Panel,
schema::{AxisPlacement, GradientMode, LineInterpolation, ShowPoints, StackingMode},
};
#[expect(
clippy::multiple_inherent_impl,
reason = "styling setters intentionally live in their own file, so `Panel` has a second inherent impl here"
)]
impl Panel {
/// Area fill under the line, as a percentage. Builder default:
/// [`DEFAULT_FILL_OPACITY`].
///
/// Panics if passed that default, or a value above 100.
#[must_use]
pub fn fill_opacity(mut self, opacity: u32) -> Self {
assert!(
opacity <= 100,
"fill_opacity({opacity}) is not a percentage"
);
assert_ne!(
opacity, DEFAULT_FILL_OPACITY,
"fill_opacity({DEFAULT_FILL_OPACITY}) is redundant: it is the builder's default. Omit the call.",
);
self.fill_opacity = Some(opacity);
self
}
/// Interpolation between points. Grafana default: `Linear`.
///
/// Panics if passed `Linear` — that's the default and would be redundant.
#[must_use]
pub fn line_interpolation(mut self, value: LineInterpolation) -> Self {
assert_ne!(
value,
LineInterpolation::Linear,
"line_interpolation(Linear) is redundant: `linear` is Grafana's default. Omit the call.",
);
self.line_interpolation = Some(value);
self
}
/// Whether/when to draw point markers. Grafana default: `Auto`.
///
/// Panics if passed `Auto` — that's the default and would be redundant.
#[must_use]
pub fn show_points(mut self, value: ShowPoints) -> Self {
assert_ne!(
value,
ShowPoints::Auto,
"show_points(Auto) is redundant: `auto` is Grafana's default. Omit the call.",
);
self.show_points = Some(value);
self
}
/// Area fill gradient. Grafana default: `None`.
///
/// Panics if passed `None` — that's the default and would be redundant.
#[must_use]
pub fn gradient_mode(mut self, value: GradientMode) -> Self {
assert_ne!(
value,
GradientMode::None,
"gradient_mode(None) is redundant: `none` is Grafana's default. Omit the call.",
);
self.gradient_mode = Some(value);
self
}
/// Series stacking. Grafana default: `None`.
///
/// Panics if passed `None` — that's the default and would be redundant.
#[must_use]
pub fn stacking(mut self, mode: StackingMode) -> Self {
assert_ne!(
mode,
StackingMode::None,
"stacking(None) is redundant: `none` is Grafana's default. Omit the call.",
);
self.stacking = Some(mode);
self
}
/// Y-axis placement. Grafana default: `Auto`.
///
/// Panics if passed `Auto` — that's the default and would be redundant.
#[must_use]
pub fn axis_placement(mut self, value: AxisPlacement) -> Self {
assert_ne!(
value,
AxisPlacement::Auto,
"axis_placement(Auto) is redundant: `auto` is Grafana's default. Omit the call.",
);
self.axis_placement = Some(value);
self
}
/// Y-axis label. Grafana default: `""` (no label).
///
/// Panics if passed an empty string — that's the default and would be redundant.
#[must_use]
pub fn axis_label(mut self, label: impl Into<String>) -> Self {
let label = label.into();
assert_ne!(
label, "",
"axis_label(\"\") is redundant: no label is Grafana's default. Omit the call.",
);
self.axis_label = Some(label);
self
}
}

Some files were not shown because too many files have changed in this diff Show More