diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 2e53fc8e4..082deb8e4 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -17,6 +17,15 @@ on: description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)" required: false type: string + battery: + description: "Which battery to run" + required: false + type: choice + options: + - assertoor + - commit-boost + - both + default: assertoor permissions: contents: read @@ -34,6 +43,13 @@ env: ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f ENCLAVE: ethlambda-assertoor + # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` + # wiring for ethlambda, which the commit above lacks. It has its own pin so + # the base battery stays where it is. Fork PR #22 (that wiring) is merged as + # a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on + # top. Re-pin to #23's merge commit once it lands; see the README. + COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 + COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: image: @@ -79,6 +95,7 @@ jobs: assertoor: name: Assertoor needs: image + if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost' runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -168,6 +185,83 @@ jobs: if: always() run: kurtosis enclave rm -f "$ENCLAVE" || true + # Dispatch-only for now: main's image does not have the builder flags until + # the builder PR stack (#694 to #705) lands. Once it is on main, drop the + # event condition so the check also runs on push and on pull requests. + commit-boost: + name: Commit-Boost + needs: image + if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + + - name: Free up disk space + uses: ./.github/actions/free-disk + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update + sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION" + kurtosis analytics disable + kurtosis version + + # A clone for the same reason as in the assertoor job. + - name: Fetch ethereum-package at the pinned commit + run: | + git init "$RUNNER_TEMP/ethereum-package" + cd "$RUNNER_TEMP/ethereum-package" + git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF" + + - name: Point the args at the image under test + env: + IMAGE: ${{ needs.image.outputs.image }} + run: | + cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml" + yq -i ' + (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) | + (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE) + ' "$RUNNER_TEMP/args.yaml" + cat "$RUNNER_TEMP/args.yaml" + + - name: Start the devnet + run: | + kurtosis run "$RUNNER_TEMP/ethereum-package" \ + --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ + | tee "$RUNNER_TEMP/kurtosis_output.txt" + + - name: Wait for builder blocks + run: | + tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + "$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results" + + - name: Dump the enclave + if: always() + run: | + mkdir -p "$RUNNER_TEMP/dump" + cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml" + cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true + cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true + cd "$RUNNER_TEMP/dump" + kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true + + - name: Upload the dump and results + if: always() + uses: actions/upload-artifact@v7 + with: + name: commit-boost-enclave-dump-${{ github.run_id }} + path: ${{ runner.temp }}/dump + if-no-files-found: warn + + - name: Remove the enclave + if: always() + run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true + notify: name: Post failure to Slack needs: [image, assertoor] diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 84faa773a..9570ef9ae 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -68,3 +68,108 @@ field by field and marks a node whose spec differs offline (`invalid node specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future value and ethlambda omits the key. The chain itself is healthy (all nodes share a head and finalize), so this is a spec-endpoint gap, not a harness problem. + +## Commit-Boost check + +Starts the same three participants with `mev_type: commit-boost` +(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder +builder with its own lighthouse beacon node, and one +[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module +per participant, which stands in for MEV-Boost. It also starts spamoor at a +light load, since rbuilder submits no bid for a block without transactions and +a heavy load slows the relay past Commit-Boost's getHeader timeout (see the +comment in the params file). The package gives the ethlambda +beacon node `--builder ` and the ethlambda validator client +`--builder-proposals`. There is no assertoor: the base battery's checks fail on +the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the +ethlambda services' Prometheus metrics instead, for up to the timeout it is given. + +| Service | Metric | Shows | +|---|---|---| +| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node | +| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid | +| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators | +| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node | +| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) | + +Passing therefore covers the registration path, a builder block from ethlambda's +own beacon node, and ethlambda's validator client proposing blinded blocks +through lighthouse. The script writes every `lean_beacon_builder_*` and +`ethlambda_validator_*` blinded or registration series of the three services to +`/.prom`, pass or fail, and the unmet conditions to +`failed.txt`. + +The relay, builder, PBS and spamoor images are pinned by digest in +`commit_boost_params.yaml` (the package's defaults are floating tags). + +### Run it locally + +The image needs the builder support, so build one from a checkout of the +builder PR stack (#694 to #705). From that checkout's root, `make docker-build +DOCKER_TAG=` tags it `ghcr.io/lambdaclass/ethlambda:`; set that as the +`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`. +Kurtosis uses an image that exists in the local Docker daemon and only pulls a +missing one (`kurtosis run --image-download`, default `missing`), so no setting +is needed; do not pass `--image-download always`. From the repository root: + +```bash +git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package +git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 + +kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ + --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml + +tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + ethlambda-commit-boost 40 /tmp/builder-results + +kurtosis enclave rm -f ethlambda-commit-boost +``` + +### In CI + +The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on +manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image` +the tag of an image built from the builder PR stack; the default `assertoor` +runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`, +separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-`. + +It is dispatch-only because the image that pushes and pull requests test +(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or +`--builder-proposals` until the builder PR stack is on `main`, so the check +would fail there. Once it lands, drop the event condition on the job to run it +on push and on pull requests. + +### The fork pin + +lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags` +(`--builder=` for the beacon node, `--builder-proposals` for the validator +client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is +`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from +lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on +top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the +current commit unreachable from the fork's default branch, so it may be garbage +collected. + +### Upstream workarounds + +A local run found two breakages in images and packages we do not control: + +| Breakage | Worked around in | +|---|---| +| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin | +| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) | + +Drop each once the upstream is fixed. The explicit chain must change with the +network parameters. + +### Startup noise + +The Commit-Boost PBS modules start after the consensus clients, so at startup +ethlambda's beacon node logs "Builder is not answering" and its first +registration batch fails. The validator client re-sends registrations every +epoch, so this clears within an epoch and is not a failure. + +From epoch 4 until the chain first finalizes, the beacon node builds locally and +counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like +Lighthouse, it skips the builder when the proposal is more than 3 epochs past +the finalized checkpoint, and the genesis checkpoint counts as one. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml new file mode 100644 index 000000000..ea232cdc6 --- /dev/null +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -0,0 +1,115 @@ +# The Commit-Boost check for ethlambda: the same three participants as +# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node +# and validator client build blocks through a Commit-Boost PBS module. There is +# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the +# builder metrics of every ethlambda service. See README.md ("Commit-Boost +# check") for the steps. The package comes from a clone of +# lambdaclass/ethereum-package at the commit pinned in +# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF). +# +# Participants, in the order ethereum-package launches them (the order fixes the +# service names check-builder-blocks.sh reads): +# 1. lighthouse + geth, lighthouse validator. It has to come first: the first +# participant is the network's bootnode, and ethlambda does not report an +# ENR over the beacon API. +# 2. lighthouse + geth, validated by the ethlambda validator client, so the +# validator client proposes blinded blocks through a beacon node that is +# not ethlambda. +# 3. ethlambda beacon node + geth, validated by the ethlambda validator client. +# +# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder +# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module +# per participant. The package passes each ethlambda beacon node `--builder` and +# each ethlambda validator client `--builder-proposals`, which only an image +# built with the builder support understands. The workflow rewrites the +# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not +# position) to the image under test; the values below are what a local run uses. +# If the battery ever changes `network_id`, the preset or the fork epochs, the +# explicit Commit-Boost chain under `mev_params` must change with it. +participants: + - el_type: geth + cl_type: lighthouse + - el_type: geth + cl_type: lighthouse + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable +network_params: + # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. + preset: mainnet + # Fulu is the last fork ethlambda processes; gloas stays unscheduled. + electra_fork_epoch: 0 + fulu_fork_epoch: 0 +mev_type: commit-boost +# rbuilder submits nothing for a block with no transactions, so without a +# spammer the relay never holds a bid and every getHeader answers 204. The load +# is kept light on purpose: under spamoor's defaults (a chain-load group plus +# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed +# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no +# bid. The package also always adds 100 Uniswap swaps a slot, sent to the +# builder's own execution client; that is not configurable. The blob spammer +# stays so bids carry blob commitments. +additional_services: + - spamoor +spamoor_params: + # ethpandaops/spamoor:master on 2026-10-27 + image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 + start_chainload: false + spammers: + - name: "EOA transfers" + description: "10 type-2 transfers a slot" + scenario: "eoatx" + config: + throughput: 10 + max_pending: 20 + max_wallets: 10 + base_fee: 20 + - name: "Blob transactions" + description: "1 type-4 transaction a slot with 1 sidecar" + scenario: "blob-combined" + config: + throughput: 1 + sidecars: 1 + max_pending: 2 + max_wallets: 2 + base_fee: 20 + blob_fee: 20 +# The package defaults are floating tags (`main`, `develop`, `latest`), so a +# run would change under us. These pin what the tags pointed to on 2026-10-26; +# bump them deliberately. +mev_params: + # ethpandaops/mev-boost-relay:main + mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19 + # ethpandaops/reth-rbuilder:develop + mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 + # ghcr.io/commit-boost/pbs:latest + mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 + # Commit-Boost reads the chain from the package's config.yaml by default, + # and its parser requires SECONDS_PER_SLOT, which the genesis generator no + # longer writes (it writes SLOT_DURATION_MS). So the chain is given + # explicitly: kurtosis's defaults for this network (12 s slots, genesis fork + # version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824). + # chain_id must be a hex string; a bare integer does not parse. The rest is + # the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl). + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled=false diff --git a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh new file mode 100755 index 000000000..0cd9945a6 --- /dev/null +++ b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Waits until ethlambda has built blocks through the builder (Commit-Boost) and +# fails if that does not happen in time. +# +# Usage: check-builder-blocks.sh +# +# Reads Prometheus metrics through `kurtosis port print +# metrics`, and succeeds once every condition below holds: +# +# ethlambda beacon node (participant 3) +# lean_beacon_builder_registrations_total{outcome="forwarded"} >= 1 +# lean_beacon_builder_payload_source_total{source="builder"} >= 1 +# ethlambda validator client of participant 3 +# ethlambda_validator_builder_registrations_sent_total >= 1 +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# ethlambda validator client of participant 2 (on a lighthouse beacon node) +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# +# On every exit, writes each ethlambda service's builder metric lines to +# /.prom, and the conditions still unmet to +# /failed.txt. A condition unmet at the deadline fails the check. + +set -euo pipefail + +ENCLAVE="${1:?enclave name required}" +TIMEOUT_MIN="${2:?timeout in minutes required}" +OUT="${3:?output directory required}" + +# Service names follow ethereum-package: cl---, and vc--- +# with - appended when the validator client differs from the beacon node. +CL3="cl-3-ethlambda-geth" +VC2="vc-2-geth-lighthouse-ethlambda" +VC3="vc-3-geth-ethlambda" +SERVICES=("$CL3" "$VC2" "$VC3") + +# service|series (name, with its labels when the metric is labelled)|description +CONDITIONS=( + "$CL3|lean_beacon_builder_registrations_total{outcome=\"forwarded\"}|beacon node forwarded a registration" + "$CL3|lean_beacon_builder_payload_source_total{source=\"builder\"}|beacon node served a builder payload" + "$VC3|ethlambda_validator_builder_registrations_sent_total|validator client of participant 3 sent a registration" + "$VC3|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 3 proposed a blinded block" + "$VC2|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 2 proposed a blinded block" +) + +mkdir -p "$OUT" +deadline=$((SECONDS + TIMEOUT_MIN * 60)) +: > "$OUT/failed.txt" + +# Prints a service's metrics, or nothing while it is unreachable. +scrape() { + local addr + addr=$(kurtosis port print "$ENCLAVE" "$1" metrics 2>/dev/null) || return 0 + curl -sf --max-time 10 "${addr#*://}/metrics" 2>/dev/null || true +} + +# Prints the sum of a series over the given exposition text, 0 if absent. A +# selector with labels matches that exact `name{labels}` token; a bare name +# matches the metric under any labels. +value_of() { + awk -v sel="$2" ' + index(sel, "{") { if ($1 == sel) sum += $NF; next } + { split($1, p, "{"); if (p[1] == sel) sum += $NF } + END { printf "%d\n", sum } + ' <<<"$1" +} + +# Writes each service's builder-related series to /.prom. +dump_metrics() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" \ + | grep -E '^(lean_beacon_builder_|ethlambda_validator_.*(blinded|registration))' \ + > "$OUT/$service.prom" || true + done +} + +# Prints one line per condition and returns 1 if any is unmet. +evaluate() { + local entry svc series desc value unmet=0 + for entry in "${CONDITIONS[@]}"; do + IFS='|' read -r svc series desc <<<"$entry" + value=$(value_of "$(scrape "$svc")" "$series") + if [ "$value" -ge 1 ]; then + echo "ok $svc $series = $value ($desc)" + else + echo "unmet $svc $series = $value ($desc)" + unmet=1 + fi + done + return "$unmet" +} + +finish() { + dump_metrics + echo "Builder metrics written to $OUT" +} +trap finish EXIT + +status=1 +while :; do + if summary=$(evaluate); then + status=0 + break + fi + echo "[$(date -u +%H:%M:%S)] builder conditions not met yet:" + grep '^unmet' <<<"$summary" | sed 's/^/ /' + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for builder blocks" + break + fi + sleep 30 +done + +echo +echo "$summary" +grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true +if [ "$status" -ne 0 ]; then + echo "Failed:" + cat "$OUT/failed.txt" + exit 1 +fi +echo "ethlambda built blocks through the builder"