From e345428433536f35b74f439095f7e282c320db56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:37:53 -0300 Subject: [PATCH 1/5] ci: add a Commit-Boost check to the kurtosis battery The builder PR stack (#694 to #705) lets ethlambda's beacon node and validator client build blocks through a MEV-Boost compatible builder, but nothing exercises it against a real relay and builder. Commit-Boost's PBS module is a drop-in MEV-Boost replacement, so a devnet with `mev_type: commit-boost` covers the whole path. The check reads the ethlambda services' builder metrics instead of running assertoor, because the base battery's assertoor tests currently fail on an unrelated spec-endpoint gap. It passes once registrations are forwarded, the ethlambda beacon node serves a builder payload, and the ethlambda validator client proposes blinded blocks both through that node and through lighthouse. The new `commit-boost` job runs on manual dispatch only: main's image has no `--builder` or `--builder-proposals` until the stack lands. It has its own ethereum-package pin, which needs the fork's builder-flags branch pushed. The relay, builder and PBS images are pinned by digest since the package defaults are floating tags. --- .github/workflows/kurtosis_assertoor.yml | 93 +++++++++++++ tooling/kurtosis-assertoor/README.md | 73 +++++++++++ .../commit_boost_params.yaml | 55 ++++++++ .../scripts/check-builder-blocks.sh | 123 ++++++++++++++++++ 4 files changed, 344 insertions(+) create mode 100644 tooling/kurtosis-assertoor/commit_boost_params.yaml create mode 100755 tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 2e53fc8e4..8fec34608 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -17,6 +17,15 @@ on: description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)" required: false type: string + battery: + description: "Which battery to run" + required: false + type: choice + options: + - assertoor + - commit-boost + - both + default: assertoor permissions: contents: read @@ -34,6 +43,12 @@ env: ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f ENCLAVE: ethlambda-assertoor + # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` + # wiring for ethlambda, which the commit above lacks. It has its own pin so + # the base battery stays where it is. This commit is on the fork branch + # feat/ethlambda-builder-flags; see the README before changing it. + COMMIT_BOOST_PACKAGE_REF: 37fe57ea309b954361c0f33ac6efc570313ffaf6 + COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: image: @@ -79,6 +94,7 @@ jobs: assertoor: name: Assertoor needs: image + if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost' runs-on: ubuntu-latest timeout-minutes: 120 outputs: @@ -168,6 +184,83 @@ jobs: if: always() run: kurtosis enclave rm -f "$ENCLAVE" || true + # Dispatch-only for now: main's image does not have the builder flags until + # the builder PR stack (#694 to #705) lands. Once it is on main, drop the + # event condition so the check also runs on push and on pull requests. + commit-boost: + name: Commit-Boost + needs: image + if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor' + runs-on: ubuntu-latest + timeout-minutes: 90 + steps: + - uses: actions/checkout@v6 + + - name: Free up disk space + uses: ./.github/actions/free-disk + + - name: Install kurtosis + run: | + echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \ + | sudo tee /etc/apt/sources.list.d/kurtosis.list + sudo apt-get update + sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION" + kurtosis analytics disable + kurtosis version + + # A clone for the same reason as in the assertoor job. + - name: Fetch ethereum-package at the pinned commit + run: | + git init "$RUNNER_TEMP/ethereum-package" + cd "$RUNNER_TEMP/ethereum-package" + git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF" + git checkout --detach FETCH_HEAD + test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF" + + - name: Point the args at the image under test + env: + IMAGE: ${{ needs.image.outputs.image }} + run: | + cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml" + yq -i ' + (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) | + (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE) + ' "$RUNNER_TEMP/args.yaml" + cat "$RUNNER_TEMP/args.yaml" + + - name: Start the devnet + run: | + kurtosis run "$RUNNER_TEMP/ethereum-package" \ + --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \ + | tee "$RUNNER_TEMP/kurtosis_output.txt" + + - name: Wait for builder blocks + run: | + tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + "$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results" + + - name: Dump the enclave + if: always() + run: | + mkdir -p "$RUNNER_TEMP/dump" + cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml" + cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true + cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true + cd "$RUNNER_TEMP/dump" + kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true + + - name: Upload the dump and results + if: always() + uses: actions/upload-artifact@v7 + with: + name: commit-boost-enclave-dump-${{ github.run_id }} + path: ${{ runner.temp }}/dump + if-no-files-found: warn + + - name: Remove the enclave + if: always() + run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true + notify: name: Post failure to Slack needs: [image, assertoor] diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 84faa773a..153c2e4ec 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -68,3 +68,76 @@ field by field and marks a node whose spec differs offline (`invalid node specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future value and ethlambda omits the key. The chain itself is healthy (all nodes share a head and finalize), so this is a spec-endpoint gap, not a harness problem. + +## Commit-Boost check + +Starts the same three participants with `mev_type: commit-boost` +(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder +builder with its own lighthouse beacon node, and one +[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module +per participant, which stands in for MEV-Boost. The package gives the ethlambda +beacon node `--builder ` and the ethlambda validator client +`--builder-proposals`. There is no assertoor: the base battery's checks fail on +the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the +ethlambda services' Prometheus metrics instead, for up to the timeout it is given. + +| Service | Metric | Shows | +|---|---|---| +| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node | +| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid | +| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators | +| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node | +| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) | + +Passing therefore covers the registration path, a builder block from ethlambda's +own beacon node, and ethlambda's validator client proposing blinded blocks +through lighthouse. The script writes every `lean_beacon_builder_*` and +`ethlambda_validator_*` blinded or registration series of the three services to +`/.prom`, pass or fail, and the unmet conditions to +`failed.txt`. + +The relay, builder and PBS images are pinned by digest in +`commit_boost_params.yaml` (the package's defaults are floating tags). + +### Run it locally + +The image needs the builder support, so build one from the builder PR stack +(#694 to #705) and set it as the `cl_image` and `vc_image` of the ethlambda +entries in `commit_boost_params.yaml`. From the repository root: + +```bash +git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package +git -C /tmp/ethereum-package checkout 37fe57ea309b954361c0f33ac6efc570313ffaf6 + +kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ + --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml + +tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \ + ethlambda-commit-boost 40 /tmp/builder-results + +kurtosis enclave rm -f ethlambda-commit-boost +``` + +### In CI + +The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on +manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image` +the tag of an image built from the builder PR stack; the default `assertoor` +runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`, +separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-`. + +It is dispatch-only because the image that pushes and pull requests test +(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or +`--builder-proposals` until the builder PR stack is on `main`, so the check +would fail there. Once it lands, drop the event condition on the job to run it +on push and on pull requests. + +### The fork pin + +`COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the +commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` +that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the +beacon node, `--builder-proposals` for the validator client). That branch has +to be pushed before the job can fetch the commit. If the fork PR is +squash-merged, the commit is no longer reachable from the fork's default branch +and may be garbage collected, so re-pin to the squash commit. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml new file mode 100644 index 000000000..a46fa5ca5 --- /dev/null +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -0,0 +1,55 @@ +# The Commit-Boost check for ethlambda: the same three participants as +# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node +# and validator client build blocks through a Commit-Boost PBS module. There is +# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the +# builder metrics of every ethlambda service. See README.md ("Commit-Boost +# check") for the steps. The package comes from a clone of +# lambdaclass/ethereum-package at the commit pinned in +# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF). +# +# Participants, in the order ethereum-package launches them (the order fixes the +# service names check-builder-blocks.sh reads): +# 1. lighthouse + geth, lighthouse validator. It has to come first: the first +# participant is the network's bootnode, and ethlambda does not report an +# ENR over the beacon API. +# 2. lighthouse + geth, validated by the ethlambda validator client, so the +# validator client proposes blinded blocks through a beacon node that is +# not ethlambda. +# 3. ethlambda beacon node + geth, validated by the ethlambda validator client. +# +# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder +# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module +# per participant. The package passes each ethlambda beacon node `--builder` and +# each ethlambda validator client `--builder-proposals`, which only an image +# built with the builder support understands. The workflow rewrites the +# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not +# position) to the image under test; the values below are what a local run uses. +participants: + - el_type: geth + cl_type: lighthouse + - el_type: geth + cl_type: lighthouse + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable + - el_type: geth + cl_type: ethlambda + cl_image: ghcr.io/lambdaclass/ethlambda:unstable + vc_type: ethlambda + vc_image: ghcr.io/lambdaclass/ethlambda:unstable +network_params: + # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s. + preset: mainnet + # Fulu is the last fork ethlambda processes; gloas stays unscheduled. + electra_fork_epoch: 0 + fulu_fork_epoch: 0 +mev_type: commit-boost +# The package defaults are floating tags (`main`, `develop`, `latest`), so a +# run would change under us. These pin what the tags pointed to on 2026-10-26; +# bump them deliberately. +mev_params: + # ethpandaops/mev-boost-relay:main + mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19 + # ethpandaops/reth-rbuilder:develop + mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 + # ghcr.io/commit-boost/pbs:latest + mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 diff --git a/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh new file mode 100755 index 000000000..0cd9945a6 --- /dev/null +++ b/tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# +# Waits until ethlambda has built blocks through the builder (Commit-Boost) and +# fails if that does not happen in time. +# +# Usage: check-builder-blocks.sh +# +# Reads Prometheus metrics through `kurtosis port print +# metrics`, and succeeds once every condition below holds: +# +# ethlambda beacon node (participant 3) +# lean_beacon_builder_registrations_total{outcome="forwarded"} >= 1 +# lean_beacon_builder_payload_source_total{source="builder"} >= 1 +# ethlambda validator client of participant 3 +# ethlambda_validator_builder_registrations_sent_total >= 1 +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# ethlambda validator client of participant 2 (on a lighthouse beacon node) +# ethlambda_validator_blinded_blocks_proposed_total >= 1 +# +# On every exit, writes each ethlambda service's builder metric lines to +# /.prom, and the conditions still unmet to +# /failed.txt. A condition unmet at the deadline fails the check. + +set -euo pipefail + +ENCLAVE="${1:?enclave name required}" +TIMEOUT_MIN="${2:?timeout in minutes required}" +OUT="${3:?output directory required}" + +# Service names follow ethereum-package: cl---, and vc--- +# with - appended when the validator client differs from the beacon node. +CL3="cl-3-ethlambda-geth" +VC2="vc-2-geth-lighthouse-ethlambda" +VC3="vc-3-geth-ethlambda" +SERVICES=("$CL3" "$VC2" "$VC3") + +# service|series (name, with its labels when the metric is labelled)|description +CONDITIONS=( + "$CL3|lean_beacon_builder_registrations_total{outcome=\"forwarded\"}|beacon node forwarded a registration" + "$CL3|lean_beacon_builder_payload_source_total{source=\"builder\"}|beacon node served a builder payload" + "$VC3|ethlambda_validator_builder_registrations_sent_total|validator client of participant 3 sent a registration" + "$VC3|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 3 proposed a blinded block" + "$VC2|ethlambda_validator_blinded_blocks_proposed_total|validator client of participant 2 proposed a blinded block" +) + +mkdir -p "$OUT" +deadline=$((SECONDS + TIMEOUT_MIN * 60)) +: > "$OUT/failed.txt" + +# Prints a service's metrics, or nothing while it is unreachable. +scrape() { + local addr + addr=$(kurtosis port print "$ENCLAVE" "$1" metrics 2>/dev/null) || return 0 + curl -sf --max-time 10 "${addr#*://}/metrics" 2>/dev/null || true +} + +# Prints the sum of a series over the given exposition text, 0 if absent. A +# selector with labels matches that exact `name{labels}` token; a bare name +# matches the metric under any labels. +value_of() { + awk -v sel="$2" ' + index(sel, "{") { if ($1 == sel) sum += $NF; next } + { split($1, p, "{"); if (p[1] == sel) sum += $NF } + END { printf "%d\n", sum } + ' <<<"$1" +} + +# Writes each service's builder-related series to /.prom. +dump_metrics() { + local service + for service in "${SERVICES[@]}"; do + scrape "$service" \ + | grep -E '^(lean_beacon_builder_|ethlambda_validator_.*(blinded|registration))' \ + > "$OUT/$service.prom" || true + done +} + +# Prints one line per condition and returns 1 if any is unmet. +evaluate() { + local entry svc series desc value unmet=0 + for entry in "${CONDITIONS[@]}"; do + IFS='|' read -r svc series desc <<<"$entry" + value=$(value_of "$(scrape "$svc")" "$series") + if [ "$value" -ge 1 ]; then + echo "ok $svc $series = $value ($desc)" + else + echo "unmet $svc $series = $value ($desc)" + unmet=1 + fi + done + return "$unmet" +} + +finish() { + dump_metrics + echo "Builder metrics written to $OUT" +} +trap finish EXIT + +status=1 +while :; do + if summary=$(evaluate); then + status=0 + break + fi + echo "[$(date -u +%H:%M:%S)] builder conditions not met yet:" + grep '^unmet' <<<"$summary" | sed 's/^/ /' + if [ "$SECONDS" -ge "$deadline" ]; then + echo "::error::timed out after ${TIMEOUT_MIN} minutes waiting for builder blocks" + break + fi + sleep 30 +done + +echo +echo "$summary" +grep '^unmet' <<<"$summary" > "$OUT/failed.txt" || true +if [ "$status" -ne 0 ]; then + echo "Failed:" + cat "$OUT/failed.txt" + exit 1 +fi +echo "ethlambda built blocks through the builder" From ad9ebeafb80835bb029b86960a296be5b323a941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 18:47:28 -0300 Subject: [PATCH 2/5] docs: explain local images and the fork PR for the Commit-Boost check A contributor running the check locally needs to know how to get an image with the builder flags and that kurtosis will use it without pulling, or it looks like the run needs a published image. The fork pin also now has a PR to point at instead of an unpushed branch. --- tooling/kurtosis-assertoor/README.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 153c2e4ec..f114d87c0 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -101,9 +101,13 @@ The relay, builder and PBS images are pinned by digest in ### Run it locally -The image needs the builder support, so build one from the builder PR stack -(#694 to #705) and set it as the `cl_image` and `vc_image` of the ethlambda -entries in `commit_boost_params.yaml`. From the repository root: +The image needs the builder support, so build one from a checkout of the +builder PR stack (#694 to #705). From that checkout's root, `make docker-build +DOCKER_TAG=` tags it `ghcr.io/lambdaclass/ethlambda:`; set that as the +`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`. +Kurtosis uses an image that exists in the local Docker daemon and only pulls a +missing one (`kurtosis run --image-download`, default `missing`), so no setting +is needed; do not pass `--image-download always`. From the repository root: ```bash git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package @@ -137,7 +141,7 @@ on push and on pull requests. `COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the -beacon node, `--builder-proposals` for the validator client). That branch has -to be pushed before the job can fetch the commit. If the fork PR is +beacon node, `--builder-proposals` for the validator client). That branch is +lambdaclass/ethereum-package#22. If the fork PR is squash-merged, the commit is no longer reachable from the fork's default branch and may be garbage collected, so re-pin to the squash commit. From 3a7bf07fef33ac3370864308be95340f5ec36996 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Mon, 26 Oct 2026 19:08:04 -0300 Subject: [PATCH 3/5] ci: work around two upstream breakages in the Commit-Boost check A local run did not start. The newest reth-rbuilder image parses only enode bootnodes while the package passes ENRs, and Commit-Boost's PBS requires SECONDS_PER_SLOT, which the genesis generator no longer writes. The first is fixed in the fork (PR #23), so the package pin moves to that commit. The second is worked around by giving Commit-Boost an explicit chain through `commit_boost_config`; its `chain_id` must be a hex string. The README lists both workarounds so they can be dropped when upstream is fixed, and explains the harmless "Builder is not answering" logs at startup. --- .github/workflows/kurtosis_assertoor.yml | 7 ++-- tooling/kurtosis-assertoor/README.md | 36 ++++++++++++++----- .../commit_boost_params.yaml | 27 ++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/.github/workflows/kurtosis_assertoor.yml b/.github/workflows/kurtosis_assertoor.yml index 8fec34608..082deb8e4 100644 --- a/.github/workflows/kurtosis_assertoor.yml +++ b/.github/workflows/kurtosis_assertoor.yml @@ -45,9 +45,10 @@ env: ENCLAVE: ethlambda-assertoor # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals` # wiring for ethlambda, which the commit above lacks. It has its own pin so - # the base battery stays where it is. This commit is on the fork branch - # feat/ethlambda-builder-flags; see the README before changing it. - COMMIT_BOOST_PACKAGE_REF: 37fe57ea309b954361c0f33ac6efc570313ffaf6 + # the base battery stays where it is. Fork PR #22 (that wiring) is merged as + # a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on + # top. Re-pin to #23's merge commit once it lands; see the README. + COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost jobs: diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index f114d87c0..82496fa6d 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -111,7 +111,7 @@ is needed; do not pass `--image-download always`. From the repository root: ```bash git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package -git -C /tmp/ethereum-package checkout 37fe57ea309b954361c0f33ac6efc570313ffaf6 +git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451 kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \ --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -138,10 +138,30 @@ on push and on pull requests. ### The fork pin -`COMMIT_BOOST_PACKAGE_REF` is `37fe57ea309b954361c0f33ac6efc570313ffaf6`, the -commit on the `lambdaclass/ethereum-package` branch `feat/ethlambda-builder-flags` -that adds ethlambda to `apply_external_builder_flags` (`--builder=` for the -beacon node, `--builder-proposals` for the validator client). That branch is -lambdaclass/ethereum-package#22. If the fork PR is -squash-merged, the commit is no longer reachable from the fork's default branch -and may be garbage collected, so re-pin to the squash commit. +lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags` +(`--builder=` for the beacon node, `--builder-proposals` for the validator +client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is +`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from +lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on +top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the +current commit unreachable from the fork's default branch, so it may be garbage +collected. + +### Upstream workarounds + +A local run found two breakages in images and packages we do not control: + +| Breakage | Worked around in | +|---|---| +| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin | +| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) | + +Drop each once the upstream is fixed. The explicit chain must change with the +network parameters. + +### Startup noise + +The Commit-Boost PBS modules start after the consensus clients, so at startup +ethlambda's beacon node logs "Builder is not answering" and its first +registration batch fails. The validator client re-sends registrations every +epoch, so this clears within an epoch and is not a failure. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index a46fa5ca5..ca5d60353 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -24,6 +24,8 @@ # built with the builder support understands. The workflow rewrites the # `cl_image` and `vc_image` of the ethlambda entries (selected by type, not # position) to the image under test; the values below are what a local run uses. +# If the battery ever changes `network_id`, the preset or the fork epochs, the +# explicit Commit-Boost chain under `mev_params` must change with it. participants: - el_type: geth cl_type: lighthouse @@ -53,3 +55,28 @@ mev_params: mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308 # ghcr.io/commit-boost/pbs:latest mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415 + # Commit-Boost reads the chain from the package's config.yaml by default, + # and its parser requires SECONDS_PER_SLOT, which the genesis generator no + # longer writes (it writes SLOT_DURATION_MS). So the chain is given + # explicitly: kurtosis's defaults for this network (12 s slots, genesis fork + # version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824). + # chain_id must be a hex string; a bare integer does not parse. The rest is + # the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl). + commit_boost_config: | + chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" } + + [pbs] + host = "0.0.0.0" + port = {{ .Port }} + + {{ range $index, $relay := .Relays }} + [[relays]] + id = "mev_relay_{{$index}}" + url = "{{ $relay }}" + {{- end }} + + [logs.stdout] + level = "debug" + + [logs.file] + enabled=false From 6372671f19f429493c782a09da7493224c418618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 27 Oct 2026 09:36:15 -0300 Subject: [PATCH 4/5] ci: start spamoor in the Commit-Boost check A local run registered every validator but never got a builder block: rbuilder submits no bid for a block without transactions, and the enclave had none, so the relay held no bids for any proposer and every getHeader answered 204. With transactions flowing, the same enclave served a builder payload within a few slots, and the ethlambda validator client signed the blinded block that the relay delivered. spamoor runs with the package's default spammers (transfers and blob transactions), pinned by digest like the other images. The README also explains the no_finality fallbacks a run counts before the chain first finalizes. --- tooling/kurtosis-assertoor/README.md | 10 ++++++++-- tooling/kurtosis-assertoor/commit_boost_params.yaml | 9 +++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 82496fa6d..1c5aef9ea 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -75,7 +75,8 @@ Starts the same three participants with `mev_type: commit-boost` (`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder builder with its own lighthouse beacon node, and one [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module -per participant, which stands in for MEV-Boost. The package gives the ethlambda +per participant, which stands in for MEV-Boost. It also starts spamoor, since +rbuilder submits no bid for a block without transactions. The package gives the ethlambda beacon node `--builder ` and the ethlambda validator client `--builder-proposals`. There is no assertoor: the base battery's checks fail on the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the @@ -96,7 +97,7 @@ through lighthouse. The script writes every `lean_beacon_builder_*` and `/.prom`, pass or fail, and the unmet conditions to `failed.txt`. -The relay, builder and PBS images are pinned by digest in +The relay, builder, PBS and spamoor images are pinned by digest in `commit_boost_params.yaml` (the package's defaults are floating tags). ### Run it locally @@ -165,3 +166,8 @@ The Commit-Boost PBS modules start after the consensus clients, so at startup ethlambda's beacon node logs "Builder is not answering" and its first registration batch fails. The validator client re-sends registrations every epoch, so this clears within an epoch and is not a failure. + +From epoch 4 until the chain first finalizes, the beacon node builds locally and +counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like +Lighthouse, it skips the builder when the proposal is more than 3 epochs past +the finalized checkpoint, and the genesis checkpoint counts as one. diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index ca5d60353..7f0929f23 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -45,6 +45,15 @@ network_params: electra_fork_epoch: 0 fulu_fork_epoch: 0 mev_type: commit-boost +# rbuilder submits nothing for a block with no transactions, so without a +# spammer the relay never holds a bid and every getHeader answers 204. spamoor's +# package defaults send type-2 transfers and blob transactions every slot, so +# bids also carry blob commitments. +additional_services: + - spamoor +spamoor_params: + # ethpandaops/spamoor:master on 2026-10-27 + image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 # The package defaults are floating tags (`main`, `develop`, `latest`), so a # run would change under us. These pin what the tags pointed to on 2026-10-26; # bump them deliberately. From a211f4994b117f8abd86971cf49c462a2b787936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gr=C3=BCner?= <47506558+MegaRedHand@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:32:55 -0300 Subject: [PATCH 5/5] ci: run spamoor at a light load in the Commit-Boost check With spamoor's defaults (a chain-load group, 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed to Commit-Boost's 950 ms timeout, so most proposals got no bid, and in 40 minutes the ethlambda validator client behind lighthouse never received one. 10 transfers and 1 blob transaction a slot keep bids coming without that slowdown: a local run met every condition about 12 minutes after the enclave started, and the relay delivered a builder block in 14 of the 32 slots of epoch 1. The package's own 100 Uniswap swaps a slot, which it adds whenever a builder runs, stay on; they are not configurable. --- tooling/kurtosis-assertoor/README.md | 6 ++-- .../commit_boost_params.yaml | 30 +++++++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/tooling/kurtosis-assertoor/README.md b/tooling/kurtosis-assertoor/README.md index 1c5aef9ea..9570ef9ae 100644 --- a/tooling/kurtosis-assertoor/README.md +++ b/tooling/kurtosis-assertoor/README.md @@ -75,8 +75,10 @@ Starts the same three participants with `mev_type: commit-boost` (`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder builder with its own lighthouse beacon node, and one [Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module -per participant, which stands in for MEV-Boost. It also starts spamoor, since -rbuilder submits no bid for a block without transactions. The package gives the ethlambda +per participant, which stands in for MEV-Boost. It also starts spamoor at a +light load, since rbuilder submits no bid for a block without transactions and +a heavy load slows the relay past Commit-Boost's getHeader timeout (see the +comment in the params file). The package gives the ethlambda beacon node `--builder ` and the ethlambda validator client `--builder-proposals`. There is no assertoor: the base battery's checks fail on the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the diff --git a/tooling/kurtosis-assertoor/commit_boost_params.yaml b/tooling/kurtosis-assertoor/commit_boost_params.yaml index 7f0929f23..ea232cdc6 100644 --- a/tooling/kurtosis-assertoor/commit_boost_params.yaml +++ b/tooling/kurtosis-assertoor/commit_boost_params.yaml @@ -46,14 +46,38 @@ network_params: fulu_fork_epoch: 0 mev_type: commit-boost # rbuilder submits nothing for a block with no transactions, so without a -# spammer the relay never holds a bid and every getHeader answers 204. spamoor's -# package defaults send type-2 transfers and blob transactions every slot, so -# bids also carry blob commitments. +# spammer the relay never holds a bid and every getHeader answers 204. The load +# is kept light on purpose: under spamoor's defaults (a chain-load group plus +# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed +# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no +# bid. The package also always adds 100 Uniswap swaps a slot, sent to the +# builder's own execution client; that is not configurable. The blob spammer +# stays so bids carry blob commitments. additional_services: - spamoor spamoor_params: # ethpandaops/spamoor:master on 2026-10-27 image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5 + start_chainload: false + spammers: + - name: "EOA transfers" + description: "10 type-2 transfers a slot" + scenario: "eoatx" + config: + throughput: 10 + max_pending: 20 + max_wallets: 10 + base_fee: 20 + - name: "Blob transactions" + description: "1 type-4 transaction a slot with 1 sidecar" + scenario: "blob-combined" + config: + throughput: 1 + sidecars: 1 + max_pending: 2 + max_wallets: 2 + base_fee: 20 + blob_fee: 20 # The package defaults are floating tags (`main`, `develop`, `latest`), so a # run would change under us. These pin what the tags pointed to on 2026-10-26; # bump them deliberately.