ci: add a Commit-Boost check to the kurtosis battery

.github/workflows/kurtosis_assertoor.yml

@@ -17,6 +17,15 @@ on:
         description: "ethlambda image to test (default: ghcr.io/lambdaclass/ethlambda:unstable)"
         required: false
         type: string
+      battery:
+        description: "Which battery to run"
+        required: false
+        type: choice
+        options:
+          - assertoor
+          - commit-boost
+          - both
+        default: assertoor
 
 permissions:
   contents: read
@@ -34,6 +43,13 @@ env:
   ETHEREUM_PACKAGE_REPO: https://github.com/lambdaclass/ethereum-package.git
   ETHEREUM_PACKAGE_REF: 67cb0e02dc0bd49ec9461cbf2ef7ac4a3232755f
   ENCLAVE: ethlambda-assertoor
+  # The Commit-Boost check needs the fork's `--builder` / `--builder-proposals`
+  # wiring for ethlambda, which the commit above lacks. It has its own pin so
+  # the base battery stays where it is. Fork PR #22 (that wiring) is merged as
+  # a15e7df; this commit adds PR #23's fix for the builder's enode bootnodes on
+  # top. Re-pin to #23's merge commit once it lands; see the README.
+  COMMIT_BOOST_PACKAGE_REF: 7fa7d1225fe5d975300ecc5584bf28ca6d42d451
+  COMMIT_BOOST_ENCLAVE: ethlambda-commit-boost
 
 jobs:
   image:
@@ -79,6 +95,7 @@ jobs:
   assertoor:
     name: Assertoor
     needs: image
+    if: github.event_name != 'workflow_dispatch' || inputs.battery != 'commit-boost'
     runs-on: ubuntu-latest
     timeout-minutes: 120
     outputs:
@@ -168,6 +185,83 @@ jobs:
         if: always()
         run: kurtosis enclave rm -f "$ENCLAVE" || true
 
+  # Dispatch-only for now: main's image does not have the builder flags until
+  # the builder PR stack (#694 to #705) lands. Once it is on main, drop the
+  # event condition so the check also runs on push and on pull requests.
+  commit-boost:
+    name: Commit-Boost
+    needs: image
+    if: github.event_name == 'workflow_dispatch' && inputs.battery != 'assertoor'
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Free up disk space
+        uses: ./.github/actions/free-disk
+
+      - name: Install kurtosis
+        run: |
+          echo "deb [trusted=yes] https://sdk.kurtosis.com/kurtosis-cli-release-artifacts/ /" \
+            | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt-get update
+          sudo apt-get install -y "kurtosis-cli=$KURTOSIS_VERSION"
+          kurtosis analytics disable
+          kurtosis version
+
+      # A clone for the same reason as in the assertoor job.
+      - name: Fetch ethereum-package at the pinned commit
+        run: |
+          git init "$RUNNER_TEMP/ethereum-package"
+          cd "$RUNNER_TEMP/ethereum-package"
+          git fetch --depth 1 "$ETHEREUM_PACKAGE_REPO" "$COMMIT_BOOST_PACKAGE_REF"
+          git checkout --detach FETCH_HEAD
+          test "$(git rev-parse HEAD)" = "$COMMIT_BOOST_PACKAGE_REF"
+
+      - name: Point the args at the image under test
+        env:
+          IMAGE: ${{ needs.image.outputs.image }}
+        run: |
+          cp tooling/kurtosis-assertoor/commit_boost_params.yaml "$RUNNER_TEMP/args.yaml"
+          yq -i '
+            (.participants[] | select(.cl_type == "ethlambda") | .cl_image) = strenv(IMAGE) |
+            (.participants[] | select(.vc_type == "ethlambda") | .vc_image) = strenv(IMAGE)
+          ' "$RUNNER_TEMP/args.yaml"
+          cat "$RUNNER_TEMP/args.yaml"
+
+      - name: Start the devnet
+        run: |
+          kurtosis run "$RUNNER_TEMP/ethereum-package" \
+            --enclave "$COMMIT_BOOST_ENCLAVE" --args-file "$RUNNER_TEMP/args.yaml" \
+            | tee "$RUNNER_TEMP/kurtosis_output.txt"
+
+      - name: Wait for builder blocks
+        run: |
+          tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \
+            "$COMMIT_BOOST_ENCLAVE" 40 "$RUNNER_TEMP/builder-results"
+
+      - name: Dump the enclave
+        if: always()
+        run: |
+          mkdir -p "$RUNNER_TEMP/dump"
+          cp "$RUNNER_TEMP/args.yaml" "$RUNNER_TEMP/dump/args.yaml"
+          cp "$RUNNER_TEMP/kurtosis_output.txt" "$RUNNER_TEMP/dump/" || true
+          cp -r "$RUNNER_TEMP/builder-results" "$RUNNER_TEMP/dump/" || true
+          cd "$RUNNER_TEMP/dump"
+          kurtosis enclave dump "$COMMIT_BOOST_ENCLAVE" || true
+
+      - name: Upload the dump and results
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: commit-boost-enclave-dump-${{ github.run_id }}
+          path: ${{ runner.temp }}/dump
+          if-no-files-found: warn
+
+      - name: Remove the enclave
+        if: always()
+        run: kurtosis enclave rm -f "$COMMIT_BOOST_ENCLAVE" || true
+
   notify:
     name: Post failure to Slack
     needs: [image, assertoor]

tooling/kurtosis-assertoor/README.md

@@ -68,3 +68,108 @@ field by field and marks a node whose spec differs offline (`invalid node
 specs: spec mismatch: GloasForkEpoch`); lighthouse reports the far-future
 value and ethlambda omits the key. The chain itself is healthy (all nodes share
 a head and finalize), so this is a spec-endpoint gap, not a harness problem.
+
+## Commit-Boost check
+
+Starts the same three participants with `mev_type: commit-boost`
+(`commit_boost_params.yaml`): a flashbots mev-boost-relay, a reth-rbuilder
+builder with its own lighthouse beacon node, and one
+[Commit-Boost](https://github.com/Commit-Boost/commit-boost-client) PBS module
+per participant, which stands in for MEV-Boost. It also starts spamoor at a
+light load, since rbuilder submits no bid for a block without transactions and
+a heavy load slows the relay past Commit-Boost's getHeader timeout (see the
+comment in the params file). The package gives the ethlambda
+beacon node `--builder <pbs url>` and the ethlambda validator client
+`--builder-proposals`. There is no assertoor: the base battery's checks fail on
+the spec-endpoint gap above, so `scripts/check-builder-blocks.sh` polls the
+ethlambda services' Prometheus metrics instead, for up to the timeout it is given.
+
+| Service | Metric | Shows |
+|---|---|---|
+| `cl-3-ethlambda-geth` | `lean_beacon_builder_registrations_total{outcome="forwarded"}` >= 1 | Registrations reach the builder through the beacon node |
+| `cl-3-ethlambda-geth` | `lean_beacon_builder_payload_source_total{source="builder"}` >= 1 | The beacon node's own `produceBlockV3` took a builder bid |
+| `vc-3-geth-ethlambda` | `ethlambda_validator_builder_registrations_sent_total` >= 1 | The validator client registers its validators |
+| `vc-3-geth-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It signs and publishes blinded blocks through ethlambda's beacon node |
+| `vc-2-geth-lighthouse-ethlambda` | `ethlambda_validator_blinded_blocks_proposed_total` >= 1 | It does the same through a third-party beacon node (lighthouse) |
+
+Passing therefore covers the registration path, a builder block from ethlambda's
+own beacon node, and ethlambda's validator client proposing blinded blocks
+through lighthouse. The script writes every `lean_beacon_builder_*` and
+`ethlambda_validator_*` blinded or registration series of the three services to
+`<output-dir>/<service>.prom`, pass or fail, and the unmet conditions to
+`failed.txt`.
+
+The relay, builder, PBS and spamoor images are pinned by digest in
+`commit_boost_params.yaml` (the package's defaults are floating tags).
+
+### Run it locally
+
+The image needs the builder support, so build one from a checkout of the
+builder PR stack (#694 to #705). From that checkout's root, `make docker-build
+DOCKER_TAG=<tag>` tags it `ghcr.io/lambdaclass/ethlambda:<tag>`; set that as the
+`cl_image` and `vc_image` of the ethlambda entries in `commit_boost_params.yaml`.
+Kurtosis uses an image that exists in the local Docker daemon and only pulls a
+missing one (`kurtosis run --image-download`, default `missing`), so no setting
+is needed; do not pass `--image-download always`. From the repository root:
+
+```bash
+git clone https://github.com/lambdaclass/ethereum-package.git /tmp/ethereum-package
+git -C /tmp/ethereum-package checkout 7fa7d1225fe5d975300ecc5584bf28ca6d42d451
+
+kurtosis run /tmp/ethereum-package --enclave ethlambda-commit-boost \
+  --args-file tooling/kurtosis-assertoor/commit_boost_params.yaml
+
+tooling/kurtosis-assertoor/scripts/check-builder-blocks.sh \
+  ethlambda-commit-boost 40 /tmp/builder-results
+
+kurtosis enclave rm -f ethlambda-commit-boost
+```
+
+### In CI
+
+The `commit-boost` job of `.github/workflows/kurtosis_assertoor.yml` runs on
+manual dispatch only. Pick `battery: commit-boost` (or `both`) and give `image`
+the tag of an image built from the builder PR stack; the default `assertoor`
+runs only the base battery, as before. Its pin is `COMMIT_BOOST_PACKAGE_REF`,
+separate from `ETHEREUM_PACKAGE_REF`, and it uploads `commit-boost-enclave-dump-<run id>`.
+
+It is dispatch-only because the image that pushes and pull requests test
+(`sha-<7>` of `main`, or `unstable`) does not have `--builder` or
+`--builder-proposals` until the builder PR stack is on `main`, so the check
+would fail there. Once it lands, drop the event condition on the job to run it
+on push and on pull requests.
+
+### The fork pin
+
+lambdaclass/ethereum-package#22 added ethlambda to `apply_external_builder_flags`
+(`--builder=<url>` for the beacon node, `--builder-proposals` for the validator
+client) and is merged (`a15e7df`). `COMMIT_BOOST_PACKAGE_REF` is
+`7fa7d1225fe5d975300ecc5584bf28ca6d42d451`, from
+lambdaclass/ethereum-package#23, which adds the builder's enode bootnode fix on
+top of it. Re-pin to #23's merge commit once it lands: a squash-merge leaves the
+current commit unreachable from the fork's default branch, so it may be garbage
+collected.
+
+### Upstream workarounds
+
+A local run found two breakages in images and packages we do not control:
+
+| Breakage | Worked around in |
+|---|---|
+| `ethpandaops/reth-rbuilder:develop` (newest image, 2026-07-17) parses only enode URLs as bootnodes, and the package has passed ENRs since its 08-05 refactor, so the builder's execution client refused them | fork PR #23, through the `COMMIT_BOOST_PACKAGE_REF` pin |
+| Commit-Boost's PBS requires `SECONDS_PER_SLOT` in the chain config, and the genesis generator now writes only `SLOT_DURATION_MS` | `mev_params.commit_boost_config` in `commit_boost_params.yaml`, which gives the chain explicitly (`chain_id` has to be a hex string) |
+
+Drop each once the upstream is fixed. The explicit chain must change with the
+network parameters.
+
+### Startup noise
+
+The Commit-Boost PBS modules start after the consensus clients, so at startup
+ethlambda's beacon node logs "Builder is not answering" and its first
+registration batch fails. The validator client re-sends registrations every
+epoch, so this clears within an epoch and is not a failure.
+
+From epoch 4 until the chain first finalizes, the beacon node builds locally and
+counts `lean_beacon_builder_fallback_total{reason="no_finality"}`: like
+Lighthouse, it skips the builder when the proposal is more than 3 epochs past
+the finalized checkpoint, and the genesis checkpoint counts as one.

tooling/kurtosis-assertoor/commit_boost_params.yaml

@@ -0,0 +1,115 @@
+# The Commit-Boost check for ethlambda: the same three participants as
+# network_params.yaml, plus `mev_type: commit-boost`, so ethlambda's beacon node
+# and validator client build blocks through a Commit-Boost PBS module. There is
+# no assertoor: the check is scripts/check-builder-blocks.sh, which reads the
+# builder metrics of every ethlambda service. See README.md ("Commit-Boost
+# check") for the steps. The package comes from a clone of
+# lambdaclass/ethereum-package at the commit pinned in
+# .github/workflows/kurtosis_assertoor.yml (COMMIT_BOOST_PACKAGE_REF).
+#
+# Participants, in the order ethereum-package launches them (the order fixes the
+# service names check-builder-blocks.sh reads):
+#   1. lighthouse + geth, lighthouse validator. It has to come first: the first
+#      participant is the network's bootnode, and ethlambda does not report an
+#      ENR over the beacon API.
+#   2. lighthouse + geth, validated by the ethlambda validator client, so the
+#      validator client proposes blinded blocks through a beacon node that is
+#      not ethlambda.
+#   3. ethlambda beacon node + geth, validated by the ethlambda validator client.
+#
+# `mev_type: commit-boost` adds a flashbots mev-boost-relay, a reth-rbuilder
+# builder (with its own lighthouse beacon node) and one Commit-Boost PBS module
+# per participant. The package passes each ethlambda beacon node `--builder` and
+# each ethlambda validator client `--builder-proposals`, which only an image
+# built with the builder support understands. The workflow rewrites the
+# `cl_image` and `vc_image` of the ethlambda entries (selected by type, not
+# position) to the image under test; the values below are what a local run uses.
+# If the battery ever changes `network_id`, the preset or the fork epochs, the
+# explicit Commit-Boost chain under `mev_params` must change with it.
+participants:
+  - el_type: geth
+    cl_type: lighthouse
+  - el_type: geth
+    cl_type: lighthouse
+    vc_type: ethlambda
+    vc_image: ghcr.io/lambdaclass/ethlambda:unstable
+  - el_type: geth
+    cl_type: ethlambda
+    cl_image: ghcr.io/lambdaclass/ethlambda:unstable
+    vc_type: ethlambda
+    vc_image: ghcr.io/lambdaclass/ethlambda:unstable
+network_params:
+  # ethlambda only supports the mainnet preset: 32-slot epochs of 12 s.
+  preset: mainnet
+  # Fulu is the last fork ethlambda processes; gloas stays unscheduled.
+  electra_fork_epoch: 0
+  fulu_fork_epoch: 0
+mev_type: commit-boost
+# rbuilder submits nothing for a block with no transactions, so without a
+# spammer the relay never holds a bid and every getHeader answers 204. The load
+# is kept light on purpose: under spamoor's defaults (a chain-load group plus
+# 200 transfers and 6 blob transactions a slot) the relay's getHeader slowed
+# past Commit-Boost's 950 ms timeout on a local run, so most proposals got no
+# bid. The package also always adds 100 Uniswap swaps a slot, sent to the
+# builder's own execution client; that is not configurable. The blob spammer
+# stays so bids carry blob commitments.
+additional_services:
+  - spamoor
+spamoor_params:
+  # ethpandaops/spamoor:master on 2026-10-27
+  image: ethpandaops/spamoor@sha256:e890785f5814218bbcb544c495820ff7b02dd554e6d714f82849a0ac8de680c5
+  start_chainload: false
+  spammers:
+    - name: "EOA transfers"
+      description: "10 type-2 transfers a slot"
+      scenario: "eoatx"
+      config:
+        throughput: 10
+        max_pending: 20
+        max_wallets: 10
+        base_fee: 20
+    - name: "Blob transactions"
+      description: "1 type-4 transaction a slot with 1 sidecar"
+      scenario: "blob-combined"
+      config:
+        throughput: 1
+        sidecars: 1
+        max_pending: 2
+        max_wallets: 2
+        base_fee: 20
+        blob_fee: 20
+# The package defaults are floating tags (`main`, `develop`, `latest`), so a
+# run would change under us. These pin what the tags pointed to on 2026-10-26;
+# bump them deliberately.
+mev_params:
+  # ethpandaops/mev-boost-relay:main
+  mev_relay_image: ethpandaops/mev-boost-relay@sha256:3600f04fcaf0bbb231f4737470f094fdf2314db54b4853197fa369aea90f8b19
+  # ethpandaops/reth-rbuilder:develop
+  mev_builder_image: ethpandaops/reth-rbuilder@sha256:916ba84da2690e3e368f57da5b0c73bfd919b452de8b95f64895846e0f3a8308
+  # ghcr.io/commit-boost/pbs:latest
+  mev_boost_image: ghcr.io/commit-boost/pbs@sha256:8612c03681d390ab7d0154f5db1581584cd265563b723947da90cd00ba7ca415
+  # Commit-Boost reads the chain from the package's config.yaml by default,
+  # and its parser requires SECONDS_PER_SLOT, which the genesis generator no
+  # longer writes (it writes SLOT_DURATION_MS). So the chain is given
+  # explicitly: kurtosis's defaults for this network (12 s slots, genesis fork
+  # version 0x10000038, fulu from genesis, chain id 3151908 = 0x301824).
+  # chain_id must be a hex string; a bare integer does not parse. The rest is
+  # the package's own template (static_files/mev/commit-boost/cb-config.toml.tmpl).
+  commit_boost_config: |
+    chain = { genesis_time_secs = {{ .Timestamp }}, slot_time_secs = 12, genesis_fork_version = "0x10000038", fulu_fork_slot = 0, chain_id = "0x301824" }
+
+    [pbs]
+    host = "0.0.0.0"
+    port = {{ .Port }}
+
+    {{ range $index, $relay := .Relays }}
+    [[relays]]
+    id = "mev_relay_{{$index}}"
+    url = "{{ $relay }}"
+    {{- end }}
+
+    [logs.stdout]
+    level = "debug"
+
+    [logs.file]
+    enabled=false