feat(conformance): add the openshell-conformance program

.github/actions/setup-e2e-cli/action.yml

@@ -1,6 +1,12 @@
 name: Setup E2E CLI
 description: Download architecture-matched prebuilt OpenShell host CLIs for E2E tests
 
+inputs:
+  conformance:
+    description: Also download the prebuilt openshell-conformance program and set OPENSHELL_CONFORMANCE_BIN
+    required: false
+    default: "false"
+
 runs:
   using: composite
   steps:
@@ -17,3 +23,19 @@ runs:
         chmod +x "$cli"
         "$cli" --version
         echo "OPENSHELL_BIN=$cli" >> "$GITHUB_ENV"
+
+    - name: Download prebuilt openshell-conformance
+      if: inputs.conformance == 'true'
+      uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+      with:
+        name: ${{ runner.arch == 'X64' && 'openshell-conformance-x86_64-unknown-linux-musl' || 'openshell-conformance-aarch64-unknown-linux-musl' }}
+        path: .e2e/prebuilt-conformance
+
+    - name: Configure prebuilt openshell-conformance
+      if: inputs.conformance == 'true'
+      shell: bash
+      run: | # zizmor: ignore[github-env] persists only a fixed path under the trusted workspace
+        program="$GITHUB_WORKSPACE/.e2e/prebuilt-conformance/openshell-conformance"
+        chmod +x "$program"
+        "$program" list
+        echo "OPENSHELL_CONFORMANCE_BIN=$program" >> "$GITHUB_ENV"

.github/workflows/branch-checks.yml

@@ -389,6 +389,7 @@ jobs:
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
+          tasks/scripts/check-conformance-program.sh
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

.github/workflows/branch-e2e.yml

@@ -144,6 +144,60 @@ jobs:
           cargo-version: ${{ needs.version.outputs.cargo }}
           extra-cargo-flags: --no-default-features
 
+  # The conformance program, built once per architecture for the host lanes.
+  # It lives in its own workspace and is not released, so it is not part of
+  # build-binaries.yml.
+  build-conformance:
+    name: openshell-conformance (${{ matrix.triple }})
+    needs: [pr_metadata]
+    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_any_e2e == 'true'
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        include:
+          - triple: x86_64-unknown-linux-musl
+            runner: linux-amd64-cpu8
+          - triple: aarch64-unknown-linux-musl
+            runner: linux-arm64-cpu8
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          ref: ${{ github.sha }}
+      - uses: ./.github/actions/setup-nix
+        with:
+          prepare-shell: "true"
+          cachix-auth-token: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      - uses: ./.github/actions/setup-rust
+        with:
+          cache-key: binaries-v2-openshell-conformance-${{ matrix.triple }}
+          save-cache: ${{ github.ref == 'refs/heads/main' }}
+      - name: Build openshell-conformance
+        shell: nix develop -c bash -euo pipefail {0}
+        env:
+          # Build into the root target directory, which setup-rust caches.
+          CARGO_TARGET_DIR: ${{ github.workspace }}/target
+          TRIPLE: ${{ matrix.triple }}
+        run: |
+          cargo build --release --locked --target "$TRIPLE" \
+            --manifest-path tests/suites/conformance/Cargo.toml \
+            --bin openshell-conformance
+          binary="target/$TRIPLE/release/openshell-conformance"
+          # The program must run on any lane host.
+          tasks/scripts/verify-static-binary.sh "$binary"
+          # Fails if the program has no tests linked.
+          "$binary" list
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: openshell-conformance-${{ matrix.triple }}
+          path: target/${{ matrix.triple }}/release/openshell-conformance
+          compression-level: 0
+          retention-days: 5
+          if-no-files-found: error
+
   build-external-drivers:
     name: ${{ matrix.binary }} (${{ matrix.triple }})
     needs: [pr_metadata, version]
@@ -316,73 +370,80 @@ jobs:
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
   docker-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-docker-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       runner: linux-arm64-cpu8
 
   podman-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-podman-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       suite-matrix: >-
         [{"suite":"ci","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:ci"}]
 
   podman-additional-ca-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-podman-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       suite-matrix: >-
         [{"suite":"additional-ca","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"OPENSHELL_E2E_PODMAN_TEST=additional_ca mise run --no-deps --skip-deps e2e:podman"}]
 
   vm-e2e:
-    needs: [pr_metadata, build-binaries, build-vm-driver]
+    needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-vm-test.yml
+    with:
+      prebuilt-conformance: true
 
   vm-additional-ca-e2e:
-    needs: [pr_metadata, build-binaries, build-vm-driver]
+    needs: [pr_metadata, build-binaries, build-conformance, build-vm-driver]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-vm-test.yml
     with:
+      prebuilt-conformance: true
       suite-name: additional-ca
       test-name: additional_ca
 
   docker-external-driver-e2e:
-    needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-docker-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       runner: linux-arm64-cpu8
       gateway-artifact: openshell-gateway-plain-aarch64-unknown-linux-gnu
@@ -391,29 +452,31 @@ jobs:
         [{"suite":"external-driver","cmd":"mise run --no-deps --skip-deps e2e:docker:external-driver","apt_packages":"openssh-client","python_proto":false,"mcp":false}]
 
   podman-external-driver-e2e:
-    needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-podman-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
       external-driver-binary: openshell-driver-podman
       suite-matrix: >-
         [{"suite":"external-driver","runner":"ubuntu-26.04","podman_major":"5","podman_package_version":"5.7.0+ds2-3build1","conmon_package_version":"2.1.13+ds1-2","cmd":"mise run --no-deps --skip-deps e2e:podman:external-driver"}]
 
   vm-external-driver-e2e:
-    needs: [pr_metadata, build-binaries, build-gateway-plain, build-vm-driver]
+    needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-vm-driver]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-vm-test.yml
     with:
+      prebuilt-conformance: true
       gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
       suite-name: external-driver
       e2e-task: e2e:vm:external-driver
@@ -430,7 +493,7 @@ jobs:
       image-tag: ${{ github.sha }}
 
   kubernetes-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     strategy:
       fail-fast: false
@@ -454,13 +517,14 @@ jobs:
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: ${{ matrix.job_name }}
       agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
       test-name: ${{ matrix.test_name }}
 
   kubernetes-app-listener-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     strategy:
       fail-fast: false
@@ -488,6 +552,7 @@ jobs:
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes app listener (${{ matrix.name }})
       e2e-task: e2e:kubernetes:app-listener
@@ -497,27 +562,29 @@ jobs:
       listener-takeover: ${{ matrix.takeover }}
 
   kubernetes-workspace-managed-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes E2E (workspace managed mode)
       e2e-task: e2e:kubernetes:workspace-managed
 
   kubernetes-external-driver-e2e:
-    needs: [pr_metadata, build-binaries, build-gateway-plain, build-external-drivers, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-gateway-plain, build-external-drivers, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes E2E (external compute driver)
       e2e-task: e2e:kubernetes:external-driver
@@ -526,27 +593,29 @@ jobs:
       cluster-images: sandbox supervisor
 
   kubernetes-workspace-operator-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes E2E (workspace operator mode)
       e2e-task: e2e:kubernetes:workspace-operator
 
   kubernetes-ha-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_ha_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes HA E2E (Rust smoke)
       extra-helm-values: deploy/helm/openshell/ci/values-high-availability.yaml
@@ -556,14 +625,15 @@ jobs:
       use-envoy-gateway: true
 
   kubernetes-credential-drivers-e2e:
-    needs: [pr_metadata, build-binaries, build-images]
+    needs: [pr_metadata, build-binaries, build-conformance, build-images]
     if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_kubernetes_credential_drivers_e2e == 'true'
     permissions:
       actions: read
       contents: read
       packages: read
     uses: ./.github/workflows/e2e-kubernetes-test.yml
     with:
+      prebuilt-conformance: true
       image-tag: ${{ github.sha }}
       job-name: Kubernetes Credential Drivers E2E
       e2e-task: e2e:kubernetes:credential-drivers

.github/workflows/e2e-docker-test.yml

@@ -21,6 +21,10 @@ on:
         required: false
         type: string
         default: ""
+      prebuilt-conformance:
+        required: false
+        type: boolean
+        default: false
       external-driver-binary:
         required: false
         type: string
@@ -67,6 +71,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance: ${{ inputs.prebuilt-conformance }}
 
       - uses: ./.github/actions/setup-e2e-gateway
         with:

.github/workflows/e2e-kubernetes-test.yml

@@ -67,6 +67,11 @@ on:
         required: false
         type: string
         default: ""
+      prebuilt-conformance:
+        description: "Run conformance with the openshell-conformance artifact instead of building it"
+        required: false
+        type: boolean
+        default: false
       external-driver-binary:
         description: "Optional standalone compute driver binary used to compose a local test image"
         required: false
@@ -124,6 +129,8 @@ jobs:
 
       - name: Use prebuilt OpenShell CLI
         uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance: ${{ inputs.prebuilt-conformance }}
 
       - name: Use prebuilt OpenShell gateway
         if: inputs.gateway-artifact != ''

.github/workflows/e2e-podman-test.yml

@@ -17,6 +17,10 @@ on:
         required: false
         type: string
         default: ""
+      prebuilt-conformance:
+        required: false
+        type: boolean
+        default: false
       external-driver-binary:
         required: false
         type: string
@@ -52,6 +56,8 @@ jobs:
           persist-credentials: false
 
       - uses: ./.github/actions/setup-e2e-cli
+        with:
+          conformance: ${{ inputs.prebuilt-conformance }}
 
       - uses: ./.github/actions/setup-e2e-gateway
         with: