test(e2e): migrate the features and drivers suites to testkit

.github/workflows/branch-checks.yml

@@ -233,8 +233,9 @@ jobs:
           cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
-          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p conformance -- --check
-          cargo fmt --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh -- --check
+          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/drivers/podman/Cargo.toml --all -- --check
+          cargo fmt --manifest-path tests/suites/features/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/support/Cargo.toml --all -- --check
 
   cargo-lockfiles:
@@ -297,8 +298,9 @@ jobs:
           cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
-          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p conformance --all-targets -- -D warnings
-          cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml -p keycloak-provider-refresh --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml --workspace --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/drivers/podman/Cargo.toml --workspace --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml --workspace --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/support/Cargo.toml --all-targets -- -D warnings
 
   rust-test:
@@ -389,6 +391,7 @@ jobs:
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/testkit/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/tmachine/Cargo.toml
           cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/runner/Cargo.toml
+          cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path tests/support/Cargo.toml
 
   rust-postgres-test:
     name: Rust PostgreSQL tests

.github/workflows/branch-e2e.yml

@@ -255,65 +255,55 @@ jobs:
     secrets:
       CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
-  # Run feature-specific integration tests:
+  # Run the testkit feature suites (tests/suites/features) on tmachine. The
+  # environments and suites are listed in e2e-suites.yml.
   feature-specific-integration:
     needs: prepare-integration
     permissions:
       actions: read
       contents: read
       packages: read
-    uses: ./.github/workflows/integration-runner.yml
+    uses: ./.github/workflows/e2e-suites.yml
     with:
-      category: feature-specific
+      family: features
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
-      test-matrix: >-
-        [
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"}
-        ]
+    secrets:
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
-  # Run driver-specific integration tests:
-  # These suites require the binary installer's CLI path, local HTTP gateway,
-  # and system service configuration for their Podman user-namespace fixtures.
+  # Run the testkit driver suites (tests/suites/drivers) on tmachine. The
+  # environments and suites are listed in e2e-suites.yml.
   driver-specific-integration:
     needs: prepare-integration
     permissions:
       actions: read
       contents: read
       packages: read
-    uses: ./.github/workflows/integration-runner.yml
+    uses: ./.github/workflows/e2e-suites.yml
     with:
-      category: driver-specific
+      family: drivers
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
-      test-matrix: >-
-        [
-          {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"driver-podman"},
-          {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"},
-          {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"driver-podman"}
-        ]
+    secrets:
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
-  # Run the testkit feature suites in tests/suites/ on tmachine. The
-  # environments and suites are listed in e2e-suites.yml; the conformance rows
-  # run in conformance-integration.
-  e2e-suites:
-    needs: [pr_metadata, prepare-integration]
-    if: needs.pr_metadata.outputs.should_run == 'true' && needs.pr_metadata.outputs.run_core_e2e == 'true'
+  # Run the Podman e2e suite on tmachine. It needs the binary installer's CLI
+  # path and local HTTP gateway.
+  podman-e2e-integration:
+    needs: prepare-integration
     permissions:
       actions: read
       contents: read
       packages: read
-    uses: ./.github/workflows/e2e-suites.yml
+    uses: ./.github/workflows/integration-runner.yml
     with:
-      family: features
+      category: driver-specific
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
-    secrets:
-      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
+      test-matrix: >-
+        [
+          {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"}
+        ]
 
   docker-e2e:
     needs: [pr_metadata, build-binaries, build-images]
@@ -573,9 +563,9 @@ jobs:
     needs:
       - pr_metadata
       - conformance-integration
-      - e2e-suites
       - feature-specific-integration
       - driver-specific-integration
+      - podman-e2e-integration
       - docker-e2e
       - podman-e2e
       - podman-additional-ca-e2e

.github/workflows/e2e-suites.yml

@@ -2,20 +2,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Runs testkit suites in tmachine environments on CI. Each row of `ROWS` below
-# names a tmachine environment and installer, the family (a workspace under
-# tests/suites/) and the package in it, the gateway name the installer
-# registers, and the runner label for the environment's architecture.
-# `openshell-test-runner` builds the tests, boots the environment, runs them
-# inside it and checks the results. The `family` input selects the rows of one
-# family; leave it empty to run every row.
+# names a tmachine environment and installer, the family (the path of a
+# workspace under tests/suites/, such as conformance or drivers/podman) and,
+# optionally, the package in it, the gateway name the installer registers, and
+# the runner label for the environment's architecture. `openshell-test-runner`
+# builds the tests, boots the environment, runs them inside it and checks the
+# results. The `family` input selects the rows of one family, including the
+# workspaces under it (drivers selects drivers/podman); leave it empty to run
+# every row.
 
 name: E2E Suites
 
 on:
   workflow_call:
     inputs:
       family:
-        description: Run only the rows of this family (for example conformance); empty runs every row
+        description: Run only the rows of this family and the workspaces under it (for example conformance or drivers); empty runs every row
         required: false
         type: string
         default: ""
@@ -62,17 +64,37 @@ jobs:
           # Only amd64 for now: CI has no arm64 runner with KVM for tmachine,
           # and the caller only builds amd64 inputs. The gateway is the name the
           # installer registers with the CLI: `openshell` for the packages,
-          # `tmachine` for the k3s installers.
+          # `tmachine` for the k3s and binaries installers. A row without a
+          # package runs every package in the workspace; each driver suite is a
+          # workspace of its own.
           rows='[
             {"environment":"ubuntu-docker-rootful","installer":"deb","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"fedora-podman-rootful","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"fedora-podman-rootless","installer":"rpm","family":"conformance","package":"conformance","gateway":"openshell","runner":"ubuntu-24.04"},
             {"environment":"ubuntu-k3s","installer":"k3s","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
             {"environment":"ubuntu-k3s","installer":"k3s-ha","family":"conformance","package":"conformance","gateway":"tmachine","runner":"ubuntu-24.04"},
-            {"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"}
+            {"environment":"ubuntu-docker-rootful-keycloak","installer":"deb","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"ubuntu-docker-rootful","installer":"deb","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless","installer":"rpm","family":"features","package":"oci-image","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless-keycloak","installer":"rpm","family":"features","package":"keycloak-provider-refresh","gateway":"openshell","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful-userns-auto","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful-userns-keep-id","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootful-userns-private","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless-userns-default","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless-userns-auto","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless-userns-keep-id","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"},
+            {"environment":"fedora-podman-rootless-userns-private","installer":"binaries","family":"drivers/podman","gateway":"tmachine","runner":"ubuntu-24.04"}
           ]'
-          matrix=$(jq -c --arg family "${FAMILY}" \
-            '{include: map(select($family == "" or .family == $family))}' <<< "$rows")
+          # `suite` names the row's tests in the job and artifact names.
+          matrix=$(jq -c --arg family "${FAMILY}" '{include: map(
+              select($family == "" or .family == $family or (.family | startswith($family + "/")))
+              | .package //= ""
+              | .suite = (if .package == "" then .family else "\(.family)/\(.package)" end)
+              | .artifact = ([.suite, .environment, .installer] | join("-") | gsub("/"; "-"))
+            )}' <<< "$rows")
           if [ "$(jq '.include | length' <<< "$matrix")" -eq 0 ]; then
             echo "::error::No E2E suites match family '${FAMILY}'"
             exit 1
@@ -81,7 +103,7 @@ jobs:
           echo "matrix=${matrix}" >> "$GITHUB_OUTPUT"
 
   suites:
-    name: E2E suites (${{ matrix.family }}/${{ matrix.package }}, ${{ matrix.environment }}, ${{ matrix.installer }})
+    name: E2E suites (${{ matrix.suite }}, ${{ matrix.environment }}, ${{ matrix.installer }})
     needs: select
     strategy:
       fail-fast: false
@@ -145,11 +167,15 @@ jobs:
           # Retry dependency preparation, then run the suites once. The runner
           # builds the suites for the guest, so it needs no prepared archives.
           nix build --no-link .#openshell-test-runner || nix build --no-link .#openshell-test-runner
+          packages=()
+          if [ -n "${PACKAGE}" ]; then
+            packages=(--package "${PACKAGE}")
+          fi
           nix run .#openshell-test-runner -- \
             --environment "${ENVIRONMENT}" \
             --installer "${INSTALLER}" \
             --family "${FAMILY}" \
-            --package "${PACKAGE}" \
+            "${packages[@]}" \
             --gateway "${GATEWAY}" \
             --results artifacts/test-results
 
@@ -162,7 +188,7 @@ jobs:
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
-          name: e2e-results-${{ matrix.family }}-${{ matrix.package }}-${{ matrix.environment }}-${{ matrix.installer }}
+          name: e2e-results-${{ matrix.artifact }}
           path: artifacts/test-results
           if-no-files-found: ignore
           retention-days: 7

.github/workflows/integration-runner.yml

@@ -24,8 +24,7 @@ on:
         type: string
         default: >-
           [
-            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},
-            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}
+            {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"}
           ]
 
 permissions:

.github/workflows/integration-test.yml

@@ -20,15 +20,14 @@ on:
         description: Integration-test category, used for the job name and concurrency key
         required: true
         type: string
-        default: feature-specific
+        default: driver-specific
       test-matrix:
         description: JSON array of environment, installer, and testsuite combinations
         required: true
         type: string
         default: >-
           [
-            {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},
-            {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}
+            {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"e2e-podman"}
           ]
 
 permissions:

.github/workflows/release-dev.yml

@@ -132,19 +132,13 @@ jobs:
       actions: read
       contents: read
       packages: read
-    uses: ./.github/workflows/integration-runner.yml
+    uses: ./.github/workflows/e2e-suites.yml
     with:
-      category: feature-specific
+      family: features
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
-      test-matrix: >-
-        [
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"}
-        ]
+    secrets:
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
   docker-e2e:
     needs: [build-binaries, build-images]

.github/workflows/release-qualification.yml

@@ -85,19 +85,13 @@ jobs:
       actions: read
       contents: read
       packages: read
-    uses: ./.github/workflows/integration-runner.yml
+    uses: ./.github/workflows/e2e-suites.yml
     with:
-      category: feature-specific
+      family: features
       source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
       integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
-      test-matrix: >-
-        [
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"},
-          {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"oci-image"},
-          {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"oci-image"}
-        ]
+    secrets:
+      CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }}
 
   docker-e2e:
     permissions:

AGENTS.md

@@ -22,7 +22,6 @@ Do not rely on this file for a full inventory. The detailed public and contribut
 | Path | Components | Purpose |
 |------|-----------|---------|
 | `crates/openshell-cli/` | CLI binary | User-facing command-line interface |
-| `crates/openshell-conformance/` | CLI conformance library | Reusable driver-agnostic scenarios and command runner |
 | `crates/openshell-server/` | Gateway server | Control-plane API, sandbox lifecycle, auth boundary |
 | `crates/openshell-sandbox/` | Sandbox runtime | Capability-free workload launcher, process identity, and seccomp-mediated I/O |
 | `crates/openshell-supervisor/` | Supervisor runtime | Gateway session, policy evaluation, credentials, and upstream networking |

CI.md

@@ -165,10 +165,13 @@ selected. DEB and RPM packages build only when integration qualification is sele
 the DEB on Ubuntu with Docker and installs the CLI and gateway RPMs on Fedora with
 rootful and rootless Podman. These lanes run conformance using the matching runtime
 images. Release Dev, Release, and Pre-release use the same package installers.
-Fedora provider-refresh tests also use RPMs. The Podman driver-specific suites
-retain the binary installer because their fixtures configure its system service,
-local HTTP gateway, and CLI path. The manual Integration Tests workflow defaults
-to the package installers and downloads the packages selected by its matrix.
+The feature suites (`tests/suites/features`) run the same way, on the
+package installers. The Podman driver suite (`tests/suites/drivers/podman`) and the
+`e2e-podman` lane retain the binary installer because they configure its system
+service, local HTTP gateway, and CLI path. The driver suite and `e2e-podman` run
+only in `Branch E2E Checks`. The manual Integration Tests workflow runs the
+remaining tmachine testsuites, such as `e2e-podman`, and downloads the packages
+selected by its matrix.
 
 Three opt-in labels enable the long-running E2E suites: