test(suites): port the old suites onto testkit and run them from the plan

.github/workflows/branch-checks.yml

@@ -224,7 +224,8 @@ jobs:
           cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check
           cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check
-          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check
+          cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -p sandbox-image -- --check
+          cargo fmt --manifest-path tests/suites/features/Cargo.toml -p providers -p podman -- --check
 
   cargo-lockfiles:
     name: Cargo lockfiles
@@ -286,7 +287,8 @@ jobs:
           cargo clippy --locked --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings
           cargo clippy --locked --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings
-          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -p sandbox-image --all-targets -- -D warnings
+          cargo clippy --locked --manifest-path tests/suites/features/Cargo.toml -p providers -p podman --all-targets -- -D warnings
 
   rust-test:
     name: Rust test (${{ matrix.system }})

TESTING.md

@@ -229,6 +229,27 @@ The older conformance, feature and driver lanes in `.github/workflows/` are stil
 written by hand and run through the tmachine testsuites in
 `tests/tmachine/config.nix`. They move to plans as their suites move to testkit.
 
+The old suites now also exist as testkit suites, and the plan runs the ones the
+runner can set up: the `sandbox` and `sandbox-image` crates of the `conformance`
+family. The old crates and playbooks stay until the release lanes switch to
+plans. Two ports are not in a plan yet:
+
+- `features/providers` (provider OAuth refresh against Keycloak) needs the
+  `keycloak` Ansible role to run a Keycloak service on the guest with the realm in
+  `scripts/keycloak-realm.json`, and `OPENSHELL_E2E_OIDC_ISSUER`,
+  `OPENSHELL_E2E_OIDC_USERNAME` and `OPENSHELL_E2E_OIDC_PASSWORD` set for the
+  tests. The runner provisions only an environment and an installer, so it has no
+  place for suite-specific guest setup.
+- `features/podman` (user namespaces) needs the gateway reconfigured for each
+  `userns` mode with the fixtures in `tests/suites/features/podman/fixtures`, and
+  a reference UID map from native Podman in `OPENSHELL_TEST_INPUT_DIR`, as the old
+  `driver-podman` playbooks do. The runner starts one gateway configuration per
+  environment and runs a suite once.
+
+Both crates are compiled and linted by Branch Checks. The Keycloak one runs
+locally with `e2e/with-keycloak.sh e2e/with-docker-gateway.sh cargo test
+--manifest-path tests/suites/features/Cargo.toml -p providers`.
+
 ### Python E2E (`e2e/python/`)
 
 `mise run e2e:python` builds `openshell/e2e-python:dev` from

tasks/rust.toml

@@ -39,7 +39,8 @@ run = [
   "cargo clippy --manifest-path tests/testkit/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path tests/tmachine/Cargo.toml --all-targets -- -D warnings",
   "cargo clippy --manifest-path tests/runner/Cargo.toml --all-targets -- -D warnings",
-  "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -p sandbox-image --all-targets -- -D warnings",
+  "cargo clippy --manifest-path tests/suites/features/Cargo.toml -p providers -p podman --all-targets -- -D warnings",
 ]
 run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 lint native"
 hide = true
@@ -55,7 +56,8 @@ run = [
   "cargo fmt --manifest-path tests/testkit/Cargo.toml --all",
   "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all",
   "cargo fmt --manifest-path tests/runner/Cargo.toml --all",
-  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox",
+  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -p sandbox-image",
+  "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p providers -p podman",
 ]
 hide = true
 
@@ -70,7 +72,8 @@ run = [
   "cargo fmt --manifest-path tests/testkit/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path tests/tmachine/Cargo.toml --all -- --check",
   "cargo fmt --manifest-path tests/runner/Cargo.toml --all -- --check",
-  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --check",
+  "cargo fmt --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -p sandbox-image -- --check",
+  "cargo fmt --manifest-path tests/suites/features/Cargo.toml -p providers -p podman -- --check",
 ]
 hide = true
 

tasks/scripts/check_cargo_lockfile_alignment.py

@@ -38,6 +38,7 @@
 ALIGNED_WORKSPACES = (
     "tests/testkit",
     "tests/suites/conformance",
+    "tests/suites/features",
 )
 
 

tasks/test.toml

@@ -161,6 +161,7 @@ run = "e2e/rust/e2e-docker.sh"
 
 ["e2e:testkit"]
 description = "Run the testkit suites in tests/suites/conformance against a Docker-backed gateway"
+env = { OPENSHELL_TEST_CAPABILITIES = "compute_driver=docker,persistent_workspace" }
 run = "e2e/with-docker-gateway.sh cargo test --locked --manifest-path tests/suites/conformance/Cargo.toml -p sandbox -- --test-threads 1 --nocapture"
 
 ["e2e:oci-image"]

tests/environments/docker.yaml

@@ -8,7 +8,9 @@
 # that provision it. `gateway` is the name the installer registers the gateway
 # under with the CLI (`openshell gateway add --name`); the packaged installers
 # register `openshell`. `capabilities` is what the environment gives tests; see
-# tests/testkit/README.md.
+# tests/testkit/README.md. `env` adds environment variables for the test
+# processes, such as the container engine command the OCI image suite builds
+# images with.
 environments:
   - id: tmachine.ubuntu2404-amd64.docker-rootful.deb
     backend: tmachine
@@ -18,6 +20,7 @@ environments:
     tmachine: { environment: ubuntu-docker-rootful, installer: deb }
     gateway: openshell
     capabilities: { compute_driver: docker, rootless: false, gpu: false, persistent_workspace: true }
+    env: { OPENSHELL_TEST_CONTAINER_ENGINE: docker }
   - id: tmachine.ubuntu2404-arm64.docker-rootful.deb
     backend: tmachine
     machine: { os: ubuntu, version: "24.04", arch: arm64 }
@@ -26,3 +29,4 @@ environments:
     tmachine: { environment: ubuntu-docker-rootful, installer: deb }
     gateway: openshell
     capabilities: { compute_driver: docker, rootless: false, gpu: false, persistent_workspace: true }
+    env: { OPENSHELL_TEST_CONTAINER_ENGINE: docker }

tests/plans/README.md

@@ -19,7 +19,14 @@ environment does not provide is left out of that environment.
 
 | Plan | Runs |
 |---|---|
-| `cpu-tests.yaml` | The `conformance` family on Ubuntu 24.04 amd64 with rootful Docker and the Debian package |
+| `cpu-tests.yaml` | The `conformance` sandbox suites (`smoke`, `lifecycle`, `cli`, `persistent-workspace`, `file-transfer`, `policy-advisor`) on rootful Docker (Debian package) and rootful and rootless Podman (RPM), and the `oci-image` suite on rootful Docker, all Ubuntu 24.04 or Fedora 44 amd64 |
+
+The Kubernetes (k3s) environments are defined but not in a plan: the old lane
+verified that the Agent Sandbox controller reconciles before it ran tests, and
+the runner only waits for the gateway. The `features` family (`providers`
+against Keycloak, `podman` user namespaces) is not in a plan either, because both
+need guest setup the runner does not do; see
+[TESTING.md](../../TESTING.md#test-plans-and-the-runner).
 
 ## How a plan runs on CI
 
@@ -40,6 +47,10 @@ Branch E2E calls `.github/workflows/e2e-suites.yml` with the plan name when the
 4. The job uploads the JUnit reports and logs, and an aggregator job reports the
    result.
 
+An environment can add environment variables for the test processes with `env`,
+for what a suite reads that is not a capability. `docker-rootful` sets
+`OPENSHELL_TEST_CONTAINER_ENGINE=docker` for the `oci-image` suite.
+
 Adding an environment or a suite to a plan changes what CI runs with no workflow
 change. Arm64 environments are defined in `tests/environments/` but are not in a
 plan until a runner that can boot them with KVM exists.

tests/plans/cpu-tests.yaml

@@ -14,6 +14,30 @@
 # for tmachine yet. The arm64 environments in tests/environments/ are defined
 # so a plan can use them once one exists.
 matrix:
+  # Sandbox behavior through the SDK and the CLI. These tests are
+  # driver-agnostic, so every environment with a gateway the runner can wait for
+  # can run them. Only Docker is planned until the others have run on CI. The Kubernetes (k3s) environments are
+  # left out: before tests, the old lane verified that the Agent Sandbox
+  # controller reconciles (tests/tmachine/ansible/playbooks/conformance/
+  # k3s-ready.yaml), and the runner only waits for the gateway.
   - type: conformance
+    suites:
+      - smoke
+      - lifecycle
+      - cli
+      - persistent-workspace
+      - file-transfer
+      - policy-advisor
+    environments:
+      - tmachine.ubuntu2404-amd64.docker-rootful.deb
+      # The Podman rpm environments (tmachine.fedora44-amd64.podman-rootful.rpm
+      # and podman-rootless.rpm) ran the same tests in the old lane and are
+      # defined in tests/environments/, but the runner has not run them on CI.
+      # Add them here after a first run; they also need Git from test-tools.yaml.
+  # OCI image behavior. The suite builds images with the gateway's container
+  # engine, which the environment names in `env`. Only Docker names it so far.
+  - type: conformance
+    suites:
+      - oci-image
     environments:
       - tmachine.ubuntu2404-amd64.docker-rootful.deb

tests/runner/src/environments.rs

@@ -50,8 +50,21 @@ pub struct Environment {
     pub gateway: Option<String>,
     #[serde(default)]
     pub capabilities: BTreeMap<String, serde_yml::Value>,
+    /// Extra environment variables for the test processes, for what a suite
+    /// reads from the environment that is not a capability, such as the
+    /// container engine command. The runner's own `OPENSHELL_GATEWAY`,
+    /// `OPENSHELL_BIN` and `OPENSHELL_TEST_CAPABILITIES` cannot be set here.
+    #[serde(default)]
+    pub env: BTreeMap<String, String>,
 }
 
+/// Variables the runner sets from other fields of the environment.
+const RESERVED_ENV: [&str; 3] = [
+    "OPENSHELL_GATEWAY",
+    "OPENSHELL_BIN",
+    "OPENSHELL_TEST_CAPABILITIES",
+];
+
 #[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "lowercase")]
 pub enum Backend {
@@ -213,6 +226,18 @@ impl Environment {
                 );
             }
         }
+        for name in self.env.keys() {
+            ensure!(
+                !name.is_empty() && !name.contains(['=', ' ']),
+                "environment {:?} has an invalid `env` variable name {name:?}",
+                self.id
+            );
+            ensure!(
+                !RESERVED_ENV.contains(&name.as_str()),
+                "environment {:?} sets {name} in `env`, which the runner sets from other fields",
+                self.id
+            );
+        }
         Ok(())
     }
 
@@ -379,6 +404,36 @@ environments:
         assert_eq!(env.gateway.as_deref(), Some("openshell"));
     }
 
+    #[test]
+    fn reads_extra_environment_variables() {
+        let yaml = UBUNTU_DOCKER.replace(
+            "    runtime:",
+            "    env: { OPENSHELL_TEST_CONTAINER_ENGINE: sudo -n podman }\n    runtime:",
+        );
+        let catalog = catalog(&yaml).unwrap();
+        let env = catalog
+            .get("tmachine.ubuntu2404-amd64.docker-rootful.deb")
+            .unwrap();
+        assert_eq!(
+            env.env
+                .get("OPENSHELL_TEST_CONTAINER_ENGINE")
+                .map(String::as_str),
+            Some("sudo -n podman")
+        );
+    }
+
+    #[test]
+    fn rejects_environment_variables_the_runner_sets() {
+        let yaml = UBUNTU_DOCKER.replace(
+            "    runtime:",
+            "    env: { OPENSHELL_GATEWAY: other }\n    runtime:",
+        );
+        let err = catalog(&yaml).unwrap_err();
+        assert!(err.to_string().contains("OPENSHELL_GATEWAY"), "{err}");
+        let yaml = UBUNTU_DOCKER.replace("    runtime:", "    env: { \"A B\": x }\n    runtime:");
+        assert!(catalog(&yaml).is_err());
+    }
+
     #[test]
     fn rejects_id_that_does_not_match_fields() {
         let yaml = UBUNTU_DOCKER.replace("docker-rootful.deb", "docker-rootless.deb");

tests/runner/src/execute.rs

@@ -162,7 +162,7 @@ async fn provision(job: &Job, config: &Config) -> Result<Session> {
 
 /// The environment variables that point the tests at the system under test:
 /// the registered gateway (when the environment names one), the installed
-/// CLI, and what the environment provides.
+/// CLI, what the environment provides, and any variables its definition adds.
 pub fn guest_env(job: &Job, openshell_bin: &str) -> Vec<String> {
     let mut vars = Vec::new();
     if let Some(gateway) = &job.environment.gateway {
@@ -173,6 +173,12 @@ pub fn guest_env(job: &Job, openshell_bin: &str) -> Vec<String> {
         "OPENSHELL_TEST_CAPABILITIES={}",
         job.environment.capabilities_env()
     ));
+    vars.extend(
+        job.environment
+            .env
+            .iter()
+            .map(|(name, value)| format!("{name}={value}")),
+    );
     vars
 }
 
@@ -474,6 +480,17 @@ mod tests {
         );
 
         let mut job = job;
+        job.environment.env.insert(
+            "OPENSHELL_TEST_CONTAINER_ENGINE".to_string(),
+            "sudo -n podman".to_string(),
+        );
+        assert_eq!(
+            guest_env(&job, "/usr/bin/openshell")
+                .last()
+                .map(String::as_str),
+            Some("OPENSHELL_TEST_CONTAINER_ENGINE=sudo -n podman")
+        );
+
         job.environment.gateway = None;
         assert!(
             !guest_env(&job, "/usr/bin/openshell")

tests/suites/conformance/Cargo.toml

@@ -3,4 +3,4 @@
 
 [workspace]
 resolver = "2"
-members = ["cli", "sandbox"]
+members = ["cli", "sandbox", "sandbox-image"]

tests/suites/conformance/sandbox-image/Cargo.toml

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "sandbox-image"
+description = "Conformance suite: OCI image behavior on the Docker and Podman drivers"
+version = "0.0.0"
+edition = "2024"
+rust-version = "1.94"
+license = "Apache-2.0"
+repository = "https://github.com/NVIDIA/OpenShell"
+publish = false
+
+[dependencies]
+openshell-testkit = { path = "../../../testkit/testkit" }
+serde_json = "1"
+tempfile = "3"