From 721a6468892f1a26de80da1fecc7f6f9e3620576 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 15 Oct 2026 16:47:36 +0200 Subject: [PATCH 1/3] test(tmachine): publish the installed gateway context to feature tests Have the openshell client role publish the installed gateway's name, active TOML path, service scope, service user, and network, and add an openshell_test_gateway role that loads that context, resolves the CLI, restarts the gateway service, and collects its journal. A feature suite can then change the gateway's active configuration for its own run and restore it afterwards. Select the candidate runtime images for packaged installs through a qualification gateway.toml referenced by OPENSHELL_GATEWAY_CONFIG instead of OPENSHELL_* variables in the service environment file, so the gateway has a TOML file a suite can modify. Update the packaging asset check to match. Signed-off-by: Evan Lezar --- tasks/scripts/test-packaging-assets.sh | 11 ++--- .../roles/openshell_client/tasks/main.yaml | 14 +++++++ .../tasks/main.yaml | 32 ++++++++++---- .../templates/gateway.toml.j2 | 8 ++++ .../openshell_test_gateway/tasks/journal.yaml | 22 ++++++++++ .../openshell_test_gateway/tasks/main.yaml | 42 +++++++++++++++++++ .../openshell_test_gateway/tasks/publish.yaml | 23 ++++++++++ .../openshell_test_gateway/tasks/restart.yaml | 35 ++++++++++++++++ 8 files changed, 172 insertions(+), 15 deletions(-) create mode 100644 tests/ansible/roles/openshell_packaged_gateway/templates/gateway.toml.j2 create mode 100644 tests/ansible/roles/openshell_test_gateway/tasks/journal.yaml create mode 100644 tests/ansible/roles/openshell_test_gateway/tasks/main.yaml create mode 100644 tests/ansible/roles/openshell_test_gateway/tasks/publish.yaml create mode 100644 tests/ansible/roles/openshell_test_gateway/tasks/restart.yaml diff --git a/tasks/scripts/test-packaging-assets.sh b/tasks/scripts/test-packaging-assets.sh index 71baf77af4..4214662133 100755 --- a/tasks/scripts/test-packaging-assets.sh +++ b/tasks/scripts/test-packaging-assets.sh @@ -91,15 +91,10 @@ assert_contains "$spec" '%files prover' assert_contains "$spec" '%{_bindir}/%{name}-prover' assert_not_contains "$spec" '%%S/openshell/tls' -# Installed-package qualification must select the candidate trusted runtime -# images through the same environment file consumed by preflight and startup, -# without generating operator-owned gateway TOML. +# Qualification retains the package-owned service's environment file. The +# selected configuration and runtime images are exercised by tmachine's +# installed-package integration lanes. assert_contains "$package_qualification" 'dest: "{{ openshell_gateway_home }}/.config/openshell/gateway.env"' -assert_contains "$package_qualification" 'OPENSHELL_COMPUTE_DRIVER={{ openshell_gateway_driver }}' -assert_contains "$package_qualification" 'OPENSHELL_SANDBOX_RUNTIME_IMAGE=docker.io/openshell/sandbox:tmachine' -assert_contains "$package_qualification" 'OPENSHELL_SUPERVISOR_IMAGE=docker.io/openshell/supervisor:tmachine' -assert_not_contains "$package_qualification" 'OPENSHELL_GATEWAY_CONFIG=' -assert_not_contains "$package_qualification" '/var/lib/openshell-qualification/gateway.toml' # Direct tmachine gateway services select trusted runtime images through the # process environment. Driver TOML remains available for settings that do not diff --git a/tests/ansible/roles/openshell_client/tasks/main.yaml b/tests/ansible/roles/openshell_client/tasks/main.yaml index 6586f23ac2..8a60195fa1 100644 --- a/tests/ansible/roles/openshell_client/tasks/main.yaml +++ b/tests/ansible/roles/openshell_client/tasks/main.yaml @@ -12,3 +12,17 @@ - --local - --name - "{{ openshell_client_gateway_name | default('tmachine') }}" + +- name: Publish test gateway context + ansible.builtin.include_role: + name: openshell_test_gateway + tasks_from: publish.yaml + vars: + openshell_test_gateway_context: + name: "{{ openshell_client_gateway_name | default('tmachine') }}" + config_path: "{{ openshell_client_gateway_config_path | default('/etc/openshell/gateway.toml') }}" + service_scope: "{{ openshell_client_gateway_service_scope | default('system') }}" + service_user: "{{ openshell_gateway_user if openshell_client_gateway_service_scope | default('system') == 'user' else 'root' }}" + service_home: "{{ openshell_gateway_home if openshell_client_gateway_service_scope | default('system') == 'user' else '/root' }}" + service_uid: "{{ openshell_gateway_uid if openshell_client_gateway_service_scope | default('system') == 'user' else '0' }}" + network_name: "{{ openshell_client_gateway_network_name | default('tmachine') }}" diff --git a/tests/ansible/roles/openshell_packaged_gateway/tasks/main.yaml b/tests/ansible/roles/openshell_packaged_gateway/tasks/main.yaml index 4d1277f5f4..a50e238150 100644 --- a/tests/ansible/roles/openshell_packaged_gateway/tasks/main.yaml +++ b/tests/ansible/roles/openshell_packaged_gateway/tasks/main.yaml @@ -21,6 +21,26 @@ name: openshell_gateway tasks_from: "{{ openshell_gateway_driver }}.yaml" +# Package defaults select published images. Keep candidate image overrides +# separate from the operator-owned gateway.toml in both installers. +- name: Create qualification configuration directory + become: true + ansible.builtin.file: + path: /var/lib/openshell-qualification + state: directory + owner: root + group: root + mode: "0755" + +- name: Configure candidate runtime images for qualification + become: true + ansible.builtin.template: + src: gateway.toml.j2 + dest: /var/lib/openshell-qualification/gateway.toml + owner: root + group: root + mode: "0644" + - name: Create packaged gateway environment directory become: true ansible.builtin.file: @@ -30,10 +50,7 @@ group: "{{ openshell_gateway_user }}" mode: "0700" -# Exercise the package-owned service's environment-file mechanism without -# generating operator-owned gateway TOML. Preflight and startup must resolve -# the same candidate runtime images. -- name: Configure candidate runtime images for qualification +- name: Select qualification gateway configuration become: true ansible.builtin.copy: dest: "{{ openshell_gateway_home }}/.config/openshell/gateway.env" @@ -41,9 +58,7 @@ group: "{{ openshell_gateway_user }}" mode: "0600" content: | - OPENSHELL_COMPUTE_DRIVER={{ openshell_gateway_driver }} - OPENSHELL_SANDBOX_RUNTIME_IMAGE=docker.io/openshell/sandbox:tmachine - OPENSHELL_SUPERVISOR_IMAGE=docker.io/openshell/supervisor:tmachine + OPENSHELL_GATEWAY_CONFIG=/var/lib/openshell-qualification/gateway.toml - name: Start gateway user manager ansible.builtin.include_role: @@ -106,3 +121,6 @@ vars: openshell_client_gateway_endpoint: https://127.0.0.1:17670 openshell_client_gateway_name: openshell + openshell_client_gateway_config_path: /var/lib/openshell-qualification/gateway.toml + openshell_client_gateway_service_scope: user + openshell_client_gateway_network_name: openshell diff --git a/tests/ansible/roles/openshell_packaged_gateway/templates/gateway.toml.j2 b/tests/ansible/roles/openshell_packaged_gateway/templates/gateway.toml.j2 new file mode 100644 index 0000000000..cb6d524228 --- /dev/null +++ b/tests/ansible/roles/openshell_packaged_gateway/templates/gateway.toml.j2 @@ -0,0 +1,8 @@ +{# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. #} +{# SPDX-License-Identifier: Apache-2.0 #} +[openshell] +version = 2 + +[openshell.drivers.{{ openshell_gateway_driver }}] +sandbox_runtime_image = "docker.io/openshell/sandbox:tmachine" +supervisor_image = "docker.io/openshell/supervisor:tmachine" diff --git a/tests/ansible/roles/openshell_test_gateway/tasks/journal.yaml b/tests/ansible/roles/openshell_test_gateway/tasks/journal.yaml new file mode 100644 index 0000000000..2e5d7874f9 --- /dev/null +++ b/tests/ansible/roles/openshell_test_gateway/tasks/journal.yaml @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Root's packaged user unit can log to the system journal. Match journal +# fields instead of --user so both root and tmachine user units are captured. +- name: Capture installed gateway journal + become: true + ansible.builtin.command: + argv: >- + {{ ['journalctl'] + + (['_SYSTEMD_USER_UNIT=openshell-gateway.service', '_UID=' ~ openshell_test_gateway.service_uid] + if openshell_test_gateway.service_scope == 'user' + else ['--unit', 'openshell-gateway.service']) + + ['--no-pager', '--lines', '200'] }} + register: openshell_test_gateway_journal + changed_when: false + failed_when: false + +- name: Show installed gateway journal + ansible.builtin.debug: + var: openshell_test_gateway_journal.stdout_lines diff --git a/tests/ansible/roles/openshell_test_gateway/tasks/main.yaml b/tests/ansible/roles/openshell_test_gateway/tasks/main.yaml new file mode 100644 index 0000000000..55d18b310e --- /dev/null +++ b/tests/ansible/roles/openshell_test_gateway/tasks/main.yaml @@ -0,0 +1,42 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Read installed gateway context + ansible.builtin.slurp: + src: /var/lib/openshell-test/gateway.yaml + register: openshell_test_gateway_metadata + +- name: Load installed gateway context + ansible.builtin.set_fact: + openshell_test_gateway: "{{ (openshell_test_gateway_metadata.content | b64decode | from_yaml).openshell_test_gateway }}" + +- name: Validate installed gateway context + ansible.builtin.assert: + that: + - openshell_test_gateway.service_scope in ['system', 'user'] + - openshell_test_gateway.config_path is match('^/') + - openshell_test_gateway.name | length > 0 + +- name: Resolve installed OpenShell CLI from PATH + ansible.builtin.command: + argv: [/bin/sh, -c, command -v openshell] + register: openshell_test_cli + changed_when: false + +- name: Check active gateway configuration + become: true + ansible.builtin.stat: + path: "{{ openshell_test_gateway.config_path }}" + register: openshell_test_gateway_config + +- name: Require active gateway configuration + ansible.builtin.assert: + that: [openshell_test_gateway_config.stat.isreg | default(false)] + +- name: Resolve gateway service environment + ansible.builtin.set_fact: + openshell_test_gateway_service_environment: + HOME: "{{ openshell_test_gateway.service_home }}" + XDG_RUNTIME_DIR: "/run/user/{{ openshell_test_gateway.service_uid }}" + DBUS_SESSION_BUS_ADDRESS: "unix:path=/run/user/{{ openshell_test_gateway.service_uid }}/bus" diff --git a/tests/ansible/roles/openshell_test_gateway/tasks/publish.yaml b/tests/ansible/roles/openshell_test_gateway/tasks/publish.yaml new file mode 100644 index 0000000000..1e04d266c2 --- /dev/null +++ b/tests/ansible/roles/openshell_test_gateway/tasks/publish.yaml @@ -0,0 +1,23 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Installation and suite execution are separate Ansible processes. Persist only +# non-secret metadata; the CLI keeps registration and mTLS credentials itself. +- name: Create test gateway metadata directory + become: true + ansible.builtin.file: + path: /var/lib/openshell-test + state: directory + owner: root + group: root + mode: "0755" + +- name: Publish installed gateway context + become: true + ansible.builtin.copy: + dest: /var/lib/openshell-test/gateway.yaml + owner: root + group: root + mode: "0644" + content: "{{ {'openshell_test_gateway': openshell_test_gateway_context} | to_nice_yaml }}" diff --git a/tests/ansible/roles/openshell_test_gateway/tasks/restart.yaml b/tests/ansible/roles/openshell_test_gateway/tasks/restart.yaml new file mode 100644 index 0000000000..56b56123f9 --- /dev/null +++ b/tests/ansible/roles/openshell_test_gateway/tasks/restart.yaml @@ -0,0 +1,35 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Restart installed gateway + block: + - name: Restart gateway service with the active configuration + become: true + become_user: "{{ openshell_test_gateway.service_user }}" + ansible.builtin.systemd_service: + name: openshell-gateway.service + scope: "{{ openshell_test_gateway.service_scope }}" + state: restarted + environment: "{{ openshell_test_gateway_service_environment }}" + + - name: Wait for authenticated gateway readiness + ansible.builtin.command: + argv: ["{{ openshell_test_cli.stdout }}", gateway, info, --output, json] + environment: + OPENSHELL_GATEWAY: "{{ openshell_test_gateway.name }}" + changed_when: false + register: openshell_test_gateway_ready + retries: 30 + delay: 2 + until: >- + openshell_test_gateway_ready.rc == 0 and + (openshell_test_gateway_ready.stdout | from_json).status == 'healthy' + + rescue: + - name: Collect gateway restart diagnostics + ansible.builtin.include_tasks: journal.yaml + + - name: Report gateway restart failure + ansible.builtin.fail: + msg: Installed gateway failed to restart with the driver fixture From f6f179cd7139db1dd8a60dd8dc792b659e804ce1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 15 Oct 2026 16:48:45 +0200 Subject: [PATCH 2/3] test(tmachine): add CDI scenario fixtures to the Docker environment Define each CDI scenario once in vars/scenarios.yaml: the host fixtures it needs (directories, files, a character device, a socket), the CDI edits that reference them, and the behavior the workload is expected to observe. The cdi_fixtures role creates the fixtures, renders one tmachine.example/cdi-scenario spec with a device per scenario, and publishes the expectations as scenarios.json. Install the spec into /var/run/cdi from systemd-tmpfiles on every boot so it survives tmachine's setup, installation, and test disk layers, and run the role as part of the ubuntu-docker-rootful setup. Signed-off-by: Evan Lezar --- tests/ansible/playbooks/cdi.yaml | 12 + .../roles/cdi_fixtures/defaults/main.yaml | 10 + .../files/openshell-cdi-echo.socket | 13 ++ .../files/openshell-cdi-echo@.service | 14 ++ .../roles/cdi_fixtures/tasks/main.yaml | 49 ++++ .../roles/cdi_fixtures/tasks/scenarios.yaml | 121 ++++++++++ .../cdi_fixtures/templates/scenarios.yaml.j2 | 14 ++ .../cdi_fixtures/templates/tmpfiles.conf.j2 | 5 + .../roles/cdi_fixtures/vars/scenarios.yaml | 209 ++++++++++++++++++ tests/config.nix | 1 + 10 files changed, 448 insertions(+) create mode 100644 tests/ansible/playbooks/cdi.yaml create mode 100644 tests/ansible/roles/cdi_fixtures/defaults/main.yaml create mode 100644 tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo.socket create mode 100644 tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo@.service create mode 100644 tests/ansible/roles/cdi_fixtures/tasks/main.yaml create mode 100644 tests/ansible/roles/cdi_fixtures/tasks/scenarios.yaml create mode 100644 tests/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 create mode 100644 tests/ansible/roles/cdi_fixtures/templates/tmpfiles.conf.j2 create mode 100644 tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml diff --git a/tests/ansible/playbooks/cdi.yaml b/tests/ansible/playbooks/cdi.yaml new file mode 100644 index 0000000000..9e8ccdeecf --- /dev/null +++ b/tests/ansible/playbooks/cdi.yaml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Prepare hardware-free CDI fixtures in the Docker environment + hosts: all + gather_facts: false + pre_tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + roles: + - cdi_fixtures diff --git a/tests/ansible/roles/cdi_fixtures/defaults/main.yaml b/tests/ansible/roles/cdi_fixtures/defaults/main.yaml new file mode 100644 index 0000000000..2be644aef4 --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/defaults/main.yaml @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +cdi_fixture_root: /var/lib/openshell-test-inputs/cdi + +# Specs the role installs into /var/run/cdi on every boot. The CDI feature +# playbook checks the same list, so a new spec is added here only. +cdi_fixture_spec_files: + - openshell-tmachine-scenarios.yaml diff --git a/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo.socket b/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo.socket new file mode 100644 index 0000000000..f5e32b4b43 --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo.socket @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[Unit] +Description=OpenShell tmachine CDI Unix socket fixture + +[Socket] +ListenStream=/var/lib/openshell-test-inputs/cdi/scenarios/socket/echo.sock +SocketMode=0666 +Accept=yes +RemoveOnStop=yes + +[Install] +WantedBy=sockets.target diff --git a/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo@.service b/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo@.service new file mode 100644 index 0000000000..dae445edbf --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/files/openshell-cdi-echo@.service @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +[Unit] +Description=OpenShell tmachine CDI echo connection + +[Service] +ExecStart=/bin/cat +StandardInput=socket +StandardOutput=socket +DynamicUser=yes +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes diff --git a/tests/ansible/roles/cdi_fixtures/tasks/main.yaml b/tests/ansible/roles/cdi_fixtures/tasks/main.yaml new file mode 100644 index 0000000000..f91d5831a3 --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/tasks/main.yaml @@ -0,0 +1,49 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Persistent sources are copied into /var/run/cdi by systemd-tmpfiles on every +# boot, including the setup, installation, and test disk layers. +- name: Check fixture paths before creating CDI specs + become: true + ansible.builtin.stat: + path: "{{ item }}" + follow: false + loop: "{{ cdi_fixture_spec_files | map('regex_replace', '^', '/var/run/cdi/') }}" + register: cdi_fixture_paths + +- name: Require a guest without existing fixture paths + ansible.builtin.assert: + that: [not item.stat.exists] + fail_msg: "Refusing to replace existing CDI fixture path {{ item.item }}" + loop: "{{ cdi_fixture_paths.results }}" + +- name: Create CDI spec directories + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ cdi_fixture_root }}" + - "{{ cdi_fixture_root }}/specs" + +- name: Prepare named CDI scenarios + ansible.builtin.include_tasks: scenarios.yaml + +- name: Recreate volatile CDI specs on every VM boot + become: true + ansible.builtin.template: + src: tmpfiles.conf.j2 + dest: /etc/tmpfiles.d/openshell-tmachine-cdi.conf + owner: root + group: root + mode: "0644" + +- name: Populate CDI specs for the current boot + become: true + ansible.builtin.command: + argv: [systemd-tmpfiles, --create, /etc/tmpfiles.d/openshell-tmachine-cdi.conf] + changed_when: true diff --git a/tests/ansible/roles/cdi_fixtures/tasks/scenarios.yaml b/tests/ansible/roles/cdi_fixtures/tasks/scenarios.yaml new file mode 100644 index 0000000000..90502c9267 --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/tasks/scenarios.yaml @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Allocate a supplementary group for the CDI device fixture + become: true + ansible.builtin.group: + name: openshell-cdi-fixture + system: true + register: cdi_fixture_group + +- name: Load named CDI scenarios + ansible.builtin.include_vars: + file: scenarios.yaml + +- name: Collect scenario fixtures + ansible.builtin.set_fact: + cdi_scenario_fixtures: >- + {{ cdi_scenario_common_fixtures + + (cdi_scenarios | map(attribute='fixtures', default=[]) | flatten) }} + +- name: Create the scenario fixture root + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}" + state: directory + owner: root + group: root + mode: "0755" + +- name: Create scenario fixture directories + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}/{{ item.path }}" + state: directory + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'dir') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Write scenario fixture files + become: true + ansible.builtin.copy: + dest: "{{ cdi_scenario_dir }}/{{ item.path }}" + content: "{{ item.content }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'file') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Create scenario fixture named pipes + become: true + ansible.builtin.command: + argv: [mkfifo, "--mode={{ item.mode }}", "{{ cdi_scenario_dir }}/{{ item.path }}"] + creates: "{{ cdi_scenario_dir }}/{{ item.path }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'fifo') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Create scenario fixture character devices + become: true + ansible.builtin.command: + argv: [mknod, "{{ cdi_scenario_dir }}/{{ item.path }}", c, "{{ item.major }}", "{{ item.minor }}"] + creates: "{{ cdi_scenario_dir }}/{{ item.path }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'eq', 'chardev') | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Set scenario fixture device ownership + become: true + ansible.builtin.file: + path: "{{ cdi_scenario_dir }}/{{ item.path }}" + owner: "{{ item.owner | default('root') }}" + group: "{{ item.group | default('root') }}" + mode: "{{ item.mode }}" + loop: "{{ cdi_scenario_fixtures | selectattr('type', 'in', ['chardev', 'fifo']) | list }}" + loop_control: + label: "{{ item.path }}" + +- name: Install socket fixture units + become: true + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: [openshell-cdi-echo.socket, 'openshell-cdi-echo@.service'] + +- name: Start the persistent socket fixture + become: true + ansible.builtin.systemd_service: + name: openshell-cdi-echo.socket + daemon_reload: true + enabled: true + state: started + +- name: Generate named CDI scenario devices + become: true + ansible.builtin.template: + src: scenarios.yaml.j2 + dest: "{{ cdi_fixture_root }}/specs/openshell-tmachine-scenarios.yaml" + owner: root + group: root + mode: "0644" + +- name: Publish scenario expectations for the CDI feature tests + become: true + ansible.builtin.copy: + dest: "{{ cdi_fixture_root }}/scenarios.json" + content: >- + {{ {'common_checks': cdi_scenario_common_checks, + 'scenarios': cdi_scenarios | items2dict(key_name='name', value_name='expect')} + | to_nice_json }} + owner: root + group: root + mode: "0644" diff --git a/tests/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 b/tests/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 new file mode 100644 index 0000000000..1d558f5a4b --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/templates/scenarios.yaml.j2 @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# Generated from vars/scenarios.yaml. Scenarios are selected individually, so +# negative scenarios never combine with positive ones. +cdiVersion: "0.8.0" +kind: tmachine.example/cdi-scenario +containerEdits: +{{ cdi_scenario_spec_edits | to_nice_yaml(indent=2) | indent(2, true) }} +devices: +{% for case in cdi_scenarios %} + - name: {{ case.name }} + containerEdits: +{{ case.edits | to_nice_yaml(indent=2) | indent(6, true) }} +{% endfor %} diff --git a/tests/ansible/roles/cdi_fixtures/templates/tmpfiles.conf.j2 b/tests/ansible/roles/cdi_fixtures/templates/tmpfiles.conf.j2 new file mode 100644 index 0000000000..1ea2aa5ffb --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/templates/tmpfiles.conf.j2 @@ -0,0 +1,5 @@ +# Hardware-free CDI fixtures for the disposable Docker test environment. +d /run/cdi 0755 root root - +{% for spec in cdi_fixture_spec_files %} +C /run/cdi/{{ spec }} 0644 root root - {{ cdi_fixture_root }}/specs/{{ spec }} +{% endfor %} diff --git a/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml b/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml new file mode 100644 index 0000000000..1a2aac81d8 --- /dev/null +++ b/tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml @@ -0,0 +1,209 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +# Each case is one device of the tmachine.example/cdi-scenario class, selected as +# `tmachine.example/cdi-scenario=`. A case declares the host fixtures it +# needs and the CDI edits that reference them; tasks/scenarios.yaml creates the +# fixtures and templates/scenarios.yaml.j2 renders the spec. +# +# Fixture `path` is relative to {{ cdi_fixture_root }}/scenarios. Types: +# dir directory (parents must be declared first) +# file regular file with `content` +# fifo named pipe +# chardev character device with `major` and `minor` +# All accept `mode`, `owner` and `group`. +# +# `expect` records the behavior the CDI feature tests assert for the scenario, +# and is rendered to {{ cdi_fixture_root }}/scenarios.json for them. It states +# what the sandbox does today; a change that alters that behavior updates the +# values here. Either: +# rejected: sandbox creation fails; must appear in the +# diagnostic (empty matches any failure). `checks` are then +# observed but not asserted. +# checks: [...] sandbox creation succeeds and each check holds, where a +# check is one of +# {type: contents, path, expect} file contents, null when it must not exist +# {type: read, path, expect} `ok` or `denied` +# {type: write, path, expect} `ok` or `denied` (nothing is written) +# {type: env, name, expect} value, null when unset +# {type: non-root, expect} whether the workload is not uid 0 (true) +# {type: group-of, path, expect} whether the workload's groups include +# path's group (true) +cdi_scenario_dir: "{{ cdi_fixture_root }}/scenarios" + +# Edits applied by every case, to check that spec and device edits merge. The +# marker is mounted under /usr/share, which the sandbox already allows reading, +# so these checks do not depend on grants derived from the CDI spec. +cdi_scenario_spec_edits: + env: + - TMACHINE_CDI_SCENARIO_SPEC=fixture + mounts: + - hostPath: "{{ cdi_scenario_dir }}/spec-marker" + containerPath: /usr/share/tmachine-cdi/common/spec-marker + options: [bind, ro] + +# Checked in every scenario that is created, to show that spec and device edits +# merge. +cdi_scenario_common_checks: + - {type: env, name: TMACHINE_CDI_SCENARIO_SPEC, expect: fixture} + - {type: contents, path: /usr/share/tmachine-cdi/common/spec-marker, expect: tmachine-cdi-spec-marker} + - {type: write, path: /usr/share/tmachine-cdi/common/spec-marker, expect: denied} + +cdi_scenario_common_fixtures: + - {path: spec-marker, type: file, content: "tmachine-cdi-spec-marker\n", mode: "0644"} + +cdi_scenarios: + # The mount point is under a directory the sandbox baseline already makes + # read-only, so nothing needs to be derived for the mount to be readable. + - name: mount-under-read-only-baseline + fixtures: + - {path: baseline-read-only-file, type: file, content: "tmachine-cdi-baseline-read-only-file\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/baseline-read-only-file" + containerPath: /usr/share/tmachine-cdi/baseline-read-only-file + options: [bind, ro] + expect: + checks: + - {type: contents, path: /usr/share/tmachine-cdi/baseline-read-only-file, expect: tmachine-cdi-baseline-read-only-file} + - {type: write, path: /usr/share/tmachine-cdi/baseline-read-only-file, expect: denied} + + - name: read-only-file + fixtures: + - {path: read-only-file, type: file, content: "tmachine-cdi-read-only-file\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/read-only-file" + containerPath: /tmachine-cdi/read-only/file + options: [bind, ro] + expect: + checks: + - {type: read, path: /tmachine-cdi/read-only/file, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/file, expect: denied} + + - name: read-only-directory + fixtures: + - {path: read-only-directory, type: dir, mode: "0755"} + - {path: read-only-directory/marker, type: file, content: "tmachine-cdi-read-only-directory/marker\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/read-only-directory" + containerPath: /tmachine-cdi/read-only/directory + options: [bind, ro] + expect: + checks: + - {type: read, path: /tmachine-cdi/read-only/directory/marker, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/directory/marker, expect: denied} + - {type: write, path: /tmachine-cdi/read-only/directory/new-file, expect: denied} + + # Both library cases bind the same file; they differ in whether its parent + # directory is narrow or the container root. + + - name: library-parent + fixtures: + - {path: libfixture.so.1, type: file, content: "tmachine-cdi-libfixture.so.1\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/libfixture.so.1" + containerPath: /tmachine-cdi/library/libfixture.so.1 + options: [bind, ro] + expect: + checks: + - {type: read, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} + - {type: write, path: /tmachine-cdi/library/libfixture.so.1, expect: denied} + + - name: broad-library-parent + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/libfixture.so.1" + containerPath: /libtmachinefixture.so.1 + options: [bind, ro] + expect: + checks: + - {type: read, path: /libtmachinefixture.so.1, expect: denied} + - {type: write, path: /libtmachinefixture.so.1, expect: denied} + + - name: writable-file + fixtures: + - {path: writable-file, type: file, content: "tmachine-cdi-original\n", mode: "0666"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-file" + containerPath: /tmachine-cdi/writable/file + options: [bind, rw] + expect: + checks: + - {type: read, path: /tmachine-cdi/writable/file, expect: denied} + - {type: write, path: /tmachine-cdi/writable/file, expect: denied} + + - name: writable-directory + fixtures: + - {path: writable-directory, type: dir, mode: "0777"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-directory" + containerPath: /tmachine-cdi/writable/directory + options: [bind, rw] + expect: + checks: + - {type: write, path: /tmachine-cdi/writable/directory/probe, expect: denied} + + - name: omitted-access-file + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/writable-file" + containerPath: /tmachine-cdi/writable/omitted-access + options: [bind] + expect: + checks: + - {type: read, path: /tmachine-cdi/writable/omitted-access, expect: denied} + - {type: write, path: /tmachine-cdi/writable/omitted-access, expect: denied} + + - name: device-node + edits: + deviceNodes: + - path: /dev/tmachine-cdi-null + hostPath: /dev/null + permissions: rw + expect: + checks: + - {type: non-root} + - {type: write, path: /dev/tmachine-cdi-null, expect: denied} + + # The device is mode 0660 with a group that only `additionalGids` grants. + - name: device-node-with-gid + fixtures: + - {path: group-device, type: chardev, major: 1, minor: 3, mode: "0660", group: openshell-cdi-fixture} + edits: + additionalGids: ["{{ cdi_fixture_group.gid | int }}"] + deviceNodes: + - path: /dev/tmachine-cdi-group-null + hostPath: "{{ cdi_scenario_dir }}/group-device" + permissions: rw + uid: 0 + gid: "{{ cdi_fixture_group.gid | int }}" + fileMode: 432 + expect: + checks: + - {type: non-root} + - {type: group-of, path: /dev/tmachine-cdi-group-null} + - {type: write, path: /dev/tmachine-cdi-group-null, expect: denied} + + # The echo socket is served by the systemd units installed in scenarios.yaml. + # Its sibling file checks that only the socket path is exposed. + + - name: unix-socket + fixtures: + - {path: socket, type: dir, mode: "0755"} + - {path: socket/sibling, type: file, content: "tmachine-cdi-socket/sibling\n", mode: "0644"} + edits: + mounts: + - hostPath: "{{ cdi_scenario_dir }}/socket/echo.sock" + containerPath: /tmachine-cdi/socket/echo.sock + options: [bind] + expect: + checks: + # Only the socket is mounted, so its sibling in the host directory is + # not visible. Connecting to the socket is not checked yet. + - {type: contents, path: /tmachine-cdi/socket/sibling, expect: null} diff --git a/tests/config.nix b/tests/config.nix index a5318cb875..f22205d101 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -48,6 +48,7 @@ let playbooks = [ "ansible/playbooks/nextest.yaml" "ansible/playbooks/docker.yaml" + "ansible/playbooks/cdi.yaml" ]; }; } From fb39ac14bb625ddc57402770850077e37635d267 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 15 Oct 2026 16:49:42 +0200 Subject: [PATCH 3/3] test(tmachine): assert CDI scenario behavior in a Docker guest Add a cdi feature suite that creates a sandbox for each CDI scenario by selecting its device explicitly with --gpu 1 and the Docker cdi_devices driver option, then checks what the workload observes or that creation is rejected. The sandbox only reports facts; the test makes every assertion, so a failure names the check that disagreed. Each scenario is its own test. The expectations record the behavior the sandbox has today, loaded from the scenarios.json published by the fixtures role. A OPENSHELL_CDI_CAPTURE mode prints observations instead of asserting, to record that behavior for a new scenario. The test playbook enables caller driver configuration in the installed gateway's active TOML for the run and restores it afterwards. Build the suite's nextest archive in tests/artifacts.nix, expose it as build-artifacts-cdi-tests, and run the suite on the Docker environment with the deb installer in the branch and release matrices. Signed-off-by: Evan Lezar --- .github/workflows/branch-e2e.yml | 1 + .github/workflows/release-dev.yml | 1 + .github/workflows/release-qualification.yml | 1 + TESTING.md | 71 +++ flake.nix | 4 + .../ansible/playbooks/features/cdi/tests.yaml | 105 ++++ tests/artifacts.nix | 11 + tests/config.nix | 7 + tests/suites/features/Cargo.lock | 10 + tests/suites/features/Cargo.toml | 2 +- tests/suites/features/cdi/Cargo.toml | 12 + tests/suites/features/cdi/tests/injection.rs | 545 ++++++++++++++++++ 12 files changed, 769 insertions(+), 1 deletion(-) create mode 100644 tests/ansible/playbooks/features/cdi/tests.yaml create mode 100644 tests/suites/features/cdi/Cargo.toml create mode 100644 tests/suites/features/cdi/tests/injection.rs diff --git a/.github/workflows/branch-e2e.yml b/.github/workflows/branch-e2e.yml index db4c96b936..015de7cf47 100644 --- a/.github/workflows/branch-e2e.yml +++ b/.github/workflows/branch-e2e.yml @@ -275,6 +275,7 @@ jobs: integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} test-matrix: >- [ + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"cdi"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, diff --git a/.github/workflows/release-dev.yml b/.github/workflows/release-dev.yml index aec77972ee..524b0aa914 100644 --- a/.github/workflows/release-dev.yml +++ b/.github/workflows/release-dev.yml @@ -145,6 +145,7 @@ jobs: integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} test-matrix: >- [ + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"cdi"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, diff --git a/.github/workflows/release-qualification.yml b/.github/workflows/release-qualification.yml index e038fc040d..112c0088bb 100644 --- a/.github/workflows/release-qualification.yml +++ b/.github/workflows/release-qualification.yml @@ -105,6 +105,7 @@ jobs: integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }} test-matrix: >- [ + {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"cdi"}, {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"provider-refresh"}, {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"oci-image"}, diff --git a/TESTING.md b/TESTING.md index 771b602103..96d29c001e 100644 --- a/TESTING.md +++ b/TESTING.md @@ -330,6 +330,77 @@ nix run .#tmachine -- test fedora-podman-rootless binaries e2e-podman nix run .#tmachine -- test fedora-podman-rootless binaries driver-podman ``` +The `cdi` feature suite tests device injection without GPU hardware in the +`ubuntu-docker-rootful` environment. Build the candidate artifacts, then run: + +```shell +nix run .#build-artifacts +nix run .#tmachine -- test ubuntu-docker-rootful deb cdi +``` + +If candidate binaries, images, and packages are already built, rebuild just the +CDI test archive with `nix run .#build-artifacts-cdi-tests`. + +The `binaries` installer also supports this suite. Docker environment setup +creates the named scenario fixtures and a CDI spec that defines one device per +scenario, selected as `tmachine.example/cdi-scenario=`. Persistent +spec sources live under `/var/lib/openshell-test-inputs/cdi/specs`; a +systemd-tmpfiles configuration recreates them in `/var/run/cdi` on every VM +boot. This preserves the fixtures across tmachine's setup, installation, and +test disk layers and makes them available to every suite using the Docker +environment. The tests need no GPU hardware. + +Each test creates a sandbox that selects one scenario explicitly with `--gpu 1` +and the Docker `cdi_devices` driver option, then checks what the workload +observes or that creation is rejected. Selection by count or default, and the +`nvidia.com/gpu` inventory, are not covered here. Caller driver configuration +is enabled temporarily in the installed gateway's active TOML. The test +playbook restores the configuration and restarts the gateway even after test +failure; the CDI fixtures remain part of the environment. Docker must support +CDI. Branch and release feature-test matrices run this suite on Docker. + +| Scenario | Injected resource | Asserted today | Expected with PR #4204 | +| --- | --- | --- | --- | +| `mount-under-read-only-baseline` | File at `/usr/share/tmachine-cdi/baseline-read-only-file`, under a directory the sandbox already makes read-only | Read succeeds, write denied. | Unchanged. | +| `read-only-file` | File at `/tmachine-cdi/read-only/file` | Read and write denied. | Read succeeds outside the baseline grants; write fails. | +| `read-only-directory` | Directory at `/tmachine-cdi/read-only/directory` | Reading its marker and creating files denied. | Reading its marker succeeds; write fails. | +| `library-parent` | Marker named `/tmachine-cdi/library/libfixture.so.1` | Read and write denied. | Resolver derives the library parent; the marker is not an actual shared library. | +| `broad-library-parent` | Marker named `/libtmachinefixture.so.1` | Read and write denied. | Launch rejects expansion to `/`. | +| `writable-file` | Writable file at `/tmachine-cdi/writable/file` | Read and write denied. | Launch rejects it even with exact or ancestor write grants. | +| `writable-directory` | Writable directory at `/tmachine-cdi/writable/directory` | Write denied. | Launch rejects it even with exact or ancestor write grants. | +| `omitted-access-file` | Bind without `ro` or `rw` | Read and write denied. | Effective writable access causes launch rejection. | +| `device-node` | Null character device at `/dev/tmachine-cdi-null` | Non-root, write denied. | Non-root read/write succeeds; an exact authored read-only entry fails launch. | +| `device-node-with-gid` | Group-restricted null device at `/dev/tmachine-cdi-group-null` | Non-root with the device's group, write denied. | The generated supplementary GID permits non-root access. | +| `unix-socket` | Echo socket at `/tmachine-cdi/socket/echo.sock` | Sibling file in the host directory is not visible. | Socket communication succeeds without deriving parent filesystem grants. | + +Every scenario also supplies a spec-level environment marker and a read-only +mount at `/usr/share/tmachine-cdi/common/spec-marker`, allowing checks of +spec/device edit merging without depending on derived grants. The device group +is allocated in the disposable guest. A systemd socket unit restores the echo +endpoint after reboot; the existing tmpfiles configuration restores the +scenario spec. Negative scenarios must be selected individually, not combined +with positive cases. + +The role publishes each scenario's expectations as `scenarios.json` next to the +fixtures. The `scenario_*` tests load them and assert the behavior recorded in +`vars/scenarios.yaml`, which is the sandbox's behavior today. A change that +alters a scenario's behavior, such as #4204 deriving grants from the spec, +updates those values in the same diff. To record the current behavior, run the +tests with `OPENSHELL_CDI_CAPTURE=1` set for the test process; they then print +what each scenario observed instead of asserting. + +To add a scenario, add an entry to `cdi_scenarios` in +`tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml` with the host fixtures +it needs, its CDI edits, and its `expect` (the checks on the workload, or a +rejection), then list it in `cdi_scenarios!` in +`tests/suites/features/cdi/tests/injection.rs`. A new spec file belongs in +`cdi_fixture_spec_files` in the role defaults, which the setup and test +playbooks share. + +Directory absence, daemon configuration refresh, and restart tests require +orchestration around these fixtures. Spec projection secrecy is a separate +workload assertion. Hardware CUDA initialization remains outside this suite. + Print the exact tmachine archive selection as a shell `PODMAN_CI_TESTS` array: ```shell diff --git a/flake.nix b/flake.nix index 21b2e1b091..b25a3c0f5e 100644 --- a/flake.nix +++ b/flake.nix @@ -169,6 +169,10 @@ type = "app"; program = "${artifacts.testArchives}/bin/build-artifacts-test-archives"; }; + build-artifacts-cdi-tests = { + type = "app"; + program = "${artifacts.cdiArchive}/bin/build-openshell-cdi-test-archive"; + }; build-artifacts-test-images = { type = "app"; program = "${artifacts.testImages}/bin/build-artifacts-test-images"; diff --git a/tests/ansible/playbooks/features/cdi/tests.yaml b/tests/ansible/playbooks/features/cdi/tests.yaml new file mode 100644 index 0000000000..e93711e38c --- /dev/null +++ b/tests/ansible/playbooks/features/cdi/tests.yaml @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +--- +- name: Test CDI injection without GPU hardware + hosts: all + gather_facts: false + vars_files: + - ../../../roles/cdi_fixtures/defaults/main.yaml + vars: + cdi_test_root: /var/lib/openshell-feature-tests/cdi + tasks: + - name: Wait for SSH + ansible.builtin.wait_for_connection: + + - name: Resolve installed gateway context + ansible.builtin.include_role: + name: openshell_test_gateway + + - name: Require environment CDI specs after boot + ansible.builtin.stat: + path: "/var/run/cdi/{{ item }}" + loop: "{{ cdi_fixture_spec_files }}" + register: cdi_environment_specs + + - name: Require CDI fixtures supplied by the Docker environment + ansible.builtin.assert: + that: [item.stat.isreg | default(false)] + fail_msg: "Docker environment did not recreate CDI spec {{ item.item }} after boot" + loop: "{{ cdi_environment_specs.results }}" + + - name: Save gateway configuration before enabling caller device selection + become: true + ansible.builtin.slurp: + src: "{{ openshell_test_gateway.config_path }}" + register: cdi_gateway_config + + - name: Run CDI tests with caller device selection enabled + block: + - name: Enable caller driver configuration for CDI tests + become: true + ansible.builtin.blockinfile: + path: "{{ openshell_test_gateway.config_path }}" + insertafter: '^\[openshell\.drivers\.docker\]$' + marker: "# {mark} OpenShell CDI test fixture" + block: allow_driver_config = true + + - name: Restart gateway with caller device selection enabled + ansible.builtin.include_role: + name: openshell_test_gateway + tasks_from: restart.yaml + + - name: Create CDI test directory + become: true + ansible.builtin.file: + path: "{{ cdi_test_root }}" + state: directory + owner: tmachine + group: tmachine + mode: "0700" + + - name: Install CDI test archive + become: true + ansible.builtin.unarchive: + src: "{{ openshell_cdi_test_bundle }}" + dest: "{{ cdi_test_root }}" + owner: tmachine + group: tmachine + + - name: Run CDI feature tests + ansible.builtin.command: + argv: [cargo-nextest, nextest, run, --archive-file, "{{ cdi_test_root }}/tests.tar.zst", --workspace-remap, "{{ cdi_test_root }}", --no-fail-fast, --no-capture] + environment: + OPENSHELL_GATEWAY: "{{ openshell_test_gateway.name }}" + OPENSHELL_BIN: "{{ openshell_test_cli.stdout }}" + register: cdi_test_result + changed_when: false + failed_when: false + + - name: Show CDI test result + ansible.builtin.debug: + var: cdi_test_result + + - name: Collect gateway diagnostics after test failure + ansible.builtin.include_role: + name: openshell_test_gateway + tasks_from: journal.yaml + when: cdi_test_result.rc != 0 + + - name: Require CDI test success + ansible.builtin.assert: + that: [cdi_test_result.rc == 0] + fail_msg: CDI feature tests failed + + always: + - name: Restore gateway configuration + become: true + ansible.builtin.copy: + content: "{{ cdi_gateway_config.content | b64decode }}" + dest: "{{ openshell_test_gateway.config_path }}" + + - name: Restart gateway with its original configuration + ansible.builtin.include_role: + name: openshell_test_gateway + tasks_from: restart.yaml diff --git a/tests/artifacts.nix b/tests/artifacts.nix index ef5e26988c..422e0c7492 100644 --- a/tests/artifacts.nix +++ b/tests/artifacts.nix @@ -98,6 +98,14 @@ let target = muslToolchain.target; output = "artifacts/test-archives/${muslToolchain.target}/provider-refresh-keycloak-tests.tar"; }; + cdiArchive = mkTestArchive { + name = "openshell-cdi"; + workspacePath = "tests/suites/features"; + manifestPath = "tests/suites/features/Cargo.toml"; + package = "openshell-test-feature-cdi"; + target = muslToolchain.target; + output = "artifacts/test-archives/${muslToolchain.target}/openshell-cdi-tests.tar"; + }; ociImageArchive = mkTestArchive { name = "oci-image"; workspacePath = "tests/suites/features"; @@ -205,6 +213,7 @@ rec { inherit conformanceTestArchive providerRefreshKeycloakArchive + cdiArchive ociImageArchive podmanDriverArchive podmanE2eArchive @@ -304,6 +313,7 @@ rec { runtimeInputs = [ conformanceTestArchive providerRefreshKeycloakArchive + cdiArchive ociImageArchive podmanDriverArchive podmanE2eArchive @@ -311,6 +321,7 @@ rec { text = '' build-openshell-conformance-test-archive build-provider-refresh-keycloak-test-archive + build-openshell-cdi-test-archive build-oci-image-test-archive build-podman-driver-test-archive build-podman-e2e-test-archive diff --git a/tests/config.nix b/tests/config.nix index f22205d101..6b1ab3ee16 100644 --- a/tests/config.nix +++ b/tests/config.nix @@ -170,6 +170,13 @@ let openshell_conformance_test_bundle = "../artifacts/test-archives/${muslTarget}/openshell-conformance-tests.tar"; }; } + { + name = "cdi"; + playbooks = [ "ansible/playbooks/features/cdi/tests.yaml" ]; + inputs = { + openshell_cdi_test_bundle = "../artifacts/test-archives/${muslTarget}/openshell-cdi-tests.tar"; + }; + } { name = "provider-refresh"; playbooks = [ "ansible/playbooks/features/provider-refresh/keycloak.yaml" ]; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index b92ccd05f5..3ff44ba944 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -918,6 +918,16 @@ dependencies = [ "url", ] +[[package]] +name = "openshell-test-feature-cdi" +version = "0.0.0" +dependencies = [ + "openshell-conformance", + "serde_json", + "tokio", +] + + [[package]] name = "openshell-test-feature-oci-image" version = "0.0.0" diff --git a/tests/suites/features/Cargo.toml b/tests/suites/features/Cargo.toml index 9259450a16..835da4f8e6 100644 --- a/tests/suites/features/Cargo.toml +++ b/tests/suites/features/Cargo.toml @@ -3,4 +3,4 @@ [workspace] resolver = "2" -members = ["oci-image", "provider-refresh/keycloak"] +members = ["cdi", "oci-image", "provider-refresh/keycloak"] diff --git a/tests/suites/features/cdi/Cargo.toml b/tests/suites/features/cdi/Cargo.toml new file mode 100644 index 0000000000..bd7801ec6d --- /dev/null +++ b/tests/suites/features/cdi/Cargo.toml @@ -0,0 +1,12 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "openshell-test-feature-cdi" +version = "0.0.0" +edition = "2024" + +[dependencies] +openshell-conformance = { path = "../../../../crates/openshell-conformance" } +serde_json = "1" +tokio = { version = "1.43", features = ["macros", "process", "io-util", "rt"] } diff --git a/tests/suites/features/cdi/tests/injection.rs b/tests/suites/features/cdi/tests/injection.rs new file mode 100644 index 0000000000..814e1920d5 --- /dev/null +++ b/tests/suites/features/cdi/tests/injection.rs @@ -0,0 +1,545 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Hardware-free CDI injection checks against the installed tmachine gateway. +//! +//! Each test creates a sandbox that explicitly selects one named scenario +//! device, then checks what the workload observes (`Check`) or that creation is +//! rejected. The sandbox only reports what it observes; every assertion is made +//! here, so a failure names the check that disagreed. +//! +//! The scenarios' fixtures, CDI edits and expectations are all defined in the +//! `cdi_fixtures` Ansible role +//! (`tests/ansible/roles/cdi_fixtures/vars/scenarios.yaml`), which publishes the +//! expectations as `scenarios.json` on the guest. + +use openshell_conformance::OpenShellRunner; +use serde_json::{Value, json}; +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::time::Duration; + +/// Where the `cdi_fixtures` role publishes scenario expectations. +const SCENARIOS_PATH: &str = "/var/lib/openshell-test-inputs/cdi/scenarios.json"; +const SCENARIO_KIND: &str = "tmachine.example/cdi-scenario"; + +/// Whether an operation on a path is expected to work. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Access { + Ok, + Denied, +} + +impl Access { + fn result(self) -> &'static str { + match self { + Self::Ok => "ok", + Self::Denied => "denied", + } + } +} + +/// A fact observed inside the sandbox together with its expected value. +#[derive(Clone, Debug)] +enum Check { + /// An environment variable, `None` when it must be unset. + Env { + name: String, + expect: Option, + }, + /// File contents, `None` when the path must not exist. + Contents { + path: String, + expect: Option, + }, + /// Opening `path` for reading works. + Read { path: String, expect: Access }, + /// Opening `path` for append works. Nothing is written, so a writable host + /// fixture is left as it was. + Write { path: String, expect: Access }, + /// Whether the workload runs as a non-root user. + NonRoot { expect: bool }, + /// Whether the workload's groups include the group that owns `path`. + HasGroupOf { path: String, expect: bool }, +} + +impl Check { + fn env(name: &str, expect: Option<&str>) -> Self { + Self::Env { + name: name.to_string(), + expect: expect.map(str::to_string), + } + } + + fn contents(path: &str, expect: Option<&str>) -> Self { + Self::Contents { + path: path.to_string(), + expect: expect.map(str::to_string), + } + } + + /// Parse a check as written in `scenarios.yaml`. + fn from_json(value: &Value) -> Result { + let text = |key: &str| { + value[key] + .as_str() + .map(str::to_string) + .ok_or_else(|| format!("check {value} needs a string `{key}`")) + }; + let optional = |key: &str| match &value[key] { + Value::Null => Ok(None), + Value::String(text) => Ok(Some(text.clone())), + _ => Err(format!("check {value} needs a string or null `{key}`")), + }; + // Defaults to true when omitted. + let flag = |key: &str| match &value[key] { + Value::Null => Ok(true), + Value::Bool(flag) => Ok(*flag), + _ => Err(format!("check {value} needs a boolean `{key}`")), + }; + match value["type"].as_str() { + Some("env") => Ok(Self::Env { + name: text("name")?, + expect: optional("expect")?, + }), + Some("contents") => Ok(Self::Contents { + path: text("path")?, + expect: optional("expect")?, + }), + Some(kind @ ("read" | "write")) => { + let path = text("path")?; + let expect = match text("expect")?.as_str() { + "ok" => Access::Ok, + "denied" => Access::Denied, + other => { + return Err(format!( + "{kind} expectation must be ok or denied, got {other:?}" + )); + } + }; + Ok(if kind == "read" { + Self::Read { path, expect } + } else { + Self::Write { path, expect } + }) + } + Some("non-root") => Ok(Self::NonRoot { + expect: flag("expect")?, + }), + Some("group-of") => Ok(Self::HasGroupOf { + path: text("path")?, + expect: flag("expect")?, + }), + other => Err(format!("unknown check type {other:?} in {value}")), + } + } + + /// Stable identifier used in the probe output and in failure reports. + fn id(&self) -> String { + match self { + Self::Env { name, .. } => format!("env:{name}"), + Self::Contents { path, .. } => format!("contents:{path}"), + Self::Read { path, .. } => format!("read:{path}"), + Self::Write { path, .. } => format!("write:{path}"), + Self::NonRoot { .. } => "non-root".to_string(), + Self::HasGroupOf { path, .. } => format!("group-of:{path}"), + } + } + + /// Shell that prints the observed result for this check. + fn observe(&self) -> String { + match self { + Self::Env { name, .. } => format!( + r#"if [ "${{{name}+x}}" ]; then printf 'set:%s' "${name}"; else printf unset; fi"# + ), + Self::Contents { path, .. } => format!( + r#"if [ -e '{path}' ]; then printf 'content:%s' "$(cat '{path}' 2>&1)"; else printf missing; fi"# + ), + Self::Read { path, .. } => { + format!(r"if ( : < '{path}' ) 2>/dev/null; then printf ok; else printf denied; fi") + } + Self::Write { path, .. } => { + format!(r"if ( : >> '{path}' ) 2>/dev/null; then printf ok; else printf denied; fi") + } + Self::NonRoot { .. } => { + r#"if [ "$(id -u)" != 0 ]; then printf ok; else printf root; fi"#.to_string() + } + Self::HasGroupOf { path, .. } => format!( + r#"set -- $(ls -ln '{path}' 2>/dev/null); case " $(id -G) " in *" $4 "*) printf ok ;; *) printf missing-group ;; esac"# + ), + } + } + + fn expected(&self) -> String { + match self { + Self::Env { expect, .. } => expect + .as_ref() + .map_or_else(|| "unset".to_string(), |value| format!("set:{value}")), + Self::Contents { expect, .. } => expect.as_ref().map_or_else( + || "missing".to_string(), + |content| format!("content:{content}"), + ), + Self::Read { expect, .. } | Self::Write { expect, .. } => expect.result().to_string(), + Self::NonRoot { expect } => if *expect { "ok" } else { "root" }.to_string(), + Self::HasGroupOf { expect, .. } => { + if *expect { "ok" } else { "missing-group" }.to_string() + } + } + } +} + +/// Shell that prints one `check= result=` line per check. +fn probe_script(checks: &[Check]) -> String { + let mut script = String::new(); + for check in checks { + writeln!( + script, + "printf 'check=%s result=' '{}'; {}; printf '\\n'", + check.id(), + check.observe() + ) + .unwrap(); + } + script +} + +fn parse_observations(stdout: &str) -> BTreeMap<&str, &str> { + stdout + .lines() + .filter_map(|line| line.strip_prefix("check=")) + .filter_map(|line| line.split_once(" result=")) + .collect() +} + +/// Describe every check whose observed result differs from its expectation. +fn mismatches(checks: &[Check], observed: &BTreeMap<&str, &str>) -> Vec { + checks + .iter() + .filter_map(|check| { + let expected = check.expected(); + let actual = observed.get(check.id().as_str()).copied(); + (actual != Some(expected.as_str())).then(|| { + format!( + "{}\n expected: {expected}\n observed: {}", + check.id(), + actual.unwrap_or("") + ) + }) + }) + .collect() +} + +/// What a scenario expects of sandbox creation and the workload. +struct Scenario { + /// Facts the workload must observe once the sandbox is created. Observed, + /// but not asserted, when creation is expected to fail. + checks: Vec, + /// Creation must fail with a diagnostic containing this text. + rejected: Option, +} + +/// Read a scenario's expectations from the published document. +fn scenario_expectations(document: &Value, name: &str) -> Result { + let expect = document["scenarios"] + .get(name) + .ok_or_else(|| format!("scenario {name:?} is not defined in the published expectations"))?; + let checks_from = |value: &Value| -> Result, String> { + value.as_array().map_or_else( + || Ok(Vec::new()), + |checks| checks.iter().map(Check::from_json).collect(), + ) + }; + let mut checks = checks_from(&document["common_checks"])?; + checks.extend(checks_from(&expect["checks"])?); + Ok(Scenario { + checks, + rejected: expect["rejected"].as_str().map(str::to_string), + }) +} + +fn load_scenario(name: &str) -> Result { + let path = std::env::var("OPENSHELL_CDI_SCENARIOS").unwrap_or_else(|_| SCENARIOS_PATH.into()); + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("read scenario expectations {path}: {error}"))?; + let document: Value = serde_json::from_str(&text) + .map_err(|error| format!("parse scenario expectations {path}: {error}"))?; + scenario_expectations(&document, name) +} + +async fn run_scenario(name: &str, scenario: Scenario) { + let Scenario { checks, rejected } = scenario; + let case = format!("cdi-scenario-{name}"); + let mut runner = + OpenShellRunner::from_env(&case).expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let info = runner + .step("preflight/driver") + .description("gateway has one Docker compute driver") + .with_timeout(Duration::from_secs(10)) + .run(&["gateway", "info", "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + info.require_success()?; + let info_json: Value = info.json().map_err(|error| error.to_string())?; + let drivers = info_json["compute_drivers"] + .as_array() + .ok_or_else(|| "gateway did not report compute drivers".to_string())?; + let driver = match drivers.as_slice() { + [driver] => driver["name"].as_str().unwrap_or_default(), + _ => return Err("CDI suite requires exactly one compute driver".to_string()), + }; + if driver != "docker" { + return Err(format!("CDI suite requires Docker, got {driver:?}")); + } + + // Capture mode reports what happens instead of asserting, to record the + // behavior a scenario currently has. + let capture = std::env::var_os("OPENSHELL_CDI_CAPTURE").is_some(); + let probe = if rejected.is_some() && !capture { + "exit 0".to_string() + } else { + probe_script(&checks) + }; + let sandbox = format!("cdi-{}", runner.id()); + runner.track_sandbox(&sandbox); + let config = + json!({driver: {"cdi_devices": [format!("{SCENARIO_KIND}={name}")]}}).to_string(); + let run = runner + .step("cdi/injection") + .description(&case) + .with_timeout(Duration::from_secs(300)) + .run(&[ + "sandbox", + "create", + "--name", + &sandbox, + "--no-tty", + "--gpu", + "1", + "--driver-config-json", + &config, + "--", + "sh", + "-c", + &probe, + ]) + .await + .map_err(|error| error.to_string())?; + + if capture { + println!("CAPTURE scenario={name} created={}", run.success()); + if !run.success() { + let diagnostic = run + .stderr() + .lines() + .rev() + .find(|line| !line.trim().is_empty()); + println!( + "CAPTURE diagnostic={:?}", + diagnostic.unwrap_or_default().trim() + ); + } + let observed = parse_observations(run.stdout()); + for check in &checks { + let id = check.id(); + println!( + "CAPTURE check={id} result={}", + observed.get(id.as_str()).copied().unwrap_or("") + ); + } + return Ok(()); + } + + if let Some(expected_error) = &rejected { + if run.success() + || !format!("{}\n{}", run.stdout(), run.stderr()).contains(expected_error.as_str()) + { + return Err(run.failure_diagnostic(&format!( + "creation fails with a CDI diagnostic containing {expected_error:?}" + ))); + } + return Ok(()); + } + + run.require_success()?; + let failures = mismatches(&checks, &parse_observations(run.stdout())); + if failures.is_empty() { + Ok(()) + } else { + Err(run.failure_diagnostic(&format!( + "all sandbox checks hold; failing:\n FAIL {}", + failures.join("\n FAIL ") + ))) + } + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("CDI test failed:\n{error}"); + } +} + +async fn check_scenario(name: &str) { + match load_scenario(name) { + Ok(scenario) => run_scenario(name, scenario).await, + Err(error) => panic!("CDI scenario {name}: {error}"), + } +} + +/// Defines one `#[tokio::test]` per scenario in `scenarios.yaml`, so nextest +/// runs and reports each independently. +macro_rules! cdi_scenarios { + ($($name:ident => $scenario:literal,)*) => { + $( + #[tokio::test] + async fn $name() { + check_scenario($scenario).await; + } + )* + }; +} + +cdi_scenarios! { + scenario_mount_under_read_only_baseline => "mount-under-read-only-baseline", + scenario_read_only_file => "read-only-file", + scenario_read_only_directory => "read-only-directory", + scenario_library_parent => "library-parent", + scenario_broad_library_parent => "broad-library-parent", + scenario_writable_file => "writable-file", + scenario_writable_directory => "writable-directory", + scenario_omitted_access_file => "omitted-access-file", + scenario_device_node => "device-node", + scenario_device_node_with_gid => "device-node-with-gid", + scenario_unix_socket => "unix-socket", +} + +#[cfg(test)] +mod probe { + use super::*; + use std::process::Command; + + fn run_probe(checks: &[Check]) -> String { + let output = Command::new("sh") + .arg("-c") + .arg(probe_script(checks)) + .env("PROBE_SET", "value") + .env_remove("PROBE_UNSET") + .output() + .expect("sh runs"); + String::from_utf8(output.stdout).expect("probe output is UTF-8") + } + + #[test] + fn probe_observes_environment_and_files() { + let dir = std::env::temp_dir().join(format!("cdi-probe-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("marker"); + std::fs::write(&file, "marker-content\n").unwrap(); + let path = file.to_str().unwrap(); + let missing = dir.join("missing"); + let unwritable = dir.join("no-such-directory/file"); + + let checks = [ + Check::env("PROBE_SET", Some("value")), + Check::env("PROBE_UNSET", None), + Check::contents(path, Some("marker-content")), + Check::contents(missing.to_str().unwrap(), None), + Check::Read { + path: path.to_string(), + expect: Access::Ok, + }, + Check::Read { + path: missing.to_str().unwrap().to_string(), + expect: Access::Denied, + }, + Check::Write { + path: path.to_string(), + expect: Access::Ok, + }, + Check::Write { + path: unwritable.to_str().unwrap().to_string(), + expect: Access::Denied, + }, + // A file the test just created belongs to one of the user's groups. + Check::HasGroupOf { + path: path.to_string(), + expect: true, + }, + ]; + let stdout = run_probe(&checks); + let observed = parse_observations(&stdout); + let failures = mismatches(&checks, &observed); + assert!(failures.is_empty(), "unexpected failures: {failures:?}"); + + let wrong = [Check::env("PROBE_SET", None)]; + let failures = mismatches(&wrong, &parse_observations(&run_probe(&wrong))); + assert_eq!(failures.len(), 1); + assert!(failures[0].contains("expected: unset")); + assert!(failures[0].contains("observed: set:value")); + std::fs::remove_dir_all(dir).unwrap(); + } + + #[test] + fn missing_output_is_reported_as_a_mismatch() { + let checks = [Check::NonRoot { expect: true }]; + let failures = mismatches(&checks, &BTreeMap::new()); + assert_eq!(failures.len(), 1); + assert!(failures[0].contains("")); + } + + #[test] + fn scenario_expectations_are_read_from_the_document() { + let document = json!({ + "common_checks": [{"type": "env", "name": "SPEC", "expect": "fixture"}], + "scenarios": { + "read-only": {"checks": [ + {"type": "contents", "path": "/p", "expect": "x"}, + {"type": "write", "path": "/p", "expect": "denied"}, + {"type": "contents", "path": "/q", "expect": null}, + {"type": "non-root", "expect": false}, + {"type": "group-of", "path": "/d"}, + ]}, + "refused": {"rejected": "writable", "checks": [{"type": "non-root"}]}, + }, + }); + + let scenario = scenario_expectations(&document, "read-only").unwrap(); + assert!(scenario.rejected.is_none()); + let ids: Vec<_> = scenario.checks.iter().map(Check::id).collect(); + assert_eq!( + ids, + [ + "env:SPEC", + "contents:/p", + "write:/p", + "contents:/q", + "non-root", + "group-of:/d" + ] + ); + assert_eq!(scenario.checks[3].expected(), "missing"); + assert_eq!(scenario.checks[4].expected(), "root"); + assert_eq!(scenario.checks[5].expected(), "ok"); + + let refused = scenario_expectations(&document, "refused").unwrap(); + assert_eq!(refused.rejected.as_deref(), Some("writable")); + // Observed even though creation is expected to fail, for capture mode. + assert_eq!(refused.checks.len(), 2); + + assert!(scenario_expectations(&document, "absent").is_err()); + } + + #[test] + fn malformed_checks_are_rejected() { + for check in [ + json!({"type": "nonsense"}), + json!({"type": "write", "path": "/p", "expect": "maybe"}), + json!({"type": "contents", "expect": "x"}), + ] { + assert!( + Check::from_json(&check).is_err(), + "{check} should not parse" + ); + } + } +}