From 41143b9eca8e0a3973e24854e27914f817c6514f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 14 Oct 2026 11:13:16 +0200 Subject: [PATCH] test(conformance): organize stories into area modules Signed-off-by: Evan Lezar --- .github/workflows/branch-checks.yml | 2 +- e2e/suites/conformance/README.md | 134 ++- .../conformance/cli/tests/file_transfer.rs | 776 ------------------ .../cli/tests/file_transfer/git_filtering.rs | 215 +++++ .../cli/tests/file_transfer/helpers.rs | 215 +++++ .../cli/tests/file_transfer/main.rs | 9 + .../cli/tests/file_transfer/path_safety.rs | 110 +++ .../cli/tests/file_transfer/round_trip.rs | 266 ++++++ .../cli/tests/lifecycle/delete_stopped.rs | 38 + .../{lifecycle.rs => lifecycle/helpers.rs} | 156 +--- .../conformance/cli/tests/lifecycle/main.rs | 8 + .../cli/tests/lifecycle/stop_start.rs | 93 +++ .../conformance/cli/tests/policy_advisor.rs | 591 ------------- .../tests/policy_advisor/draft_assertion.rs | 74 ++ .../cli/tests/policy_advisor/helpers.rs | 188 +++++ .../cli/tests/policy_advisor/main.rs | 10 + .../policy_advisor/mechanistic_proposal.rs | 97 +++ .../policy_advisor/new_hostname_proposal.rs | 71 ++ .../cli/tests/policy_advisor/policy_local.rs | 193 +++++ .../conformance/cli/tests/smoke/main.rs | 6 + .../{smoke.rs => smoke/sandbox_lifecycle.rs} | 184 ++--- tasks/test.toml | 2 +- 22 files changed, 1815 insertions(+), 1623 deletions(-) delete mode 100644 e2e/suites/conformance/cli/tests/file_transfer.rs create mode 100644 e2e/suites/conformance/cli/tests/file_transfer/git_filtering.rs create mode 100644 e2e/suites/conformance/cli/tests/file_transfer/helpers.rs create mode 100644 e2e/suites/conformance/cli/tests/file_transfer/main.rs create mode 100644 e2e/suites/conformance/cli/tests/file_transfer/path_safety.rs create mode 100644 e2e/suites/conformance/cli/tests/file_transfer/round_trip.rs create mode 100644 e2e/suites/conformance/cli/tests/lifecycle/delete_stopped.rs rename e2e/suites/conformance/cli/tests/{lifecycle.rs => lifecycle/helpers.rs} (62%) create mode 100644 e2e/suites/conformance/cli/tests/lifecycle/main.rs create mode 100644 e2e/suites/conformance/cli/tests/lifecycle/stop_start.rs delete mode 100644 e2e/suites/conformance/cli/tests/policy_advisor.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/draft_assertion.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/helpers.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/main.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/mechanistic_proposal.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/new_hostname_proposal.rs create mode 100644 e2e/suites/conformance/cli/tests/policy_advisor/policy_local.rs create mode 100644 e2e/suites/conformance/cli/tests/smoke/main.rs rename e2e/suites/conformance/cli/tests/{smoke.rs => smoke/sandbox_lifecycle.rs} (52%) diff --git a/.github/workflows/branch-checks.yml b/.github/workflows/branch-checks.yml index 002cd440c4..5437eddd32 100644 --- a/.github/workflows/branch-checks.yml +++ b/.github/workflows/branch-checks.yml @@ -237,7 +237,7 @@ jobs: OPENSHELL_TELEMETRY_ENABLED: "false" run: | cargo nextest run --locked --profile ci --workspace --features openshell-server/test-support - cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path e2e/suites/conformance/Cargo.toml --package openshell-test-conformance-cli -E 'binary(policy_advisor) & test(draft_assertion)' + cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path e2e/suites/conformance/Cargo.toml --package openshell-test-conformance-cli -E 'binary(policy_advisor) & test(draft_assertion::tests::)' cargo nextest run --locked --config-file .config/nextest.toml --profile ci --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml rust-build-modes: diff --git a/e2e/suites/conformance/README.md b/e2e/suites/conformance/README.md index bb335013ea..63a8023b55 100644 --- a/e2e/suites/conformance/README.md +++ b/e2e/suites/conformance/README.md @@ -1,43 +1,127 @@ # OpenShell conformance suite -Conformance tests exercise required public behavior against a prepared gateway. -Cargo discovers and runs the tests directly. Each area retains its existing -scenario implementation and helpers in one integration-test file: -`file_transfer.rs`, `lifecycle.rs`, `policy_advisor.rs`, and `smoke.rs`. +Conformance is an end-to-end test type for required public behavior against an +already configured gateway. This workspace contains Cargo tests organized by +conformance area and user story. The CLI is the current client interface; lifecycle, file transfer, and policy behavior define the test groups. -The runner, process executor, polling, diagnostics, and cleanup remain in -`crates/openshell-conformance`. That crate no longer defines scenarios or a -registry. Existing Rust e2e, driver, and feature consumers retain their harnesses -and locations. Story-module organization and shared-runner consolidation are -separate follow-ups. +## Layout -## Run +- `cli/tests//main.rs`: declares story modules for one Cargo test target. +- `cli/tests//.rs`: test functions, steps, and assertions for a story. +- `cli/tests//helpers.rs`: setup and assertions shared by that area's stories. -Supply the matching candidate CLI and a configured gateway: +The current areas are `file_transfer`, `lifecycle`, `policy_advisor`, and `smoke`. +Each area compiles into one test binary; individual stories remain selectable by +their module prefix. `main.rs` contains only module declarations. + +CLI execution, polling, diagnostics, and cleanup remain in +`crates/openshell-conformance`. Story modules implement their tests directly; +Cargo and nextest provide discovery and selection. The runner crate contains no +scenario registry. Existing Rust e2e tests retain their separate harness. + +For example, run only file-transfer round trips against a prepared gateway: ```shell OPENSHELL_BIN=/absolute/path/to/openshell \ cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml \ - --package openshell-test-conformance-cli -- --test-threads=1 --nocapture + --test file_transfer round_trip:: ``` -Use `--test file_transfer`, `--test lifecycle`, `--test policy_advisor`, or -`--test smoke` to select an area. Existing test names remain available for -focused selection. The lifecycle test retains combined stop/start persistence -and stopped-sandbox deletion checks. +## Run without a gateway -## Source checks +Run shared tooling and scenario unit tests: ```shell cargo test --locked --package openshell-conformance cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml \ - --package openshell-test-conformance-cli --test policy_advisor draft_assertion + --package openshell-test-conformance-cli --test policy_advisor draft_assertion::tests:: +``` + +Compile the gateway-backed entry points without executing them: + +```shell +cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml \ + --package openshell-test-conformance-cli --no-run +``` + +Repository formatting, Clippy, and unit-test checks include this workspace. +The unit-test commands select only the pure policy assertion tests. Running every +workspace test also executes the gateway-backed cases. + +## Run against a prepared gateway + +Install, register, and select the gateway before starting the suite. Supply the +matching candidate CLI explicitly: + +```shell +OPENSHELL_BIN=/absolute/path/to/openshell \ + cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml \ + --package openshell-test-conformance-cli --no-fail-fast -- \ + --test-threads=1 --nocapture ``` -The runner unit tests and three policy assertions run without a gateway. Other -integration tests require target preparation and run in the existing driver E2E -and installed nextest archive lanes. Tests register resources before creation, -use unique names, and always call `OpenShellRunner::finish` so cleanup failures -remain visible. Bounded polling observes asynchronous state; failed assertions -are not retried. Live tests and public internet dependencies remain outside -source unit checks. +For development, use a suite, CLI, and gateway from the same candidate build. +Cross-version qualification is not defined. The suite does not install OpenShell, start a gateway, or select a compute driver. The target +must supply the workload image and tools used by the selected scenarios. Missing +CLI or gateway prerequisites fail the run rather than silently passing. + +Run one group with `--test file_transfer`, `--test lifecycle`, +`--test policy_advisor`, or `--test smoke`, before the `--` separator. A group or +individual-test filter exercises only part of the suite. + +The tmachine conformance testsuite runs these same entry points from the +nextest archive built by `build-openshell-conformance-test-archive`. See +[CI.md](../../../CI.md) for the implemented integration lanes and +[the test-guest guide](../../../nix/test-guest/README.md) for preparation +and artifact execution. Archive construction selects the CLI test package; +harness unit tests run separately in source checks. + +## Add or change a test + +Document the expected public behavior, preconditions, required permissions, and +significant side effects beside each scenario implementation. Use descriptive, +stable test names so results can be tracked across runs. Add the Cargo entry point +in the behavior group it exercises. Preserve existing test names when moving code so +Cargo and nextest filters keep working. + +Use structured CLI output and sandbox operations for behavioral assertions. +Separate end-user and administrative interactions into different tests. Tests +may modify resources and settings through public APIs within their declared +permissions; deployment configuration belongs to target preparation. Keep +runtime inspection and host-specific assertions in driver suites. Behavioral +assertions must not depend on public internet services. + +Tests must clean up their changes. Use unique resource names, register resources +before creation, and call `OpenShellRunner::finish` even when the scenario fails so it +attempts cleanup. Cleanup failures must remain visible. Current policy-advisor +cases set the proposal setting on their own sandbox; the suite runs serially. + +## Failures and evidence + +Every conformance test checks required behavior. A failing test on a supported +target records a conformance gap; missing support must not turn the result into +a passing test. Cargo currently reports ordinary test success or failure. + +Investigate failures and track their product, test, or infrastructure causes. +Retain the result and failure logs with the revision, target configuration, and +run duration. Notify the responsible maintainers of new failures. Do not +configure automatic retries for failed tests. + +An explicit waiver records its scope, rationale, and tracking issue while +retaining the failed result. Waivers are review decisions, not changes to test +assertions or successful results. Automated evidence retention, notifications, +and waiver handling remain implementation work. + +## Migration scope + +The conformance workspace lives under `e2e/suites/conformance/`; feature and +driver suites remain under `tests/suites/`, alongside target provisioning. Existing +assertions and test identities are preserved. The legacy mixed suites under +`e2e/rust/` still need classification and migration by behavioral intent. + +[RFC 0016](https://github.com/NVIDIA/OpenShell/pull/3460) also proposes independent +conformance, feature, and driver PR selection, release qualification, and later +restart/upgrade disruption tests. Those execution changes and packaging for runs +without a source checkout are separate follow-ups in +[#3954](https://github.com/NVIDIA/OpenShell/issues/3954). See [CI.md](../../../CI.md) for the +implemented gates; this layout change does not establish those proposed gates. diff --git a/e2e/suites/conformance/cli/tests/file_transfer.rs b/e2e/suites/conformance/cli/tests/file_transfer.rs deleted file mode 100644 index 305f22f17d..0000000000 --- a/e2e/suites/conformance/cli/tests/file_transfer.rs +++ /dev/null @@ -1,776 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Portable CLI file-transfer conformance scenario. - -use std::fs; -use std::path::Path; -use std::process::Stdio; -use std::time::Duration; - -use tokio::process::Command; - -use openshell_conformance::{CommandResult, OpenShellRunner}; - -const CREATE_TIMEOUT: Duration = Duration::from_mins(10); -const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); -const TRANSFER_TIMEOUT: Duration = Duration::from_mins(5); -const LARGE_FILE_SIZE: usize = 512 * 1024; - -/// Certify basic file and directory upload and download behavior. -#[tokio::test] -async fn round_trip() { - let mut runner = OpenShellRunner::from_env("file-transfer/round-trip") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "round-trip").await?; - upload_existing_directories(runner, &sandbox_name, &remote_root, local.path()).await?; - transfer_round_trip(runner, &sandbox_name, &remote_root, local.path()).await?; - download_file(runner, &sandbox_name, &remote_root, local.path()).await?; - download_directory(runner, &sandbox_name, &remote_root, local.path()).await?; - delete_sandbox(runner, &sandbox_name).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("file-transfer/round-trip conformance scenario failed:\n{error}"); - } -} - -/// Certify Git-aware upload filtering and explicit unfiltered uploads. -#[tokio::test] -async fn git_filtering() { - let mut runner = OpenShellRunner::from_env("file-transfer/git-filtering") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "git-filtering").await?; - gitignore_filtering(runner, &sandbox_name, &remote_root, local.path()).await?; - single_file_from_git_repo(runner, &sandbox_name, &remote_root, local.path()).await?; - gitignored_directory_requires_override(runner, &sandbox_name, &remote_root, local.path()) - .await?; - delete_sandbox(runner, &sandbox_name).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("file-transfer/git-filtering conformance scenario failed:\n{error}"); - } -} - -/// Certify file-transfer workspace boundary and filename safety behavior. -#[tokio::test] -async fn path_safety() { - let mut runner = OpenShellRunner::from_env("file-transfer/path-safety") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "path-safety").await?; - reject_workspace_escape(runner, &sandbox_name, &remote_root, local.path()).await?; - download_dash_leading_name(runner, &sandbox_name, &remote_root, local.path()).await?; - delete_sandbox(runner, &sandbox_name).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("file-transfer/path-safety conformance scenario failed:\n{error}"); - } -} - -async fn prepare_sandbox( - runner: &mut OpenShellRunner, - group: &str, -) -> Result<(String, String, tempfile::TempDir), String> { - let suffix = match group { - "round-trip" => "fr", - "git-filtering" => "fg", - "path-safety" => "fs", - _ => return Err(format!("unknown file-transfer group {group:?}")), - }; - let sandbox_name = format!("ct-{}-{suffix}", runner.id()); - let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", runner.id()); - let local = - tempfile::tempdir().map_err(|error| format!("create temporary directory: {error}"))?; - - runner.track_sandbox(&sandbox_name); - let create = runner - .step(format!("{group}/create")) - .description(format!("sandbox '{sandbox_name}' is created")) - .with_timeout(CREATE_TIMEOUT) - .run(&["sandbox", "create", "--name", &sandbox_name, "--detach"]) - .await - .map_err(|error| error.to_string())?; - create.require_success()?; - - exec( - runner, - &sandbox_name, - &format!("{group}/prepare"), - &format!("mkdir -p '{remote_root}'"), - ) - .await?; - - Ok((sandbox_name, remote_root, local)) -} - -async fn delete_sandbox(runner: &mut OpenShellRunner, sandbox_name: &str) -> Result<(), String> { - let delete = runner - .step("delete") - .description(format!("sandbox '{sandbox_name}' is deleted")) - .with_timeout(COMMAND_TIMEOUT) - .run(&["sandbox", "delete", sandbox_name]) - .await - .map_err(|error| error.to_string())?; - delete.require_success()?; - runner.forget_sandbox(sandbox_name); - Ok(()) -} - -/// Check remote destination types, not just successful transfer status: tar -/// can report success after replacing an empty directory with a regular file. -async fn upload_existing_directories( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let source = local_root.join("config.json"); - fs::write(&source, "upload-destination-payload") - .map_err(fs_error("write destination fixture"))?; - let root = format!("{remote_root}/upload destinations"); - exec( - runner, - sandbox, - "upload-destinations/prepare", - &format!( - "mkdir -p '{root}/empty' '{root}/populated' '{root}/target' '{root}/slash' && \ - printf keep > '{root}/populated/keep.txt' && \ - ln -s target '{root}/link'" - ), - ) - .await?; - - for directory in ["empty", "populated", "link", "slash/"] { - let destination = format!("{root}/{directory}"); - upload( - runner, - sandbox, - &format!("upload-destinations/{directory}/upload"), - &source, - &destination, - true, - ) - .await?; - exec( - runner, - sandbox, - &format!("upload-destinations/{directory}/verify"), - &format!( - "test -d '{destination}' && \ - test \"$(cat '{destination}/config.json')\" = upload-destination-payload && \ - test \"$(cat '{root}/populated/keep.txt')\" = keep && \ - test -L '{root}/link'" - ), - ) - .await?; - } - - // A missing destination remains an intentional rename; uploading again - // must replace that file rather than treating it as a directory. - let renamed = format!("{root}/renamed.json"); - for pass in ["rename", "overwrite"] { - upload( - runner, - sandbox, - &format!("upload-destinations/{pass}/upload"), - &source, - &renamed, - true, - ) - .await?; - exec( - runner, - sandbox, - &format!("upload-destinations/{pass}/verify"), - &format!( - "test -f '{renamed}' && test \"$(cat '{renamed}')\" = upload-destination-payload" - ), - ) - .await?; - } - Ok(()) -} - -async fn transfer_round_trip( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let source = local_root.join("roundtrip-upload"); - fs::create_dir_all(source.join("subdir")).map_err(fs_error("create round-trip source"))?; - fs::write(source.join("greeting.txt"), "hello-from-local") - .map_err(fs_error("write greeting.txt"))?; - fs::write(source.join("subdir/nested.txt"), "nested-content") - .map_err(fs_error("write nested.txt"))?; - - let large = (0u8..=250) - .cycle() - .take(LARGE_FILE_SIZE) - .collect::>(); - fs::write(source.join("large.bin"), &large).map_err(fs_error("write large.bin"))?; - - let remote = format!("{remote_root}/roundtrip"); - let result = - upload_result(runner, sandbox, "roundtrip/upload", &source, &remote, false).await?; - result.require_success()?; - if !result.stderr().contains("outside a Git work tree") - || !result.stderr().contains(".gitignore rules are not applied") - { - return Err( - result.failure_diagnostic("upload outside Git warns that filtering is disabled") - ); - } - - let destination = local_root.join("roundtrip-download"); - fs::create_dir(&destination).map_err(fs_error("create round-trip destination"))?; - let remote_source = format!("{remote}/roundtrip-upload"); - download( - runner, - sandbox, - "roundtrip/download", - &remote_source, - &destination, - ) - .await?; - - require_text( - &destination.join("greeting.txt"), - "hello-from-local", - "round-trip greeting", - )?; - require_text( - &destination.join("subdir/nested.txt"), - "nested-content", - "round-trip nested file", - )?; - let actual = fs::read(destination.join("large.bin")).map_err(fs_error("read large.bin"))?; - if actual != large { - return Err(format!( - "large file changed during round trip: expected {} bytes, received {} bytes", - large.len(), - actual.len() - )); - } - - let single = local_root.join("single.txt"); - fs::write(&single, "single-file-payload").map_err(fs_error("write single.txt"))?; - let remote_single = format!("{remote_root}/single.txt"); - upload( - runner, - sandbox, - "single/upload", - &single, - &remote_single, - false, - ) - .await?; - let single_destination = local_root.join("single-download"); - fs::create_dir(&single_destination).map_err(fs_error("create single-file destination"))?; - download( - runner, - sandbox, - "single/download", - &remote_single, - &single_destination, - ) - .await?; - require_text( - &single_destination.join("single.txt"), - "single-file-payload", - "single-file round trip", - ) -} - -async fn gitignore_filtering( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let repository = local_root.join("filter-repo"); - fs::create_dir(&repository).map_err(fs_error("create filter repository"))?; - git_init(&repository).await?; - fs::write(repository.join(".gitignore"), "*.log\nbuild/\n") - .map_err(fs_error("write filter .gitignore"))?; - fs::write(repository.join("tracked.txt"), "i-am-tracked") - .map_err(fs_error("write tracked.txt"))?; - fs::write(repository.join("ignored.log"), "i-should-be-filtered") - .map_err(fs_error("write ignored.log"))?; - fs::create_dir(repository.join("build")).map_err(fs_error("create ignored build directory"))?; - fs::write(repository.join("build/output.bin"), "build-artifact") - .map_err(fs_error("write ignored build artifact"))?; - git(&repository, &["add", "."]).await?; - - let remote = format!("{remote_root}/filtered"); - upload( - runner, - sandbox, - "gitignore/upload", - &repository, - &remote, - false, - ) - .await?; - - let destination = local_root.join("filter-download"); - fs::create_dir(&destination).map_err(fs_error("create filter destination"))?; - download(runner, sandbox, "gitignore/download", &remote, &destination).await?; - let uploaded = destination.join("filter-repo"); - require_text( - &uploaded.join("tracked.txt"), - "i-am-tracked", - "Git-aware tracked file", - )?; - require_exists(&uploaded.join(".gitignore"), "uploaded .gitignore")?; - require_absent(&uploaded.join("ignored.log"), "Git-ignored file")?; - require_absent(&uploaded.join("build"), "Git-ignored directory") -} - -async fn single_file_from_git_repo( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let repository = local_root.join("single-repo"); - fs::create_dir_all(repository.join("nested")) - .map_err(fs_error("create single-file repository"))?; - git_init(&repository).await?; - fs::write(repository.join(".gitignore"), "*.log\n") - .map_err(fs_error("write single-file .gitignore"))?; - fs::write( - repository.join("nested/config.txt"), - "single-file-from-repo", - ) - .map_err(fs_error("write repository config.txt"))?; - fs::write(repository.join("tracked.txt"), "should-not-upload") - .map_err(fs_error("write unrelated tracked.txt"))?; - fs::write(repository.join("ignored.log"), "ignored") - .map_err(fs_error("write repository ignored.log"))?; - - let remote = format!("{remote_root}/single-from-repo"); - upload( - runner, - sandbox, - "single-from-repo/upload", - &repository.join("nested/config.txt"), - &remote, - false, - ) - .await?; - let destination = local_root.join("single-repo-download"); - fs::create_dir(&destination).map_err(fs_error("create single-repo destination"))?; - download( - runner, - sandbox, - "single-from-repo/download", - &remote, - &destination, - ) - .await?; - require_text( - &destination.join("config.txt"), - "single-file-from-repo", - "single file selected from repository", - )?; - require_absent( - &destination.join("tracked.txt"), - "unselected repository file", - )?; - require_absent(&destination.join("ignored.log"), "ignored repository file") -} - -async fn download_file( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let remote = format!("{remote_root}/download-file.txt"); - exec( - runner, - sandbox, - "download-file/seed", - &format!("printf greeting-payload > '{remote}'"), - ) - .await?; - let destination = local_root.join("download-file"); - fs::create_dir(&destination).map_err(fs_error("create file-download destination"))?; - download( - runner, - sandbox, - "download-file/download", - &remote, - &destination, - ) - .await?; - require_text( - &destination.join("download-file.txt"), - "greeting-payload", - "downloaded file", - ) -} - -async fn download_directory( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let remote = format!("{remote_root}/tree"); - exec( - runner, - sandbox, - "download-directory/seed", - &format!( - "mkdir -p '{remote}/sub' && printf top-level > '{remote}/root.txt' && printf nested > '{remote}/sub/child.txt'" - ), - ) - .await?; - let destination = local_root.join("download-directory"); - fs::create_dir(&destination).map_err(fs_error("create directory-download destination"))?; - download( - runner, - sandbox, - "download-directory/download", - &remote, - &destination, - ) - .await?; - require_text( - &destination.join("root.txt"), - "top-level", - "downloaded directory root file", - )?; - require_text( - &destination.join("sub/child.txt"), - "nested", - "downloaded directory nested file", - ) -} - -async fn reject_workspace_escape( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let etc_link = format!("{remote_root}/etc-link"); - let passwd_link = format!("{remote_root}/passwd-link"); - exec( - runner, - sandbox, - "workspace-escape/seed", - &format!("ln -s /etc '{etc_link}' && ln -s /etc/passwd '{passwd_link}'"), - ) - .await?; - let destination = local_root.join("workspace-escape"); - fs::create_dir(&destination).map_err(fs_error("create workspace-escape destination"))?; - - for (step, source) in [ - ("directory-link", etc_link.clone()), - ("file-link", passwd_link), - ("linked-component", format!("{etc_link}/passwd")), - ] { - let result = download_result( - runner, - sandbox, - &format!("workspace-escape/{step}"), - &source, - &destination, - ) - .await?; - if result.success() { - return Err(format!( - "download unexpectedly accepted sandbox path {source:?} that resolves outside the workspace" - )); - } - let diagnostic = format!("{}\n{}", result.stdout(), result.stderr()); - if !diagnostic.contains("resolves to") - || !diagnostic.contains("outside the") - || !diagnostic.contains("sandbox workspace") - { - return Err(result.failure_diagnostic( - "download is rejected because the resolved source is outside the sandbox workspace", - )); - } - } - require_absent(&destination.join("passwd"), "escaped passwd file")?; - require_absent(&destination.join("etc-link"), "escaped /etc directory") -} - -async fn download_dash_leading_name( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let remote = format!("{remote_root}/--checkpoint-action=evil"); - exec( - runner, - sandbox, - "dash-leading/seed", - &format!("printf dash-payload > '{remote}'"), - ) - .await?; - let destination = local_root.join("dash-leading"); - fs::create_dir(&destination).map_err(fs_error("create dash-leading destination"))?; - download( - runner, - sandbox, - "dash-leading/download", - &remote, - &destination, - ) - .await?; - require_text( - &destination.join("--checkpoint-action=evil"), - "dash-payload", - "dash-leading file", - ) -} - -async fn gitignored_directory_requires_override( - runner: &OpenShellRunner, - sandbox: &str, - remote_root: &str, - local_root: &Path, -) -> Result<(), String> { - let remote_seed = format!("{remote_root}/runs/test.json"); - exec( - runner, - sandbox, - "gitignored-override/seed", - &format!("mkdir -p '{remote_root}/runs' && printf downloaded-payload > '{remote_seed}'"), - ) - .await?; - - let repository = local_root.join("override-repo"); - fs::create_dir(&repository).map_err(fs_error("create override repository"))?; - git_init(&repository).await?; - fs::write(repository.join(".gitignore"), "runs/\n") - .map_err(fs_error("write override .gitignore"))?; - let runs = repository.join("runs"); - fs::create_dir(&runs).map_err(fs_error("create ignored runs directory"))?; - download( - runner, - sandbox, - "gitignored-override/download-seed", - &remote_seed, - &runs, - ) - .await?; - require_exists(&runs.join("test.json"), "downloaded ignored file")?; - - let remote = format!("{remote_root}/reuploaded"); - let rejected = upload_result( - runner, - sandbox, - "gitignored-override/reject-upload", - &runs, - &remote, - false, - ) - .await?; - let output = format!("{}\n{}", rejected.stdout(), rejected.stderr()); - if rejected.success() - || !output.contains("filtering selected no files") - || !output.contains("--no-git-ignore") - { - return Err(rejected.failure_diagnostic( - "upload rejects an empty Git selection and explains the explicit override", - )); - } - exec( - runner, - sandbox, - "gitignored-override/no-transfer", - &format!("test ! -e '{remote}'"), - ) - .await?; - upload( - runner, - sandbox, - "gitignored-override/upload", - &runs, - &remote, - true, - ) - .await?; - - let destination = local_root.join("override-download"); - fs::create_dir(&destination).map_err(fs_error("create override destination"))?; - download( - runner, - sandbox, - "gitignored-override/download", - &remote, - &destination, - ) - .await?; - require_text( - &destination.join("runs/test.json"), - "downloaded-payload", - "re-uploaded Git-ignored file", - ) -} - -async fn upload( - runner: &OpenShellRunner, - sandbox: &str, - step: &str, - source: &Path, - destination: &str, - no_git_ignore: bool, -) -> Result<(), String> { - let result = upload_result(runner, sandbox, step, source, destination, no_git_ignore).await?; - result.require_success() -} - -async fn upload_result( - runner: &OpenShellRunner, - sandbox: &str, - step: &str, - source: &Path, - destination: &str, - no_git_ignore: bool, -) -> Result { - let source = source - .to_str() - .ok_or_else(|| format!("local upload path is not UTF-8: {}", source.display()))?; - let mut args = vec!["sandbox", "upload", sandbox, source, destination]; - if no_git_ignore { - args.push("--no-git-ignore"); - } - runner - .step(step) - .description(format!("upload {source:?} to {destination:?}")) - .with_timeout(TRANSFER_TIMEOUT) - .run(&args) - .await - .map_err(|error| error.to_string()) -} - -async fn download( - runner: &OpenShellRunner, - sandbox: &str, - step: &str, - source: &str, - destination: &Path, -) -> Result<(), String> { - let result = download_result(runner, sandbox, step, source, destination).await?; - result.require_success() -} - -async fn download_result( - runner: &OpenShellRunner, - sandbox: &str, - step: &str, - source: &str, - destination: &Path, -) -> Result { - let destination = destination.to_str().ok_or_else(|| { - format!( - "local download path is not UTF-8: {}", - destination.display() - ) - })?; - runner - .step(step) - .description(format!( - "download {source:?} to {destination:?} has the expected disposition" - )) - .with_timeout(TRANSFER_TIMEOUT) - .run(&["sandbox", "download", sandbox, source, destination]) - .await - .map_err(|error| error.to_string()) -} - -async fn exec( - runner: &OpenShellRunner, - sandbox: &str, - step: &str, - script: &str, -) -> Result<(), String> { - let result = runner - .step(step) - .description(format!("sandbox fixture setup for {step} succeeds")) - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", "exec", "--name", sandbox, "--no-tty", "--", "sh", "-c", script, - ]) - .await - .map_err(|error| error.to_string())?; - result.require_success() -} - -async fn git_init(repository: &Path) -> Result<(), String> { - git(repository, &["init", "--quiet"]).await -} - -async fn git(repository: &Path, args: &[&str]) -> Result<(), String> { - let output = Command::new("git") - .args(args) - .current_dir(repository) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .output() - .await - .map_err(|error| format!("failed to run git in {}: {error}", repository.display()))?; - if output.status.success() { - return Ok(()); - } - Err(format!( - "git {} failed in {} with status {}\nstdout:\n{}\nstderr:\n{}", - args.join(" "), - repository.display(), - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr), - )) -} - -fn require_text(path: &Path, expected: &str, label: &str) -> Result<(), String> { - let actual = fs::read_to_string(path) - .map_err(|error| format!("read {label} at {}: {error}", path.display()))?; - if actual == expected { - Ok(()) - } else { - Err(format!( - "{label} content mismatch at {}: expected {expected:?}, received {actual:?}", - path.display() - )) - } -} - -fn require_exists(path: &Path, label: &str) -> Result<(), String> { - if path.exists() { - Ok(()) - } else { - Err(format!("{label} does not exist at {}", path.display())) - } -} - -fn require_absent(path: &Path, label: &str) -> Result<(), String> { - if path.exists() { - Err(format!("{label} unexpectedly exists at {}", path.display())) - } else { - Ok(()) - } -} - -fn fs_error(context: &'static str) -> impl FnOnce(std::io::Error) -> String { - move |error| format!("{context}: {error}") -} diff --git a/e2e/suites/conformance/cli/tests/file_transfer/git_filtering.rs b/e2e/suites/conformance/cli/tests/file_transfer/git_filtering.rs new file mode 100644 index 0000000000..8a1d0bb6e1 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/file_transfer/git_filtering.rs @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + delete_sandbox, download, exec, fs_error, git, git_init, prepare_sandbox, require_absent, + require_exists, require_text, upload, upload_result, +}; +use openshell_conformance::OpenShellRunner; +use std::fs; +use std::path::Path; + +/// Verify Git-aware upload selection and explicit unfiltered uploads. +#[tokio::test] +async fn respects_git_selection_and_explicit_override() { + let mut runner = OpenShellRunner::from_env("file-transfer/git-filtering") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "git-filtering").await?; + gitignore_filtering(runner, &sandbox_name, &remote_root, local.path()).await?; + single_file_from_git_repo(runner, &sandbox_name, &remote_root, local.path()).await?; + gitignored_directory_requires_override(runner, &sandbox_name, &remote_root, local.path()) + .await?; + delete_sandbox(runner, &sandbox_name).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("file-transfer/git-filtering conformance story failed:\n{error}"); + } +} + +async fn gitignore_filtering( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let repository = local_root.join("filter-repo"); + fs::create_dir(&repository).map_err(fs_error("create filter repository"))?; + git_init(&repository).await?; + fs::write(repository.join(".gitignore"), "*.log\nbuild/\n") + .map_err(fs_error("write filter .gitignore"))?; + fs::write(repository.join("tracked.txt"), "i-am-tracked") + .map_err(fs_error("write tracked.txt"))?; + fs::write(repository.join("ignored.log"), "i-should-be-filtered") + .map_err(fs_error("write ignored.log"))?; + fs::create_dir(repository.join("build")).map_err(fs_error("create ignored build directory"))?; + fs::write(repository.join("build/output.bin"), "build-artifact") + .map_err(fs_error("write ignored build artifact"))?; + git(&repository, &["add", "."]).await?; + + let remote = format!("{remote_root}/filtered"); + upload( + runner, + sandbox, + "gitignore/upload", + &repository, + &remote, + false, + ) + .await?; + + let destination = local_root.join("filter-download"); + fs::create_dir(&destination).map_err(fs_error("create filter destination"))?; + download(runner, sandbox, "gitignore/download", &remote, &destination).await?; + let uploaded = destination.join("filter-repo"); + require_text( + &uploaded.join("tracked.txt"), + "i-am-tracked", + "Git-aware tracked file", + )?; + require_exists(&uploaded.join(".gitignore"), "uploaded .gitignore")?; + require_absent(&uploaded.join("ignored.log"), "Git-ignored file")?; + require_absent(&uploaded.join("build"), "Git-ignored directory") +} + +async fn single_file_from_git_repo( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let repository = local_root.join("single-repo"); + fs::create_dir_all(repository.join("nested")) + .map_err(fs_error("create single-file repository"))?; + git_init(&repository).await?; + fs::write(repository.join(".gitignore"), "*.log\n") + .map_err(fs_error("write single-file .gitignore"))?; + fs::write( + repository.join("nested/config.txt"), + "single-file-from-repo", + ) + .map_err(fs_error("write repository config.txt"))?; + fs::write(repository.join("tracked.txt"), "should-not-upload") + .map_err(fs_error("write unrelated tracked.txt"))?; + fs::write(repository.join("ignored.log"), "ignored") + .map_err(fs_error("write repository ignored.log"))?; + + let remote = format!("{remote_root}/single-from-repo"); + upload( + runner, + sandbox, + "single-from-repo/upload", + &repository.join("nested/config.txt"), + &remote, + false, + ) + .await?; + let destination = local_root.join("single-repo-download"); + fs::create_dir(&destination).map_err(fs_error("create single-repo destination"))?; + download( + runner, + sandbox, + "single-from-repo/download", + &remote, + &destination, + ) + .await?; + require_text( + &destination.join("config.txt"), + "single-file-from-repo", + "single file selected from repository", + )?; + require_absent( + &destination.join("tracked.txt"), + "unselected repository file", + )?; + require_absent(&destination.join("ignored.log"), "ignored repository file") +} + +async fn gitignored_directory_requires_override( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let remote_seed = format!("{remote_root}/runs/test.json"); + exec( + runner, + sandbox, + "gitignored-override/seed", + &format!("mkdir -p '{remote_root}/runs' && printf downloaded-payload > '{remote_seed}'"), + ) + .await?; + + let repository = local_root.join("override-repo"); + fs::create_dir(&repository).map_err(fs_error("create override repository"))?; + git_init(&repository).await?; + fs::write(repository.join(".gitignore"), "runs/\n") + .map_err(fs_error("write override .gitignore"))?; + let runs = repository.join("runs"); + fs::create_dir(&runs).map_err(fs_error("create ignored runs directory"))?; + download( + runner, + sandbox, + "gitignored-override/download-seed", + &remote_seed, + &runs, + ) + .await?; + require_exists(&runs.join("test.json"), "downloaded ignored file")?; + + let remote = format!("{remote_root}/reuploaded"); + let rejected = upload_result( + runner, + sandbox, + "gitignored-override/reject-upload", + &runs, + &remote, + false, + ) + .await?; + let output = format!("{}\n{}", rejected.stdout(), rejected.stderr()); + if rejected.success() + || !output.contains("filtering selected no files") + || !output.contains("--no-git-ignore") + { + return Err(rejected.failure_diagnostic( + "upload rejects an empty Git selection and explains the explicit override", + )); + } + exec( + runner, + sandbox, + "gitignored-override/no-transfer", + &format!("test ! -e '{remote}'"), + ) + .await?; + upload( + runner, + sandbox, + "gitignored-override/upload", + &runs, + &remote, + true, + ) + .await?; + + let destination = local_root.join("override-download"); + fs::create_dir(&destination).map_err(fs_error("create override destination"))?; + download( + runner, + sandbox, + "gitignored-override/download", + &remote, + &destination, + ) + .await?; + require_text( + &destination.join("runs/test.json"), + "downloaded-payload", + "re-uploaded Git-ignored file", + ) +} diff --git a/e2e/suites/conformance/cli/tests/file_transfer/helpers.rs b/e2e/suites/conformance/cli/tests/file_transfer/helpers.rs new file mode 100644 index 0000000000..407419e592 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/file_transfer/helpers.rs @@ -0,0 +1,215 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use openshell_conformance::CommandResult; +use openshell_conformance::OpenShellRunner; +use std::fs; +use std::path::Path; +use std::process::Stdio; +use std::time::Duration; +use tokio::process::Command; + +pub const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +pub const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); +pub const TRANSFER_TIMEOUT: Duration = Duration::from_mins(5); +pub const LARGE_FILE_SIZE: usize = 512 * 1024; + +pub async fn prepare_sandbox( + runner: &mut OpenShellRunner, + group: &str, +) -> Result<(String, String, tempfile::TempDir), String> { + let suffix = match group { + "round-trip" => "fr", + "git-filtering" => "fg", + "path-safety" => "fs", + _ => return Err(format!("unknown file-transfer group {group:?}")), + }; + let sandbox_name = format!("ct-{}-{suffix}", runner.id()); + let remote_root = format!("/sandbox/file-transfer-{}-{suffix}", runner.id()); + let local = + tempfile::tempdir().map_err(|error| format!("create temporary directory: {error}"))?; + + runner.track_sandbox(&sandbox_name); + let create = runner + .step(format!("{group}/create")) + .description(format!("sandbox '{sandbox_name}' is created")) + .with_timeout(CREATE_TIMEOUT) + .run(&["sandbox", "create", "--name", &sandbox_name, "--detach"]) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + + exec( + runner, + &sandbox_name, + &format!("{group}/prepare"), + &format!("mkdir -p '{remote_root}'"), + ) + .await?; + + Ok((sandbox_name, remote_root, local)) +} + +pub async fn delete_sandbox( + runner: &mut OpenShellRunner, + sandbox_name: &str, +) -> Result<(), String> { + let delete = runner + .step("delete") + .description(format!("sandbox '{sandbox_name}' is deleted")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "delete", sandbox_name]) + .await + .map_err(|error| error.to_string())?; + delete.require_success()?; + runner.forget_sandbox(sandbox_name); + Ok(()) +} + +pub async fn upload( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + source: &Path, + destination: &str, + no_git_ignore: bool, +) -> Result<(), String> { + let result = upload_result(runner, sandbox, step, source, destination, no_git_ignore).await?; + result.require_success() +} + +pub async fn upload_result( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + source: &Path, + destination: &str, + no_git_ignore: bool, +) -> Result { + let source = source + .to_str() + .ok_or_else(|| format!("local upload path is not UTF-8: {}", source.display()))?; + let mut args = vec!["sandbox", "upload", sandbox, source, destination]; + if no_git_ignore { + args.push("--no-git-ignore"); + } + runner + .step(step) + .description(format!("upload {source:?} to {destination:?}")) + .with_timeout(TRANSFER_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string()) +} + +pub async fn download( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + source: &str, + destination: &Path, +) -> Result<(), String> { + let result = download_result(runner, sandbox, step, source, destination).await?; + result.require_success() +} + +pub async fn download_result( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + source: &str, + destination: &Path, +) -> Result { + let destination = destination.to_str().ok_or_else(|| { + format!( + "local download path is not UTF-8: {}", + destination.display() + ) + })?; + runner + .step(step) + .description(format!( + "download {source:?} to {destination:?} has the expected disposition" + )) + .with_timeout(TRANSFER_TIMEOUT) + .run(&["sandbox", "download", sandbox, source, destination]) + .await + .map_err(|error| error.to_string()) +} + +pub async fn exec( + runner: &OpenShellRunner, + sandbox: &str, + step: &str, + script: &str, +) -> Result<(), String> { + let result = runner + .step(step) + .description(format!("sandbox fixture setup for {step} succeeds")) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", "exec", "--name", sandbox, "--no-tty", "--", "sh", "-c", script, + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success() +} + +pub async fn git_init(repository: &Path) -> Result<(), String> { + git(repository, &["init", "--quiet"]).await +} + +pub async fn git(repository: &Path, args: &[&str]) -> Result<(), String> { + let output = Command::new("git") + .args(args) + .current_dir(repository) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .await + .map_err(|error| format!("failed to run git in {}: {error}", repository.display()))?; + if output.status.success() { + return Ok(()); + } + Err(format!( + "git {} failed in {} with status {}\nstdout:\n{}\nstderr:\n{}", + args.join(" "), + repository.display(), + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + )) +} + +pub fn require_text(path: &Path, expected: &str, label: &str) -> Result<(), String> { + let actual = fs::read_to_string(path) + .map_err(|error| format!("read {label} at {}: {error}", path.display()))?; + if actual == expected { + Ok(()) + } else { + Err(format!( + "{label} content mismatch at {}: expected {expected:?}, received {actual:?}", + path.display() + )) + } +} + +pub fn require_exists(path: &Path, label: &str) -> Result<(), String> { + if path.exists() { + Ok(()) + } else { + Err(format!("{label} does not exist at {}", path.display())) + } +} + +pub fn require_absent(path: &Path, label: &str) -> Result<(), String> { + if path.exists() { + Err(format!("{label} unexpectedly exists at {}", path.display())) + } else { + Ok(()) + } +} + +pub fn fs_error(context: &'static str) -> impl FnOnce(std::io::Error) -> String { + move |error| format!("{context}: {error}") +} diff --git a/e2e/suites/conformance/cli/tests/file_transfer/main.rs b/e2e/suites/conformance/cli/tests/file_transfer/main.rs new file mode 100644 index 0000000000..ccf0da9468 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/file_transfer/main.rs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! File transfer conformance stories. + +mod git_filtering; +mod helpers; +mod path_safety; +mod round_trip; diff --git a/e2e/suites/conformance/cli/tests/file_transfer/path_safety.rs b/e2e/suites/conformance/cli/tests/file_transfer/path_safety.rs new file mode 100644 index 0000000000..c408fa7ea7 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/file_transfer/path_safety.rs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + delete_sandbox, download, download_result, exec, fs_error, prepare_sandbox, require_absent, + require_text, +}; +use openshell_conformance::OpenShellRunner; +use std::fs; +use std::path::Path; + +/// Verify workspace boundary enforcement and safe filename handling. +#[tokio::test] +async fn enforces_workspace_boundary_and_handles_filenames() { + let mut runner = OpenShellRunner::from_env("file-transfer/path-safety") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "path-safety").await?; + reject_workspace_escape(runner, &sandbox_name, &remote_root, local.path()).await?; + download_dash_leading_name(runner, &sandbox_name, &remote_root, local.path()).await?; + delete_sandbox(runner, &sandbox_name).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("file-transfer/path-safety conformance story failed:\n{error}"); + } +} + +async fn reject_workspace_escape( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let etc_link = format!("{remote_root}/etc-link"); + let passwd_link = format!("{remote_root}/passwd-link"); + exec( + runner, + sandbox, + "workspace-escape/seed", + &format!("ln -s /etc '{etc_link}' && ln -s /etc/passwd '{passwd_link}'"), + ) + .await?; + let destination = local_root.join("workspace-escape"); + fs::create_dir(&destination).map_err(fs_error("create workspace-escape destination"))?; + + for (step, source) in [ + ("directory-link", etc_link.clone()), + ("file-link", passwd_link), + ("linked-component", format!("{etc_link}/passwd")), + ] { + let result = download_result( + runner, + sandbox, + &format!("workspace-escape/{step}"), + &source, + &destination, + ) + .await?; + if result.success() { + return Err(format!( + "download unexpectedly accepted sandbox path {source:?} that resolves outside the workspace" + )); + } + let diagnostic = format!("{}\n{}", result.stdout(), result.stderr()); + if !diagnostic.contains("resolves to") + || !diagnostic.contains("outside the") + || !diagnostic.contains("sandbox workspace") + { + return Err(result.failure_diagnostic( + "download is rejected because the resolved source is outside the sandbox workspace", + )); + } + } + require_absent(&destination.join("passwd"), "escaped passwd file")?; + require_absent(&destination.join("etc-link"), "escaped /etc directory") +} + +async fn download_dash_leading_name( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let remote = format!("{remote_root}/--checkpoint-action=evil"); + exec( + runner, + sandbox, + "dash-leading/seed", + &format!("printf dash-payload > '{remote}'"), + ) + .await?; + let destination = local_root.join("dash-leading"); + fs::create_dir(&destination).map_err(fs_error("create dash-leading destination"))?; + download( + runner, + sandbox, + "dash-leading/download", + &remote, + &destination, + ) + .await?; + require_text( + &destination.join("--checkpoint-action=evil"), + "dash-payload", + "dash-leading file", + ) +} diff --git a/e2e/suites/conformance/cli/tests/file_transfer/round_trip.rs b/e2e/suites/conformance/cli/tests/file_transfer/round_trip.rs new file mode 100644 index 0000000000..74046fe5b2 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/file_transfer/round_trip.rs @@ -0,0 +1,266 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + LARGE_FILE_SIZE, delete_sandbox, download, exec, fs_error, prepare_sandbox, require_text, + upload, upload_result, +}; +use openshell_conformance::OpenShellRunner; +use std::fs; +use std::path::Path; + +/// Verify file and directory upload and download round trips. +#[tokio::test] +async fn transfers_files_and_directories() { + let mut runner = OpenShellRunner::from_env("file-transfer/round-trip") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let (sandbox_name, remote_root, local) = prepare_sandbox(runner, "round-trip").await?; + upload_existing_directories(runner, &sandbox_name, &remote_root, local.path()).await?; + round_trip(runner, &sandbox_name, &remote_root, local.path()).await?; + download_file(runner, &sandbox_name, &remote_root, local.path()).await?; + download_directory(runner, &sandbox_name, &remote_root, local.path()).await?; + delete_sandbox(runner, &sandbox_name).await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("file-transfer/round-trip conformance story failed:\n{error}"); + } +} + +async fn round_trip( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let source = local_root.join("roundtrip-upload"); + fs::create_dir_all(source.join("subdir")).map_err(fs_error("create round-trip source"))?; + fs::write(source.join("greeting.txt"), "hello-from-local") + .map_err(fs_error("write greeting.txt"))?; + fs::write(source.join("subdir/nested.txt"), "nested-content") + .map_err(fs_error("write nested.txt"))?; + + let large = (0u8..=250) + .cycle() + .take(LARGE_FILE_SIZE) + .collect::>(); + fs::write(source.join("large.bin"), &large).map_err(fs_error("write large.bin"))?; + + let remote = format!("{remote_root}/roundtrip"); + let result = + upload_result(runner, sandbox, "roundtrip/upload", &source, &remote, false).await?; + result.require_success()?; + if !result.stderr().contains("outside a Git work tree") + || !result.stderr().contains(".gitignore rules are not applied") + { + return Err( + result.failure_diagnostic("upload outside Git warns that filtering is disabled") + ); + } + + let destination = local_root.join("roundtrip-download"); + fs::create_dir(&destination).map_err(fs_error("create round-trip destination"))?; + let remote_source = format!("{remote}/roundtrip-upload"); + download( + runner, + sandbox, + "roundtrip/download", + &remote_source, + &destination, + ) + .await?; + + require_text( + &destination.join("greeting.txt"), + "hello-from-local", + "round-trip greeting", + )?; + require_text( + &destination.join("subdir/nested.txt"), + "nested-content", + "round-trip nested file", + )?; + let actual = fs::read(destination.join("large.bin")).map_err(fs_error("read large.bin"))?; + if actual != large { + return Err(format!( + "large file changed during round trip: expected {} bytes, received {} bytes", + large.len(), + actual.len() + )); + } + + let single = local_root.join("single.txt"); + fs::write(&single, "single-file-payload").map_err(fs_error("write single.txt"))?; + let remote_single = format!("{remote_root}/single.txt"); + upload( + runner, + sandbox, + "single/upload", + &single, + &remote_single, + false, + ) + .await?; + let single_destination = local_root.join("single-download"); + fs::create_dir(&single_destination).map_err(fs_error("create single-file destination"))?; + download( + runner, + sandbox, + "single/download", + &remote_single, + &single_destination, + ) + .await?; + require_text( + &single_destination.join("single.txt"), + "single-file-payload", + "single-file round trip", + ) +} + +async fn download_file( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let remote = format!("{remote_root}/download-file.txt"); + exec( + runner, + sandbox, + "download-file/seed", + &format!("printf greeting-payload > '{remote}'"), + ) + .await?; + let destination = local_root.join("download-file"); + fs::create_dir(&destination).map_err(fs_error("create file-download destination"))?; + download( + runner, + sandbox, + "download-file/download", + &remote, + &destination, + ) + .await?; + require_text( + &destination.join("download-file.txt"), + "greeting-payload", + "downloaded file", + ) +} + +async fn download_directory( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let remote = format!("{remote_root}/tree"); + exec( + runner, + sandbox, + "download-directory/seed", + &format!( + "mkdir -p '{remote}/sub' && printf top-level > '{remote}/root.txt' && printf nested > '{remote}/sub/child.txt'" + ), + ) + .await?; + let destination = local_root.join("download-directory"); + fs::create_dir(&destination).map_err(fs_error("create directory-download destination"))?; + download( + runner, + sandbox, + "download-directory/download", + &remote, + &destination, + ) + .await?; + require_text( + &destination.join("root.txt"), + "top-level", + "downloaded directory root file", + )?; + require_text( + &destination.join("sub/child.txt"), + "nested", + "downloaded directory nested file", + ) +} + +/// Check remote destination types, not just successful transfer status: tar +/// can report success after replacing an empty directory with a regular file. +async fn upload_existing_directories( + runner: &OpenShellRunner, + sandbox: &str, + remote_root: &str, + local_root: &Path, +) -> Result<(), String> { + let source = local_root.join("config.json"); + fs::write(&source, "upload-destination-payload") + .map_err(fs_error("write destination fixture"))?; + let root = format!("{remote_root}/upload destinations"); + exec( + runner, + sandbox, + "upload-destinations/prepare", + &format!( + "mkdir -p '{root}/empty' '{root}/populated' '{root}/target' '{root}/slash' && \ + printf keep > '{root}/populated/keep.txt' && \ + ln -s target '{root}/link'" + ), + ) + .await?; + + for directory in ["empty", "populated", "link", "slash/"] { + let destination = format!("{root}/{directory}"); + upload( + runner, + sandbox, + &format!("upload-destinations/{directory}/upload"), + &source, + &destination, + true, + ) + .await?; + exec( + runner, + sandbox, + &format!("upload-destinations/{directory}/verify"), + &format!( + "test -d '{destination}' && \ + test \"$(cat '{destination}/config.json')\" = upload-destination-payload && \ + test \"$(cat '{root}/populated/keep.txt')\" = keep && \ + test -L '{root}/link'" + ), + ) + .await?; + } + + // A missing destination remains an intentional rename; uploading again + // must replace that file rather than treating it as a directory. + let renamed = format!("{root}/renamed.json"); + for pass in ["rename", "overwrite"] { + upload( + runner, + sandbox, + &format!("upload-destinations/{pass}/upload"), + &source, + &renamed, + true, + ) + .await?; + exec( + runner, + sandbox, + &format!("upload-destinations/{pass}/verify"), + &format!( + "test -f '{renamed}' && test \"$(cat '{renamed}')\" = upload-destination-payload" + ), + ) + .await?; + } + Ok(()) +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle/delete_stopped.rs b/e2e/suites/conformance/cli/tests/lifecycle/delete_stopped.rs new file mode 100644 index 0000000000..22f0c531b7 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/delete_stopped.rs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + create_running_sandbox, run_lifecycle_command, wait_for_absence, wait_for_phase, +}; +use openshell_conformance::OpenShellRunner; + +/// Verify a stopped sandbox can be deleted. +#[tokio::test] +async fn deletes_a_stopped_sandbox() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/delete-stopped") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-sd", runner.id()); + create_running_sandbox( + runner, + &sandbox_name, + "exec sleep infinity", + "stopped-delete", + ) + .await?; + + run_lifecycle_command(runner, "stop", &sandbox_name, "stopped-delete/stop").await?; + let sandbox = + wait_for_phase(runner, &sandbox_name, "Stopped", "stopped-delete/stopped").await?; + run_lifecycle_command(runner, "delete", &sandbox_name, "stopped-delete/delete").await?; + wait_for_absence(runner, &sandbox.id, &sandbox_name, "stopped-delete/deleted").await?; + runner.forget_sandbox(&sandbox_name); + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/delete-stopped conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/lifecycle.rs b/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs similarity index 62% rename from e2e/suites/conformance/cli/tests/lifecycle.rs rename to e2e/suites/conformance/cli/tests/lifecycle/helpers.rs index 5e8350b3b1..41987074a0 100644 --- a/e2e/suites/conformance/cli/tests/lifecycle.rs +++ b/e2e/suites/conformance/cli/tests/lifecycle/helpers.rs @@ -1,144 +1,32 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Portable sandbox lifecycle conformance scenarios. - -use std::collections::HashSet; -use std::time::{Duration, Instant}; - +use openshell_conformance::OpenShellRunner; +use openshell_conformance::Poll; use serde::Deserialize; +use std::collections::HashSet; +use std::time::Duration; +use std::time::Instant; -use openshell_conformance::{OpenShellRunner, Poll}; - -const CREATE_TIMEOUT: Duration = Duration::from_mins(10); -const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); -const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4); -const TRANSITION_INTERVAL: Duration = Duration::from_secs(2); +pub const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +pub const COMMAND_TIMEOUT: Duration = Duration::from_mins(2); +pub const TRANSITION_TIMEOUT: Duration = Duration::from_mins(4); +pub const TRANSITION_INTERVAL: Duration = Duration::from_secs(2); #[derive(Debug, Deserialize)] -struct SandboxState { - id: String, - name: String, - phase: String, +pub struct SandboxState { + pub id: String, + pub name: String, + pub phase: String, } #[derive(Debug, Deserialize)] -struct SandboxListPage { - sandboxes: Vec, - next_page_token: String, -} - -/// Certify sandbox stop, start, and deletion lifecycle behavior. - -#[tokio::test] -async fn sandbox_lifecycle() { - let mut runner = OpenShellRunner::from_env("sandbox-lifecycle") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - stop_start_preserves_workspace(runner).await?; - stopped_can_be_deleted(runner).await - } - .await; - if let Err(error) = runner.finish(result).await { - panic!("sandbox-lifecycle conformance scenario failed:\n{error}"); - } -} - -async fn stop_start_preserves_workspace(runner: &mut OpenShellRunner) -> Result<(), String> { - let sandbox_name = format!("ct-{}-ss", runner.id()); - let sentinel = format!("openshell-stop-start-{}", runner.id()); - let sentinel_path = "/sandbox/.openshell-stop-start-sentinel"; - let run_count_path = "/sandbox/.openshell-main-run-count"; - let main = format!( - "count=0; test ! -f '{run_count_path}' || count=$(cat '{run_count_path}'); \ - count=$((count + 1)); printf '%s\\n' \"$count\" > '{run_count_path}'; \ - exec sleep infinity" - ); - - create_running_sandbox(runner, &sandbox_name, &main, "stop-start").await?; - exec_expect_exact( - runner, - &sandbox_name, - "write-sentinel", - &[ - "sh", - "-lc", - &format!("printf '%s\\n' '{sentinel}' > '{sentinel_path}' && sync"), - ], - "", - ) - .await?; - - run_lifecycle_command(runner, "stop", &sandbox_name, "stop-start/stop").await?; - wait_for_phase(runner, &sandbox_name, "Stopped", "stop-start/stopped").await?; - - let stopped_exec = runner - .step("stop-start/exec-while-stopped") - .description(format!( - "sandbox '{sandbox_name}' rejects exec while stopped" - )) - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - &sandbox_name, - "--no-tty", - "--", - "cat", - sentinel_path, - ]) - .await - .map_err(|error| error.to_string())?; - if stopped_exec.success() { - return Err( - stopped_exec.failure_diagnostic("sandbox exec fails while the sandbox is stopped") - ); - } - - run_lifecycle_command(runner, "start", &sandbox_name, "stop-start/start").await?; - wait_for_phase(runner, &sandbox_name, "Ready", "stop-start/restarted").await?; - - exec_expect_exact( - runner, - &sandbox_name, - "read-sentinel", - &["cat", sentinel_path], - &format!("{sentinel}\n"), - ) - .await?; - exec_expect_exact( - runner, - &sandbox_name, - "read-main-run-count", - &["cat", run_count_path], - "2\n", - ) - .await -} - -async fn stopped_can_be_deleted(runner: &mut OpenShellRunner) -> Result<(), String> { - let sandbox_name = format!("ct-{}-sd", runner.id()); - create_running_sandbox( - runner, - &sandbox_name, - "exec sleep infinity", - "stopped-delete", - ) - .await?; - - run_lifecycle_command(runner, "stop", &sandbox_name, "stopped-delete/stop").await?; - let sandbox = - wait_for_phase(runner, &sandbox_name, "Stopped", "stopped-delete/stopped").await?; - run_lifecycle_command(runner, "delete", &sandbox_name, "stopped-delete/delete").await?; - wait_for_absence(runner, &sandbox.id, &sandbox_name, "stopped-delete/deleted").await?; - runner.forget_sandbox(&sandbox_name); - Ok(()) +pub struct SandboxListPage { + pub sandboxes: Vec, + pub next_page_token: String, } -async fn create_running_sandbox( +pub async fn create_running_sandbox( runner: &mut OpenShellRunner, sandbox_name: &str, main: &str, @@ -169,7 +57,7 @@ async fn create_running_sandbox( .map(|_| ()) } -async fn run_lifecycle_command( +pub async fn run_lifecycle_command( runner: &OpenShellRunner, operation: &str, sandbox_name: &str, @@ -185,7 +73,7 @@ async fn run_lifecycle_command( result.require_success() } -async fn exec_expect_exact( +pub async fn exec_expect_exact( runner: &OpenShellRunner, sandbox_name: &str, step: &str, @@ -209,7 +97,7 @@ async fn exec_expect_exact( } } -async fn wait_for_phase( +pub async fn wait_for_phase( runner: &mut OpenShellRunner, sandbox_name: &str, expected_phase: &str, @@ -259,7 +147,7 @@ async fn wait_for_phase( .map_err(|error| error.to_string()) } -async fn wait_for_absence( +pub async fn wait_for_absence( runner: &mut OpenShellRunner, sandbox_id: &str, sandbox_name: &str, @@ -288,7 +176,7 @@ async fn wait_for_absence( .map_err(|error| error.to_string()) } -async fn sandbox_is_listed( +pub async fn sandbox_is_listed( runner: &OpenShellRunner, sandbox_id: &str, sandbox_name: &str, diff --git a/e2e/suites/conformance/cli/tests/lifecycle/main.rs b/e2e/suites/conformance/cli/tests/lifecycle/main.rs new file mode 100644 index 0000000000..a7958f54fb --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/main.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Lifecycle conformance stories. + +mod delete_stopped; +mod helpers; +mod stop_start; diff --git a/e2e/suites/conformance/cli/tests/lifecycle/stop_start.rs b/e2e/suites/conformance/cli/tests/lifecycle/stop_start.rs new file mode 100644 index 0000000000..03773f1894 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/lifecycle/stop_start.rs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + COMMAND_TIMEOUT, create_running_sandbox, exec_expect_exact, run_lifecycle_command, + wait_for_phase, +}; +use openshell_conformance::OpenShellRunner; + +/// Verify stop/start preserves the workspace and restarts the main process. +#[tokio::test] +async fn preserves_workspace_and_restarts_main_process() { + let mut runner = OpenShellRunner::from_env("sandbox-lifecycle/stop-start") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let sandbox_name = format!("ct-{}-ss", runner.id()); + let sentinel = format!("openshell-stop-start-{}", runner.id()); + let sentinel_path = "/sandbox/.openshell-stop-start-sentinel"; + let run_count_path = "/sandbox/.openshell-main-run-count"; + let main = format!( + "count=0; test ! -f '{run_count_path}' || count=$(cat '{run_count_path}'); \ + count=$((count + 1)); printf '%s\\n' \"$count\" > '{run_count_path}'; \ + exec sleep infinity" + ); + + create_running_sandbox(runner, &sandbox_name, &main, "stop-start").await?; + exec_expect_exact( + runner, + &sandbox_name, + "write-sentinel", + &[ + "sh", + "-lc", + &format!("printf '%s\\n' '{sentinel}' > '{sentinel_path}' && sync"), + ], + "", + ) + .await?; + + run_lifecycle_command(runner, "stop", &sandbox_name, "stop-start/stop").await?; + wait_for_phase(runner, &sandbox_name, "Stopped", "stop-start/stopped").await?; + + let stopped_exec = runner + .step("stop-start/exec-while-stopped") + .description(format!( + "sandbox '{sandbox_name}' rejects exec while stopped" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &sandbox_name, + "--no-tty", + "--", + "cat", + sentinel_path, + ]) + .await + .map_err(|error| error.to_string())?; + if stopped_exec.success() { + return Err( + stopped_exec.failure_diagnostic("sandbox exec fails while the sandbox is stopped") + ); + } + + run_lifecycle_command(runner, "start", &sandbox_name, "stop-start/start").await?; + wait_for_phase(runner, &sandbox_name, "Ready", "stop-start/restarted").await?; + + exec_expect_exact( + runner, + &sandbox_name, + "read-sentinel", + &["cat", sentinel_path], + &format!("{sentinel}\n"), + ) + .await?; + exec_expect_exact( + runner, + &sandbox_name, + "read-main-run-count", + &["cat", run_count_path], + "2\n", + ) + .await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("sandbox-lifecycle/stop-start conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor.rs b/e2e/suites/conformance/cli/tests/policy_advisor.rs deleted file mode 100644 index 3fbd1bb997..0000000000 --- a/e2e/suites/conformance/cli/tests/policy_advisor.rs +++ /dev/null @@ -1,591 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Portable policy-local and mechanistic proposal checks. - -use std::io::Write as _; -use std::time::{Duration, Instant}; - -use serde::Deserialize; -use serde_json::Value; -use tempfile::NamedTempFile; -use tokio::time::sleep; - -use openshell_conformance::OpenShellRunner; - -const CREATE_TIMEOUT: Duration = Duration::from_mins(10); -const COMMAND_TIMEOUT: Duration = Duration::from_secs(45); -const READY_TIMEOUT: Duration = Duration::from_mins(4); -const POLL_INTERVAL: Duration = Duration::from_secs(2); -const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90); - -#[derive(Deserialize)] -struct SandboxState { - name: String, - phase: String, -} - -const EMPTY_NETWORK_POLICY: &[u8] = br"version: 1 -filesystem_policy: - include_workdir: true - read_only: [/usr, /bin, /lib, /lib64, /proc, /dev/urandom, /app, /etc, /var/log] - read_write: [/sandbox, /tmp, /dev/null] -landlock: { compatibility: best_effort } -network_policies: {} -"; - -#[tokio::test] -async fn policy_local() { - let mut runner = - OpenShellRunner::from_env("policy-local").expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let name = format!("ct-{}-pl", runner.id()); - create_sandbox(runner, &name, None).await?; - enable_proposals(runner, &name).await?; - let binary = sandbox_bash_path(runner, &name).await?; - - let started = Instant::now(); - let readiness_path = format!("/v1/proposals/ct-{}-readiness", runner.id()); - loop { - match request_policy_local(runner, &name, "/v1/policy/current").await { - Ok(response) - if response["format"] == "yaml" - && response["policy_yaml"] - .as_str() - .is_some_and(|yaml| yaml.contains("version: 1")) => - { - // The current-policy route is local; proposal submission also - // needs the supervisor's workspace and gateway lookup session. - match request_policy_local_http(runner, &name, "GET", &readiness_path, "", 404) - .await - { - Ok(lookup) if lookup["error"] == "chunk_not_found" => break, - Ok(lookup) => { - return Err(format!( - "policy.local proposal lookup returned an invalid readiness response: {lookup}" - )); - } - Err(error) if started.elapsed() >= READY_TIMEOUT => return Err(error), - Err(_) => {} - } - } - Ok(response) => { - return Err(format!( - "policy.local returned an invalid current policy: {response}" - )); - } - Err(error) => { - if started.elapsed() >= READY_TIMEOUT { - return Err(error); - } - } - } - sleep(POLL_INTERVAL).await; - } - let denials = request_policy_local(runner, &name, "/v1/denials?last=1").await?; - if !denials["denials"].is_array() || !denials["log_available"].is_boolean() { - return Err(format!( - "policy.local returned an invalid denials response: {denials}" - )); - } - - let rule_name = format!("conformance_local_{}", runner.id()); - let payload = serde_json::json!({ - "intent_summary": "Allow Bash to read the conformance path on example.invalid.", - "operations": [{ - "addRule": { - "ruleName": &rule_name, - "rule": { - "name": &rule_name, - "endpoints": [{ - "host": "example.invalid", - "port": 443, - "protocol": "rest", - "enforcement": "enforce", - "rules": [{"allow": {"method": "GET", "path": "/conformance"}}] - }], - "binaries": [{"path": &binary}] - } - } - }] - }) - .to_string(); - let submitted = - request_policy_local_http(runner, &name, "POST", "/v1/proposals", &payload, 202) - .await?; - let chunk_id = submitted["accepted_chunk_ids"] - .as_array() - .filter(|ids| ids.len() == 1) - .and_then(|ids| ids[0].as_str()) - .filter(|id| !id.is_empty()) - .ok_or_else(|| format!("policy.local did not accept one proposal: {submitted}"))?; - if submitted["status"] != "submitted" - || submitted["accepted_chunks"] != 1 - || submitted["rejected_chunks"] != 0 - { - return Err(format!("policy.local did not submit one rule: {submitted}")); - } - - let state = - request_policy_local(runner, &name, &format!("/v1/proposals/{chunk_id}")).await?; - if state["chunk_id"] != chunk_id - || state["rule_name"] != rule_name - || state["binary"] != binary - || !matches!(state["status"].as_str(), Some("pending" | "approved")) - { - return Err(format!("policy.local returned the wrong proposal: {state}")); - } - - let review = runner - .step("reviewer-inbox") - .description("the requested rule is visible to the reviewer") - .with_timeout(COMMAND_TIMEOUT) - .run(&["rule", "get", &name]) - .await - .map_err(|error| error.to_string())?; - review.require_success()?; - if !review.stdout().contains(&format!("Chunk: {chunk_id}")) - || !review.stdout().contains(&format!("Rule: {rule_name}")) - { - return Err(review.failure_diagnostic("the submitted rule is in the reviewer inbox")); - } - Ok(()) - - }.await; - if let Err(error) = runner.finish(result).await { - panic!("policy-local conformance scenario failed:\n{error}"); - } -} - -async fn sandbox_bash_path(runner: &OpenShellRunner, name: &str) -> Result { - let result = runner - .step("bash-binary") - .description("the sandbox's Bash executable has a canonical path") - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - name, - "--no-tty", - "--", - "bash", - "-c", - "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"", - ]) - .await - .map_err(|error| error.to_string())?; - result.require_success()?; - let binary = result.stdout().trim(); - if !binary.starts_with('/') - || binary.contains('\n') - || binary.rsplit('/').next() != Some("bash") - { - return Err(result.failure_diagnostic("one absolute Bash executable path ending in /bash")); - } - Ok(binary.to_string()) -} - -async fn enable_proposals(runner: &OpenShellRunner, name: &str) -> Result<(), String> { - let set = runner - .step("enable-policy-advisor") - .description("sandbox-scoped policy advisor setting is enabled") - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "settings", - "set", - name, - "--key", - "agent_policy_proposals_enabled", - "--value", - "true", - ]) - .await - .map_err(|error| error.to_string())?; - set.require_success()?; - - let settings = runner - .step("effective-settings") - .description("policy advisor setting is effectively enabled") - .with_timeout(COMMAND_TIMEOUT) - .run(&["settings", "get", name, "--json"]) - .await - .map_err(|error| error.to_string())?; - settings.require_success()?; - let value: Value = settings.json().map_err(|error| error.to_string())?; - if value["settings"]["agent_policy_proposals_enabled"]["value"] != "true" { - return Err(settings.failure_diagnostic( - "effective agent_policy_proposals_enabled is true; check for a global override", - )); - } - Ok(()) -} - -async fn request_policy_local( - runner: &OpenShellRunner, - sandbox: &str, - path: &str, -) -> Result { - request_policy_local_http(runner, sandbox, "GET", path, "", 200).await -} - -async fn request_policy_local_http( - runner: &OpenShellRunner, - sandbox: &str, - method: &str, - path: &str, - body: &str, - expected_status: u16, -) -> Result { - let script = "method=$1; path=$2; body=$3; exec 3<>/dev/tcp/policy.local/80 || exit 1; printf '%s %s HTTP/1.1\\r\\nHost: policy.local\\r\\nContent-Type: application/json\\r\\nContent-Length: %s\\r\\nConnection: close\\r\\n\\r\\n' \"$method\" \"$path\" \"${#body}\" >&3; printf '%s' \"$body\" >&3; cat <&3"; - let result = runner - .step(format!("policy-local-{method}{path}")) - .description(format!( - "{method} http://policy.local{path} succeeds from the sandbox" - )) - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - sandbox, - "--no-tty", - "--", - "bash", - "-c", - script, - "policy-local-http", - method, - path, - body, - ]) - .await - .map_err(|error| error.to_string())?; - result.require_success()?; - let (headers, body) = result.stdout().split_once("\r\n\r\n").ok_or_else(|| { - result.failure_diagnostic("a complete HTTP response with headers and JSON body") - })?; - if !headers.starts_with(&format!("HTTP/1.1 {expected_status} ")) - || !headers.lines().any(|line| { - line.to_ascii_lowercase() - .starts_with("content-type: application/json") - }) - { - return Err( - result.failure_diagnostic(&format!("HTTP {expected_status} with JSON Content-Type")) - ); - } - serde_json::from_str(body) - .map_err(|error| result.failure_diagnostic(&format!("valid JSON body: {error}"))) -} - -#[tokio::test] -async fn mechanistic_proposal() { - let mut runner = OpenShellRunner::from_env("mechanistic-proposal") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let mut policy = NamedTempFile::new().map_err(|error| error.to_string())?; - policy - .write_all(EMPTY_NETWORK_POLICY) - .map_err(|error| error.to_string())?; - let policy_path = policy - .path() - .to_str() - .ok_or("temporary policy path is not UTF-8")?; - let name = format!("ct-{}-mp", runner.id()); - create_sandbox(runner, &name, Some(policy_path)).await?; - enable_proposals(runner, &name).await?; - - let effective = runner - .step("effective-policy") - .description("sandbox has no network allow rules before the probe") - .with_timeout(COMMAND_TIMEOUT) - .run(&["policy", "get", &name, "--full", "--output", "json"]) - .await - .map_err(|error| error.to_string())?; - effective.require_success()?; - let value: Value = effective.json().map_err(|error| error.to_string())?; - let network_rules = &value["policy"]["network_policies"]; - if !network_rules.is_null() - && !network_rules - .as_object() - .is_some_and(serde_json::Map::is_empty) - { - return Err(effective.failure_diagnostic("effective network_policies is empty")); - } - - let probe = runner - .step("denied-tcp-open") - .description("one Bash TCP open to 1.1.1.1:443 is denied by policy") - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - &name, - "--no-tty", - "--", - "bash", - "-c", - "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", - ]) - .await - .map_err(|error| error.to_string())?; - probe.require_success()?; - let binary = probe - .stdout() - .lines() - .find_map(|line| line.strip_prefix("BINARY=")) - .filter(|binary| binary.starts_with('/') && binary.rsplit('/').next() == Some("bash")) - .ok_or_else(|| probe.failure_diagnostic("canonical Bash executable path is reported"))? - .to_string(); - if !probe.stdout().lines().any(|line| line == "DENIED") { - return Err(probe.failure_diagnostic("TCP open is denied before any upstream dial")); - } - - await_mechanistic_draft( - runner, - &name, - &ExpectedDraft { - rule: "allow_1_1_1_1_443", - endpoint: "1.1.1.1:443", - binary: &binary, - }, - probe.stderr(), - ) - .await - - }.await; - if let Err(error) = runner.finish(result).await { - panic!("mechanistic-proposal conformance scenario failed:\n{error}"); - } -} - -#[tokio::test] -async fn new_hostname_proposal() { - let mut runner = OpenShellRunner::from_env("new-hostname-proposal") - .expect("candidate openshell CLI is available"); - let result = async { - runner.check_gateway_status().await?; - let runner = &mut runner; - let mut policy = NamedTempFile::new().map_err(|error| error.to_string())?; - policy - .write_all(EMPTY_NETWORK_POLICY) - .map_err(|error| error.to_string())?; - let policy_path = policy - .path() - .to_str() - .ok_or("temporary policy path is not UTF-8")?; - let name = format!("ct-{}-nh", runner.id()); - create_sandbox(runner, &name, Some(policy_path)).await?; - let binary = sandbox_bash_path(runner, &name).await?; - - let probe = runner - .step("denied-new-hostname") - .description("Bash cannot connect to pypi.org:80 before approval") - .with_timeout(COMMAND_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - &name, - "--no-tty", - "--", - "bash", - "-c", - "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", - ]) - .await - .map_err(|error| error.to_string())?; - probe.require_success()?; - if !probe.stdout().lines().any(|line| line == "DENIED") { - return Err(probe.failure_diagnostic("new hostname stays denied before approval")); - } - - await_mechanistic_draft( - runner, - &name, - &ExpectedDraft { - rule: "allow_pypi_org_80", - endpoint: "pypi.org:80", - binary: &binary, - }, - probe.stderr(), - ) - .await - - }.await; - if let Err(error) = runner.finish(result).await { - panic!("new-hostname-proposal conformance scenario failed:\n{error}"); - } -} - -/// The single L4 mechanistic draft expected for one denied endpoint. -struct ExpectedDraft<'a> { - rule: &'a str, - endpoint: &'a str, - binary: &'a str, -} - -/// Poll the reviewer inbox until a draft appears, tolerating transient read -/// failures, then require that it is the single expected draft. -async fn await_mechanistic_draft( - runner: &OpenShellRunner, - sandbox: &str, - expected: &ExpectedDraft<'_>, - probe_stderr: &str, -) -> Result<(), String> { - let started = Instant::now(); - loop { - let draft = runner - .step("mechanistic-draft") - .description("a single scoped mechanistic draft appears") - .with_timeout(COMMAND_TIMEOUT) - .run(&["rule", "get", sandbox]) - .await - .map_err(|error| error.to_string())?; - if !draft.success() { - if started.elapsed() >= PROPOSAL_TIMEOUT { - return Err(draft.failure_diagnostic("the reviewer inbox is readable")); - } - sleep(POLL_INTERVAL).await; - continue; - } - if !draft.stdout().contains("Chunk:") { - if started.elapsed() >= PROPOSAL_TIMEOUT { - return Err(draft.failure_diagnostic(&format!( - "one mechanistic draft for {} and {}; probe stderr:\n{probe_stderr}", - expected.endpoint, expected.binary - ))); - } - sleep(POLL_INTERVAL).await; - continue; - } - return assert_mechanistic_draft(draft.stdout(), expected) - .map_err(|error| draft.failure_diagnostic(&error)); - } -} - -fn assert_mechanistic_draft(output: &str, expected: &ExpectedDraft<'_>) -> Result<(), String> { - let fields = output.lines().map(str::trim).collect::>(); - let field = |name: &str| { - fields - .iter() - .find_map(|line| line.strip_prefix(name).map(str::trim)) - }; - let endpoints = format!("{} [L4]", expected.endpoint); - if fields - .iter() - .filter(|line| line.starts_with("Chunk:")) - .count() - != 1 - || !matches!(field("Status:"), Some("pending" | "approved")) - || field("Rule:") != Some(expected.rule) - || field("Binary:") != Some(expected.binary) - || field("Binaries:") != Some(expected.binary) - || field("Endpoints:") != Some(endpoints.as_str()) - || !field("Rationale:").is_some_and(|value| value.contains(expected.endpoint)) - { - return Err(format!( - "expected one pending or approved L4 mechanistic draft scoped to {} and {}", - expected.binary, expected.endpoint - )); - } - Ok(()) -} - -async fn create_sandbox( - runner: &mut OpenShellRunner, - name: &str, - policy_path: Option<&str>, -) -> Result<(), String> { - runner.track_sandbox(name); - let mut args = vec![ - "sandbox", - "create", - "--name", - name, - "--detach", - "--no-tty", - "--no-auto-providers", - ]; - if let Some(path) = policy_path { - args.extend(["--policy", path]); - } - args.extend(["--", "sh", "-c", "exec sleep infinity"]); - let create = runner - .step("create") - .description(format!("sandbox '{name}' is created")) - .with_timeout(CREATE_TIMEOUT) - .run(&args) - .await - .map_err(|error| error.to_string())?; - create.require_success()?; - - let started = Instant::now(); - loop { - let get = runner - .step("ready") - .description(format!("sandbox '{name}' reaches Ready")) - .with_timeout(COMMAND_TIMEOUT) - .run(&["sandbox", "get", name, "--output", "json"]) - .await - .map_err(|error| error.to_string())?; - if get.success() { - let state: SandboxState = get.json().map_err(|error| error.to_string())?; - if state.name != name { - return Err(get.failure_diagnostic("sandbox get returns the created name")); - } - if state.phase == "Ready" { - return Ok(()); - } - if state.phase == "Failed" { - return Err(get.failure_diagnostic("sandbox reaches Ready instead of Failed")); - } - } - if started.elapsed() >= READY_TIMEOUT { - return Err(get.failure_diagnostic("sandbox reaches Ready before timeout")); - } - sleep(POLL_INTERVAL).await; - } -} - -#[cfg(test)] -mod tests { - use super::{ExpectedDraft, assert_mechanistic_draft}; - - const EXPECTED: ExpectedDraft<'static> = ExpectedDraft { - rule: "allow_pypi_org_80", - endpoint: "pypi.org:80", - binary: "/usr/bin/bash", - }; - - fn draft(binary: &str) -> String { - format!( - "Chunk: id\nStatus: pending\nRule: allow_pypi_org_80\nBinary: {binary}\nRationale: Allow {binary} to connect to pypi.org:80 (HTTP).\nEndpoints: pypi.org:80 [L4]\nBinaries: {binary}\n" - ) - } - - #[test] - fn draft_assertion_accepts_a_hostname_scoped_draft() { - assert!(assert_mechanistic_draft(&draft("/usr/bin/bash"), &EXPECTED).is_ok()); - } - - #[test] - fn draft_assertion_rejects_unrelated_binary() { - assert!(assert_mechanistic_draft(&draft("/usr/bin/sh"), &EXPECTED).is_err()); - } - - #[test] - fn draft_assertion_rejects_fields_spread_across_drafts() { - let drafts = format!( - "{}Chunk: other\nStatus: pending\nRule: allow_1_1_1_1_443\n", - draft("/usr/bin/bash") - ); - assert!(assert_mechanistic_draft(&drafts, &EXPECTED).is_err()); - } -} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/draft_assertion.rs b/e2e/suites/conformance/cli/tests/policy_advisor/draft_assertion.rs new file mode 100644 index 0000000000..b680f0744d --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/draft_assertion.rs @@ -0,0 +1,74 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Pure assertions for a single scoped mechanistic policy draft. + +pub struct ExpectedDraft<'a> { + pub rule: &'a str, + pub endpoint: &'a str, + pub binary: &'a str, +} + +pub fn assert_mechanistic_draft(output: &str, expected: &ExpectedDraft<'_>) -> Result<(), String> { + let fields = output.lines().map(str::trim).collect::>(); + let field = |name: &str| { + fields + .iter() + .find_map(|line| line.strip_prefix(name).map(str::trim)) + }; + let endpoints = format!("{} [L4]", expected.endpoint); + if fields + .iter() + .filter(|line| line.starts_with("Chunk:")) + .count() + != 1 + || !matches!(field("Status:"), Some("pending" | "approved")) + || field("Rule:") != Some(expected.rule) + || field("Binary:") != Some(expected.binary) + || field("Binaries:") != Some(expected.binary) + || field("Endpoints:") != Some(endpoints.as_str()) + || !field("Rationale:").is_some_and(|value| value.contains(expected.endpoint)) + { + return Err(format!( + "expected one pending or approved L4 mechanistic draft scoped to {} and {}", + expected.binary, expected.endpoint + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ExpectedDraft, assert_mechanistic_draft}; + + const EXPECTED: ExpectedDraft<'static> = ExpectedDraft { + rule: "allow_pypi_org_80", + endpoint: "pypi.org:80", + binary: "/usr/bin/bash", + }; + + fn draft(binary: &str) -> String { + format!( + "Chunk: id\nStatus: pending\nRule: allow_pypi_org_80\nBinary: {binary}\nRationale: Allow {binary} to connect to pypi.org:80 (HTTP).\nEndpoints: pypi.org:80 [L4]\nBinaries: {binary}\n" + ) + } + + #[test] + fn draft_assertion_accepts_a_hostname_scoped_draft() { + assert!(assert_mechanistic_draft(&draft("/usr/bin/bash"), &EXPECTED).is_ok()); + } + + #[test] + fn draft_assertion_rejects_unrelated_binary() { + assert!(assert_mechanistic_draft(&draft("/usr/bin/sh"), &EXPECTED).is_err()); + } + + #[test] + fn draft_assertion_rejects_fields_spread_across_drafts() { + let drafts = format!( + "{}Chunk: other\nStatus: pending\nRule: allow_1_1_1_1_443\n", + draft("/usr/bin/bash") + ); + assert!(assert_mechanistic_draft(&drafts, &EXPECTED).is_err()); + } +} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/helpers.rs b/e2e/suites/conformance/cli/tests/policy_advisor/helpers.rs new file mode 100644 index 0000000000..52f9757e76 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/helpers.rs @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::draft_assertion::{ExpectedDraft, assert_mechanistic_draft}; +use openshell_conformance::OpenShellRunner; +use serde::Deserialize; +use serde_json::Value; +use std::time::Duration; +use std::time::Instant; +use tokio::time::sleep; + +pub const CREATE_TIMEOUT: Duration = Duration::from_mins(10); +pub const COMMAND_TIMEOUT: Duration = Duration::from_secs(45); +pub const READY_TIMEOUT: Duration = Duration::from_mins(4); +pub const POLL_INTERVAL: Duration = Duration::from_secs(2); +pub const PROPOSAL_TIMEOUT: Duration = Duration::from_secs(90); + +#[derive(Deserialize)] +pub struct SandboxState { + pub name: String, + pub phase: String, +} + +pub const EMPTY_NETWORK_POLICY: &[u8] = br"version: 1 +filesystem_policy: + include_workdir: true + read_only: [/usr, /bin, /lib, /lib64, /proc, /dev/urandom, /app, /etc, /var/log] + read_write: [/sandbox, /tmp, /dev/null] +landlock: { compatibility: best_effort } +network_policies: {} +"; + +pub async fn sandbox_bash_path(runner: &OpenShellRunner, name: &str) -> Result { + let result = runner + .step("bash-binary") + .description("the sandbox's Bash executable has a canonical path") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + name, + "--no-tty", + "--", + "bash", + "-c", + "printf '%s\\n' \"$(readlink -f /proc/$$/exe)\"", + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + let binary = result.stdout().trim(); + if !binary.starts_with('/') + || binary.contains('\n') + || binary.rsplit('/').next() != Some("bash") + { + return Err(result.failure_diagnostic("one absolute Bash executable path ending in /bash")); + } + Ok(binary.to_string()) +} + +pub async fn enable_proposals(runner: &OpenShellRunner, name: &str) -> Result<(), String> { + let set = runner + .step("enable-policy-advisor") + .description("sandbox-scoped policy advisor setting is enabled") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "settings", + "set", + name, + "--key", + "agent_policy_proposals_enabled", + "--value", + "true", + ]) + .await + .map_err(|error| error.to_string())?; + set.require_success()?; + + let settings = runner + .step("effective-settings") + .description("policy advisor setting is effectively enabled") + .with_timeout(COMMAND_TIMEOUT) + .run(&["settings", "get", name, "--json"]) + .await + .map_err(|error| error.to_string())?; + settings.require_success()?; + let value: Value = settings.json().map_err(|error| error.to_string())?; + if value["settings"]["agent_policy_proposals_enabled"]["value"] != "true" { + return Err(settings.failure_diagnostic( + "effective agent_policy_proposals_enabled is true; check for a global override", + )); + } + Ok(()) +} + +pub async fn await_mechanistic_draft( + runner: &OpenShellRunner, + sandbox: &str, + expected: &ExpectedDraft<'_>, + probe_stderr: &str, +) -> Result<(), String> { + let started = Instant::now(); + loop { + let draft = runner + .step("mechanistic-draft") + .description("a single scoped mechanistic draft appears") + .with_timeout(COMMAND_TIMEOUT) + .run(&["rule", "get", sandbox]) + .await + .map_err(|error| error.to_string())?; + if !draft.success() { + if started.elapsed() >= PROPOSAL_TIMEOUT { + return Err(draft.failure_diagnostic("the reviewer inbox is readable")); + } + sleep(POLL_INTERVAL).await; + continue; + } + if !draft.stdout().contains("Chunk:") { + if started.elapsed() >= PROPOSAL_TIMEOUT { + return Err(draft.failure_diagnostic(&format!( + "one mechanistic draft for {} and {}; probe stderr:\n{probe_stderr}", + expected.endpoint, expected.binary + ))); + } + sleep(POLL_INTERVAL).await; + continue; + } + return assert_mechanistic_draft(draft.stdout(), expected) + .map_err(|error| draft.failure_diagnostic(&error)); + } +} + +pub async fn create_sandbox( + runner: &mut OpenShellRunner, + name: &str, + policy_path: Option<&str>, +) -> Result<(), String> { + runner.track_sandbox(name); + let mut args = vec![ + "sandbox", + "create", + "--name", + name, + "--detach", + "--no-tty", + "--no-auto-providers", + ]; + if let Some(path) = policy_path { + args.extend(["--policy", path]); + } + args.extend(["--", "sh", "-c", "exec sleep infinity"]); + let create = runner + .step("create") + .description(format!("sandbox '{name}' is created")) + .with_timeout(CREATE_TIMEOUT) + .run(&args) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + + let started = Instant::now(); + loop { + let get = runner + .step("ready") + .description(format!("sandbox '{name}' reaches Ready")) + .with_timeout(COMMAND_TIMEOUT) + .run(&["sandbox", "get", name, "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + if get.success() { + let state: SandboxState = get.json().map_err(|error| error.to_string())?; + if state.name != name { + return Err(get.failure_diagnostic("sandbox get returns the created name")); + } + if state.phase == "Ready" { + return Ok(()); + } + if state.phase == "Failed" { + return Err(get.failure_diagnostic("sandbox reaches Ready instead of Failed")); + } + } + if started.elapsed() >= READY_TIMEOUT { + return Err(get.failure_diagnostic("sandbox reaches Ready before timeout")); + } + sleep(POLL_INTERVAL).await; + } +} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/main.rs b/e2e/suites/conformance/cli/tests/policy_advisor/main.rs new file mode 100644 index 0000000000..c9a449fbaa --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/main.rs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Policy advisor conformance stories. + +mod draft_assertion; +mod helpers; +mod mechanistic_proposal; +mod new_hostname_proposal; +mod policy_local; diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/mechanistic_proposal.rs b/e2e/suites/conformance/cli/tests/policy_advisor/mechanistic_proposal.rs new file mode 100644 index 0000000000..e7eb01e385 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/mechanistic_proposal.rs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::draft_assertion::ExpectedDraft; +use super::helpers::{ + COMMAND_TIMEOUT, EMPTY_NETWORK_POLICY, await_mechanistic_draft, create_sandbox, + enable_proposals, +}; +use openshell_conformance::OpenShellRunner; +use serde_json::Value; +use std::io::Write as _; +use tempfile::NamedTempFile; + +/// Turn a denied transparent TCP open into a scoped policy draft. +#[tokio::test] +async fn creates_a_scoped_draft_for_a_denied_endpoint() { + let mut runner = OpenShellRunner::from_env("mechanistic-proposal") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let mut policy = NamedTempFile::new().map_err(|error| error.to_string())?; + policy + .write_all(EMPTY_NETWORK_POLICY) + .map_err(|error| error.to_string())?; + let policy_path = policy + .path() + .to_str() + .ok_or("temporary policy path is not UTF-8")?; + let name = format!("ct-{}-mp", runner.id()); + create_sandbox(runner, &name, Some(policy_path)).await?; + enable_proposals(runner, &name).await?; + + let effective = runner + .step("effective-policy") + .description("sandbox has no network allow rules before the probe") + .with_timeout(COMMAND_TIMEOUT) + .run(&["policy", "get", &name, "--full", "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + effective.require_success()?; + let value: Value = effective.json().map_err(|error| error.to_string())?; + let network_rules = &value["policy"]["network_policies"]; + if !network_rules.is_null() + && !network_rules + .as_object() + .is_some_and(serde_json::Map::is_empty) + { + return Err(effective.failure_diagnostic("effective network_policies is empty")); + } + + let probe = runner + .step("denied-tcp-open") + .description("one Bash TCP open to 1.1.1.1:443 is denied by policy") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &name, + "--no-tty", + "--", + "bash", + "-c", + "printf 'BINARY=%s\\n' \"$(readlink -f /proc/$$/exe)\"; if exec 3<>/dev/tcp/1.1.1.1/443; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", + ]) + .await + .map_err(|error| error.to_string())?; + probe.require_success()?; + let binary = probe + .stdout() + .lines() + .find_map(|line| line.strip_prefix("BINARY=")) + .filter(|binary| binary.starts_with('/') && binary.rsplit('/').next() == Some("bash")) + .ok_or_else(|| probe.failure_diagnostic("canonical Bash executable path is reported"))? + .to_string(); + if !probe.stdout().lines().any(|line| line == "DENIED") { + return Err(probe.failure_diagnostic("TCP open is denied before any upstream dial")); + } + + await_mechanistic_draft( + runner, + &name, + &ExpectedDraft { + rule: "allow_1_1_1_1_443", + endpoint: "1.1.1.1:443", + binary: &binary, + }, + probe.stderr(), + ) + .await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("mechanistic-proposal conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/new_hostname_proposal.rs b/e2e/suites/conformance/cli/tests/policy_advisor/new_hostname_proposal.rs new file mode 100644 index 0000000000..283f26db81 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/new_hostname_proposal.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::draft_assertion::ExpectedDraft; +use super::helpers::{ + COMMAND_TIMEOUT, EMPTY_NETWORK_POLICY, await_mechanistic_draft, create_sandbox, + sandbox_bash_path, +}; +use openshell_conformance::OpenShellRunner; +use std::io::Write as _; +use tempfile::NamedTempFile; + +/// Turn a denied TCP open to a hostname absent from policy into a scoped draft. +#[tokio::test] +async fn creates_a_scoped_draft_for_a_new_hostname() { + let mut runner = OpenShellRunner::from_env("new-hostname-proposal") + .expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let mut policy = NamedTempFile::new().map_err(|error| error.to_string())?; + policy + .write_all(EMPTY_NETWORK_POLICY) + .map_err(|error| error.to_string())?; + let policy_path = policy + .path() + .to_str() + .ok_or("temporary policy path is not UTF-8")?; + let name = format!("ct-{}-nh", runner.id()); + create_sandbox(runner, &name, Some(policy_path)).await?; + let binary = sandbox_bash_path(runner, &name).await?; + + let probe = runner + .step("denied-new-hostname") + .description("Bash cannot connect to pypi.org:80 before approval") + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &name, + "--no-tty", + "--", + "bash", + "-c", + "if exec 3<>/dev/tcp/pypi.org/80; then echo UNEXPECTED_ALLOWED; exit 1; else echo DENIED; fi", + ]) + .await + .map_err(|error| error.to_string())?; + probe.require_success()?; + if !probe.stdout().lines().any(|line| line == "DENIED") { + return Err(probe.failure_diagnostic("new hostname stays denied before approval")); + } + + await_mechanistic_draft( + runner, + &name, + &ExpectedDraft { + rule: "allow_pypi_org_80", + endpoint: "pypi.org:80", + binary: &binary, + }, + probe.stderr(), + ) + .await + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("new-hostname-proposal conformance story failed:\n{error}"); + } +} diff --git a/e2e/suites/conformance/cli/tests/policy_advisor/policy_local.rs b/e2e/suites/conformance/cli/tests/policy_advisor/policy_local.rs new file mode 100644 index 0000000000..0767bae1b2 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/policy_advisor/policy_local.rs @@ -0,0 +1,193 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::helpers::{ + COMMAND_TIMEOUT, POLL_INTERVAL, READY_TIMEOUT, create_sandbox, enable_proposals, + sandbox_bash_path, +}; +use openshell_conformance::OpenShellRunner; +use serde_json::Value; +use std::time::Instant; +use tokio::time::sleep; + +/// Read and request a rule through the sandbox-local policy HTTP API. +#[tokio::test] +async fn submits_a_rule_for_review() { + let mut runner = + OpenShellRunner::from_env("policy-local").expect("candidate openshell CLI is available"); + let result = async { + runner.check_gateway_status().await?; + let runner = &mut runner; + let name = format!("ct-{}-pl", runner.id()); + create_sandbox(runner, &name, None).await?; + enable_proposals(runner, &name).await?; + let binary = sandbox_bash_path(runner, &name).await?; + + let started = Instant::now(); + let readiness_path = format!("/v1/proposals/ct-{}-readiness", runner.id()); + loop { + match request_policy_local(runner, &name, "/v1/policy/current").await { + Ok(response) + if response["format"] == "yaml" + && response["policy_yaml"] + .as_str() + .is_some_and(|yaml| yaml.contains("version: 1")) => + { + // The current-policy route is local; proposal submission also + // needs the supervisor's workspace and gateway lookup session. + match request_policy_local_http(runner, &name, "GET", &readiness_path, "", 404) + .await + { + Ok(lookup) if lookup["error"] == "chunk_not_found" => break, + Ok(lookup) => { + return Err(format!( + "policy.local proposal lookup returned an invalid readiness response: {lookup}" + )); + } + Err(error) if started.elapsed() >= READY_TIMEOUT => return Err(error), + Err(_) => {} + } + } + Ok(response) => { + return Err(format!( + "policy.local returned an invalid current policy: {response}" + )); + } + Err(error) => { + if started.elapsed() >= READY_TIMEOUT { + return Err(error); + } + } + } + sleep(POLL_INTERVAL).await; + } + let denials = request_policy_local(runner, &name, "/v1/denials?last=1").await?; + if !denials["denials"].is_array() || !denials["log_available"].is_boolean() { + return Err(format!( + "policy.local returned an invalid denials response: {denials}" + )); + } + + let rule_name = format!("conformance_local_{}", runner.id()); + let payload = serde_json::json!({ + "intent_summary": "Allow Bash to read the conformance path on example.invalid.", + "operations": [{ + "addRule": { + "ruleName": &rule_name, + "rule": { + "name": &rule_name, + "endpoints": [{ + "host": "example.invalid", + "port": 443, + "protocol": "rest", + "enforcement": "enforce", + "rules": [{"allow": {"method": "GET", "path": "/conformance"}}] + }], + "binaries": [{"path": &binary}] + } + } + }] + }) + .to_string(); + let submitted = + request_policy_local_http(runner, &name, "POST", "/v1/proposals", &payload, 202).await?; + let chunk_id = submitted["accepted_chunk_ids"] + .as_array() + .filter(|ids| ids.len() == 1) + .and_then(|ids| ids[0].as_str()) + .filter(|id| !id.is_empty()) + .ok_or_else(|| format!("policy.local did not accept one proposal: {submitted}"))?; + if submitted["status"] != "submitted" + || submitted["accepted_chunks"] != 1 + || submitted["rejected_chunks"] != 0 + { + return Err(format!("policy.local did not submit one rule: {submitted}")); + } + + let state = request_policy_local(runner, &name, &format!("/v1/proposals/{chunk_id}")).await?; + if state["chunk_id"] != chunk_id + || state["rule_name"] != rule_name + || state["binary"] != binary + || !matches!(state["status"].as_str(), Some("pending" | "approved")) + { + return Err(format!("policy.local returned the wrong proposal: {state}")); + } + + let review = runner + .step("reviewer-inbox") + .description("the requested rule is visible to the reviewer") + .with_timeout(COMMAND_TIMEOUT) + .run(&["rule", "get", &name]) + .await + .map_err(|error| error.to_string())?; + review.require_success()?; + if !review.stdout().contains(&format!("Chunk: {chunk_id}")) + || !review.stdout().contains(&format!("Rule: {rule_name}")) + { + return Err(review.failure_diagnostic("the submitted rule is in the reviewer inbox")); + } + Ok(()) + } + .await; + if let Err(error) = runner.finish(result).await { + panic!("policy-local conformance story failed:\n{error}"); + } +} + +async fn request_policy_local( + runner: &OpenShellRunner, + sandbox: &str, + path: &str, +) -> Result { + request_policy_local_http(runner, sandbox, "GET", path, "", 200).await +} + +async fn request_policy_local_http( + runner: &OpenShellRunner, + sandbox: &str, + method: &str, + path: &str, + body: &str, + expected_status: u16, +) -> Result { + let script = "method=$1; path=$2; body=$3; exec 3<>/dev/tcp/policy.local/80 || exit 1; printf '%s %s HTTP/1.1\\r\\nHost: policy.local\\r\\nContent-Type: application/json\\r\\nContent-Length: %s\\r\\nConnection: close\\r\\n\\r\\n' \"$method\" \"$path\" \"${#body}\" >&3; printf '%s' \"$body\" >&3; cat <&3"; + let result = runner + .step(format!("policy-local-{method}{path}")) + .description(format!( + "{method} http://policy.local{path} succeeds from the sandbox" + )) + .with_timeout(COMMAND_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + sandbox, + "--no-tty", + "--", + "bash", + "-c", + script, + "policy-local-http", + method, + path, + body, + ]) + .await + .map_err(|error| error.to_string())?; + result.require_success()?; + let (headers, body) = result.stdout().split_once("\r\n\r\n").ok_or_else(|| { + result.failure_diagnostic("a complete HTTP response with headers and JSON body") + })?; + if !headers.starts_with(&format!("HTTP/1.1 {expected_status} ")) + || !headers.lines().any(|line| { + line.to_ascii_lowercase() + .starts_with("content-type: application/json") + }) + { + return Err( + result.failure_diagnostic(&format!("HTTP {expected_status} with JSON Content-Type")) + ); + } + serde_json::from_str(body) + .map_err(|error| result.failure_diagnostic(&format!("valid JSON body: {error}"))) +} diff --git a/e2e/suites/conformance/cli/tests/smoke/main.rs b/e2e/suites/conformance/cli/tests/smoke/main.rs new file mode 100644 index 0000000000..85e17395b3 --- /dev/null +++ b/e2e/suites/conformance/cli/tests/smoke/main.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Smoke conformance stories. + +mod sandbox_lifecycle; diff --git a/e2e/suites/conformance/cli/tests/smoke.rs b/e2e/suites/conformance/cli/tests/smoke/sandbox_lifecycle.rs similarity index 52% rename from e2e/suites/conformance/cli/tests/smoke.rs rename to e2e/suites/conformance/cli/tests/smoke/sandbox_lifecycle.rs index 5d8b376e0a..09b91107d8 100644 --- a/e2e/suites/conformance/cli/tests/smoke.rs +++ b/e2e/suites/conformance/cli/tests/smoke/sandbox_lifecycle.rs @@ -1,12 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Portable phase-1 CLI conformance scenario. - -use std::time::{Duration, Instant}; - -use openshell_conformance::{OpenShellRunner, STATUS_TIMEOUT}; +use openshell_conformance::OpenShellRunner; +use openshell_conformance::STATUS_TIMEOUT; use serde::Deserialize; +use std::time::Duration; +use std::time::Instant; use tokio::time::sleep; const CREATE_TIMEOUT: Duration = Duration::from_mins(10); @@ -28,106 +27,101 @@ struct SandboxListPage { next_page_token: String, } -/// Certify status -> create -> list Ready -> exec -> delete -> list empty. - +/// Verify status, creation, listing, execution, and deletion through the CLI. #[tokio::test] -async fn smoke() { +async fn creates_executes_and_deletes_a_sandbox() { let mut runner = OpenShellRunner::from_env("smoke").expect("candidate openshell CLI is available"); let result = async { runner.check_gateway_status().await?; let runner = &mut runner; - run_smoke_inner(runner).await + let status = runner + .step("status") + .description("openshell status succeeds") + .with_timeout(STATUS_TIMEOUT) + .run(&["status"]) + .await + .map_err(|error| error.to_string())?; + status.require_success()?; + + let sandbox_name = format!("ct-{}-01", runner.id()); + runner.track_sandbox(&sandbox_name); + let create = runner + .step("create") + .description("sandbox creation succeeds") + .with_timeout(CREATE_TIMEOUT) + .run(&["sandbox", "create", "--name", &sandbox_name, "--detach"]) + .await + .map_err(|error| error.to_string())?; + create.require_success()?; + + let get = runner + .step("get-ready") + .description(format!("sandbox '{sandbox_name}' can be retrieved")) + .with_timeout(LIST_ATTEMPT_TIMEOUT) + .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) + .await + .map_err(|error| error.to_string())?; + get.require_success()?; + + let sandbox = get + .json::() + .map_err(|error| error.to_string())?; + if sandbox.name != sandbox_name { + return Err(format!( + "sandbox get returned {:?}; expected sandbox '{sandbox_name}'", + sandbox.name + )); + } + if sandbox.phase != "Ready" { + return Err(format!( + "sandbox '{sandbox_name}' is in phase {:?}; expected Ready", + sandbox.phase + )); + } + + check_sandbox_listed(runner, &sandbox_name).await?; + + let marker = format!("openshell-conformance-{}", runner.id()); + let exec = runner + .step("exec") + .description("sandbox exec exits successfully") + .with_timeout(EXEC_TIMEOUT) + .run(&[ + "sandbox", + "exec", + "--name", + &sandbox_name, + "--no-tty", + "--", + "echo", + &marker, + ]) + .await + .map_err(|error| error.to_string())?; + exec.require_success()?; + let expected_stdout = format!("{marker}\n"); + if exec.stdout() != expected_stdout { + return Err(exec.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}"))); + } + + let delete = runner + .step("delete") + .description("sandbox deletion succeeds") + .with_timeout(DELETE_TIMEOUT) + .run(&["sandbox", "delete", &sandbox_name]) + .await + .map_err(|error| error.to_string())?; + delete.require_success()?; + + check_empty_list(runner, &sandbox_name).await?; + runner.forget_sandbox(&sandbox_name); + Ok(()) } .await; if let Err(error) = runner.finish(result).await { - panic!("smoke conformance scenario failed:\n{error}"); - } -} - -async fn run_smoke_inner(runner: &mut OpenShellRunner) -> Result<(), String> { - let status = runner - .step("status") - .description("openshell status succeeds") - .with_timeout(STATUS_TIMEOUT) - .run(&["status"]) - .await - .map_err(|error| error.to_string())?; - status.require_success()?; - - let sandbox_name = format!("ct-{}-01", runner.id()); - runner.track_sandbox(&sandbox_name); - let create = runner - .step("create") - .description("sandbox creation succeeds") - .with_timeout(CREATE_TIMEOUT) - .run(&["sandbox", "create", "--name", &sandbox_name, "--detach"]) - .await - .map_err(|error| error.to_string())?; - create.require_success()?; - - let get = runner - .step("get-ready") - .description(format!("sandbox '{sandbox_name}' can be retrieved")) - .with_timeout(LIST_ATTEMPT_TIMEOUT) - .run(&["sandbox", "get", &sandbox_name, "--output", "json"]) - .await - .map_err(|error| error.to_string())?; - get.require_success()?; - - let sandbox = get - .json::() - .map_err(|error| error.to_string())?; - if sandbox.name != sandbox_name { - return Err(format!( - "sandbox get returned {:?}; expected sandbox '{sandbox_name}'", - sandbox.name - )); + panic!("smoke conformance story failed:\n{error}"); } - if sandbox.phase != "Ready" { - return Err(format!( - "sandbox '{sandbox_name}' is in phase {:?}; expected Ready", - sandbox.phase - )); - } - - check_sandbox_listed(runner, &sandbox_name).await?; - - let marker = format!("openshell-conformance-{}", runner.id()); - let exec = runner - .step("exec") - .description("sandbox exec exits successfully") - .with_timeout(EXEC_TIMEOUT) - .run(&[ - "sandbox", - "exec", - "--name", - &sandbox_name, - "--no-tty", - "--", - "echo", - &marker, - ]) - .await - .map_err(|error| error.to_string())?; - exec.require_success()?; - let expected_stdout = format!("{marker}\n"); - if exec.stdout() != expected_stdout { - return Err(exec.failure_diagnostic(&format!("stdout is exactly {expected_stdout:?}"))); - } - - let delete = runner - .step("delete") - .description("sandbox deletion succeeds") - .with_timeout(DELETE_TIMEOUT) - .run(&["sandbox", "delete", &sandbox_name]) - .await - .map_err(|error| error.to_string())?; - delete.require_success()?; - - check_empty_list(runner, &sandbox_name).await?; - runner.forget_sandbox(&sandbox_name); - Ok(()) } async fn check_sandbox_listed(runner: &OpenShellRunner, sandbox_name: &str) -> Result<(), String> { diff --git a/tasks/test.toml b/tasks/test.toml index 9cd5755d44..a72dfc08ac 100644 --- a/tasks/test.toml +++ b/tasks/test.toml @@ -115,7 +115,7 @@ run = [ # with test-only helpers enabled. "cargo test --workspace --exclude openshell-server", "cargo test -p openshell-server --features test-support", - "cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml --package openshell-test-conformance-cli --test policy_advisor draft_assertion", + "cargo test --locked --manifest-path e2e/suites/conformance/Cargo.toml --package openshell-test-conformance-cli --test policy_advisor draft_assertion::tests::", "cargo nextest run --config-file .config/nextest.toml --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml", ] run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native"