diff --git a/Cargo.lock b/Cargo.lock index 6a2c214e2c..b342ef1a0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4505,6 +4505,7 @@ dependencies = [ "opentelemetry", "opentelemetry_sdk", "prost-types", + "rcgen", "rustix 1.1.4", "serde", "serde_json", diff --git a/crates/openshell-core/src/driver_utils.rs b/crates/openshell-core/src/driver_utils.rs index 71fb790b11..bc2f97273f 100644 --- a/crates/openshell-core/src/driver_utils.rs +++ b/crates/openshell-core/src/driver_utils.rs @@ -134,6 +134,18 @@ pub const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = "/etc/openshell/auth/upstream-p /// mount is used rather than a driver secret. pub const PROXY_CA_MOUNT_PATH: &str = "/etc/openshell/tls/proxy/ca-bundle.pem"; +/// Container-side mount path for an operator-supplied additional destination +/// CA bundle. +/// +/// Drivers with an `additional_ca_bundle` operator setting bind-mount the +/// host PEM file here (read-only) and pass the path on the supervisor's argv +/// via `--additional-ca-bundle`. The supervisor folds it into its upstream +/// TLS trust store alongside the system CA bundle, for direct (non-proxied) +/// inspected HTTPS egress to destinations signed by a private CA — unlike +/// [`PROXY_CA_MOUNT_PATH`], this applies regardless of whether a corporate +/// forward proxy is configured. +pub const ADDITIONAL_CA_MOUNT_PATH: &str = "/etc/openshell/tls/additional/ca-bundle.pem"; + /// A validated corporate upstream-proxy address. /// /// Produced by [`parse_upstream_proxy_url`], which is the single source of diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index 18b1928c33..cd9b3cc570 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -50,6 +50,7 @@ url = { workspace = true } openshell-otel-test-support = { path = "../openshell-otel-test-support" } opentelemetry_sdk = { workspace = true, features = ["testing"] } prost-types = { workspace = true } +rcgen = { workspace = true } temp-env = "0.3" tempfile = "3" tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/openshell-driver-podman/src/config.rs b/crates/openshell-driver-podman/src/config.rs index 9e4e9b571b..f3739aeae0 100644 --- a/crates/openshell-driver-podman/src/config.rs +++ b/crates/openshell-driver-podman/src/config.rs @@ -178,6 +178,23 @@ pub struct PodmanComputeConfig { /// and upstream verification. Only meaningful with `https_proxy` set; the /// bundle must exist and contain at least one certificate. pub proxy_ca_bundle: Option, + /// Path (on the gateway host) to a PEM CA bundle trusted for direct + /// (non-proxied), policy-inspected HTTPS egress to destinations signed by + /// a private CA. + /// + /// Unlike [`proxy_ca_bundle`](Self::proxy_ca_bundle), this applies + /// regardless of whether `https_proxy` is configured: it addresses + /// supervisors that run as their own container, separate from the + /// workload image, and therefore cannot inherit a private CA baked into + /// the workload's system trust store. The gateway bind-mounts this file + /// read-only into the supervisor container (at + /// [`ADDITIONAL_CA_MOUNT_PATH`](openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH)) + /// and passes its path via `--additional-ca-bundle`. The supervisor folds + /// it into its upstream TLS trust store alongside (not instead of) the + /// system CA bundle. The bundle must exist and contain at least one + /// certificate; this is validated on the gateway host at sandbox-create + /// time, not deferred until the supervisor container starts. + pub additional_ca_bundle: Option, /// User namespace mode for sandbox containers (e.g. `auto`, `private`). /// When unset, containers use the default user namespace. pub userns: Option, @@ -332,6 +349,21 @@ impl PodmanComputeConfig { )); } } + // additional_ca_bundle is independent of the corporate proxy setting + // above: it applies to direct, inspected egress and has no + // https_proxy dependency. Read and validate the actual host file + // here (this path is not yet mounted into any container), rather + // than only checking it is non-empty: an operator with a mistyped + // path or a certificate-free bundle should see a config-validation + // error when the driver starts, not a failed/crash-looping sandbox + // once the supervisor container starts and tries to read it. + if let Some(path) = self.additional_ca_bundle.as_deref() { + openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file( + path, + "additional_ca_bundle", + ) + .map_err(crate::client::PodmanApiError::InvalidInput)?; + } Ok(()) } @@ -519,6 +551,7 @@ impl Default for PodmanComputeConfig { proxy_auth_allow_insecure: None, proxy_connect_by_hostname: None, proxy_ca_bundle: None, + additional_ca_bundle: None, userns: None, uidmap: Vec::new(), gidmap: Vec::new(), @@ -832,6 +865,65 @@ mod tests { assert!(err.to_string().contains("proxy_ca_bundle"), "{err}"); } + #[test] + fn validate_configuration_accepts_additional_ca_bundle_without_proxy() { + // #3781: unlike proxy_ca_bundle, this must work with no corporate + // proxy configured at all. + let directory = tempfile::tempdir().unwrap(); + let ca_path = directory.path().join("additional-ca.pem"); + std::fs::write(&ca_path, generate_test_ca_pem()).unwrap(); + + let cfg = PodmanComputeConfig { + additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()), + ..PodmanComputeConfig::default() + }; + assert!(cfg.validate_proxy_config().is_ok()); + assert!(cfg.https_proxy.is_none()); + } + + #[test] + fn validate_configuration_rejects_empty_additional_ca_bundle() { + let cfg = PodmanComputeConfig { + additional_ca_bundle: Some(" ".to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg.validate_proxy_config().unwrap_err(); + assert!(err.to_string().contains("additional_ca_bundle"), "{err}"); + } + + #[test] + fn validate_configuration_rejects_nonexistent_additional_ca_bundle() { + // #3781 fix requirement: an operator with a mistyped path must see + // this at config-validation time, not a failed sandbox later. + let cfg = PodmanComputeConfig { + additional_ca_bundle: Some("/nonexistent/additional-ca.pem".to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg.validate_proxy_config().unwrap_err(); + assert!(err.to_string().contains("additional_ca_bundle"), "{err}"); + } + + #[test] + fn validate_configuration_rejects_certificate_free_additional_ca_bundle() { + let directory = tempfile::tempdir().unwrap(); + let ca_path = directory.path().join("not-a-cert.pem"); + std::fs::write(&ca_path, "not a certificate\n").unwrap(); + + let cfg = PodmanComputeConfig { + additional_ca_bundle: Some(ca_path.to_str().unwrap().to_string()), + ..PodmanComputeConfig::default() + }; + let err = cfg.validate_proxy_config().unwrap_err(); + assert!(err.to_string().contains("additional_ca_bundle"), "{err}"); + } + + /// Generate a real, valid self-signed CA certificate PEM for tests. + fn generate_test_ca_pem() -> String { + let key_pair = rcgen::KeyPair::generate().unwrap(); + let params = rcgen::CertificateParams::new(Vec::new()).unwrap(); + params.self_signed(&key_pair).unwrap().pem() + } + #[test] fn validate_proxy_config_rejects_url_components() { for url in [ diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 3091ac978c..ff2b821ec5 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -69,6 +69,7 @@ const UPSTREAM_PROXY_AUTH_MOUNT_PATH: &str = openshell_core::driver_utils::UPSTREAM_PROXY_AUTH_MOUNT_PATH; const RESOLV_CONF_PATH: &str = "/etc/resolv.conf"; const PROXY_CA_MOUNT_PATH: &str = openshell_core::driver_utils::PROXY_CA_MOUNT_PATH; +const ADDITIONAL_CA_MOUNT_PATH: &str = openshell_core::driver_utils::ADDITIONAL_CA_MOUNT_PATH; const PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR: &str = openshell_core::driver_utils::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR; @@ -612,6 +613,19 @@ fn upstream_proxy_cli_args(config: &PodmanComputeConfig) -> Vec { args } +/// Build the additional-destination-CA command-line argument passed to the +/// supervisor. Independent of `upstream_proxy_cli_args`: this applies to +/// direct, inspected egress regardless of whether a corporate proxy is +/// configured. +fn additional_ca_cli_args(config: &PodmanComputeConfig) -> Vec { + let mut args = Vec::new(); + if config.additional_ca_bundle.is_some() { + args.push("--additional-ca-bundle".to_string()); + args.push(ADDITIONAL_CA_MOUNT_PATH.to_string()); + } + args +} + fn build_env( sandbox: &DriverSandbox, config: &PodmanComputeConfig, @@ -1272,6 +1286,7 @@ fn build_base_spec( image_volumes.extend(user_mounts.image_volumes); let mut command = vec!["--workdir".to_string(), image.workspace_root.clone()]; command.extend(upstream_proxy_cli_args(config)); + command.extend(additional_ca_cli_args(config)); let container_spec = ContainerSpec { name, @@ -1438,6 +1453,24 @@ fn build_base_spec( options: ro, }); } + // Bind-mount an operator-supplied additional destination CA + // bundle read-only when configured. Independent of the corporate + // proxy CA above: this applies to direct, inspected egress. The + // supervisor reads it via the --additional-ca-bundle argv path + // (see additional_ca_cli_args) and folds it into its upstream + // trust store alongside the system bundle. + if let Some(ca_bundle) = &config.additional_ca_bundle { + let mut ro = vec!["ro".into(), "rbind".into()]; + if is_selinux_enabled() { + ro.push("z".into()); + } + m.push(Mount { + kind: "bind".into(), + source: ca_bundle.clone(), + destination: ADDITIONAL_CA_MOUNT_PATH.into(), + options: ro, + }); + } if let Some(bin_path) = supervisor_bin_path { let mut opts = vec!["ro".into(), "rbind".into()]; if is_selinux_enabled() { @@ -1729,7 +1762,10 @@ pub fn build_isolation_specs( fn trusted_mount(destination: &str) -> bool { matches!( destination, - TLS_CA_MOUNT_PATH | PROXY_CA_MOUNT_PATH | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR + TLS_CA_MOUNT_PATH + | PROXY_CA_MOUNT_PATH + | ADDITIONAL_CA_MOUNT_PATH + | PROVIDER_SPIFFE_WORKLOAD_API_SOCKET_MOUNT_DIR ) || destination == openshell_core::container_paths::NETNS_MOUNT_ROOT } @@ -2870,6 +2906,128 @@ mod tests { ); } + #[test] + fn container_spec_binds_additional_ca_bundle_and_passes_argv_without_proxy() { + // #3781: the additional CA must work for direct, inspected egress + // with no corporate proxy configured at all — unlike proxy_ca_bundle, + // which requires https_proxy. + let sandbox = test_sandbox("additional-ca-id", "additional-ca-name"); + let mut config = test_config(); + assert!( + config.https_proxy.is_none(), + "test must exercise the no-proxy case" + ); + config.additional_ca_bundle = Some("/host/additional-ca.pem".to_string()); + + let spec = build_container_spec(&sandbox, &config); + let command = spec_command(&spec); + + let idx = command + .iter() + .position(|a| a == "--additional-ca-bundle") + .expect("additional CA bundle flag present"); + assert_eq!( + command.get(idx + 1).map(String::as_str), + Some("/etc/openshell/tls/additional/ca-bundle.pem") + ); + + let mounts = spec["mounts"] + .as_array() + .expect("mounts should be an array"); + let ca_mount = mounts + .iter() + .find(|m| { + m["type"].as_str() == Some("bind") + && m["destination"].as_str() + == Some("/etc/openshell/tls/additional/ca-bundle.pem") + }) + .expect("additional CA bundle bind mount present"); + assert_eq!(ca_mount["source"].as_str(), Some("/host/additional-ca.pem")); + assert!( + ca_mount["options"] + .as_array() + .expect("options array") + .iter() + .any(|o| o.as_str() == Some("ro")), + "additional CA bundle mount must be read-only" + ); + } + + #[test] + fn isolated_pair_mounts_additional_ca_bundle_only_on_supervisor() { + // build_isolation_specs() filters the base spec's mounts per role, so + // the base-spec test above does not prove the final supervisor keeps + // the CA the --additional-ca-bundle argv points at. + let sandbox = test_sandbox("additional-ca-pair", "additional-ca-pair"); + let mut config = test_config(); + config.additional_ca_bundle = Some("/host/additional-ca.pem".to_string()); + let identity = openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "default".into(), + "sha256:image".into(), + ) + .unwrap(); + let specs = build_isolation_specs(IsolationSpecInput { + sandbox: &sandbox, + config: &config, + token_secret: Some("jwt"), + resolver_secret: "resolver", + gpu_devices: None, + image: &ResolvedPodmanImage::new("sha256:image"), + supervisor_bin: None, + tls_secrets: None, + identity: &identity, + }) + .unwrap(); + + let ca_mount = specs + .supervisor + .mounts + .iter() + .find(|mount| mount.destination == ADDITIONAL_CA_MOUNT_PATH) + .expect("supervisor keeps the additional CA bind mount"); + assert_eq!(ca_mount.kind, "bind"); + assert_eq!(ca_mount.source, "/host/additional-ca.pem"); + assert!(ca_mount.options.contains(&"ro".to_string())); + let idx = specs + .supervisor + .command + .iter() + .position(|arg| arg == "--additional-ca-bundle") + .expect("supervisor argv carries the additional CA flag"); + assert_eq!( + specs.supervisor.command.get(idx + 1).map(String::as_str), + Some(ADDITIONAL_CA_MOUNT_PATH) + ); + assert!( + specs + .workload + .mounts + .iter() + .all(|mount| mount.destination != ADDITIONAL_CA_MOUNT_PATH), + "workload must not receive the additional CA mount" + ); + } + + #[test] + fn container_spec_omits_additional_ca_bundle_when_unconfigured() { + let sandbox = test_sandbox("test-id", "test-name"); + let spec = build_container_spec(&sandbox, &test_config()); + let command = spec_command(&spec); + assert!( + !command.iter().any(|a| a == "--additional-ca-bundle"), + "no additional-ca-bundle flag without operator config" + ); + let mounts = spec["mounts"].as_array().expect("mounts array"); + assert!( + !mounts.iter().any(|m| m["destination"].as_str() + == Some("/etc/openshell/tls/additional/ca-bundle.pem")), + "no additional CA bundle mount without operator config" + ); + } + #[test] fn container_spec_sandbox_env_cannot_influence_proxy_argv() { use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate}; diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 4fe83e0c8a..e8f9e80ae9 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -174,6 +174,14 @@ struct Args { #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CA_BUNDLE")] sandbox_proxy_ca_bundle: Option, + /// Path (on the gateway host) to a PEM CA bundle trusted for direct + /// (non-proxied), policy-inspected HTTPS egress to destinations signed by + /// a private CA. Unlike `--sandbox-proxy-ca-bundle`, this applies + /// regardless of whether a corporate proxy is configured. Bind-mounted + /// read-only into the supervisor container. + #[arg(long, env = "OPENSHELL_SANDBOX_ADDITIONAL_CA_BUNDLE")] + sandbox_additional_ca_bundle: Option, + /// User namespace mode for sandbox containers (e.g. `auto`). /// When unset, containers use the default user namespace. #[arg(long, env = "OPENSHELL_PODMAN_USERNS")] @@ -241,6 +249,7 @@ async fn main() -> Result<()> { proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, proxy_ca_bundle: args.sandbox_proxy_ca_bundle, + additional_ca_bundle: args.sandbox_additional_ca_bundle, userns: args.userns, uidmap: args.uidmap, gidmap: args.gidmap, diff --git a/crates/openshell-supervisor-network/src/l7/tls.rs b/crates/openshell-supervisor-network/src/l7/tls.rs index d106eb2060..aae99ba6c3 100644 --- a/crates/openshell-supervisor-network/src/l7/tls.rs +++ b/crates/openshell-supervisor-network/src/l7/tls.rs @@ -585,11 +585,80 @@ mod tests { assert_eq!(config.alpn_protocols, vec![b"http/1.1".to_vec()]); } + /// Reproduces issue #3781 end to end with a real TLS handshake: an + /// upstream client trying to reach a destination signed by an + /// operator-owned private CA fails when that CA is unknown, and succeeds + /// once it is appended to the bundle passed to + /// `build_upstream_client_config` — the same mechanism the Podman driver + /// fix (`additional_ca_bundle`) feeds into. No fakes: a real rustls + /// `TlsAcceptor`/`TlsConnector` pair over a real loopback TCP connection. + #[tokio::test] + async fn additional_ca_enables_trust_of_privately_signed_upstream() { + let hostname = "private.example"; + + // A CA the system bundle has never heard of, standing in for the + // operator-owned private CA from #3781. + let private_ca = SandboxCa::generate().unwrap(); + let private_ca_pem = private_ca.cert_pem().to_string(); + let cert_cache = CertCache::new(private_ca); + let leaf = cert_cache.get_or_generate(hostname).unwrap(); + + let server_config = Arc::new( + ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(leaf.cert_chain.clone(), leaf.private_key.clone_key()) + .unwrap(), + ); + let acceptor = TlsAcceptor::from(server_config); + + // --- Without the private CA: must fail. This is the literal bug in + // #3781 — "Upstream TLS establishment failed" against a + // policy-allowed, private-CA-signed destination. --- + let addr = serve_one(acceptor.clone()).await; + let client_without = build_upstream_client_config("").unwrap(); + let tcp = tokio::net::TcpStream::connect(addr).await.unwrap(); + let result = tls_connect_upstream(tcp, hostname, &client_without).await; + assert!( + result.is_err(), + "connection to a privately-signed destination must fail when the \ + private CA is not trusted, reproducing #3781" + ); + + // --- With the private CA appended to the bundle: must succeed. This + // is what additional_ca_bundle plumbs through in the Podman driver + // fix. --- + let addr = serve_one(acceptor).await; + let client_with = build_upstream_client_config(&private_ca_pem).unwrap(); + let tcp = tokio::net::TcpStream::connect(addr).await.unwrap(); + let result = tls_connect_upstream(tcp, hostname, &client_with).await; + assert!( + result.is_ok(), + "connection must succeed once the private CA is trusted: {:?}", + result.err() + ); + } + /// Helper: generate a self-signed CA and return its PEM string. fn generate_ca_pem() -> String { SandboxCa::generate().unwrap().ca_cert_pem } + /// Accept exactly one TLS connection on a fresh loopback listener and + /// return its address. Used by `additional_ca_enables_trust_of_...` to + /// give each connection attempt its own listener. + async fn serve_one(acceptor: TlsAcceptor) -> std::net::SocketAddr { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + // Don't unwrap: when the client (correctly) rejects an untrusted + // certificate, the server observes a failed handshake too. Only + // the client's outcome matters here. + let _ = acceptor.accept(stream).await; + }); + addr + } + #[test] fn load_pem_certs_single_ca() { let pem = generate_ca_pem(); diff --git a/crates/openshell-supervisor-network/src/run.rs b/crates/openshell-supervisor-network/src/run.rs index 1eba8a0742..5cfddacbf7 100644 --- a/crates/openshell-supervisor-network/src/run.rs +++ b/crates/openshell-supervisor-network/src/run.rs @@ -142,6 +142,34 @@ fn advance_allocation_epoch(path: &std::path::Path, sandbox_id: Option<&str>) -> Ok(epoch) } +/// Read an operator-supplied additional destination CA bundle (#3781) and +/// fold it into `system_ca_bundle`, so it reaches both the sandbox trust +/// bundle (`write_ca_files`) and the supervisor's own upstream TLS +/// verification (`build_upstream_client_config`) alongside the system CA +/// bundle. A no-op when `additional_ca_bundle` is `None`. +/// +/// # Errors +/// +/// Fails closed on an unreadable or certificate-free bundle, matching the +/// corporate-proxy CA bundle's validation. +fn append_additional_ca_bundle( + system_ca_bundle: &mut String, + additional_ca_bundle: Option<&str>, +) -> Result<(), String> { + let Some(path) = additional_ca_bundle else { + return Ok(()); + }; + let pem = openshell_core::driver_utils::read_upstream_proxy_ca_bundle_file( + path, + "--additional-ca-bundle", + )?; + if !system_ca_bundle.is_empty() && !system_ca_bundle.ends_with('\n') { + system_ca_bundle.push('\n'); + } + system_ca_bundle.push_str(&pem); + Ok(()) +} + /// Handles and values produced by [`run_networking`] that the rest of /// `run_sandbox` consumes. /// @@ -197,6 +225,7 @@ pub async fn run_networking( agent_proposals: AgentProposals, workspace_rx: tokio::sync::watch::Receiver, upstream_proxy_args: &crate::upstream_proxy::UpstreamProxyArgs, + additional_ca_bundle: Option<&str>, proxy_tls_dir: Option<&std::path::Path>, host_gateway_ip: Option, #[cfg(target_os = "linux")] transparent_runtime: Option, @@ -370,6 +399,16 @@ pub async fn run_networking( } system_ca_bundle.push_str(&pem); } + // An operator-supplied additional destination CA (#3781): + // unlike the corporate proxy CA above, this applies + // regardless of whether a proxy is configured, so a private + // CA can be trusted for direct, inspected egress without + // rebuilding the supervisor image. Folded into the same + // bundle for the same reason (sandbox trust + upstream + // verification must agree), and validated the same way + // (fail closed on an unreadable or certificate-free bundle). + append_additional_ca_bundle(&mut system_ca_bundle, additional_ca_bundle) + .map_err(|err| miette::miette!("{err}"))?; match write_ca_files(&ca, tls_dir, &system_ca_bundle) { Ok(paths) => { // /etc/openshell-tls is subsumed by the /etc baseline @@ -566,4 +605,70 @@ mod transparent_runtime_tests { let error = advance_allocation_epoch(&path, Some("sandbox-a")).unwrap_err(); assert!(error.to_string().contains("allocation epoch is invalid")); } + + #[test] + fn append_additional_ca_bundle_is_noop_when_unset() { + let mut bundle = "existing-system-bundle".to_string(); + append_additional_ca_bundle(&mut bundle, None).unwrap(); + assert_eq!(bundle, "existing-system-bundle"); + } + + #[test] + fn append_additional_ca_bundle_appends_to_nonempty_system_bundle() { + let directory = tempfile::tempdir().unwrap(); + let ca_path = directory.path().join("additional-ca.pem"); + let ca_pem = generate_test_ca_pem(); + std::fs::write(&ca_path, &ca_pem).unwrap(); + + let mut bundle = "system-bundle-without-trailing-newline".to_string(); + append_additional_ca_bundle(&mut bundle, Some(ca_path.to_str().unwrap())).unwrap(); + + // The two bundles must both be present, newline-separated — this is + // the exact property build_upstream_client_config and write_ca_files + // rely on: a #3781 fix that silently dropped or overwrote the system + // bundle instead of appending would pass a naive "contains the CA" + // check but break every other destination's upstream verification. + assert!(bundle.starts_with("system-bundle-without-trailing-newline\n")); + assert!(bundle.trim_end().ends_with(ca_pem.trim_end())); + } + + #[test] + fn append_additional_ca_bundle_works_with_empty_system_bundle() { + let directory = tempfile::tempdir().unwrap(); + let ca_path = directory.path().join("additional-ca.pem"); + let ca_pem = generate_test_ca_pem(); + std::fs::write(&ca_path, &ca_pem).unwrap(); + + let mut bundle = String::new(); + append_additional_ca_bundle(&mut bundle, Some(ca_path.to_str().unwrap())).unwrap(); + assert_eq!(bundle.trim_end(), ca_pem.trim_end()); + } + + #[test] + fn append_additional_ca_bundle_fails_closed_on_missing_file() { + let mut bundle = String::new(); + let error = + append_additional_ca_bundle(&mut bundle, Some("/nonexistent/additional-ca.pem")) + .unwrap_err(); + assert!(error.contains("--additional-ca-bundle"), "{error}"); + // Fail closed: the bundle must not be silently left unmodified/partial. + assert!(bundle.is_empty()); + } + + #[test] + fn append_additional_ca_bundle_fails_closed_on_certificate_free_file() { + let directory = tempfile::tempdir().unwrap(); + let ca_path = directory.path().join("not-a-cert.pem"); + std::fs::write(&ca_path, "not a certificate\n").unwrap(); + + let mut bundle = String::new(); + let error = + append_additional_ca_bundle(&mut bundle, Some(ca_path.to_str().unwrap())).unwrap_err(); + assert!(error.contains("--additional-ca-bundle"), "{error}"); + } + + /// Generate a real, valid self-signed CA certificate PEM for tests. + fn generate_test_ca_pem() -> String { + SandboxCa::generate().unwrap().cert_pem().to_string() + } } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 52bf73e150..ebd45f7b7c 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -694,6 +694,7 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { ocsf_enabled: Arc::new(AtomicBool::new(false)), ocsf_schema_version: Arc::new(Mutex::new(String::new())), upstream_proxy_args: UpstreamProxyArgs::default(), + additional_ca_bundle: None, backend_descriptor: TestSetup::descriptor(), auth_bundle: setup.auth.clone(), admitted_isolation_backend: Some(TEST_BACKEND.into()), diff --git a/crates/openshell-supervisor/src/lib.rs b/crates/openshell-supervisor/src/lib.rs index aa6d8e36e5..bc7ca5d514 100644 --- a/crates/openshell-supervisor/src/lib.rs +++ b/crates/openshell-supervisor/src/lib.rs @@ -503,6 +503,7 @@ pub async fn run_network_proxy( policy_data: String, tls_dir: Option, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + additional_ca_bundle: Option, ) -> Result { if !listen.ip().is_loopback() { return Err(miette::miette!( @@ -570,6 +571,7 @@ pub async fn run_network_proxy( AgentProposals::new(initial_agent_proposals_enabled), workspace_rx, &upstream_proxy_args, + additional_ca_bundle.as_deref(), Some(&tls_dir.path), None, #[cfg(target_os = "linux")] @@ -635,6 +637,7 @@ pub async fn run_sandbox( ocsf_enabled: Arc, ocsf_schema_version: Arc>, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + additional_ca_bundle: Option, backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, auth_bundle: openshell_core::jwt::SupervisorAuthBundle, admitted_isolation_backend: Option, @@ -660,6 +663,7 @@ pub async fn run_sandbox( ocsf_enabled, ocsf_schema_version, upstream_proxy_args, + additional_ca_bundle, backend_descriptor, auth_bundle, admitted_isolation_backend, @@ -686,6 +690,7 @@ struct SandboxRunConfig { ocsf_enabled: Arc, ocsf_schema_version: Arc>, upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, + additional_ca_bundle: Option, backend_descriptor: openshell_isolation_interface::contract::BackendDescriptor, auth_bundle: openshell_core::jwt::SupervisorAuthBundle, admitted_isolation_backend: Option, @@ -716,6 +721,7 @@ async fn run_sandbox_with_backend( ocsf_enabled, ocsf_schema_version, upstream_proxy_args, + additional_ca_bundle, backend_descriptor, auth_bundle, admitted_isolation_backend, @@ -1041,6 +1047,7 @@ async fn run_sandbox_with_backend( agent_proposals.clone(), workspace_rx.clone(), &upstream_proxy_args, + additional_ca_bundle.as_deref(), None, remote_host_gateway_ip, #[cfg(target_os = "linux")] @@ -5138,6 +5145,7 @@ mod tests { Arc::new(AtomicBool::new(false)), Arc::new(std::sync::Mutex::new(String::new())), openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs::default(), + None, openshell_isolation_interface::contract::BackendDescriptor { backend_name: "in-process-test".to_string(), payload: b"in-process-only-launch-data".to_vec(), diff --git a/crates/openshell-supervisor/src/main.rs b/crates/openshell-supervisor/src/main.rs index 7c062b5f2b..fbc6f5c761 100644 --- a/crates/openshell-supervisor/src/main.rs +++ b/crates/openshell-supervisor/src/main.rs @@ -118,6 +118,12 @@ struct Args { #[arg(long)] upstream_proxy_ca_bundle: Option, + /// Path to a PEM CA bundle trusted for direct (non-proxied), inspected + /// upstream HTTPS egress. Independent of `--upstream-proxy-ca-bundle`: + /// applies regardless of whether a corporate proxy is configured. + #[arg(long)] + additional_ca_bundle: Option, + #[arg(long)] backend_descriptor_file: Option, @@ -431,6 +437,7 @@ fn main() -> Result<()> { ocsf_enabled, ocsf_schema_version, upstream_proxy_args, + args.additional_ca_bundle, backend_descriptor, auth_bundle, admitted_isolation_backend, @@ -452,6 +459,7 @@ fn main() -> Result<()> { policy_data, args.tls_dir, upstream_proxy_args, + args.additional_ca_bundle, ) .await } diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index e14131264e..ec1b90dad7 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -1168,6 +1168,22 @@ health_check_interval_secs = 10 # proxy_connect_by_hostname = true # Corporate CA trusted for an https:// proxy and TLS-intercepting proxies. # proxy_ca_bundle = "/etc/openshell/tls/proxy-ca.pem" +# additional_ca_bundle points at a PEM CA bundle (on the gateway host) trusted +# for direct, policy-inspected HTTPS egress to a destination signed by a +# private CA — independent of proxy_ca_bundle above, this applies whether or +# not https_proxy is set. The Podman supervisor runs as its own container, +# separate from the workload image, so it cannot inherit a private CA baked +# into the workload's own trust store; this setting restores that trust +# without rebuilding the supervisor image. The gateway bind-mounts the file +# read-only into the supervisor container and folds it into the upstream +# trust store alongside (not instead of) the system CA bundle. The CA is also +# added to the trust bundle inside each sandbox, so workloads trust it directly. +# It is trusted for every destination, not only the one that needs it, so use a +# CA you control or one with name constraints. The file must +# exist and contain at least one certificate; the gateway validates this on +# the gateway host when the Podman driver starts, so a bad path or a +# certificate-free bundle fails gateway startup rather than a later sandbox. +# additional_ca_bundle = "/etc/openshell/tls/additional-ca.pem" # Project a host Workload API Unix socket into the supervisor, or use an # explicit container-reachable TCP endpoint, for provider token exchange. # provider_spiffe_workload_api_socket = "/run/spire/agent.sock" diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index b577497bd7..90061134c7 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -196,6 +196,8 @@ On macOS, set `host_gateway_ip` only if your Podman machine uses a non-standard For networks that require a corporate proxy, set `https_proxy`, `no_proxy`, and related `proxy_*` options. See the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. +To trust a private CA for direct, policy-inspected HTTPS egress without a corporate proxy — for example, because the supervisor container cannot inherit a private CA baked into the workload image — set `additional_ca_bundle`. Both the supervisor and the workload trust the CA for all destinations, so use a CA you control. See the [Gateway Configuration File](/how-it-works/gateways/configuration) reference for details. + ### Podman Mounts Podman supports the same `volume`, `tmpfs`, and `bind` mounts as [Docker](#docker-mounts), using the `podman` key in driver config, and the same bind-mount warning applies. Podman `volume` mounts do not support `subpath`. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ad331a2bda..d896d8b7b5 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -394,6 +394,17 @@ Common findings: probe and the companion supervisor's private health check. Do not add capabilities, attach a workload network, or disable the runtime seccomp profile. There is no sandbox nftables or nested-network setup to repair. +- Gateway fails to start with an `additional_ca_bundle` error: the Podman + driver validates the setting at startup. The file must exist on the gateway + host, be non-empty, and contain at least one PEM certificate. +- Inspected HTTPS to a private-CA destination fails (for example + `curl: (52) Empty reply from server`), or the supervisor exits at startup, + with `additional_ca_bundle` set: the supervisor container needs the CA, not + just the workload. Check + `podman inspect -supervisor --format '{{json .Mounts}}'` for a + read-only bind of the bundle at `/etc/openshell/tls/additional/ca-bundle.pem`, + and `--format '{{json .Config.Cmd}}'` for `--additional-ca-bundle` with that + path. The `sandbox` role container must not have the mount. - On Linux, verify that the host-networked Podman supervisor can reach the gateway's primary loopback endpoint. On macOS, verify Podman Machine's host-loopback forwarding or configure an explicit `grpc_endpoint`.