From 679e9aabab0c7810ac11f382f79ad93f0a7bc7de Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 11:15:44 -0700 Subject: [PATCH 1/9] feat(sandbox): support sandbox-initiated supervisor connections Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 7 +- .../src/isolation_registration.rs | 12 + crates/openshell-driver-kubernetes/README.md | 60 +- .../openshell-driver-kubernetes/src/config.rs | 66 +- .../openshell-driver-kubernetes/src/driver.rs | 663 ++++++++++++++++-- .../src/isolation.rs | 364 +++++++++- crates/openshell-driver-kubernetes/src/lib.rs | 5 +- .../src/lifecycle_tests.rs | 17 + .../openshell-driver-kubernetes/src/main.rs | 14 +- .../src/sandbox_runtime.rs | 126 +++- crates/openshell-driver-vm/src/driver.rs | 6 + crates/openshell-sandbox-backend/README.md | 25 +- .../src/boundary_protocol.rs | 70 ++ crates/openshell-sandbox-backend/src/lib.rs | 3 + .../openshell-sandbox-backend/src/runtime.rs | 300 +++++++- .../src/runtime_initiated.rs | 496 +++++++++++++ crates/openshell-sandbox/README.md | 9 + .../openshell-sandbox/src/boundary_server.rs | 366 +++++++++- .../openshell-supervisor/src/backend_setup.rs | 76 +- .../src/backend_setup/tests.rs | 55 ++ docs/how-it-works/gateways/configuration.mdx | 37 +- docs/how-it-works/sandboxes/runtimes.mdx | 2 + skills/debug-openshell-cluster/SKILL.md | 40 +- 23 files changed, 2693 insertions(+), 126 deletions(-) create mode 100644 crates/openshell-sandbox-backend/src/runtime_initiated.rs diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 9fdf315c3c..64881ea44f 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -81,7 +81,12 @@ The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a capability-free workload Pod and a directly managed capability-free supervisor Pod. One namespace-wide NetworkPolicy denies direct egress from every OpenShell -workload Pod. The +workload Pod. New sandboxes default to `runtime-initiated` control: the sandbox +runtime dials its supervisor Pod, and the driver adds a per-sandbox +`os-control-` NetworkPolicy for that path only. Set +`gatewayConfig."openshell.drivers.kubernetes".sandbox_runtime.control_connection` +to `supervisor-initiated` when testing with an older supervisor or sandbox +runtime image. Existing sandboxes keep the mode recorded at creation. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway generate-certs`) generates gateway and CLI TLS secrets on first install. Supervisor Pods project only `ca.crt` and authenticate gateway RPCs with sandbox bearer tokens. User diff --git a/crates/openshell-core/src/isolation_registration.rs b/crates/openshell-core/src/isolation_registration.rs index 1633dbb742..873c39f265 100644 --- a/crates/openshell-core/src/isolation_registration.rs +++ b/crates/openshell-core/src/isolation_registration.rs @@ -29,6 +29,12 @@ pub enum SandboxTransport { guest_cid: u32, port: u32, }, + /// The supervisor accepts a connection from its sandbox runtime. TLS and + /// gRPC roles stay unchanged. + RuntimeInitiatedTcp { + authority: String, + listen_address: SocketAddr, + }, } impl SandboxTransport { @@ -45,6 +51,9 @@ impl SandboxTransport { .iter() .all(|address| address.port() != 0 && !address.ip().is_unspecified()) } + Self::RuntimeInitiatedTcp { authority, listen_address } => { + !authority.is_empty() && listen_address.port() != 0 + } Self::Vsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } @@ -101,6 +110,9 @@ impl TryFrom<&BackendRegistration> for crate::proto::compute::v1::IsolationBacke validate_backend_name(®istration.name)?; registration.endpoint.validate()?; let endpoint = match ®istration.endpoint { + SandboxTransport::RuntimeInitiatedTcp { .. } => { + return Err("runtime-initiated control is not a registered backend endpoint"); + } SandboxTransport::Unix { socket_path } => Endpoint::UnixSocketPath( socket_path .to_str() diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 8c2f24a7a4..0405aa1301 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -103,6 +103,63 @@ them. The cluster CNI must enforce ingress and egress `NetworkPolicy` for every sandbox namespace. Kubernetes accepts policy objects without confirming enforcement, so operators must verify CNI support before running sandboxes. +### Control connection direction + +Each sandbox records which side opens its Sandbox Protocol TCP connection in +the Sandbox annotation `openshell.ai/sandbox-runtime-control-connection`, +written once at creation from `sandbox_runtime.control_connection` (default +`runtime-initiated`). A sandbox without the annotation was created before the +mode existed and is `supervisor-initiated`. Start, gateway-restart recovery, +and image upgrades read the annotation and never apply the current default, so +both modes run side by side in one namespace. An unknown value fails closed. + +Only the dial direction differs. In both modes the sandbox runtime presents the +generation-pinned TLS server identity and serves gRPC, and the supervisor +verifies that identity, acts as the gRPC client, and sends the Sandbox Protocol +JWT. The per-sandbox Service `os-boundary-` keeps one stable ClusterIP for +the sandbox's lifetime; it selects the paired workload Pod for +`supervisor-initiated` and the paired supervisor Pod for `runtime-initiated`. + +For `runtime-initiated`: + +- The bootstrap Secret gives the runtime the Service ClusterIP and port. The + runtime dials it, keeps one unclaimed spare connection for reconnects and + credential rotation, and retries with capped backoff until the supervisor + listens, so neither Pod waits on the other during bootstrap. The workload + still cannot reach the control port. +- The supervisor listens on `boundary_port`. It completes TLS on each accepted + socket before use, limits concurrent unauthenticated handshakes and their + duration, and queues at most two authenticated spares. A new connection + cannot displace a live authenticated channel. +- The driver adds a per-sandbox `NetworkPolicy` `os-control-` that selects + only this sandbox's two Pods and admits one flow, from its workload Pod to its + supervisor Pod on `boundary_port` over TCP, as both workload egress and + supervisor ingress. Ingress matters because supervisors can already be + ingress-isolated, for example by the managed SSH ingress policy, which selects + every OpenShell-managed Pod. This also makes the supervisor ingress-isolated + when that SSH policy is disabled; exec, file sync, and logs need no other + supervisor ingress because the supervisor opens those connections. The CNI + must still allow kubelet readiness probes. The namespace-wide policies are + unchanged. +- The boundary Service sets `publishNotReadyAddresses`, because the supervisor + becomes Ready only after the runtime has attached through that Service. + The policy's UID and resourceVersion are bound into the boundary resource + claims and outer-fence evidence. Reconciliation suspends the workload if the + policy disappears, diverges, or changes generation, or if the boundary + Service no longer selects the role the recorded mode requires. +- The Service and control policy are owned by the Sandbox resource and are + deleted only with that sandbox. + +Deploy a gateway, supervisor image, and sandbox runtime image that all support +`runtime-initiated` before using it. Older binaries reject the new descriptor +shapes and fail closed. + +Rollback is unsupported for `runtime-initiated` sandboxes. Already running Pods +keep their established connection, but older binaries cannot start or restart +these sandboxes: an older driver ignores the recorded mode while the boundary +Service selects the supervisor, and older supervisors and runtimes reject the +new descriptor variants. Delete them before rolling back, or roll forward. + Each sandbox generation uses two immutable bootstrap Secrets. A trusted init container stages the sandbox bootstrap into memory, and the sandbox removes it before starting untrusted code. The other Secret is mounted only by the @@ -110,7 +167,8 @@ supervisor; when `proxy_ca_bundle` is configured it also carries the operator's corporate proxy CA bundle, which the gateway reads from its own filesystem so the anchor stays in the gateway's trust domain rather than the sandbox namespace. The TLS channel binds the namespace, Sandbox CR, workload Pod, -supervisor Pod, and shared network-policy identities. Stop deletes the workload +supervisor Pod, and shared network-policy identities, plus the per-sandbox +control policy for `runtime-initiated` sandboxes. Stop deletes the workload and supervisor Pods. Start rotates both Secrets and creates a new supervisor Pod before releasing a new workload Pod. It also renders the trusted init container again from gateway config, so a sandbox created by an older gateway diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index ec37ed61b1..a0b5f8354a 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -74,18 +74,63 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default"; /// Default storage size for the workspace PVC. pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; +/// Which side opens the Sandbox Protocol TCP connection. +/// +/// Recorded on each Sandbox resource at creation. Existing sandboxes keep their +/// recorded mode across stop/start, restarts, and upgrades; a sandbox created +/// before this setting existed is `SupervisorInitiated`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum KubernetesControlConnection { + /// The sandbox runtime dials its supervisor's per-sandbox Service. + #[default] + RuntimeInitiated, + /// The supervisor dials the sandbox runtime's listener (original mode). + SupervisorInitiated, +} + +impl KubernetesControlConnection { + /// Stable spelling used in configuration and Sandbox annotations. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::RuntimeInitiated => "runtime-initiated", + Self::SupervisorInitiated => "supervisor-initiated", + } + } +} + +impl FromStr for KubernetesControlConnection { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "runtime-initiated" => Ok(Self::RuntimeInitiated), + "supervisor-initiated" => Ok(Self::SupervisorInitiated), + other => Err(format!( + "invalid sandbox control connection '{other}'; expected runtime-initiated or supervisor-initiated" + )), + } + } +} + /// Driver-owned requirements for the Kubernetes sandbox runtime. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] pub struct KubernetesSandboxRuntimeConfig { - /// TCP port exposed by the workload boundary to its paired control pod. + /// TCP port of the Sandbox Protocol listener: the workload boundary for + /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, + /// Connection mode recorded on sandboxes created from now on. Changing it + /// never migrates an existing sandbox. + pub control_connection: KubernetesControlConnection, } impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { boundary_port: 5500, + control_connection: KubernetesControlConnection::default(), } } } @@ -952,6 +997,25 @@ fn validate_provider_spiffe_workload_api_socket_path_value( #[cfg(test)] mod tests { use super::*; + + #[test] + fn control_connection_creation_default_is_runtime_initiated_and_overridable() { + assert_eq!( + KubernetesSandboxRuntimeConfig::default().control_connection, + KubernetesControlConnection::RuntimeInitiated + ); + let parsed: KubernetesSandboxRuntimeConfig = + toml::from_str("control_connection = \"supervisor-initiated\"").unwrap(); + assert_eq!( + parsed.control_connection, + KubernetesControlConnection::SupervisorInitiated + ); + assert_eq!(parsed.boundary_port, 5500); + assert!( + toml::from_str::("control_connection = \"both\"") + .is_err() + ); + } use std::collections::BTreeMap as HashMap; #[test] diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index f30b49f501..d3a977febc 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -5,17 +5,20 @@ use crate::config::{ DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_SANDBOX_UID, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, OperatorNamespaceAllowlist, WorkspaceMode, is_dns_1123_label, - managed_namespace, managed_namespace_prefix, validate_managed_namespace_name, + KubernetesComputeConfig, KubernetesControlConnection, OperatorNamespaceAllowlist, + WorkspaceMode, is_dns_1123_label, managed_namespace, managed_namespace_prefix, + validate_managed_namespace_name, }; use crate::isolation::{ - BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeBoundarySpec, + BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesControlPath, + KubernetesSandboxRuntimeBoundarySpec, }; use crate::lifecycle::LifecycleGates; use crate::sandbox_runtime::{ BOUNDARY_CERTIFICATE_PATH, BOUNDARY_CONFIG_PATH, BOUNDARY_PRIVATE_KEY_PATH, ClientTlsMaterial, SUPERVISOR_TERMINATION_GRACE_PERIOD_SECONDS, SandboxRuntimeNames, SupervisorClientTls, - boundary_service, generate_proxy_ca_material, image_pull_secret_copy, sandbox_bootstrap_secret, + boundary_service, boundary_service_matches_mode, control_policy, generate_proxy_ca_material, + image_pull_secret_copy, sandbox_bootstrap_secret, sandbox_owner_reference as sandbox_runtime_sandbox_owner_reference, supervisor_bootstrap_secret, supervisor_pod, workload_fence, }; @@ -104,6 +107,14 @@ const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: &str = "openshell.ai/sandbox-runtime-network-policy-uid"; const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: &str = "openshell.ai/sandbox-runtime-network-policy-generation"; +/// Durable, creation-time Sandbox Protocol connection mode. Absent on +/// sandboxes created before the mode existed, which are supervisor-initiated. +const ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION: &str = + "openshell.ai/sandbox-runtime-control-connection"; +const ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID: &str = + "openshell.ai/sandbox-runtime-control-policy-uid"; +const ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION: &str = + "openshell.ai/sandbox-runtime-control-policy-generation"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum SandboxRuntimeBootstrapPhase { @@ -145,6 +156,72 @@ fn boundary_service_authority( ) } +/// Read the connection mode recorded when the sandbox was created. Starting a +/// sandbox never consults the current creation default. +fn sandbox_runtime_control_connection( + object: &DynamicObject, +) -> Result { + object + .metadata + .annotations + .as_ref() + .and_then(|annotations| annotations.get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION)) + .map_or( + Ok(KubernetesControlConnection::SupervisorInitiated), + |value| value.parse().map_err(KubernetesDriverError::Precondition), + ) +} + +/// Stored coordinates of a validated runtime-initiated control policy. +struct VerifiedControlPolicy { + uid: String, + resource_version: String, + generation: i64, +} + +impl VerifiedControlPolicy { + fn from_policy(policy: &NetworkPolicy) -> Result { + let missing = |field: &str| { + KubernetesDriverError::Message(format!("control NetworkPolicy has no {field}")) + }; + Ok(Self { + uid: policy.metadata.uid.clone().ok_or_else(|| missing("UID"))?, + resource_version: policy + .metadata + .resource_version + .clone() + .ok_or_else(|| missing("resourceVersion"))?, + generation: policy + .metadata + .generation + .ok_or_else(|| missing("generation"))?, + }) + } +} + +/// Runtime annotations recorded once both Pods have UIDs, plus the control +/// policy coordinates reconciliation compares for runtime-initiated control. +fn sandbox_runtime_binding_annotations( + workload_pod_uid: &str, + supervisor_uid: &str, + fence_uid: &str, + fence_generation: i64, + control_policy: Option<&VerifiedControlPolicy>, +) -> serde_json::Value { + let mut annotations = serde_json::json!({ + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid, + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), + }); + if let Some(policy) = control_policy { + annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID] = policy.uid.clone().into(); + annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION] = + policy.generation.to_string().into(); + } + annotations +} + #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { #[error("sandbox already exists")] @@ -2120,6 +2197,11 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL.to_string(), log_level.clone(), ); + let control_connection = self.config.sandbox_runtime.control_connection; + annotations.insert( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION.to_string(), + control_connection.as_str().to_string(), + ); for key in [ crate::config::ANNOTATION_SCC_UID_RANGE, crate::config::ANNOTATION_SCC_SUPPLEMENTAL_GROUPS, @@ -2182,6 +2264,7 @@ impl KubernetesComputeDriver { &created, &proxy_names, &generation, + control_connection, resolved_user_id, resolved_group_id, &main_process_spec, @@ -2202,6 +2285,139 @@ impl KubernetesComputeDriver { Ok(runtime_identity) } + /// Create or validate this sandbox's control egress policy and return the + /// stored coordinates bound into its boundary evidence. + async fn ensure_sandbox_runtime_control_policy( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + owner: OwnerReference, + ) -> Result { + let policies: Api = Api::namespaced(self.client.clone(), namespace); + let expected = control_policy( + namespace, + names, + sandbox_id, + self.config.sandbox_runtime.boundary_port, + owner, + ); + create_or_validate_sandbox_runtime_fence(&policies, &expected).await?; + // Bind the coordinates of the object that was actually validated. + let stored = tokio::time::timeout(KUBE_API_TIMEOUT, policies.get(&names.control_policy)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading sandbox-runtime control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + validate_sandbox_runtime_fence(&stored, &expected)?; + VerifiedControlPolicy::from_policy(&stored) + } + + fn sandbox_runtime_control_path( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + service_ip: std::net::IpAddr, + control_policy: Option, + ) -> KubernetesControlPath { + let port = self.config.sandbox_runtime.boundary_port; + let unspecified = std::net::SocketAddr::new( + if service_ip.is_ipv6() { + std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) + } else { + std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) + }, + port, + ); + let control_authority = boundary_service_authority(namespace, names, port); + let service_address = std::net::SocketAddr::new(service_ip, port); + match control_policy { + None => KubernetesControlPath::SupervisorInitiated { + boundary_listener: unspecified, + control_authority, + control_address: service_address, + }, + Some(policy) => KubernetesControlPath::RuntimeInitiated { + control_authority, + supervisor_address: service_address, + supervisor_listen: unspecified, + control_policy_uid: policy.uid, + control_policy_resource_version: policy.resource_version, + }, + } + } + + /// Fail closed when a running sandbox's endpoint or control policy no + /// longer matches the mode recorded at creation. Returns `Precondition` + /// for drift that must suspend the workload. + async fn verify_sandbox_runtime_control_path( + &self, + object: &DynamicObject, + sandbox_id: &str, + namespace: &str, + ) -> Result<(), KubernetesDriverError> { + let mode = sandbox_runtime_control_connection(object)?; + let names = SandboxRuntimeNames::new(sandbox_id); + let services = Api::::namespaced(self.client.clone(), namespace); + let service = + tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(&names.boundary_service)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading sandbox-runtime boundary Service".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)?; + // A missing Service is reported by supervisor reconciliation. + if let Some(service) = service + && !boundary_service_matches_mode(&service, sandbox_id, mode) + { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} no longer selects the {} listener", + names.boundary_service, + mode.as_str() + ))); + } + if mode == KubernetesControlConnection::SupervisorInitiated { + return Ok(()); + } + let policies = Api::::namespaced(self.client.clone(), namespace); + let stored = + tokio::time::timeout(KUBE_API_TIMEOUT, policies.get_opt(&names.control_policy)) + .await + .map_err(|_| { + KubernetesDriverError::Message( + "timed out reading sandbox-runtime control NetworkPolicy".to_string(), + ) + })? + .map_err(KubernetesDriverError::from_kube)? + .ok_or_else(|| { + KubernetesDriverError::Precondition(format!( + "control NetworkPolicy {} is missing", + names.control_policy + )) + })?; + // Ownership is not part of enforcement; compare the spec and labels. + let expected = control_policy( + namespace, + &names, + sandbox_id, + self.config.sandbox_runtime.boundary_port, + OwnerReference::default(), + ); + validate_sandbox_runtime_fence(&stored, &expected)?; + if !sandbox_runtime_control_policy_generation_matches(&stored, object) { + return Err(KubernetesDriverError::Precondition(format!( + "control NetworkPolicy {} changed after it was bound", + names.control_policy + ))); + } + Ok(()) + } + async fn create_sandbox_runtime_fence( &self, namespace: &str, @@ -2482,6 +2698,7 @@ impl KubernetesComputeDriver { sandbox_cr: &DynamicObject, names: &SandboxRuntimeNames, _generation: &str, + control_connection: KubernetesControlConnection, agent_uid: u32, agent_gid: u32, main_process_spec: &str, @@ -2520,6 +2737,7 @@ impl KubernetesComputeDriver { names, &sandbox.id, self.config.sandbox_runtime.boundary_port, + control_connection, dependent_owner.clone(), ), ) @@ -2601,6 +2819,18 @@ impl KubernetesComputeDriver { "workload NetworkPolicy has no resourceVersion".to_string(), ) })?; + let control_policy = match control_connection { + KubernetesControlConnection::SupervisorInitiated => None, + KubernetesControlConnection::RuntimeInitiated => Some( + self.ensure_sandbox_runtime_control_policy( + namespace, + names, + &sandbox.id, + dependent_owner.clone(), + ) + .await?, + ), + }; // Release the Sandbox CR only far enough for the controller to create // the workload Pod. The Pod remains unschedulable because its template @@ -2637,16 +2867,18 @@ impl KubernetesComputeDriver { let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { KubernetesDriverError::Message("workload Pod has no name".to_string()) })?; + let binding_annotations = sandbox_runtime_binding_annotations( + &workload_pod_uid, + &supervisor_uid, + &fence_uid, + fence_generation, + control_policy.as_ref(), + ); patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { serde_json::json!({ "metadata": { "resourceVersion": version, - "annotations": { - ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), - } + "annotations": binding_annotations.clone(), } }) }) @@ -2706,22 +2938,11 @@ impl KubernetesComputeDriver { supervisor_pod_uid: supervisor_uid.clone(), egress_policy_uid: fence_uid.clone(), egress_policy_resource_version: fence_resource_version.clone(), - boundary_listener: std::net::SocketAddr::new( - if service_ip.is_ipv6() { - std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) - } else { - std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) - }, - self.config.sandbox_runtime.boundary_port, - ), - control_authority: boundary_service_authority( + control_path: self.sandbox_runtime_control_path( namespace, names, - self.config.sandbox_runtime.boundary_port, - ), - control_address: std::net::SocketAddr::new( service_ip, - self.config.sandbox_runtime.boundary_port, + control_policy, ), sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), @@ -2834,6 +3055,7 @@ impl KubernetesComputeDriver { cr_uid: &str, names: &SandboxRuntimeNames, _generation: &str, + control_connection: KubernetesControlConnection, supervisor_uid: &str, agent_uid: u32, agent_gid: u32, @@ -2856,10 +3078,20 @@ impl KubernetesComputeDriver { KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) })?; let services = Api::::namespaced(self.client.clone(), namespace); - let service_ip: std::net::IpAddr = services + let service = services .get(&names.boundary_service) .await - .map_err(KubernetesDriverError::from_kube)? + .map_err(KubernetesDriverError::from_kube)?; + // The Service and its selected role were fixed when the sandbox was + // created; restart must not reinterpret them under another mode. + if !boundary_service_matches_mode(&service, sandbox_id, control_connection) { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} does not select the {} listener recorded for this sandbox", + names.boundary_service, + control_connection.as_str() + ))); + } + let service_ip: std::net::IpAddr = service .spec .and_then(|spec| spec.cluster_ip) .filter(|ip| ip != "None") @@ -2888,6 +3120,23 @@ impl KubernetesComputeDriver { "workload NetworkPolicy has no resourceVersion".to_string(), ) })?; + let control_policy = match control_connection { + KubernetesControlConnection::SupervisorInitiated => None, + KubernetesControlConnection::RuntimeInitiated => Some( + self.ensure_sandbox_runtime_control_policy( + namespace, + names, + sandbox_id, + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + ) + .await?, + ), + }; let pods = Api::::namespaced(self.client.clone(), namespace); let workload_pod = self @@ -2916,16 +3165,18 @@ impl KubernetesComputeDriver { let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { KubernetesDriverError::Message("workload Pod has no name".to_string()) })?; + let binding_annotations = sandbox_runtime_binding_annotations( + &workload_pod_uid, + supervisor_uid, + &fence_uid, + fence_generation, + control_policy.as_ref(), + ); patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { serde_json::json!({ "metadata": { "resourceVersion": version, - "annotations": { - ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), - } + "annotations": binding_annotations.clone(), } }) }) @@ -2963,22 +3214,11 @@ impl KubernetesComputeDriver { supervisor_pod_uid: supervisor_uid.to_string(), egress_policy_uid: fence_uid.clone(), egress_policy_resource_version: fence_resource_version.clone(), - boundary_listener: std::net::SocketAddr::new( - if service_ip.is_ipv6() { - std::net::IpAddr::V6(std::net::Ipv6Addr::UNSPECIFIED) - } else { - std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED) - }, - self.config.sandbox_runtime.boundary_port, - ), - control_authority: boundary_service_authority( + control_path: self.sandbox_runtime_control_path( namespace, names, - self.config.sandbox_runtime.boundary_port, - ), - control_address: std::net::SocketAddr::new( service_ip, - self.config.sandbox_runtime.boundary_port, + control_policy, ), sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), @@ -3325,6 +3565,7 @@ impl KubernetesComputeDriver { )); } + let control_connection = sandbox_runtime_control_connection(&object)?; let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); self.create_sandbox_runtime_fence(&namespace, &names) .await?; @@ -3466,6 +3707,7 @@ impl KubernetesComputeDriver { cr_uid, &names, generation.as_str(), + control_connection, &supervisor_uid, agent_uid, agent_gid, @@ -3992,6 +4234,24 @@ impl KubernetesComputeDriver { .await; continue; } + if workload_may_run { + match self + .verify_sandbox_runtime_control_path(&object, &sandbox_id, namespace) + .await + { + Ok(()) => {} + Err(KubernetesDriverError::Precondition(error)) => { + warn!(sandbox_id, %error, "sandbox-runtime control path drifted; suspending workload"); + self.suspend_sandbox_runtime_after_dependency_failure(&lookup_api, &object) + .await; + continue; + } + Err(error) => { + warn!(sandbox_id, %error, "could not verify sandbox-runtime control path; reconciliation will retry"); + continue; + } + } + } if sandbox_runtime_bootstrap_in_progress(&object) { if sandbox_runtime_control_availability(&self.client, namespace, &sandbox_id).await == SandboxRuntimeControlAvailability::Available @@ -7118,7 +7378,7 @@ fn validate_sandbox_runtime_fence( } else { let name = expected.metadata.name.as_deref().unwrap_or_default(); Err(KubernetesDriverError::Precondition(format!( - "sandbox-runtime workload fence {name} exists but does not match the intended enforcement" + "sandbox-runtime NetworkPolicy {name} exists but does not match the intended enforcement" ))) } } @@ -7177,6 +7437,23 @@ fn sandbox_runtime_namespace_fence_generation_matches( == policy.metadata.generation } +fn sandbox_runtime_control_policy_generation_matches( + policy: &NetworkPolicy, + sandbox: &DynamicObject, +) -> bool { + let Some(annotations) = sandbox.metadata.annotations.as_ref() else { + return false; + }; + policy.metadata.uid.as_deref() + == annotations + .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID) + .map(String::as_str) + && annotations + .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION) + .and_then(|generation| generation.parse::().ok()) + == policy.metadata.generation +} + fn kubernetes_sandbox_has_stopped_condition(obj: &DynamicObject) -> bool { obj.data .get("status") @@ -8932,6 +9209,302 @@ mod tests { assert!(steps.lock().unwrap().is_empty()); } + fn scripted_control_driver( + steps: Vec, + ) -> ( + KubernetesComputeDriver, + Arc>>, + ) { + let (driver, steps, _) = scripted_driver(KubernetesComputeConfig::default(), steps); + (driver, steps) + } + + fn control_mode_object(annotations: &[(&str, &str)]) -> DynamicObject { + let mut object = sandbox_object_for_test("resource-uid", "sandbox-1"); + object.metadata.annotations = Some( + annotations + .iter() + .map(|(key, value)| ((*key).to_string(), (*value).to_string())) + .collect(), + ); + object + } + + fn boundary_service_json(role: &str, publish_not_ready: bool) -> serde_json::Value { + let mut service = serde_json::json!({ + "apiVersion": "v1", + "kind": "Service", + "metadata": {"name": "os-boundary-sandbox-1", "namespace": "openshell"}, + "spec": { + "clusterIP": "10.43.0.7", + "selector": {BOUNDARY_PAIR_LABEL: "sandbox-1", BOUNDARY_ROLE_LABEL: role} + } + }); + if publish_not_ready { + service["spec"]["publishNotReadyAddresses"] = true.into(); + } + service + } + + /// The Service each mode renders: runtime-initiated publishes the + /// supervisor before it is Ready. + fn boundary_service_response(role: &str) -> http::Response { + kube_test_response( + http::StatusCode::OK, + boundary_service_json(role, role == "supervisor"), + ) + } + + fn stored_control_policy(generation: i64) -> NetworkPolicy { + let names = SandboxRuntimeNames::new("sandbox-1"); + let mut policy = control_policy( + "openshell", + &names, + "sandbox-1", + 5500, + OwnerReference::default(), + ); + policy.metadata.uid = Some("control-uid".to_string()); + policy.metadata.generation = Some(generation); + policy.metadata.resource_version = Some("77".to_string()); + policy + } + + type Annotations<'a> = &'a [(&'a str, &'a str)]; + + const SERVICE_PATH: &str = "/api/v1/namespaces/openshell/services/os-boundary-sandbox-1"; + const CONTROL_POLICY_PATH: &str = + "/apis/networking.k8s.io/v1/namespaces/openshell/networkpolicies/os-control-sandbox-1"; + + #[test] + fn control_connection_is_read_from_the_sandbox_and_defaults_to_legacy() { + assert_eq!( + sandbox_runtime_control_connection(&control_mode_object(&[])).unwrap(), + KubernetesControlConnection::SupervisorInitiated + ); + assert_eq!( + sandbox_runtime_control_connection(&control_mode_object(&[( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, + "runtime-initiated" + )])) + .unwrap(), + KubernetesControlConnection::RuntimeInitiated + ); + assert!(matches!( + sandbox_runtime_control_connection(&control_mode_object(&[( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, + "Runtime-Initiated" + )])), + Err(KubernetesDriverError::Precondition(_)) + )); + } + + #[test] + fn binding_annotations_add_control_policy_only_for_runtime_initiated() { + let legacy = sandbox_runtime_binding_annotations("w", "s", "f", 3, None); + assert!( + legacy + .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID) + .is_none() + ); + let policy = VerifiedControlPolicy::from_policy(&stored_control_policy(4)).unwrap(); + let runtime = sandbox_runtime_binding_annotations("w", "s", "f", 3, Some(&policy)); + assert_eq!( + runtime[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID], + "control-uid" + ); + assert_eq!( + runtime[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION], + "4" + ); + assert_eq!( + runtime[ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION], + "3" + ); + } + + #[tokio::test] + async fn reconcile_accepts_each_mode_with_its_own_endpoint() { + let (driver, steps) = scripted_control_driver(vec![( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("workload"), + )]); + driver + .verify_sandbox_runtime_control_path( + &control_mode_object(&[]), + "sandbox-1", + "openshell", + ) + .await + .expect("legacy sandbox with workload Service"); + assert!(steps.lock().unwrap().is_empty()); + + let (driver, steps) = scripted_control_driver(vec![ + ( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("supervisor"), + ), + ( + http::Method::GET, + CONTROL_POLICY_PATH, + kube_test_response( + http::StatusCode::OK, + serde_json::to_value(stored_control_policy(4)).unwrap(), + ), + ), + ]); + driver + .verify_sandbox_runtime_control_path( + &control_mode_object(&[ + ( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, + "runtime-initiated", + ), + (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, "control-uid"), + (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, "4"), + ]), + "sandbox-1", + "openshell", + ) + .await + .expect("runtime-initiated sandbox with bound control policy"); + assert!(steps.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn reconcile_fails_closed_on_service_or_control_policy_drift() { + let runtime_annotations = [ + ( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, + "runtime-initiated", + ), + (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, "control-uid"), + (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, "4"), + ]; + let mut broadened = stored_control_policy(4); + broadened + .spec + .as_mut() + .unwrap() + .egress + .as_mut() + .unwrap() + .push(k8s_openapi::api::networking::v1::NetworkPolicyEgressRule::default()); + let cases: Vec<(Annotations<'_>, Vec)> = vec![ + // A legacy sandbox's Service must keep selecting its workload. + ( + &[], + vec![( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("supervisor"), + )], + ), + // A runtime-initiated Service must keep selecting its supervisor. + ( + &runtime_annotations, + vec![( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("workload"), + )], + ), + // A runtime-initiated Service that withholds the not-Ready + // supervisor would deadlock bootstrap and recovery. + ( + &runtime_annotations, + vec![( + http::Method::GET, + SERVICE_PATH, + kube_test_response( + http::StatusCode::OK, + boundary_service_json("supervisor", false), + ), + )], + ), + // The control policy was deleted. + ( + &runtime_annotations, + vec![ + ( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("supervisor"), + ), + ( + http::Method::GET, + CONTROL_POLICY_PATH, + kube_test_not_found("networkpolicies", "os-control-sandbox-1"), + ), + ], + ), + // The control policy was broadened in place. + ( + &runtime_annotations, + vec![ + ( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("supervisor"), + ), + ( + http::Method::GET, + CONTROL_POLICY_PATH, + kube_test_response( + http::StatusCode::OK, + serde_json::to_value(&broadened).unwrap(), + ), + ), + ], + ), + // The control policy changed after it was bound. + ( + &runtime_annotations, + vec![ + ( + http::Method::GET, + SERVICE_PATH, + boundary_service_response("supervisor"), + ), + ( + http::Method::GET, + CONTROL_POLICY_PATH, + kube_test_response( + http::StatusCode::OK, + serde_json::to_value(stored_control_policy(5)).unwrap(), + ), + ), + ], + ), + // An unknown recorded mode is never reinterpreted. + ( + &[( + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, + "bidirectional", + )], + vec![], + ), + ]; + for (index, (annotations, steps)) in cases.into_iter().enumerate() { + let (driver, remaining) = scripted_control_driver(steps); + let result = driver + .verify_sandbox_runtime_control_path( + &control_mode_object(annotations), + "sandbox-1", + "openshell", + ) + .await; + assert!( + matches!(result, Err(KubernetesDriverError::Precondition(_))), + "case {index}: {:?}", + result.err() + ); + assert!(remaining.lock().unwrap().is_empty(), "case {index}"); + } + } + #[test] fn bootstrap_phase_round_trips_releasing() { let phase = SandboxRuntimeBootstrapPhase::Releasing; diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index f4ae89b38b..ed095f4e56 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -29,6 +29,8 @@ use openshell_sandbox_backend::boundary_protocol::{ }; use serde::Serialize; +use crate::config::KubernetesControlConnection; + #[derive(Serialize)] struct KubernetesOuterFenceEvidence<'a> { network_policy_uid: &'a str, @@ -36,6 +38,36 @@ struct KubernetesOuterFenceEvidence<'a> { ingress_isolated: bool, egress_isolated: bool, egress_rule_count: u32, + /// Present only for runtime-initiated control. Omitted otherwise so the + /// supervisor-initiated evidence encoding and digest are unchanged. + #[serde(skip_serializing_if = "Option::is_none")] + runtime_control_egress: Option>, +} + +/// Facts about the per-sandbox policy that admits only the runtime's dial to +/// its own supervisor. The driver records them only after verifying the +/// stored policy matches [`runtime_control_policy`]. +#[derive(Serialize)] +struct KubernetesControlEgressEvidence<'a> { + network_policy_uid: &'a str, + network_policy_resource_version: &'a str, + /// Egress rules in the policy; the only one targets the paired supervisor. + egress_rule_count: u32, + /// Ingress rules in the policy; the only one admits the paired workload. + ingress_rule_count: u32, + paired_supervisor_only: bool, + port: u16, +} + +impl KubernetesControlEgressEvidence<'_> { + fn is_managed_control_path(&self) -> bool { + !self.network_policy_uid.is_empty() + && !self.network_policy_resource_version.is_empty() + && self.egress_rule_count == 1 + && self.ingress_rule_count == 1 + && self.paired_supervisor_only + && self.port != 0 + } } impl KubernetesOuterFenceEvidence<'_> { @@ -50,6 +82,9 @@ impl KubernetesOuterFenceEvidence<'_> { "Kubernetes provisioning evidence is incomplete".to_string(), )); } + if self.runtime_control_egress.as_ref().is_some_and(|control| !control.is_managed_control_path()) { + return Err(BackendError::Descriptor("Kubernetes control policy evidence is incomplete".into())); + } let encoded = serde_json::to_vec(self).map_err(|error| { BackendError::Descriptor(format!("encode Kubernetes outer fence evidence: {error}")) })?; @@ -174,6 +209,101 @@ fn role_labels(role: &str) -> BTreeMap { BTreeMap::from([(BOUNDARY_ROLE_LABEL.to_string(), role.to_string())]) } +/// Labels selecting one role of one workload/supervisor pair. +#[must_use] +pub fn pair_role_labels(pair: &str, role: &str) -> BTreeMap { + BTreeMap::from([ + (BOUNDARY_PAIR_LABEL.to_string(), pair.to_string()), + (BOUNDARY_ROLE_LABEL.to_string(), role.to_string()), + ]) +} + +/// Pod role the per-sandbox boundary Service selects for a connection mode. +#[must_use] +pub const fn boundary_service_role(mode: KubernetesControlConnection) -> &'static str { + match mode { + KubernetesControlConnection::SupervisorInitiated => WORKLOAD_ROLE, + KubernetesControlConnection::RuntimeInitiated => SUPERVISOR_ROLE, + } +} + +/// Per-sandbox allowance for runtime-initiated control. +/// +/// Policies are additive: this admits exactly one flow, from this pair's +/// workload Pod to this pair's supervisor Pod on the control port, at both +/// enforcement points. Egress is required because the namespace fence denies +/// all workload egress; ingress is required because supervisors may be +/// ingress-isolated (for example by the managed SSH ingress policy). The +/// selector matches only this pair, so the extra rules can name only the pair +/// itself. It never adds DNS, gateway, API-server, or address-block access and +/// leaves the namespace-wide policies untouched. +#[must_use] +pub fn runtime_control_policy(namespace: &str, name: &str, pair: &str, port: u16) -> NetworkPolicy { + let pair_role = |role| { + Some(vec![NetworkPolicyPeer { + pod_selector: Some(LabelSelector { + match_labels: Some(pair_role_labels(pair, role)), + ..Default::default() + }), + ..Default::default() + }]) + }; + let control_port = || { + Some(vec![NetworkPolicyPort { + port: Some(IntOrString::Int(i32::from(port))), + protocol: Some("TCP".to_string()), + ..Default::default() + }]) + }; + NetworkPolicy { + metadata: ObjectMeta { + name: Some(name.to_string()), + namespace: Some(namespace.to_string()), + ..Default::default() + }, + spec: Some(NetworkPolicySpec { + pod_selector: LabelSelector { + match_labels: Some(BTreeMap::from([( + BOUNDARY_PAIR_LABEL.to_string(), + pair.to_string(), + )])), + ..Default::default() + }, + policy_types: Some(vec!["Ingress".to_string(), "Egress".to_string()]), + ingress: Some(vec![NetworkPolicyIngressRule { + from: pair_role(WORKLOAD_ROLE), + ports: control_port(), + }]), + egress: Some(vec![NetworkPolicyEgressRule { + to: pair_role(SUPERVISOR_ROLE), + ports: control_port(), + }]), + }), + } +} + +/// Which side opens the Sandbox Protocol connection, with the verified +/// Kubernetes coordinates that mode needs. +pub enum KubernetesControlPath { + /// The supervisor dials the workload's listener through the Service. + SupervisorInitiated { + boundary_listener: SocketAddr, + control_authority: String, + control_address: SocketAddr, + }, + /// The workload runtime dials the supervisor through the Service. + RuntimeInitiated { + control_authority: String, + /// Service address the runtime dials. + supervisor_address: SocketAddr, + /// Supervisor bind address. + supervisor_listen: SocketAddr, + /// Verified per-sandbox control policy coordinates. + control_policy_uid: String, + control_policy_resource_version: String, + }, +} + /// Driver-owned inputs that bind one workload/supervisor pair to one boundary. /// /// The driver constructs this only after Kubernetes has assigned every UID and @@ -195,9 +325,7 @@ pub struct KubernetesSandboxRuntimeBoundarySpec { pub supervisor_pod_uid: String, pub egress_policy_uid: String, pub egress_policy_resource_version: String, - pub boundary_listener: SocketAddr, - pub control_authority: String, - pub control_address: SocketAddr, + pub control_path: KubernetesControlPath, pub sandbox_tls: SandboxTlsServerConfig, pub supervisor_tls: SandboxTlsClientConfig, pub host_gateway_ip: Option, @@ -215,7 +343,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { /// Produce both sides of the common protocol from one observed Kubernetes /// resource set so a stale or recreated object cannot be attached. pub fn provision(self) -> Result { - let resource_claims = BTreeMap::from([ + let mut resource_claims = BTreeMap::from([ ("kubernetes.namespace_uid".to_string(), self.namespace_uid), ( "kubernetes.sandbox_resource_uid".to_string(), @@ -238,6 +366,55 @@ impl KubernetesSandboxRuntimeBoundarySpec { self.egress_policy_resource_version, ), ]); + let (listener, transport) = match self.control_path { + KubernetesControlPath::SupervisorInitiated { + boundary_listener, + control_authority, + control_address, + } => ( + BoundaryListener::TlsTcp { + address: boundary_listener, + tls: self.sandbox_tls, + }, + SandboxTransport::Tcp { + authority: control_authority, + addresses: vec![control_address], + }, + ), + KubernetesControlPath::RuntimeInitiated { + control_authority, + supervisor_address, + supervisor_listen, + control_policy_uid, + control_policy_resource_version, + } => { + resource_claims.insert( + "kubernetes.control_policy_uid".to_string(), + control_policy_uid, + ); + resource_claims.insert( + "kubernetes.control_policy_resource_version".to_string(), + control_policy_resource_version, + ); + ( + BoundaryListener::RuntimeInitiatedTlsTcp { + authority: control_authority.clone(), + addresses: vec![supervisor_address], + tls: self.sandbox_tls, + }, + SandboxTransport::RuntimeInitiatedTcp { + authority: control_authority, + listen_address: supervisor_listen, + }, + ) + } + }; + let control_port = match &transport { + SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } => { + Some(listen_address.port()) + } + _ => None, + }; let provisioning_evidence = KubernetesOuterFenceEvidence { network_policy_uid: &resource_claims["kubernetes.egress_policy_uid"], network_policy_resource_version: &resource_claims @@ -245,6 +422,15 @@ impl KubernetesSandboxRuntimeBoundarySpec { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, + runtime_control_egress: control_port.map(|port| KubernetesControlEgressEvidence { + network_policy_uid: &resource_claims["kubernetes.control_policy_uid"], + network_policy_resource_version: &resource_claims + ["kubernetes.control_policy_resource_version"], + egress_rule_count: 1, + ingress_rule_count: 1, + paired_supervisor_only: true, + port, + }), } .project(&self.generation)?; Ok(KubernetesSandboxRuntimeBoundaryProvisioning { @@ -256,10 +442,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { auth_epoch: self.auth_epoch, gateway_id: self.gateway_id, verification_keys: self.verification_keys, - listener: BoundaryListener::TlsTcp { - address: self.boundary_listener, - tls: self.sandbox_tls, - }, + listener, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::from([( "kubernetes.workload_pod_uid".to_string(), @@ -274,10 +457,7 @@ impl KubernetesSandboxRuntimeBoundarySpec { generation: self.generation, session_id: self.session_id, workload_identity: self.workload_identity, - transport: SandboxTransport::Tcp { - authority: self.control_authority, - addresses: vec![self.control_address], - }, + transport, tls: self.supervisor_tls, host_gateway_ip: self.host_gateway_ip, resource_claims, @@ -300,6 +480,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, + runtime_control_egress: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -307,6 +488,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, + runtime_control_egress: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -314,6 +496,7 @@ mod tests { ingress_isolated: false, egress_isolated: true, egress_rule_count: 0, + runtime_control_egress: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -321,6 +504,7 @@ mod tests { ingress_isolated: true, egress_isolated: false, egress_rule_count: 0, + runtime_control_egress: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -328,6 +512,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 1, + runtime_control_egress: None, }, ] { assert!(evidence.project("generation-1").is_err()); @@ -353,9 +538,11 @@ mod tests { supervisor_pod_uid: "supervisor-pod-uid".to_string(), egress_policy_uid: "network-policy-uid".to_string(), egress_policy_resource_version: "1945".to_string(), - boundary_listener: "0.0.0.0:5500".parse().expect("valid listener"), - control_authority: "os-boundary-sandbox.default.svc:5500".to_string(), - control_address: "10.42.0.7:5500".parse().expect("valid target"), + control_path: KubernetesControlPath::SupervisorInitiated { + boundary_listener: "0.0.0.0:5500".parse().expect("valid listener"), + control_authority: "os-boundary-sandbox.default.svc:5500".to_string(), + control_address: "10.42.0.7:5500".parse().expect("valid target"), + }, sandbox_tls: SandboxTlsServerConfig { certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), private_key_path: PathBuf::from("/run/boundary/tls.key"), @@ -436,6 +623,153 @@ mod tests { ); } + fn runtime_initiated_spec() -> KubernetesSandboxRuntimeBoundarySpec { + let mut spec = spec(); + spec.control_path = KubernetesControlPath::RuntimeInitiated { + control_authority: "os-boundary-sandbox.default.svc:5500".to_string(), + supervisor_address: "10.43.0.7:5500".parse().expect("valid target"), + supervisor_listen: "0.0.0.0:5500".parse().expect("valid listener"), + control_policy_uid: "control-policy-uid".to_string(), + control_policy_resource_version: "2001".to_string(), + }; + spec + } + + #[test] + fn supervisor_initiated_evidence_encoding_is_unchanged() { + let evidence = KubernetesOuterFenceEvidence { + network_policy_uid: "uid", + network_policy_resource_version: "1", + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + runtime_control_egress: None, + }; + assert_eq!( + serde_json::to_string(&evidence).unwrap(), + r#"{"network_policy_uid":"uid","network_policy_resource_version":"1","ingress_isolated":true,"egress_isolated":true,"egress_rule_count":0}"# + ); + } + + #[test] + fn runtime_initiated_provisioning_reverses_only_the_dial_direction() { + let provisioned = runtime_initiated_spec().provision().unwrap(); + assert_eq!( + provisioned.boundary_config.listener, + BoundaryListener::RuntimeInitiatedTlsTcp { + authority: "os-boundary-sandbox.default.svc:5500".to_string(), + addresses: vec!["10.43.0.7:5500".parse().expect("valid target")], + tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), + private_key_path: PathBuf::from("/run/boundary/tls.key"), + }, + } + ); + assert_eq!( + provisioned.runtime_descriptor.transport, + SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox.default.svc:5500".to_string(), + listen_address: "0.0.0.0:5500".parse().expect("valid listener"), + } + ); + // TLS identity and every legacy binding are unchanged; the control + // policy is bound in addition. + let legacy = spec().provision().unwrap(); + assert_eq!( + provisioned.runtime_descriptor.tls, + legacy.runtime_descriptor.tls + ); + let claims = &provisioned.runtime_descriptor.resource_claims; + assert_eq!(claims, &provisioned.boundary_config.resource_claims); + for (key, value) in &legacy.runtime_descriptor.resource_claims { + assert_eq!(claims.get(key), Some(value)); + } + assert_eq!( + claims["kubernetes.control_policy_uid"], + "control-policy-uid" + ); + assert_eq!(claims["kubernetes.control_policy_resource_version"], "2001"); + assert!( + provisioned + .runtime_descriptor + .outer_fence + .validate("generation-1") + .is_ok() + ); + assert_ne!( + provisioned.runtime_descriptor.outer_fence.evidence_digest, + legacy.runtime_descriptor.outer_fence.evidence_digest + ); + } + + #[test] + fn runtime_control_evidence_rejects_any_unmanaged_egress() { + let control = |uid, rules, paired| KubernetesControlEgressEvidence { + network_policy_uid: uid, + network_policy_resource_version: "1", + egress_rule_count: rules, + ingress_rule_count: 1, + paired_supervisor_only: paired, + port: 5500, + }; + for runtime_control_egress in [ + control("", 1, true), + control("uid", 2, true), + control("uid", 1, false), + ] { + let evidence = KubernetesOuterFenceEvidence { + network_policy_uid: "uid", + network_policy_resource_version: "1", + ingress_isolated: true, + egress_isolated: true, + egress_rule_count: 0, + runtime_control_egress: Some(runtime_control_egress), + }; + assert!(evidence.project("generation-1").is_err()); + } + } + + #[test] + fn runtime_control_policy_admits_only_the_paired_supervisor_port() { + let policy = runtime_control_policy("sandbox-ns", "os-control-pair", "pair", 5500); + let spec = policy.spec.expect("policy has a spec"); + // Supervisors may already be ingress-isolated (managed SSH policy), + // so the one flow is admitted at both enforcement points. + assert_eq!( + spec.policy_types, + Some(vec!["Ingress".to_string(), "Egress".to_string()]) + ); + assert_eq!( + spec.pod_selector.match_labels, + Some(BTreeMap::from([( + BOUNDARY_PAIR_LABEL.to_string(), + "pair".to_string() + )])) + ); + let only_peer_and_port = |peers: &Option>, + ports: &Option>, + role: &str| { + let peers = peers.as_ref().expect("rule has peers"); + assert_eq!(peers.len(), 1); + assert!(peers[0].ip_block.is_none()); + assert!(peers[0].namespace_selector.is_none()); + assert_eq!( + peers[0].pod_selector.as_ref().unwrap().match_labels, + Some(pair_role_labels("pair", role)) + ); + let ports = ports.as_ref().expect("rule has ports"); + assert_eq!(ports.len(), 1); + assert_eq!(ports[0].protocol.as_deref(), Some("TCP")); + assert_eq!(ports[0].port, Some(IntOrString::Int(5500))); + }; + let egress = spec.egress.expect("policy has egress rules"); + assert_eq!(egress.len(), 1); + only_peer_and_port(&egress[0].to, &egress[0].ports, SUPERVISOR_ROLE); + let ingress = spec.ingress.expect("policy has ingress rules"); + assert_eq!(ingress.len(), 1); + only_peer_and_port(&ingress[0].from, &ingress[0].ports, WORKLOAD_ROLE); + } + #[test] fn network_fence_denies_all_workload_initiated_egress() { let fence = KubernetesSandboxRuntimeNetworkFenceSpec { diff --git a/crates/openshell-driver-kubernetes/src/lib.rs b/crates/openshell-driver-kubernetes/src/lib.rs index aa980db44f..da17212a39 100644 --- a/crates/openshell-driver-kubernetes/src/lib.rs +++ b/crates/openshell-driver-kubernetes/src/lib.rs @@ -12,8 +12,9 @@ mod sandbox_runtime; pub use config::{ DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE, - KubernetesComputeConfig, KubernetesImagePullPolicy, KubernetesSandboxRuntimeConfig, - ManagedSshIngressConfig, WorkspaceMode, managed_namespace_prefix, + KubernetesComputeConfig, KubernetesControlConnection, KubernetesImagePullPolicy, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, + managed_namespace_prefix, }; pub use driver::{KubernetesComputeDriver, KubernetesDriverError}; pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs index 9d70b4340e..16c8f82981 100644 --- a/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs +++ b/crates/openshell-driver-kubernetes/src/lifecycle_tests.rs @@ -175,6 +175,23 @@ mod lifecycle_reconciliation { workload(), supervisor(), workload(), + // The recorded mode (none: supervisor-initiated) must still match + // the role the boundary Service selects. + ( + http::Method::GET, + "/api/v1/namespaces/openshell/services/os-boundary-sandbox-1", + kube_test_response( + http::StatusCode::OK, + serde_json::json!({ + "apiVersion": "v1", "kind": "Service", + "metadata": {"name": "os-boundary-sandbox-1"}, + "spec": {"selector": { + BOUNDARY_PAIR_LABEL: "sandbox-1", + BOUNDARY_ROLE_LABEL: "workload" + }} + }), + ), + ), ( http::Method::GET, "/api/v1/namespaces/openshell/pods/os-supervisor-sandbox-1", diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index ab6e0bf1b8..9be1e6cccc 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -12,8 +12,9 @@ use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesImagePullPolicy, - KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesControlConnection, + KubernetesImagePullPolicy, KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, + WorkspaceMode, }; #[derive(Parser, Debug)] @@ -157,6 +158,14 @@ struct Args { )] sandbox_runtime_boundary_port: u16, + /// Sandbox Protocol connection mode recorded on newly created sandboxes. + #[arg( + long, + env = "OPENSHELL_K8S_SANDBOX_RUNTIME_CONTROL_CONNECTION", + default_value = "runtime-initiated" + )] + sandbox_runtime_control_connection: KubernetesControlConnection, + /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] https_proxy: Option, @@ -286,6 +295,7 @@ async fn main() -> Result<()> { supervisor_image_pull_policy: args.supervisor_image_pull_policy, sandbox_runtime: KubernetesSandboxRuntimeConfig { boundary_port: args.sandbox_runtime_boundary_port, + control_connection: args.sandbox_runtime_control_connection, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index d3bc1b27ec..d4295253ef 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -19,9 +19,14 @@ use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString; use kube::core::ObjectMeta; use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose}; +use k8s_openapi::api::networking::v1::NetworkPolicy; + +use crate::config::KubernetesControlConnection; +#[cfg(test)] +use crate::isolation::{BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL}; use crate::isolation::{ - BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeNetworkFence, - KubernetesSandboxRuntimeNetworkFenceSpec, + KubernetesSandboxRuntimeNetworkFence, KubernetesSandboxRuntimeNetworkFenceSpec, + boundary_service_role, runtime_control_policy, }; pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap"; @@ -108,6 +113,8 @@ pub struct SandboxRuntimeNames { pub sandbox_secret: String, pub supervisor_secret: String, pub boundary_service: String, + /// Per-sandbox egress allowance used only by runtime-initiated control. + pub control_policy: String, pub supervisor_pod: String, pub workload_policy: String, pub supervisor_policy: String, @@ -122,6 +129,7 @@ impl SandboxRuntimeNames { sandbox_secret: format!("os-sandbox-{suffix}"), supervisor_secret: format!("os-supervisor-{suffix}"), boundary_service: format!("os-boundary-{suffix}"), + control_policy: format!("os-control-{suffix}"), supervisor_pod: format!("os-supervisor-{suffix}"), workload_policy: NAMESPACE_WORKLOAD_POLICY_NAME.to_string(), supervisor_policy: NAMESPACE_SUPERVISOR_EGRESS_POLICY_NAME.to_string(), @@ -176,12 +184,16 @@ pub fn workload_fence( .provision() } +/// Per-sandbox Sandbox Protocol endpoint. Its stable `ClusterIP` is allocated +/// before either Pod runs; the selected role follows the sandbox's recorded +/// connection mode and never changes for the sandbox's lifetime. #[must_use] pub fn boundary_service( namespace: &str, names: &SandboxRuntimeNames, sandbox_id: &str, boundary_port: u16, + mode: KubernetesControlConnection, owner: OwnerReference, ) -> Service { Service { @@ -193,7 +205,7 @@ pub fn boundary_service( ..Default::default() }, spec: Some(ServiceSpec { - selector: Some(pair_labels(sandbox_id, "workload")), + selector: Some(pair_labels(sandbox_id, boundary_service_role(mode))), ports: Some(vec![ServicePort { name: Some("boundary".to_string()), protocol: Some("TCP".to_string()), @@ -201,12 +213,61 @@ pub fn boundary_service( target_port: Some(IntOrString::Int(i32::from(boundary_port))), ..Default::default() }]), + publish_not_ready_addresses: publishes_not_ready_supervisor(mode), ..Default::default() }), ..Default::default() } } +/// A runtime-initiated supervisor becomes Ready only after the runtime has +/// attached through this Service, so the Service must route to it before it is +/// Ready. TLS and the Sandbox Protocol bearer, not readiness, authenticate the +/// endpoint. Supervisor-initiated Services keep their original encoding. +const fn publishes_not_ready_supervisor(mode: KubernetesControlConnection) -> Option { + match mode { + KubernetesControlConnection::RuntimeInitiated => Some(true), + KubernetesControlConnection::SupervisorInitiated => None, + } +} + +/// Whether an existing boundary Service still selects the role required by +/// `mode` for this sandbox's pair. +#[must_use] +pub fn boundary_service_matches_mode( + service: &Service, + sandbox_id: &str, + mode: KubernetesControlConnection, +) -> bool { + let Some(spec) = service.spec.as_ref() else { + return false; + }; + let publishes_not_ready = spec.publish_not_ready_addresses == Some(true); + spec.selector.as_ref() == Some(&pair_labels(sandbox_id, boundary_service_role(mode))) + && publishes_not_ready == (publishes_not_ready_supervisor(mode) == Some(true)) +} + +/// Per-sandbox control egress policy, owned by the Sandbox resource so it is +/// garbage-collected with that sandbox only. +#[must_use] +pub fn control_policy( + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + boundary_port: u16, + owner: OwnerReference, +) -> NetworkPolicy { + let mut policy = runtime_control_policy( + namespace, + &names.control_policy, + &pair_label_value(sandbox_id), + boundary_port, + ); + policy.metadata.owner_references = Some(vec![owner]); + policy.metadata.labels = Some(common_labels(sandbox_id, "sandbox-control-egress")); + policy +} + #[allow(clippy::too_many_arguments, clippy::similar_names)] pub fn supervisor_pod( namespace: &str, @@ -632,13 +693,7 @@ pub fn sandbox_owner_reference( } fn pair_labels(sandbox_id: &str, role: &str) -> BTreeMap { - BTreeMap::from([ - ( - BOUNDARY_PAIR_LABEL.to_string(), - pair_label_value(sandbox_id), - ), - (BOUNDARY_ROLE_LABEL.to_string(), role.to_string()), - ]) + crate::isolation::pair_role_labels(&pair_label_value(sandbox_id), role) } fn common_labels(sandbox_id: &str, component: &str) -> BTreeMap { @@ -724,12 +779,55 @@ mod tests { } #[test] - fn service_selects_only_the_workload_boundary() { + fn service_selects_the_listener_of_the_recorded_mode() { let names = SandboxRuntimeNames::new("4b67c0d0-1111-2222-3333-444444444444"); - let service = boundary_service("sandbox", &names, "pair", 5500, owner()); + for (mode, role, other) in [ + ( + KubernetesControlConnection::SupervisorInitiated, + "workload", + KubernetesControlConnection::RuntimeInitiated, + ), + ( + KubernetesControlConnection::RuntimeInitiated, + "supervisor", + KubernetesControlConnection::SupervisorInitiated, + ), + ] { + let service = boundary_service("sandbox", &names, "Pair", 5500, mode, owner()); + let selector = service.spec.as_ref().unwrap().selector.clone().unwrap(); + assert_eq!(selector[BOUNDARY_ROLE_LABEL], role); + // The runtime must reach a not-yet-Ready supervisor; the legacy + // Service encoding is unchanged. + let serialized = serde_json::to_value(&service).unwrap(); + assert_eq!( + serialized["spec"].get("publishNotReadyAddresses"), + (mode == KubernetesControlConnection::RuntimeInitiated) + .then_some(&serde_json::Value::Bool(true)) + ); + assert_eq!(selector[BOUNDARY_PAIR_LABEL], "pair"); + assert!(boundary_service_matches_mode(&service, "Pair", mode)); + assert!(!boundary_service_matches_mode(&service, "Pair", other)); + assert!(!boundary_service_matches_mode(&service, "other", mode)); + let mut flipped = service.clone(); + let spec = flipped.spec.as_mut().unwrap(); + spec.publish_not_ready_addresses = match spec.publish_not_ready_addresses { + Some(true) => None, + _ => Some(true), + }; + assert!(!boundary_service_matches_mode(&flipped, "Pair", mode)); + } + } + + #[test] + fn control_policy_is_owned_and_scoped_to_one_pair() { + let names = SandboxRuntimeNames::for_generation("Pair", "generation"); + let policy = control_policy("sandbox", &names, "Pair", 5500, owner()); + assert_eq!(policy.metadata.name.as_deref(), Some("os-control-pair")); + assert_eq!(policy.metadata.owner_references, Some(vec![owner()])); + let spec = policy.spec.unwrap(); assert_eq!( - service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL], - "workload" + spec.pod_selector.match_labels.unwrap()[BOUNDARY_PAIR_LABEL], + "pair" ); } diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d0309a2efd..a25ad5ab07 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -7112,6 +7112,12 @@ fn inject_guest_boundary_bundle( BoundaryListener::Unix { tls, .. } | BoundaryListener::TlsTcp { tls, .. } | BoundaryListener::Vsock { tls, .. } => tls.clone(), + // The VM driver provisions only listener transports. + BoundaryListener::RuntimeInitiatedTlsTcp { .. } => { + return Err( + "VM sandboxes do not support runtime-initiated boundary control".to_string(), + ); + } }; let encoded_config = config .encode() diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index 534c4c4589..caeb671249 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -102,6 +102,25 @@ retain generation, session, resource, identity, and provisioning evidence checks ## Boundary transport establishment -The client connects to the configured Unix socket, TCP, or vsock endpoint. It -owns TLS peer verification, session bearer authentication, generation checks, -and reconnection handling for policy discovery and attachment. +Backends may supply a `BoundaryTransportConnector` for raw stream establishment. +The shared client retains TLS peer verification, session bearer authentication, +generation checks, and reconnection handling. The connector is reused for policy +discovery and attachment; it does not define platform launch configuration or +replace the isolation interface. + +### Runtime-initiated transport + +`SandboxTransport::RuntimeInitiatedTcp` and +`BoundaryListener::RuntimeInitiatedTlsTcp` reverse only the TCP dial: the +sandbox runtime connects to the supervisor, then remains the TLS server and gRPC +server. The supervisor binds one `RuntimeInitiatedListener` per sandbox session +and shares it between policy discovery and attachment. The listener completes +TLS with the descriptor's pinned identity before it queues a stream, bounds +concurrent unauthenticated handshakes and their duration, and keeps at most two +unclaimed authenticated streams. The client wraps each stream in the same +abortable transport as dialed connections, so cancellation closes the socket. +The runtime keeps one unclaimed spare connection open so reconnects and +credential rotation never wait for a new dial. + +The new variants leave existing encodings unchanged. Binaries that predate them +reject runtime-initiated descriptors instead of falling back to another mode. diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 4bbcaca879..2cf6431c60 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -354,6 +354,15 @@ pub enum BoundaryListener { control_port: u32, tls: SandboxTlsServerConfig, }, + /// The boundary dials the supervisor's listener and serves TLS and gRPC + /// over that outbound connection. Only driver-resolved addresses are used. + RuntimeInitiatedTlsTcp { + /// Stable logical Service authority, for diagnostics. + authority: String, + /// Driver-resolved supervisor endpoints, tried in order. + addresses: Vec, + tls: SandboxTlsServerConfig, + }, } /// Public verification key staged in the sandbox's immutable auth bundle. @@ -1765,6 +1774,67 @@ mod tests { assert_eq!(decoded, transport); } + #[test] + fn legacy_control_encodings_are_unchanged_by_runtime_initiated_variants() { + // Binaries from before runtime-initiated control decode exactly these + // shapes; the new variants must not alter them. + let transport = SandboxTransport::Tcp { + authority: "os-boundary-a.ns.svc:5500".to_string(), + addresses: vec!["10.43.0.9:5500".parse().expect("test address")], + }; + assert_eq!( + serde_json::to_string(&transport).unwrap(), + r#"{"kind":"tcp","authority":"os-boundary-a.ns.svc:5500","addresses":["10.43.0.9:5500"]}"# + ); + let listener = BoundaryListener::TlsTcp { + address: "0.0.0.0:5500".parse().expect("test address"), + tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/run/tls.crt"), + private_key_path: PathBuf::from("/run/tls.key"), + }, + }; + assert_eq!( + serde_json::to_string(&listener).unwrap(), + r#"{"kind":"tls-tcp","address":"0.0.0.0:5500","tls":{"certificate_chain_path":"/run/tls.crt","private_key_path":"/run/tls.key"}}"# + ); + } + + #[test] + fn runtime_initiated_control_variants_round_trip_and_reject_unknown_fields() { + let transport = SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-a.ns.svc:5500".to_string(), + listen_address: "0.0.0.0:5500".parse().expect("test address"), + }; + let encoded = serde_json::to_string(&transport).unwrap(); + assert_eq!( + encoded, + r#"{"kind":"runtime-initiated-tcp","authority":"os-boundary-a.ns.svc:5500","listen_address":"0.0.0.0:5500"}"# + ); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + transport + ); + let listener = BoundaryListener::RuntimeInitiatedTlsTcp { + authority: "os-boundary-a.ns.svc:5500".to_string(), + addresses: vec!["10.43.0.9:5500".parse().expect("test address")], + tls: SandboxTlsServerConfig { + certificate_chain_path: PathBuf::from("/run/tls.crt"), + private_key_path: PathBuf::from("/run/tls.key"), + }, + }; + let encoded = serde_json::to_string(&listener).unwrap(); + assert_eq!( + serde_json::from_str::(&encoded).unwrap(), + listener + ); + assert!( + serde_json::from_str::( + r#"{"kind":"runtime-initiated-tcp","authority":"a","listen_address":"0.0.0.0:5500","target":"10.0.0.1:1"}"# + ) + .is_err() + ); + } + #[test] fn exec_wire_accepts_legacy_direct_exec_without_shell_intent() { let wire: ExecSpecWire = serde_json::from_str( diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index dd25b36f4b..0baad01772 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -17,6 +17,7 @@ pub mod isolation_protocol; pub mod mediation; pub mod protobuf; mod runtime; +mod runtime_initiated; pub mod sandbox_auth; pub mod stream_adapter; @@ -26,6 +27,8 @@ pub use runtime::{ ConfirmedBoundary, ExecSession, NetworkMediationSource, OpenShellRuntimeBackend, ProcessAttachment, ReadyBoundary, RunningBoundary, }; +pub use runtime::BoundaryTransportConnector; +pub use runtime_initiated::RuntimeInitiatedListener; /// Stable isolation backend name implemented by `openshell-sandbox`. pub const BACKEND_NAME: &str = openshell_core::isolation_registration::BUILTIN_BACKEND_NAME; diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index bcda27830c..4d5354fc72 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -72,7 +72,7 @@ fn begin_recovery_window( /// Internal raw transport establishment; shared code still authenticates /// the resulting stream with generation-pinned TLS and Sandbox Protocol JWTs. #[async_trait] -trait BoundaryTransportConnector: std::fmt::Debug + Send + Sync { +pub trait BoundaryTransportConnector: std::fmt::Debug + Send + Sync { async fn connect( &self, descriptor: &SandboxRuntimeDescriptor, @@ -83,6 +83,7 @@ trait BoundaryTransportConnector: std::fmt::Debug + Send + Sync { #[derive(Debug)] pub struct OpenShellRuntimeBackend { connector: Option>, + runtime_initiated: Option>, ca_file_paths: Arc>>, provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, sandbox_bearer: openshell_core::jwt::SessionBearerTokenSlot, @@ -111,9 +112,40 @@ impl OpenShellRuntimeBackend { descriptor: SandboxRuntimeDescriptor, bearer: openshell_core::jwt::SessionBearerTokenSlot, ) -> Result<(Option, bool), BackendError> { - BoundaryClient::new(descriptor, bearer) - .discover_policy() - .await + Self::discover_policy_with_connector(descriptor, bearer, None).await + } + + pub async fn discover_policy_with_connector( + descriptor: SandboxRuntimeDescriptor, + bearer: openshell_core::jwt::SessionBearerTokenSlot, + connector: Option>, + ) -> Result<(Option, bool), BackendError> { + let mut client = BoundaryClient::new(descriptor, bearer); + client.connector = connector; + Self::discover_policy_with_client(client).await + } + + /// Read image policy over a runtime-initiated connection accepted by + /// `listener`. Share the same listener with the attached backend. + pub async fn discover_policy_with_runtime_initiated_listener( + descriptor: SandboxRuntimeDescriptor, + bearer: openshell_core::jwt::SessionBearerTokenSlot, + listener: Arc, + ) -> Result<(Option, bool), BackendError> { + let mut client = BoundaryClient::new(descriptor, bearer); + client.runtime_initiated = Some(listener); + Self::discover_policy_with_client(client).await + } + + async fn discover_policy_with_client( + client: BoundaryClient, + ) -> Result<(Option, bool), BackendError> { + match client.call_idempotent(Request::DiscoverPolicy).await? { + Response::ImagePolicy { yaml, invalid } => Ok((yaml, invalid)), + _ => Err(BackendError::Descriptor( + "expected image policy discovery response".to_string(), + )), + } } pub fn new( @@ -123,6 +155,7 @@ impl OpenShellRuntimeBackend { ) -> Self { Self { connector: None, + runtime_initiated: None, ca_file_paths, provider_credentials, sandbox_bearer, @@ -164,6 +197,18 @@ impl OpenShellRuntimeBackend { client.discover_policy().await } + /// Accept runtime-initiated connections through `listener`, which must be + /// bound for the same descriptor this backend will attach. + #[must_use] + pub fn with_runtime_initiated_listener( + mut self, + listener: Arc, + ) -> Self { + self.runtime_initiated = Some(listener); + self + } + } + pub fn backend_name(&self) -> &str { self.delegated .as_ref() @@ -205,6 +250,7 @@ impl OpenShellRuntimeBackend { descriptor.payload.clone(), ); client.connector = self.connector.clone(); + client.runtime_initiated = self.runtime_initiated.clone(); let client = Arc::new(client); let response = client .call_idempotent(Request::Attach { @@ -304,7 +350,9 @@ fn validate_client_tls(tls: &SandboxTlsClientConfig) -> Result<(), BackendError> tls_client_config(tls).map(|_| ()) } -fn tls_client_config(tls: &SandboxTlsClientConfig) -> Result { +pub fn tls_client_config( + tls: &SandboxTlsClientConfig, +) -> Result { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let certificates = rustls_pemfile::certs(&mut tls.trust_anchor_pem.as_bytes()) .collect::, _>>() @@ -1059,6 +1107,7 @@ async fn dispatch_client_mediation_frame( struct BoundaryClient { connector: Option>, + runtime_initiated: Option>, runtime_descriptor: SandboxRuntimeDescriptor, backend_name: String, driver_descriptor: Vec, @@ -1251,6 +1300,7 @@ impl BoundaryClient { ) -> Self { Self { connector: None, + runtime_initiated: None, runtime_descriptor, backend_name, driver_descriptor, @@ -1881,6 +1931,7 @@ impl BoundaryClient { ) -> Result<(tonic::transport::Channel, Arc), BackendError> { let runtime_descriptor = self.runtime_descriptor.clone(); let connector = self.connector.clone(); + let runtime_initiated = self.runtime_initiated.clone(); let transport = Arc::new(TransportAbort::default()); let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { @@ -1903,6 +1954,7 @@ impl BoundaryClient { .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { let runtime_descriptor = runtime_descriptor.clone(); let connector = connector.clone(); + let runtime_initiated = runtime_initiated.clone(); let abort = connector_transport.clone(); async move { // A channel is one authenticated connection. Never let it @@ -1910,7 +1962,11 @@ impl BoundaryClient { if abort.is_aborted() || abort.connected.swap(true, Ordering::AcqRel) { return Err(AbortableTransport::aborted_error()); } - connect_boundary_with_retry(&runtime_descriptor, connector.clone()) + let source = runtime_initiated.as_deref().map_or( + TransportSource::Dial(connector.as_deref()), + TransportSource::RuntimeInitiated, + ); + connect_boundary_with_retry(&runtime_descriptor, source) .await .map(|inner| TokioIo::new(AbortableTransport { inner, abort })) .map_err(|error| std::io::Error::other(error.to_string())) @@ -2004,13 +2060,28 @@ impl BoundaryClient { } } +/// Where a client obtains each physical connection. +#[derive(Clone, Copy)] +enum TransportSource<'a> { + /// Dial the descriptor's endpoint, optionally through a backend connector. + Dial(Option<&'a dyn BoundaryTransportConnector>), + /// Take a connection the runtime opened to the supervisor's listener. + RuntimeInitiated(&'a crate::RuntimeInitiatedListener), +} + async fn connect_boundary_with_retry( runtime_descriptor: &SandboxRuntimeDescriptor, - connector: Option>, + source: TransportSource<'_>, ) -> Result { + let connector = match source { + TransportSource::Dial(connector) => connector, + TransportSource::RuntimeInitiated(listener) => { + return connect_runtime_initiated(runtime_descriptor, listener).await; + } + }; let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; loop { - match connect_boundary_using_connector(runtime_descriptor, connector.as_deref()).await { + match connect_boundary_using_connector(runtime_descriptor, connector).await { Ok(stream) => return Ok(stream), Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, @@ -2025,6 +2096,27 @@ async fn connect_boundary_once( connect_boundary_using_connector(runtime_descriptor, None).await } +/// The listener completes TLS with the descriptor's pinned identity before a +/// stream is queued, so the shared TLS step below is not applied twice. +async fn connect_runtime_initiated( + runtime_descriptor: &SandboxRuntimeDescriptor, + listener: &crate::RuntimeInitiatedListener, +) -> Result { + let SandboxTransport::RuntimeInitiatedTcp { authority, .. } = &runtime_descriptor.transport + else { + return Err(BackendError::Descriptor( + "runtime-initiated listener supplied for a supervisor-initiated transport".to_string(), + )); + }; + tokio::time::timeout(CONNECT_RETRY_TIMEOUT, listener.accept(runtime_descriptor)) + .await + .map_err(|_| { + BackendError::Unavailable(format!( + "timed out waiting for the sandbox runtime to connect through {authority}" + )) + })? +} + async fn connect_boundary_using_connector( runtime_descriptor: &SandboxRuntimeDescriptor, connector: Option<&dyn BoundaryTransportConnector>, @@ -2064,6 +2156,12 @@ async fn connect_boundary_using_connector( Box::new(stream) } SandboxTransport::Vsock { guest_cid, port } => connect_host_vsock(*guest_cid, *port)?, + SandboxTransport::RuntimeInitiatedTcp { .. } => { + return Err(BackendError::Descriptor( + "runtime-initiated transport requires a supervisor control listener" + .to_string(), + )); + } } }; let tls = &runtime_descriptor.tls; @@ -2227,7 +2325,7 @@ fn control_read_error(error: std::io::Error) -> BackendError { } } -fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { +pub fn enable_boundary_tcp_keepalive(stream: &tokio::net::TcpStream) { let keepalive = socket2::TcpKeepalive::new() .with_time(Duration::from_secs(30)) .with_interval(Duration::from_secs(10)); @@ -3615,6 +3713,190 @@ mod tests { server.await.unwrap(); } + /// Records when the runtime side of a connection observes EOF or failure. + struct ClosureProbe { + inner: BoundaryDuplexStream, + closed: Arc, + } + + impl tokio::io::AsyncRead for ClosureProbe { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let before = buffer.filled().len(); + let result = Pin::new(&mut self.inner).poll_read(context, buffer); + match &result { + Poll::Ready(Ok(())) if buffer.filled().len() == before => { + self.closed.store(true, Ordering::Release); + } + Poll::Ready(Err(_)) => self.closed.store(true, Ordering::Release), + _ => {} + } + result + } + } + + impl tokio::io::AsyncWrite for ClosureProbe { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + Pin::new(&mut self.inner).poll_write(context, buffer) + } + + fn poll_flush( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_flush(context) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(context) + } + } + + /// Dial the supervisor listener as `openshell-sandbox` does and serve the + /// protocol over the outbound connection. + async fn serve_as_runtime_initiated_boundary( + address: std::net::SocketAddr, + server_config: Arc, + expected_token: String, + closed: Arc, + ) { + let stream = tokio::net::TcpStream::connect(address).await.unwrap(); + let stream = tokio_rustls::TlsAcceptor::from(server_config) + .accept(stream) + .await + .unwrap(); + serve_test_grpc( + Box::new(ClosureProbe { + inner: Box::new(stream), + closed, + }), + expected_token, + ) + .await; + } + + fn runtime_initiated_descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { + let mut descriptor = tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), tls); + descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), + listen_address: "127.0.0.1:5500".parse().unwrap(), + }; + descriptor + } + + #[tokio::test] + async fn runtime_initiated_transport_preserves_tls_bearer_and_physical_abort() { + let certificate = test_certificate(); + let descriptor = runtime_initiated_descriptor(certificate.client_tls); + let listener = crate::RuntimeInitiatedListener::bind_address( + "127.0.0.1:0".parse().unwrap(), + &descriptor.tls, + ) + .unwrap(); + let address = listener.local_address(); + let first_closed = Arc::new(AtomicBool::new(false)); + let second_closed = Arc::new(AtomicBool::new(false)); + let first = tokio::spawn(serve_as_runtime_initiated_boundary( + address, + certificate.server_config.clone(), + "a".repeat(32), + first_closed.clone(), + )); + let mut client = BoundaryClient::new(descriptor, test_bearer(&"a".repeat(32))); + client.runtime_initiated = Some(listener); + let response = + tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap() + .unwrap(); + assert!(matches!(response, Response::Confirmed { .. })); + + // Aborting the cached channel closes the runtime's physical socket, + // and the next request uses a fresh runtime-initiated connection. + let second = tokio::spawn(serve_as_runtime_initiated_boundary( + address, + certificate.server_config, + "a".repeat(32), + second_closed.clone(), + )); + client + .grpc_channel + .lock() + .await + .take() + .unwrap() + .transport + .abort(); + tokio::time::timeout(Duration::from_secs(5), async { + while !first_closed.load(Ordering::Acquire) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("aborting the channel must close the runtime-initiated socket"); + let response = + tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap() + .unwrap(); + assert!(matches!(response, Response::Confirmed { .. })); + assert!(!second_closed.load(Ordering::Acquire)); + first.abort(); + second.abort(); + } + + #[tokio::test] + async fn runtime_initiated_transport_rejects_a_wrong_bearer() { + let certificate = test_certificate(); + let descriptor = runtime_initiated_descriptor(certificate.client_tls); + let listener = crate::RuntimeInitiatedListener::bind_address( + "127.0.0.1:0".parse().unwrap(), + &descriptor.tls, + ) + .unwrap(); + let runtime = tokio::spawn(serve_as_runtime_initiated_boundary( + listener.local_address(), + certificate.server_config, + "a".repeat(32), + Arc::new(AtomicBool::new(false)), + )); + let mut client = BoundaryClient::new(descriptor, test_bearer(&"b".repeat(32))); + client.runtime_initiated = Some(listener); + let result = + tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap(); + assert!(result.is_err()); + runtime.abort(); + } + + #[test] + fn runtime_initiated_descriptor_requires_a_listen_port_and_authority() { + let certificate = test_certificate(); + let mut descriptor = runtime_initiated_descriptor(certificate.client_tls); + assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_ok()); + descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { + authority: String::new(), + listen_address: "0.0.0.0:5500".parse().unwrap(), + }; + assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); + descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), + listen_address: "0.0.0.0:0".parse().unwrap(), + }; + assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); + } + #[tokio::test] async fn tls_tcp_rejects_tls12_only_server() { let certificate = test_certificate_with_protocol_versions(&[&rustls::version::TLS12]); diff --git a/crates/openshell-sandbox-backend/src/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime_initiated.rs new file mode 100644 index 0000000000..ecdf911a2c --- /dev/null +++ b/crates/openshell-sandbox-backend/src/runtime_initiated.rs @@ -0,0 +1,496 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Supervisor listener for runtime-initiated Sandbox Protocol connections. +//! +//! Only the TCP dial direction is reversed. The supervisor remains the TLS +//! client and gRPC client: it verifies the runtime's generation-pinned server +//! identity on every accepted socket before the socket can carry a request, and +//! every request still carries the Sandbox Protocol bearer token. A peer +//! address is never trusted; it is only the source of a socket to authenticate. + +use std::collections::VecDeque; +use std::net::SocketAddr; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; + +use crate::boundary_protocol::{ + SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, +}; + +/// Unauthenticated sockets may hold at most this many TLS handshakes. +/// Further sockets are closed immediately rather than queued. +const MAX_PENDING_HANDSHAKES: usize = 8; +/// Upper bound on the time an unauthenticated peer may occupy a slot. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(5); +/// Authenticated connections not yet claimed by a client. The runtime keeps +/// one spare open, so a small bound only absorbs reconnect races. +const MAX_QUEUED_CONNECTIONS: usize = 2; + +/// Accepts runtime-initiated connections for exactly one sandbox session. +/// +/// Live channels own their streams; this listener only holds unclaimed, +/// already-authenticated spares, so no new connection can evict a live owner. +/// Dropping the listener closes the socket, every pending handshake, and every +/// queued spare. +pub struct RuntimeInitiatedListener { + tls: SandboxTlsClientConfig, + local_address: SocketAddr, + queue: Arc, + accept_task: tokio::task::AbortHandle, +} + +impl std::fmt::Debug for RuntimeInitiatedListener { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("RuntimeInitiatedListener") + .field("local_address", &self.local_address) + .field("server_name", &self.tls.server_name) + .finish_non_exhaustive() + } +} + +impl RuntimeInitiatedListener { + /// Bind the descriptor's listen address and start authenticating peers. + /// + /// Must be called inside a Tokio runtime. + pub fn bind(descriptor: &SandboxRuntimeDescriptor) -> Result, BackendError> { + let SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } = &descriptor.transport + else { + return Err(BackendError::Descriptor( + "runtime-initiated listener requires a runtime-initiated transport".to_string(), + )); + }; + if listen_address.port() == 0 { + return Err(BackendError::Descriptor( + "runtime-initiated listen address must have a nonzero port".to_string(), + )); + } + Self::bind_address(*listen_address, &descriptor.tls) + } + + pub(crate) fn bind_address( + address: SocketAddr, + tls: &SandboxTlsClientConfig, + ) -> Result, BackendError> { + let server_name = rustls::pki_types::ServerName::try_from(tls.server_name.clone()) + .map_err(|error| { + BackendError::Descriptor(format!( + "boundary TLS server name {:?} is invalid: {error}", + tls.server_name + )) + })?; + let connector = + tokio_rustls::TlsConnector::from(Arc::new(crate::runtime::tls_client_config(tls)?)); + let listener = bind_tcp(address).map_err(|error| { + BackendError::Unavailable(format!( + "bind runtime-initiated control listener {address}: {error}" + )) + })?; + let local_address = listener.local_addr().map_err(|error| { + BackendError::Unavailable(format!("read control listener address: {error}")) + })?; + let queue = Arc::new(AdmissionQueue::default()); + let accept_task = + tokio::spawn(accept_loop(listener, connector, server_name, queue.clone())) + .abort_handle(); + tracing::info!(%local_address, "Runtime-initiated Sandbox Protocol listener ready"); + Ok(Arc::new(Self { + tls: tls.clone(), + local_address, + queue, + accept_task, + })) + } + + /// Address actually bound, including an OS-assigned port. + #[must_use] + pub fn local_address(&self) -> SocketAddr { + self.local_address + } + + /// Wait for one authenticated runtime connection for `descriptor`. + /// + /// The returned stream has completed TLS with the descriptor's pinned + /// identity. A listener bound for another session is refused. + pub(crate) async fn accept( + &self, + descriptor: &SandboxRuntimeDescriptor, + ) -> Result { + if descriptor.tls != self.tls { + return Err(BackendError::Descriptor( + "runtime-initiated listener is bound to a different sandbox session".to_string(), + )); + } + Ok(self.queue.pop().await) + } +} + +impl Drop for RuntimeInitiatedListener { + fn drop(&mut self) { + self.accept_task.abort(); + } +} + +fn bind_tcp(address: SocketAddr) -> std::io::Result { + let socket = socket2::Socket::new( + socket2::Domain::for_address(address), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + socket.set_reuse_address(true)?; + socket.set_nonblocking(true)?; + socket.bind(&address.into())?; + socket.listen(64)?; + tokio::net::TcpListener::from_std(socket.into()) +} + +#[derive(Default)] +struct AdmissionQueue { + streams: Mutex>, + ready: tokio::sync::Notify, +} + +impl AdmissionQueue { + fn push(&self, stream: BoundaryDuplexStream) { + let displaced = { + let mut streams = self + .streams + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let displaced = (streams.len() >= MAX_QUEUED_CONNECTIONS) + .then(|| streams.pop_front()) + .flatten(); + streams.push_back(stream); + displaced + }; + if displaced.is_some() { + tracing::debug!("discarding an unclaimed runtime-initiated spare connection"); + } + self.ready.notify_one(); + } + + async fn pop(&self) -> BoundaryDuplexStream { + loop { + let notified = self.ready.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + let stream = self + .streams + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .pop_front(); + if let Some(stream) = stream { + return stream; + } + notified.await; + } + } + + #[cfg(test)] + fn len(&self) -> usize { + self.streams + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .len() + } +} + +async fn accept_loop( + listener: tokio::net::TcpListener, + connector: tokio_rustls::TlsConnector, + server_name: rustls::pki_types::ServerName<'static>, + queue: Arc, +) { + let pending = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); + // Owned here so aborting this task also cancels every pending handshake. + let mut handshakes = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let (stream, peer) = match accepted { + Ok(accepted) => accepted, + Err(error) => { + // Descriptor exhaustion and similar errors are transient. + tracing::warn!(%error, "runtime-initiated control accept failed"); + tokio::time::sleep(Duration::from_millis(100)).await; + continue; + } + }; + let Ok(permit) = pending.clone().try_acquire_owned() else { + tracing::debug!(%peer, "runtime-initiated handshake limit reached; closing connection"); + continue; + }; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + crate::runtime::enable_boundary_tcp_keepalive(&stream); + let connector = connector.clone(); + let server_name = server_name.clone(); + let queue = queue.clone(); + handshakes.spawn(async move { + let _permit = permit; + match tokio::time::timeout( + HANDSHAKE_TIMEOUT, + connector.connect(server_name, stream), + ) + .await + { + Ok(Ok(stream)) => queue.push(Box::new(stream)), + Ok(Err(error)) => { + tracing::debug!(%peer, %error, "rejected unauthenticated runtime-initiated connection"); + } + Err(_) => { + tracing::debug!(%peer, "runtime-initiated TLS handshake timed out"); + } + } + }); + } + Some(_) = handshakes.join_next(), if !handshakes.is_empty() => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + struct TestIdentity { + client: SandboxTlsClientConfig, + server: Arc, + } + + fn identity() -> TestIdentity { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let material = crate::boundary_protocol::generate_sandbox_tls_material( + openshell_core::SandboxSessionId::new(), + ) + .unwrap(); + let certificates = rustls_pemfile::certs(&mut material.certificate_chain_pem.as_bytes()) + .collect::, _>>() + .unwrap(); + let key = rustls_pemfile::private_key(&mut material.private_key_pem.as_bytes()) + .unwrap() + .unwrap(); + let mut server = + rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) + .with_no_client_auth() + .with_single_cert(certificates, key) + .unwrap(); + server.alpn_protocols = vec![b"h2".to_vec()]; + TestIdentity { + client: SandboxTlsClientConfig { + server_name: material.server_name, + trust_anchor_pem: material.trust_anchor_pem, + }, + server: Arc::new(server), + } + } + + fn descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { + SandboxRuntimeDescriptor { + boundary_id: "sandbox-1".to_string(), + generation: "generation-1".to_string(), + session_id: openshell_core::SandboxSessionId::new(), + workload_identity: + openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( + 1000, + 1000, + Vec::new(), + "test".to_string(), + "test".to_string(), + ) + .unwrap(), + transport: SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), + listen_address: "127.0.0.1:0".parse().unwrap(), + }, + tls, + host_gateway_ip: None, + resource_claims: std::collections::BTreeMap::new(), + outer_fence: openshell_isolation_interface::contract::OuterFenceGuarantees::from_enforcement_evidence( + "generation-1", + [], + b"test", + ) + .unwrap(), + } + } + + /// Production descriptors require a fixed port; tests bind an ephemeral one. + fn bind_ephemeral( + descriptor: &SandboxRuntimeDescriptor, + ) -> Result, BackendError> { + RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), &descriptor.tls) + } + + #[test] + fn production_bind_requires_a_fixed_port() { + let _runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let identity = identity(); + assert!(matches!( + RuntimeInitiatedListener::bind(&descriptor(identity.client)), + Err(BackendError::Descriptor(_)) + )); + } + + /// Dial the listener as the runtime does and serve TLS on the outbound socket. + async fn dial_as_runtime( + address: SocketAddr, + server: Arc, + ) -> std::io::Result> { + let stream = tokio::net::TcpStream::connect(address).await?; + tokio_rustls::TlsAcceptor::from(server).accept(stream).await + } + + async fn queued_within(listener: &RuntimeInitiatedListener, count: usize) { + tokio::time::timeout(Duration::from_secs(5), async { + while listener.queue.len() < count { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("authenticated connection was not queued"); + } + + #[tokio::test] + async fn authenticated_runtime_connection_is_accepted_and_carries_data() { + let identity = identity(); + let descriptor = descriptor(identity.client); + let listener = bind_ephemeral(&descriptor).unwrap(); + let runtime = tokio::spawn(dial_as_runtime(listener.local_address(), identity.server)); + + let mut supervisor_side = + tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)) + .await + .unwrap() + .unwrap(); + let mut runtime_side = runtime.await.unwrap().unwrap(); + supervisor_side.write_all(b"ping").await.unwrap(); + supervisor_side.flush().await.unwrap(); + let mut received = [0; 4]; + runtime_side.read_exact(&mut received).await.unwrap(); + assert_eq!(&received, b"ping"); + } + + #[tokio::test] + async fn untrusted_and_silent_peers_cannot_displace_the_genuine_runtime() { + let genuine = identity(); + let impostor = identity(); + let descriptor = descriptor(genuine.client); + let listener = bind_ephemeral(&descriptor).unwrap(); + let address = listener.local_address(); + + // Silent sockets occupy handshake slots but never reach the queue. + let mut silent = Vec::new(); + for _ in 0..(MAX_PENDING_HANDSHAKES - 2) { + silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + // A peer presenting another session's identity fails TLS verification. + let rejected = dial_as_runtime(address, impostor.server).await; + assert!( + rejected.is_err() || { + // TLS 1.3 servers can finish before the client's alert arrives; + // the connection must then be closed without being queued. + let mut stream = rejected.unwrap(); + let mut byte = [0; 1]; + stream.read(&mut byte).await.map_or(true, |read| read == 0) + } + ); + assert_eq!(listener.queue.len(), 0); + + let runtime = tokio::spawn(dial_as_runtime(address, genuine.server)); + let accepted = + tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)).await; + assert!(accepted.is_ok_and(|stream| stream.is_ok())); + assert!(runtime.await.unwrap().is_ok()); + drop(silent); + } + + #[tokio::test] + async fn handshake_slots_bound_unauthenticated_sockets() { + let genuine = identity(); + let descriptor = descriptor(genuine.client); + let listener = bind_ephemeral(&descriptor).unwrap(); + let address = listener.local_address(); + let mut silent = Vec::new(); + for _ in 0..MAX_PENDING_HANDSHAKES { + silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + // Let the listener assign every slot before the excess socket arrives. + tokio::time::sleep(Duration::from_millis(200)).await; + let mut excess = tokio::net::TcpStream::connect(address).await.unwrap(); + let mut byte = [0; 1]; + let closed = tokio::time::timeout(Duration::from_secs(2), excess.read(&mut byte)).await; + assert!( + matches!(closed, Ok(Ok(0) | Err(_))), + "a socket beyond the handshake limit must be closed" + ); + + // Freed slots admit the genuine runtime again after the timeout. + drop(silent); + let runtime = tokio::spawn(dial_as_runtime(address, genuine.server)); + assert!( + tokio::time::timeout(HANDSHAKE_TIMEOUT * 2, listener.accept(&descriptor)) + .await + .is_ok() + ); + assert!(runtime.await.unwrap().is_ok()); + } + + #[tokio::test] + async fn spare_queue_is_bounded_and_never_holds_more_than_the_limit() { + let genuine = identity(); + let descriptor = descriptor(genuine.client); + let listener = bind_ephemeral(&descriptor).unwrap(); + let mut runtimes = Vec::new(); + for _ in 0..=MAX_QUEUED_CONNECTIONS { + runtimes.push( + dial_as_runtime(listener.local_address(), genuine.server.clone()) + .await + .unwrap(), + ); + } + queued_within(&listener, MAX_QUEUED_CONNECTIONS).await; + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(listener.queue.len(), MAX_QUEUED_CONNECTIONS); + // The oldest spare was displaced and closed. + let mut byte = [0; 1]; + let oldest = tokio::time::timeout(Duration::from_secs(2), runtimes[0].read(&mut byte)) + .await + .unwrap(); + assert!(matches!(oldest, Ok(0) | Err(_))); + } + + #[tokio::test] + async fn listener_refuses_a_descriptor_for_another_session() { + let genuine = identity(); + let other = identity(); + let listener = bind_ephemeral(&descriptor(genuine.client)).unwrap(); + assert!(matches!( + listener.accept(&descriptor(other.client)).await, + Err(BackendError::Descriptor(_)) + )); + } + + #[tokio::test] + async fn dropping_the_listener_closes_queued_connections() { + let genuine = identity(); + let descriptor = descriptor(genuine.client); + let listener = bind_ephemeral(&descriptor).unwrap(); + let mut runtime = dial_as_runtime(listener.local_address(), genuine.server) + .await + .unwrap(); + queued_within(&listener, 1).await; + drop(listener); + let mut byte = [0; 1]; + let read = tokio::time::timeout(Duration::from_secs(2), runtime.read(&mut byte)) + .await + .unwrap(); + assert!(matches!(read, Ok(0) | Err(_))); + } +} diff --git a/crates/openshell-sandbox/README.md b/crates/openshell-sandbox/README.md index 3ee57c085f..1cde277dd8 100644 --- a/crates/openshell-sandbox/README.md +++ b/crates/openshell-sandbox/README.md @@ -17,3 +17,12 @@ the kernel then tracks the lifetime through workload-owned descriptors. Native the last workload reference to a listener removes it without waiting for registry collection. Broker source descriptors always use close-on-exec independently of the workload's requested descriptor flags. + +A boundary configured with `RuntimeInitiatedTlsTcp` binds no listener. It dials +only the driver-provided supervisor addresses, which must be concrete, +non-loopback, and share one port, and serves TLS and gRPC over that outbound +connection. It keeps one spare connection that the supervisor has not yet used +and dials a replacement when the supervisor starts HTTP/2 on it or the spare +closes. Live connections keep the same disconnect, freeze, and recovery-deadline +handling as listener mode, and the workload still cannot connect to the control +port. diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index a0c3bf5bc5..c4a5e7e0db 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -92,6 +92,15 @@ mod linux { const FORCE_KILL_REAP_TIMEOUT: Duration = Duration::from_secs(2); const MAX_PENDING_HANDSHAKES: usize = 32; const MAX_CONTROL_CONNECTIONS: usize = 128; + /// Bounds one runtime-initiated TCP connect attempt. + const RUNTIME_DIAL_TIMEOUT: Duration = Duration::from_secs(5); + const RUNTIME_DIAL_BACKOFF_MIN: Duration = Duration::from_millis(100); + const RUNTIME_DIAL_BACKOFF_MAX: Duration = Duration::from_secs(2); + /// A spare that dies this quickly without being claimed counts as a failure. + const RUNTIME_SPARE_MIN_LIFETIME: Duration = Duration::from_secs(1); + /// Consecutive dial failures are expected while the supervisor starts; + /// report a persistent failure at most this often. + const RUNTIME_DIAL_WARNING_INTERVAL: Duration = Duration::from_secs(30); const MAX_REPLAY_LEDGER_ENTRIES: usize = 4096; const MAX_RETAINED_EXEC_PROCESSES: usize = 64; // Three stdio pipes require six descriptors before fork, and Rust's @@ -239,6 +248,10 @@ mod linux { .map_err(|error| format!("start sandbox workload launcher: {error}"))?; let protected_control_port = match &config.listener { BoundaryListenerConfig::TlsTcp { address, .. } => Some(address.port()), + // Validation requires every supervisor address to share one port. + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { + addresses.first().map(std::net::SocketAddr::port) + } BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::Vsock { .. } => None, }; let network_broker = NetworkBroker::start(listener, protected_control_port) @@ -353,6 +366,11 @@ mod linux { } => { return Err("boundary control port must be nonzero".to_string()); } + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { + authority, + addresses, + tls, + } => validate_runtime_initiated_targets(authority, addresses, tls)?, BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::TlsTcp { .. } | BoundaryListenerConfig::Vsock { .. } => {} @@ -391,6 +409,39 @@ mod linux { Ok(()) } + /// The dial target is driver-resolved bootstrap state, never workload + /// input. Loopback is refused because workloads share the loopback + /// interface and could otherwise receive the runtime's connection. + fn validate_runtime_initiated_targets( + authority: &str, + addresses: &[std::net::SocketAddr], + tls: &openshell_sandbox_backend::boundary_protocol::SandboxTlsServerConfig, + ) -> Result<(), String> { + let Some(first) = addresses.first() else { + return Err("runtime-initiated control requires a supervisor address".to_string()); + }; + if authority.is_empty() || !tls_paths_are_absolute(tls) { + return Err( + "runtime-initiated control requires an authority and absolute certificate paths" + .to_string(), + ); + } + for address in addresses { + let ip = address.ip().to_canonical(); + if address.port() == 0 + || address.port() != first.port() + || ip.is_unspecified() + || ip.is_loopback() + { + return Err( + "runtime-initiated supervisor addresses must be concrete, non-loopback, and share one nonzero port" + .to_string(), + ); + } + } + Ok(()) + } + fn tls_paths_are_absolute( tls: &openshell_sandbox_backend::boundary_protocol::SandboxTlsServerConfig, ) -> bool { @@ -487,31 +538,42 @@ mod linux { Ok(()) } + /// Shut down after SIGTERM/SIGINT; `None` while no termination is pending. + fn take_termination(runtime: &BoundaryRuntime) -> Option> { + if !BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { + return None; + } + let signal = BOUNDARY_TERMINATION_SIGNAL.load(Ordering::Acquire); + let before_supervisor_confirmation = !matches!( + *lock(&runtime.supervisor_connection), + SupervisorConnectionState::Connected(_) + ); + runtime.shutdown(); + if before_supervisor_confirmation { + return Some(Err(format!( + "sandbox boundary received {} before supervisor confirmation", + boundary_termination_signal_name(signal) + ))); + } + tracing::info!( + signal = boundary_termination_signal_name(signal), + "Sandbox boundary received termination signal" + ); + Some(Ok(())) + } + fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { + if let BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, tls, .. } = config { + return serve_runtime_initiated(config, addresses.clone(), tls, runtime); + } let listener = ControlListener::bind(config) .map_err(|error| format!("bind boundary control listener: {error}"))?; let active_connections = Arc::new(AtomicUsize::new(0)); let pending_handshakes = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); tracing::info!(?config, "Boundary control listener ready"); loop { - if BOUNDARY_TERMINATION_REQUESTED.load(Ordering::Acquire) { - let signal = BOUNDARY_TERMINATION_SIGNAL.load(Ordering::Acquire); - let before_supervisor_confirmation = !matches!( - *lock(&runtime.supervisor_connection), - SupervisorConnectionState::Connected(_) - ); - runtime.shutdown(); - if before_supervisor_confirmation { - return Err(format!( - "sandbox boundary received {} before supervisor confirmation", - boundary_termination_signal_name(signal) - )); - } - tracing::info!( - signal = boundary_termination_signal_name(signal), - "Sandbox boundary received termination signal" - ); - return Ok(()); + if let Some(result) = take_termination(&runtime) { + return result; } match listener.accept() { Ok(stream) => { @@ -556,6 +618,180 @@ mod linux { } } + /// Runtime-initiated control: dial the supervisor instead of listening. + /// TLS server and gRPC server roles are the same as for a listener. + fn serve_runtime_initiated( + config: &BoundaryListenerConfig, + addresses: Vec, + tls: &openshell_sandbox_backend::boundary_protocol::SandboxTlsServerConfig, + runtime: Arc, + ) -> Result<(), String> { + let server_config = Arc::new( + load_tls_server_config(tls) + .map_err(|error| format!("load boundary TLS identity: {error}"))?, + ); + tracing::info!(?config, "Boundary control dialer ready"); + let dialer = runtime.process_runtime.spawn(run_runtime_initiated_control( + addresses, + server_config, + runtime.clone(), + Arc::new(AtomicUsize::new(0)), + )); + loop { + if let Some(result) = take_termination(&runtime) { + dialer.abort(); + return result; + } + std::thread::sleep(Duration::from_millis(10)); + } + } + + /// Keep exactly one unclaimed spare connection open to the supervisor. + /// + /// The supervisor claims a spare by starting HTTP/2 on it; then the next + /// spare is dialed so credential rotation and recovery, which open a new + /// connection before retiring the old one, never wait on this loop. Live + /// connections keep the existing disconnect, freeze, and termination path. + async fn run_runtime_initiated_control( + addresses: Vec, + server_config: Arc, + runtime: Arc, + active_connections: Arc, + ) { + let mut backoff = RUNTIME_DIAL_BACKOFF_MIN; + let mut first_failure: Option = None; + let mut last_warning: Option = None; + loop { + let Some(slot) = acquire_control_connection_slot(&active_connections) else { + tokio::time::sleep(RUNTIME_DIAL_BACKOFF_MAX).await; + continue; + }; + let dialed_at = tokio::time::Instant::now(); + let stream = match dial_runtime_initiated(&addresses, server_config.clone()).await { + Ok(stream) => { + first_failure = None; + stream + } + Err(error) => { + let now = tokio::time::Instant::now(); + let failing_since = *first_failure.get_or_insert(now); + if now.duration_since(failing_since) >= RUNTIME_DIAL_WARNING_INTERVAL + && last_warning.is_none_or(|last| { + now.duration_since(last) >= RUNTIME_DIAL_WARNING_INTERVAL + }) + { + last_warning = Some(now); + tracing::warn!( + %error, + failing_for_secs = now.duration_since(failing_since).as_secs(), + "Runtime-initiated control cannot reach the supervisor; retrying" + ); + } else { + tracing::debug!(%error, "Runtime-initiated control dial failed; retrying"); + } + drop(slot); + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(RUNTIME_DIAL_BACKOFF_MAX); + continue; + } + }; + let (claimed, claim) = tokio::sync::oneshot::channel(); + runtime.process_runtime.spawn({ + let runtime = runtime.clone(); + async move { + let _slot = slot; + let stream = Box::new(ClaimOnFirstRead { + inner: stream, + claimed: Some(claimed), + }); + if let Err(error) = + serve_grpc(stream, runtime, SandboxConnectionId::new()).await + { + tracing::debug!(%error, "Boundary control connection ended"); + } + } + }); + if claim.await.is_ok() || dialed_at.elapsed() >= RUNTIME_SPARE_MIN_LIFETIME { + backoff = RUNTIME_DIAL_BACKOFF_MIN; + } else { + tokio::time::sleep(backoff).await; + backoff = (backoff * 2).min(RUNTIME_DIAL_BACKOFF_MAX); + } + } + } + + async fn dial_runtime_initiated( + addresses: &[std::net::SocketAddr], + server_config: Arc, + ) -> io::Result { + let stream = tokio::time::timeout( + RUNTIME_DIAL_TIMEOUT, + openshell_core::net::connect_tcp_nodelay_best_effort(addresses), + ) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "supervisor dial timed out"))??; + // An idle spare must notice a silently lost supervisor well inside the + // authenticated recovery window. + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(10)) + .with_interval(Duration::from_secs(5)) + .with_retries(3); + let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); + let stream = tokio::time::timeout( + CONTROL_HANDSHAKE_TIMEOUT, + tokio_rustls::TlsAcceptor::from(server_config).accept(stream), + ) + .await + .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out"))? + .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; + Ok(Box::new(stream)) + } + + /// Signals once the supervisor sends its first application bytes. + struct ClaimOnFirstRead { + inner: openshell_isolation_interface::contract::BoundaryDuplexStream, + claimed: Option>, + } + + impl tokio::io::AsyncRead for ClaimOnFirstRead { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let before = buffer.filled().len(); + let result = Pin::new(&mut self.inner).poll_read(context, buffer); + if matches!(result, Poll::Ready(Ok(()))) + && buffer.filled().len() > before + && let Some(claimed) = self.claimed.take() + { + let _ = claimed.send(()); + } + result + } + } + + impl tokio::io::AsyncWrite for ClaimOnFirstRead { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + Pin::new(&mut self.inner).poll_write(context, buffer) + } + + fn poll_flush(mut self: Pin<&mut Self>, context: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(context) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(context) + } + } + fn boundary_termination_signal_name(signal: i32) -> &'static str { match signal { libc::SIGTERM => "SIGTERM", @@ -3364,6 +3600,10 @@ mod linux { server_config, }) } + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { .. } => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "runtime-initiated control dials the supervisor and binds no listener", + )), BoundaryListenerConfig::TlsTcp { address, tls } => { let (listener, reject_local_peers) = Self::bind_tcp(*address)?; listener.set_nonblocking(true)?; @@ -5520,6 +5760,96 @@ mod linux { server.abort(); } + #[test] + fn runtime_initiated_targets_must_be_concrete_remote_and_single_port() { + let tls = placeholder_server_tls(); + let address = |value: &str| value.parse::().unwrap(); + assert!( + validate_runtime_initiated_targets( + "os-boundary-a.ns.svc:5500", + &[address("10.43.0.9:5500"), address("[fd00::9]:5500")], + &tls + ) + .is_ok() + ); + for addresses in [ + vec![], + vec![address("127.0.0.1:5500")], + vec![address("[::ffff:127.0.0.1]:5500")], + vec![address("0.0.0.0:5500")], + vec![address("10.43.0.9:0")], + vec![address("10.43.0.9:5500"), address("10.43.0.10:5501")], + ] { + assert!( + validate_runtime_initiated_targets( + "os-boundary-a.ns.svc:5500", + &addresses, + &tls + ) + .is_err(), + "{addresses:?}" + ); + } + assert!( + validate_runtime_initiated_targets("", &[address("10.43.0.9:5500")], &tls).is_err() + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_initiated_control_keeps_one_spare_and_redials_after_loss() { + let (runtime, _token) = availability_test_runtime(); + let directory = tempfile::tempdir().expect("temporary directory"); + let (server_tls, client_tls) = stage_test_tls(directory.path(), "dial"); + let server_config = + Arc::new(load_tls_server_config(&server_tls).expect("load TLS identity")); + let supervisor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dialer = tokio::spawn(run_runtime_initiated_control( + vec![supervisor.local_addr().unwrap()], + server_config, + runtime, + Arc::new(AtomicUsize::new(0)), + )); + let connector = + tokio_rustls::TlsConnector::from(Arc::new(test_client_config(&client_tls))); + let server_name = + rustls::pki_types::ServerName::try_from(client_tls.server_name.clone()).unwrap(); + let accept_authenticated = || async { + let (stream, _) = tokio::time::timeout(Duration::from_secs(5), supervisor.accept()) + .await + .expect("runtime dials the supervisor") + .unwrap(); + connector + .connect(server_name.clone(), stream) + .await + .expect("runtime presents its pinned identity") + }; + let no_new_dial = || async { + tokio::time::timeout(Duration::from_millis(400), supervisor.accept()) + .await + .is_err() + }; + + let mut first = accept_authenticated().await; + assert!(no_new_dial().await, "an unclaimed spare suppresses dialing"); + + // Starting HTTP/2 claims the spare; the runtime dials a new one. + first + .write_all(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") + .await + .unwrap(); + first.flush().await.unwrap(); + let spare = accept_authenticated().await; + assert!(no_new_dial().await, "only one spare is kept"); + + // Losing the claimed connection does not add spares; losing the + // spare is repaired by a redial. + drop(first); + assert!(no_new_dial().await); + drop(spare); + let _replacement = accept_authenticated().await; + dialer.abort(); + } + #[test] fn pod_control_listener_rejects_loopback_peers_before_tls() { let directory = tempfile::tempdir().expect("temporary directory"); diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index fbf770b905..3e40416fb4 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -231,11 +231,48 @@ impl BackendSetup for OpenShellBackendSetup { gid: descriptor.workload_identity.gid, }), }; - Ok((identity, Box::new(OpenShellLaunch(descriptor)))) + Ok(( + identity, + Box::new(OpenShellLaunch { + descriptor, + runtime_initiated: std::sync::OnceLock::new(), + }), + )) } } -struct OpenShellLaunch(openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor); +struct OpenShellLaunch { + descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, + /// Bound once and shared by discovery and the attached backend, so a + /// runtime spare accepted during discovery remains usable for attach. + runtime_initiated: + std::sync::OnceLock>, +} + +impl OpenShellLaunch { + /// The listener exists only for a runtime-initiated transport. Binding + /// requires the supervisor's Tokio runtime and happens before attach. + fn runtime_initiated_listener( + &self, + ) -> std::result::Result< + Option>, + BackendError, + > { + if !matches!( + self.descriptor.transport, + openshell_sandbox_backend::boundary_protocol::SandboxTransport::RuntimeInitiatedTcp { .. } + ) { + return Ok(None); + } + if let Some(listener) = self.runtime_initiated.get() { + return Ok(Some(listener.clone())); + } + let listener = openshell_sandbox_backend::RuntimeInitiatedListener::bind(&self.descriptor)?; + Ok(Some( + self.runtime_initiated.get_or_init(|| listener).clone(), + )) + } +} #[tonic::async_trait] impl PreparedBackend for OpenShellLaunch { @@ -243,23 +280,42 @@ impl PreparedBackend for OpenShellLaunch { &self, bearer: SessionBearerTokenSlot, ) -> std::result::Result<(Option, bool), BackendError> { - openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy(self.0.clone(), bearer) - .await + match self.runtime_initiated_listener()? { + Some(listener) => { + openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy_with_runtime_initiated_listener( + self.descriptor.clone(), + bearer, + listener, + ) + .await + } + None => { + openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( + self.descriptor.clone(), + bearer, + ) + .await + } + } } fn build( self: Box, services: BackendServices, ) -> std::result::Result { - let payload = self.0.backend_descriptor()?.payload; + let payload = self.descriptor.backend_descriptor()?.payload; + let mut backend = openshell_sandbox_backend::OpenShellRuntimeBackend::new( + services.ca_file_paths, + services.provider_credentials, + services.sandbox_bearer, + ); + if let Some(listener) = self.runtime_initiated_listener()? { + backend = backend.with_runtime_initiated_listener(listener); + } Ok(BuiltBackend { payload, proxy_listener: None, - backend: Arc::new(openshell_sandbox_backend::OpenShellRuntimeBackend::new( - services.ca_file_paths, - services.provider_credentials, - services.sandbox_bearer, - )), + backend: Arc::new(backend), }) } } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 1025860f7e..0b5e0bf38c 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -813,3 +813,58 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .is_err() ); } + +#[tokio::test] +async fn runtime_initiated_launch_shares_one_listener_between_discovery_and_attach() { + use openshell_sandbox_backend::boundary_protocol::{ + SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, + generate_sandbox_tls_material, + }; + + let material = generate_sandbox_tls_material(SandboxSessionId::new()).unwrap(); + // Production descriptors name a fixed port; reserve a free one for the test. + let port = std::net::TcpListener::bind("127.0.0.1:0") + .unwrap() + .local_addr() + .unwrap() + .port(); + let mut descriptor = SandboxRuntimeDescriptor { + boundary_id: "sandbox-1".into(), + generation: "generation-1".into(), + session_id: SandboxSessionId::new(), + workload_identity: identity(), + transport: SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox-1.ns.svc:5500".into(), + listen_address: std::net::SocketAddr::from(([127, 0, 0, 1], port)), + }, + tls: SandboxTlsClientConfig { + server_name: material.server_name, + trust_anchor_pem: material.trust_anchor_pem, + }, + host_gateway_ip: None, + resource_claims: BTreeMap::new(), + outer_fence: OuterFenceGuarantees::from_enforcement_evidence( + "generation-1", + [], + b"test-evidence", + ) + .unwrap(), + }; + let launch = OpenShellLaunch { + descriptor: descriptor.clone(), + runtime_initiated: std::sync::OnceLock::new(), + }; + let first = launch.runtime_initiated_listener().unwrap().unwrap(); + let second = launch.runtime_initiated_listener().unwrap().unwrap(); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(first.local_address().port(), port); + + descriptor.transport = SandboxTransport::Unix { + socket_path: "/unused-test-runtime".into(), + }; + let legacy = OpenShellLaunch { + descriptor, + runtime_initiated: std::sync::OnceLock::new(), + }; + assert!(legacy.runtime_initiated_listener().unwrap().is_none()); +} diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index aab1204eb5..a5f055c461 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -989,8 +989,11 @@ gateway_namespace = "openshell" gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } [openshell.drivers.kubernetes.sandbox_runtime] -# TLS-protected boundary listener reached only by the paired control pod. +# Port of the TLS-protected Sandbox Protocol listener on the paired Pod. boundary_port = 5500 +# Which side opens the Sandbox Protocol connection for sandboxes created from +# now on: "runtime-initiated" (default) or "supervisor-initiated". +control_connection = "runtime-initiated" ``` The Kubernetes driver always creates the workload `NetworkPolicy` before @@ -998,6 +1001,38 @@ releasing a sandbox Pod. Your cluster CNI must enforce ingress and egress `NetworkPolicy` in every sandbox namespace; the Kubernetes API does not verify enforcement. +#### Sandbox Control Connection + +`control_connection` applies only when a sandbox is created. The driver records +the mode on the sandbox, and stop/start, gateway restarts, and upgrades keep +it. Sandboxes created before this setting existed stay `supervisor-initiated`. +With `runtime-initiated`, the sandbox runtime dials its own supervisor Pod +through the sandbox's boundary Service, and the driver adds one per-sandbox +`NetworkPolicy` that allows only that connection, as egress from the workload +Pod and ingress to the supervisor Pod. Authentication is the same in both modes. + +That policy makes the supervisor Pod ingress-isolated even when Helm +`networkPolicy.enabled` is `false`. The sandbox needs no other ingress to its +supervisor: exec, file transfer, and logs use connections the supervisor opens. +Your CNI must still allow kubelet readiness probes to ingress-isolated Pods, as +it already must for the default `networkPolicy.enabled` SSH policy. Do not +loosen `os-control-`; add any extra supervisor ingress your platform needs, +such as monitoring, as a separate policy. + +Use `runtime-initiated` only when the gateway, `supervisor_image`, and +`sandbox_runtime_image` all support it. If you pin an older supervisor or +sandbox runtime image, set `control_connection = "supervisor-initiated"` until +you upgrade both images. + +Rolling back is not supported for `runtime-initiated` sandboxes. Older gateways, +supervisors, and sandbox runtimes cannot start or restart them: an older driver +does not read the recorded mode, the sandbox's Service selects the supervisor +rather than the workload, and older supervisors and runtimes reject the new +launch descriptors. Running supervisor and sandbox processes can keep their +established connection, but starting a new generation with incompatible +binaries fails closed. Delete those sandboxes before rolling back, or roll +forward. + In managed workspace mode, the Kubernetes driver copies each explicitly named `image_pull_secrets` Secret from `namespace` into the managed workspace namespace on sandbox creation. Shared and operator modes require the Secret to diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 89ec2c8d44..93f953c91d 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -299,6 +299,8 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `proxy_auth_secret_name` / `proxy_auth_secret_key` | `upstreamProxy.authSecret.name` / `.key` | Secret holding the proxy `user:pass` credential. | | `proxy_ca_bundle` | `upstreamProxy.caBundle.configMapName` / `.key` | ConfigMap with the proxy's CA certificate. | +New Kubernetes sandboxes default to a runtime-initiated control connection, which needs a supervisor image and sandbox runtime image that support it. If you override either image with an older version, first set `gatewayConfig["openshell.drivers.kubernetes"].sandbox_runtime.control_connection` to `supervisor-initiated`. Existing sandboxes keep their mode. See [Sandbox Control Connection](/how-it-works/gateways/configuration#sandbox-control-connection). + For the full list, see the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. On OpenShift, copy the cluster proxy's CA into the gateway namespace and set `upstreamProxy.caBundle.configMapName` to `corporate-proxy-ca`: diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ddf042c67c..9e10e77dac 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1010,10 +1010,42 @@ per-sandbox Service, split immutable bootstrap Secrets, and a gated supervisor Pod before releasing either Pod. The supervisor Pod runs `/openshell-supervisor`. Both Pods use the same resolved non-root identity, request no capabilities, drop `ALL`, disable -privilege escalation, and use `RuntimeDefault` seccomp. The supervisor reaches -the sandbox over per-sandbox TLS with server-certificate verification plus -bootstrap-token client authentication, and owns gateway policy, provider -credentials, DNS, and mediated upstream connections. +privilege escalation, and use `RuntimeDefault` seccomp. The supervisor and +sandbox talk over per-sandbox TLS with server-certificate verification plus +bootstrap-token client authentication; the supervisor owns gateway policy, +provider credentials, DNS, and mediated upstream connections. + +Each sandbox records which side opens that connection in the Sandbox annotation +`openshell.ai/sandbox-runtime-control-connection`. A missing annotation means +`supervisor-initiated`: the `os-boundary-` Service selects the workload Pod +and the supervisor dials it. With `runtime-initiated` (the default for new +sandboxes), the Service selects the supervisor Pod, the sandbox runtime dials +it, and a per-sandbox NetworkPolicy `os-control-` admits only that +connection at both the workload (egress) and the supervisor (ingress). The +Service publishes the supervisor before it is Ready, because readiness follows +the runtime's attach. If the runtime logs `cannot reach the supervisor`, check +that the CNI applied both directions of `os-control-`. +Stop/start and gateway restarts keep the recorded mode. Check the mode first +when a sandbox never becomes ready, especially after changing images or +`sandbox_runtime.control_connection`: + +```bash +kubectl -n get sandbox \ + -o jsonpath='{.metadata.annotations.openshell\.ai/sandbox-runtime-control-connection}{"\n"}' +kubectl -n get service os-boundary- -o jsonpath='{.spec.selector}{"\n"}' +kubectl -n get networkpolicy os-control- -o yaml +``` + +A `runtime-initiated` sandbox needs a gateway, supervisor image, and sandbox +runtime image that all support the mode. An older supervisor or runtime fails +to decode its launch descriptor, so the sandbox cannot start or restart. +Rolling the gateway or images back is unsupported for `runtime-initiated` +sandboxes; running Pods keep their connection, but their next start fails. +Upgrade the images, or set `control_connection = "supervisor-initiated"` for +new sandboxes. +Do not edit the annotation, Service selector, or `os-control-` policy: +reconciliation suspends a sandbox whose Service or control policy no longer +matches its recorded mode. Inspect all driver-managed resources when a Kubernetes sandbox remains Starting or loses readiness: From 30ee27f2396bc89b9e3e0235bf53b311df5e9f15 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 12:25:40 -0700 Subject: [PATCH 2/9] refactor(sandbox): consolidate control transport setup Signed-off-by: Drew Newberry --- crates/openshell-driver-kubernetes/README.md | 35 +- .../openshell-driver-kubernetes/src/driver.rs | 678 ++++++------------ .../src/isolation.rs | 64 +- crates/openshell-sandbox-backend/README.md | 33 +- crates/openshell-sandbox-backend/src/lib.rs | 3 +- .../openshell-sandbox-backend/src/runtime.rs | 395 +++------- .../src/runtime/tests/runtime_initiated.rs | 337 +++++++++ .../src/runtime_initiated.rs | 435 ++--------- .../openshell-sandbox/src/boundary_server.rs | 119 +-- .../openshell-supervisor/src/backend_setup.rs | 76 +- .../src/backend_setup/tests.rs | 55 -- 11 files changed, 866 insertions(+), 1364 deletions(-) create mode 100644 crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index 0405aa1301..a3328315ac 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -123,27 +123,20 @@ the sandbox's lifetime; it selects the paired workload Pod for For `runtime-initiated`: - The bootstrap Secret gives the runtime the Service ClusterIP and port. The - runtime dials it, keeps one unclaimed spare connection for reconnects and - credential rotation, and retries with capped backoff until the supervisor - listens, so neither Pod waits on the other during bootstrap. The workload - still cannot reach the control port. -- The supervisor listens on `boundary_port`. It completes TLS on each accepted - socket before use, limits concurrent unauthenticated handshakes and their - duration, and queues at most two authenticated spares. A new connection - cannot displace a live authenticated channel. -- The driver adds a per-sandbox `NetworkPolicy` `os-control-` that selects - only this sandbox's two Pods and admits one flow, from its workload Pod to its - supervisor Pod on `boundary_port` over TCP, as both workload egress and - supervisor ingress. Ingress matters because supervisors can already be - ingress-isolated, for example by the managed SSH ingress policy, which selects - every OpenShell-managed Pod. This also makes the supervisor ingress-isolated - when that SSH policy is disabled; exec, file sync, and logs need no other - supervisor ingress because the supervisor opens those connections. The CNI - must still allow kubelet readiness probes. The namespace-wide policies are - unchanged. -- The boundary Service sets `publishNotReadyAddresses`, because the supervisor - becomes Ready only after the runtime has attached through that Service. - The policy's UID and resourceVersion are bound into the boundary resource + runtime retries with capped backoff until the supervisor listens, so neither + Pod waits on the other during bootstrap. The Service sets + `publishNotReadyAddresses` because the supervisor becomes Ready only after + the runtime attaches through it. The workload still cannot reach the control + port. +- A per-sandbox `NetworkPolicy` `os-control-` selects only this sandbox's + two Pods and admits one flow, from its workload Pod to its supervisor Pod on + `boundary_port` over TCP, as workload egress and supervisor ingress. + Supervisors may already be ingress-isolated by the managed SSH ingress policy; + when that policy is disabled, this one isolates them. Exec, file sync, and + logs need no other supervisor ingress because the supervisor opens those + connections. The CNI must still allow kubelet readiness probes. The + namespace-wide policies are unchanged. +- The policy's UID and resourceVersion are bound into the boundary resource claims and outer-fence evidence. Reconciliation suspends the workload if the policy disappears, diverges, or changes generation, or if the boundary Service no longer selects the role the recorded mode requires. diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index d3a977febc..496498bd29 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -172,6 +172,12 @@ fn sandbox_runtime_control_connection( ) } +/// Operator-provided bootstrap bytes read once per runtime generation. +struct StagedBootstrapMaterial { + upstream_proxy_ca_bundle: Option>, + client_tls: Option, +} + /// Stored coordinates of a validated runtime-initiated control policy. struct VerifiedControlPolicy { uid: String, @@ -199,29 +205,6 @@ impl VerifiedControlPolicy { } } -/// Runtime annotations recorded once both Pods have UIDs, plus the control -/// policy coordinates reconciliation compares for runtime-initiated control. -fn sandbox_runtime_binding_annotations( - workload_pod_uid: &str, - supervisor_uid: &str, - fence_uid: &str, - fence_generation: i64, - control_policy: Option<&VerifiedControlPolicy>, -) -> serde_json::Value { - let mut annotations = serde_json::json!({ - ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid, - ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid, - ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), - }); - if let Some(policy) = control_policy { - annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID] = policy.uid.clone().into(); - annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION] = - policy.generation.to_string().into(); - } - annotations -} - #[derive(Debug, thiserror::Error)] pub enum KubernetesDriverError { #[error("sandbox already exists")] @@ -2263,7 +2246,6 @@ impl KubernetesComputeDriver { &agent_sandbox_api, &created, &proxy_names, - &generation, control_connection, resolved_user_id, resolved_group_id, @@ -2350,14 +2332,16 @@ impl KubernetesComputeDriver { } } - /// Fail closed when a running sandbox's endpoint or control policy no - /// longer matches the mode recorded at creation. Returns `Precondition` - /// for drift that must suspend the workload. + /// Fail closed when a sandbox's boundary Service or control policy no + /// longer matches the mode recorded at creation. This is the reconcile + /// pass's only Service read; a missing Service is drift once the sandbox + /// should be running. Returns `Precondition` for drift that must suspend. async fn verify_sandbox_runtime_control_path( &self, object: &DynamicObject, sandbox_id: &str, namespace: &str, + require_service: bool, ) -> Result<(), KubernetesDriverError> { let mode = sandbox_runtime_control_connection(object)?; let names = SandboxRuntimeNames::new(sandbox_id); @@ -2371,15 +2355,21 @@ impl KubernetesComputeDriver { ) })? .map_err(KubernetesDriverError::from_kube)?; - // A missing Service is reported by supervisor reconciliation. - if let Some(service) = service - && !boundary_service_matches_mode(&service, sandbox_id, mode) - { - return Err(KubernetesDriverError::Precondition(format!( - "boundary Service {} no longer selects the {} listener", - names.boundary_service, - mode.as_str() - ))); + match service { + None if require_service => { + return Err(KubernetesDriverError::Precondition(format!( + "sandbox-runtime boundary Service {} is missing and its allocated address cannot be safely reconstructed", + names.boundary_service + ))); + } + Some(service) if !boundary_service_matches_mode(&service, sandbox_id, mode) => { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} no longer selects the {} listener", + names.boundary_service, + mode.as_str() + ))); + } + _ => {} } if mode == KubernetesControlConnection::SupervisorInitiated { return Ok(()); @@ -2409,7 +2399,12 @@ impl KubernetesComputeDriver { OwnerReference::default(), ); validate_sandbox_runtime_fence(&stored, &expected)?; - if !sandbox_runtime_control_policy_generation_matches(&stored, object) { + if !sandbox_runtime_policy_generation_matches( + &stored, + object, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, + ) { return Err(KubernetesDriverError::Precondition(format!( "control NetworkPolicy {} changed after it was bound", names.control_policy @@ -2697,31 +2692,26 @@ impl KubernetesComputeDriver { sandbox_api: &AgentSandboxApi, sandbox_cr: &DynamicObject, names: &SandboxRuntimeNames, - _generation: &str, control_connection: KubernetesControlConnection, agent_uid: u32, agent_gid: u32, main_process_spec: &str, log_level: &str, ) -> Result { - // Read once per call so the whole generation stages, mounts, and - // references the same bytes, and so a bundle that became unreadable - // fails before any Service, Pod, or Secret is created. - let upstream_proxy_ca_bundle = - read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; - let client_tls = self.read_client_tls_material().await?; + let staged = self.read_staged_bootstrap_material().await?; + let launch_authentication = sandbox + .spec + .as_ref() + .filter(|spec| !spec.launch_authentication.is_empty()) + .ok_or_else(|| { + KubernetesDriverError::Precondition( + "Kubernetes sandbox launch authentication is required".to_string(), + ) + }) + .and_then(|spec| decode_launch_authentication(&spec.launch_authentication))?; let cr_uid = sandbox_cr.metadata.uid.as_deref().ok_or_else(|| { KubernetesDriverError::Message("created Sandbox CR has no UID".to_string()) })?; - let namespace_uid = Api::::all(self.client.clone()) - .get(namespace) - .await - .map_err(KubernetesDriverError::from_kube)? - .metadata - .uid - .ok_or_else(|| { - KubernetesDriverError::Message("sandbox namespace has no UID".to_string()) - })?; let dependent_owner = sandbox_runtime_sandbox_owner_reference( cr_name, cr_uid, @@ -2729,7 +2719,7 @@ impl KubernetesComputeDriver { false, ); let services: Api = Api::namespaced(self.client.clone(), namespace); - let service = services + services .create( &PostParams::default(), &boundary_service( @@ -2743,94 +2733,20 @@ impl KubernetesComputeDriver { ) .await .map_err(KubernetesDriverError::from_kube)?; - let service_ip: std::net::IpAddr = service - .spec - .and_then(|spec| spec.cluster_ip) - .filter(|ip| ip != "None") - .ok_or_else(|| { - KubernetesDriverError::Message("boundary Service has no ClusterIP".to_string()) - })? - .parse() - .map_err(|error| { - KubernetesDriverError::Message(format!( - "invalid boundary Service ClusterIP: {error}" - )) - })?; - - let pods: Api = Api::namespaced(self.client.clone(), namespace); - let supervisor = pods - .create( - &PostParams::default(), - &supervisor_pod( - namespace, - names, - &sandbox.id, - &sandbox.name, - &self.config.gateway_id, - &self.config.supervisor_image, - self.config.supervisor_image_pull_policy, - &self.config.service_account_name, - agent_uid, - agent_gid, - &self.generation_image_pull_secret_names(names), - &self.config.grpc_endpoint, - self.config.supervisor_client_tls(), - main_process_spec, - log_level, - self.config.effective_sa_token_ttl_secs(), - self.config.https_proxy.as_deref(), - self.config.no_proxy.as_deref(), - self.config - .proxy_auth_secret_name - .as_deref() - .zip(self.config.proxy_auth_secret_key.as_deref()), - self.config.proxy_auth_allow_insecure == Some(true), - self.config.proxy_connect_by_hostname == Some(true), - upstream_proxy_ca_bundle.is_some(), - self.config.provider_spiffe_enabled().then_some( - self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - dependent_owner.clone(), - &self.supervisor_otlp_environment(), - ) - .map_err(KubernetesDriverError::Message)?, - ) - .await - .map_err(KubernetesDriverError::from_kube)?; - let supervisor_uid = supervisor.metadata.uid.ok_or_else(|| { - KubernetesDriverError::Message("supervisor Pod has no UID".to_string()) - })?; - - let policies: Api = Api::namespaced(self.client.clone(), namespace); - let fence = policies - .get(&names.workload_policy) - .await - .map_err(KubernetesDriverError::from_kube)?; - let fence_uid = fence.metadata.uid.ok_or_else(|| { - KubernetesDriverError::Message("workload NetworkPolicy has no UID".to_string()) - })?; - let fence_generation = fence.metadata.generation.ok_or_else(|| { - KubernetesDriverError::Message("workload NetworkPolicy has no generation".to_string()) - })?; - let fence_resource_version = fence.metadata.resource_version.ok_or_else(|| { - KubernetesDriverError::Message( - "workload NetworkPolicy has no resourceVersion".to_string(), + let supervisor_uid = self + .create_sandbox_runtime_supervisor( + namespace, + names, + &sandbox.id, + &sandbox.name, + dependent_owner, + agent_uid, + agent_gid, + main_process_spec, + log_level, + staged.upstream_proxy_ca_bundle.is_some(), ) - })?; - let control_policy = match control_connection { - KubernetesControlConnection::SupervisorInitiated => None, - KubernetesControlConnection::RuntimeInitiated => Some( - self.ensure_sandbox_runtime_control_policy( - namespace, - names, - &sandbox.id, - dependent_owner.clone(), - ) - .await?, - ), - }; + .await?; // Release the Sandbox CR only far enough for the controller to create // the workload Pod. The Pod remains unschedulable because its template @@ -2841,59 +2757,6 @@ impl KubernetesComputeDriver { }) .await?; - let workload_pod = self - .wait_for_bootstrap_workload_pod(&pods, cr_name, cr_uid) - .await?; - let admission_object = sandbox_api - .api - .get(cr_name) - .await - .map_err(KubernetesDriverError::from_kube)?; - self.admit_stored_resources(&admission_object) - .await - .map_err(admission_error)?; - Self::validate_capability_free_workload_pod( - &workload_pod, - cr_uid, - &sandbox.id, - agent_uid, - agent_gid, - &names.sandbox_secret, - )?; - let workload_pod_uid = - workload_pod.metadata.uid.clone().ok_or_else(|| { - KubernetesDriverError::Message("workload Pod has no UID".to_string()) - })?; - let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { - KubernetesDriverError::Message("workload Pod has no name".to_string()) - })?; - let binding_annotations = sandbox_runtime_binding_annotations( - &workload_pod_uid, - &supervisor_uid, - &fence_uid, - fence_generation, - control_policy.as_ref(), - ); - patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { - serde_json::json!({ - "metadata": { - "resourceVersion": version, - "annotations": binding_annotations.clone(), - } - }) - }) - .await?; - - let launch_authentication = sandbox - .spec - .as_ref() - .filter(|spec| !spec.launch_authentication.is_empty()) - .ok_or_else(|| { - KubernetesDriverError::Precondition( - "Kubernetes sandbox launch authentication is required".to_string(), - ) - }) - .and_then(|spec| decode_launch_authentication(&spec.launch_authentication))?; let mut child_env = sandbox .spec .as_ref() @@ -2905,133 +2768,22 @@ impl KubernetesComputeDriver { child_env.extend(spec.environment.clone()); } child_env.retain(|name, _| !name.starts_with("OPENSHELL_")); - let host_gateway_ip = self.config.host_gateway_ip.parse().ok(); - let session_id = launch_authentication.supervisor.session_id; - let tls = generate_sandbox_tls_material(session_id) - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let verification_keys = - gateway_verification_keys(&launch_authentication.verification_keys)?; - let proxy_ca = generate_proxy_ca_material().map_err(KubernetesDriverError::Message)?; - let workload_identity = ResolvedWorkloadIdentity::new( - agent_uid, - agent_gid, - Vec::new(), - "kubernetes-config".to_string(), - format!("sandbox:{cr_uid}"), - ) - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let provisioned = KubernetesSandboxRuntimeBoundarySpec { - boundary_id: sandbox.id.clone(), - generation: launch_authentication - .supervisor - .runtime_generation - .to_string(), - session_id, - session_rotation: launch_authentication.supervisor.session_rotation, - auth_epoch: launch_authentication.supervisor.auth_epoch, - gateway_id: launch_authentication.gateway_id, - verification_keys, - namespace_uid, - sandbox_resource_uid: cr_uid.to_string(), - workload_pod_uid: workload_pod_uid.clone(), - workload_pod_uid_path: PathBuf::from(SANDBOX_POD_UID_PATH), - supervisor_pod_uid: supervisor_uid.clone(), - egress_policy_uid: fence_uid.clone(), - egress_policy_resource_version: fence_resource_version.clone(), - control_path: self.sandbox_runtime_control_path( - namespace, - names, - service_ip, - control_policy, - ), - sandbox_tls: SandboxTlsServerConfig { - certificate_chain_path: PathBuf::from(BOUNDARY_CERTIFICATE_PATH), - private_key_path: PathBuf::from(BOUNDARY_PRIVATE_KEY_PATH), - }, - supervisor_tls: SandboxTlsClientConfig { - server_name: tls.server_name.clone(), - trust_anchor_pem: tls.trust_anchor_pem.clone(), - }, - host_gateway_ip, - workload_identity, - child_env, - } - .provision() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let descriptor = provisioned - .runtime_descriptor - .backend_descriptor() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?; - let workload_owner = runtime_pod_owner(&workload_pod_name, &workload_pod_uid); - let supervisor_owner = runtime_pod_owner(&names.supervisor_pod, &supervisor_uid); - let sandbox_secret = sandbox_bootstrap_secret( + self.install_sandbox_runtime_generation( namespace, - names, - &sandbox.id, - provisioned - .boundary_config - .encode() - .map_err(|error| KubernetesDriverError::Message(error.to_string()))?, - tls.certificate_chain_pem.into_bytes(), - tls.private_key_pem.into_bytes(), - workload_owner.clone(), - ); - let supervisor_secret = supervisor_bootstrap_secret( - namespace, - names, + cr_name, + sandbox_api, &sandbox.id, - descriptor.payload, - serde_json::to_vec(&launch_authentication.supervisor).map_err(|error| { - KubernetesDriverError::Message(format!("encode supervisor auth bundle: {error}")) - })?, - proxy_ca.certificate_pem.into_bytes(), - proxy_ca.private_key_pem.into_bytes(), - upstream_proxy_ca_bundle.clone(), - client_tls, - supervisor_owner.clone(), - ); - let secrets = Api::::namespaced(self.client.clone(), namespace); - secrets - .create(&PostParams::default(), &sandbox_secret) - .await - .map_err(KubernetesDriverError::from_kube)?; - secrets - .create(&PostParams::default(), &supervisor_secret) - .await - .map_err(KubernetesDriverError::from_kube)?; - self.create_generation_image_pull_secrets( - namespace, + cr_uid, names, - &sandbox.id, - vec![workload_owner, supervisor_owner], - ) - .await?; - - pods.patch( - &names.supervisor_pod, - &PatchParams::default(), - &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), - ) - .await - .map_err(KubernetesDriverError::from_kube)?; - pods.patch( - &workload_pod_name, - &PatchParams::default(), - &Patch::Merge(&serde_json::json!({"spec": {"schedulingGates": []}})), + control_connection, + &supervisor_uid, + agent_uid, + agent_gid, + child_env, + &launch_authentication, + staged, ) - .await - .map_err(KubernetesDriverError::from_kube)?; - patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { - sandbox_runtime_bootstrap_phase_patch(version, SandboxRuntimeBootstrapPhase::Released) - }) .await?; - spawn_sandbox_runtime_bootstrap_completion( - pods.clone(), - sandbox_api.api.clone(), - names.supervisor_pod.clone(), - cr_name.to_string(), - Some(cr_uid.to_string()), - ); // Return while the CR remains explicitly bootstrapping. The gateway // can now commit the sandbox configuration required by a policy-less // control process without deadlocking behind this driver call. Only @@ -3045,6 +2797,82 @@ impl KubernetesComputeDriver { )) } + /// Read operator-provided bootstrap material once per generation, so the + /// whole generation stages, mounts, and references the same bytes, and so + /// unreadable material fails before any Service, Pod, or Secret is created. + async fn read_staged_bootstrap_material( + &self, + ) -> Result { + Ok(StagedBootstrapMaterial { + upstream_proxy_ca_bundle: read_staged_upstream_proxy_ca_bundle( + self.config.proxy_ca_bundle.as_deref(), + ) + .await?, + client_tls: self.read_client_tls_material().await?, + }) + } + + /// Create one generation's gated supervisor Pod and return its UID. + #[allow(clippy::too_many_arguments, clippy::similar_names)] + async fn create_sandbox_runtime_supervisor( + &self, + namespace: &str, + names: &SandboxRuntimeNames, + sandbox_id: &str, + sandbox_name: &str, + owner: OwnerReference, + agent_uid: u32, + agent_gid: u32, + main_process_spec: &str, + log_level: &str, + upstream_proxy_ca_bundle_staged: bool, + ) -> Result { + let pods: Api = Api::namespaced(self.client.clone(), namespace); + pods.create( + &PostParams::default(), + &supervisor_pod( + namespace, + names, + sandbox_id, + sandbox_name, + &self.config.gateway_id, + &self.config.supervisor_image, + self.config.supervisor_image_pull_policy, + &self.config.service_account_name, + agent_uid, + agent_gid, + &self.generation_image_pull_secret_names(names), + &self.config.grpc_endpoint, + self.config.supervisor_client_tls(), + main_process_spec, + log_level, + self.config.effective_sa_token_ttl_secs(), + self.config.https_proxy.as_deref(), + self.config.no_proxy.as_deref(), + self.config + .proxy_auth_secret_name + .as_deref() + .zip(self.config.proxy_auth_secret_key.as_deref()), + self.config.proxy_auth_allow_insecure == Some(true), + self.config.proxy_connect_by_hostname == Some(true), + upstream_proxy_ca_bundle_staged, + self.config.provider_spiffe_enabled().then_some( + self.config + .provider_spiffe_workload_api_socket_path + .as_str(), + ), + owner, + &self.supervisor_otlp_environment(), + ) + .map_err(KubernetesDriverError::Message)?, + ) + .await + .map_err(KubernetesDriverError::from_kube)? + .metadata + .uid + .ok_or_else(|| KubernetesDriverError::Message("supervisor Pod has no UID".to_string())) + } + #[allow(clippy::too_many_arguments, clippy::similar_names)] async fn install_sandbox_runtime_generation( &self, @@ -3054,20 +2882,14 @@ impl KubernetesComputeDriver { sandbox_id: &str, cr_uid: &str, names: &SandboxRuntimeNames, - _generation: &str, control_connection: KubernetesControlConnection, supervisor_uid: &str, agent_uid: u32, agent_gid: u32, child_env: std::collections::HashMap, launch_authentication: &openshell_core::jwt::SandboxLaunchAuthentication, + staged: StagedBootstrapMaterial, ) -> Result<(), KubernetesDriverError> { - // Read once per call so the whole generation stages, mounts, and - // references the same bytes, and so a bundle that became unreadable - // fails before any Service, Pod, or Secret is created. - let upstream_proxy_ca_bundle = - read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; - let client_tls = self.read_client_tls_material().await?; let namespace_uid = Api::::all(self.client.clone()) .get(namespace) .await @@ -3165,19 +2987,20 @@ impl KubernetesComputeDriver { let workload_pod_name = workload_pod.metadata.name.clone().ok_or_else(|| { KubernetesDriverError::Message("workload Pod has no name".to_string()) })?; - let binding_annotations = sandbox_runtime_binding_annotations( - &workload_pod_uid, - supervisor_uid, - &fence_uid, - fence_generation, - control_policy.as_ref(), - ); + let mut annotations = serde_json::json!({ + ANNOTATION_SANDBOX_RUNTIME_WORKLOAD_UID: workload_pod_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: fence_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: fence_generation.to_string(), + }); + if let Some(policy) = &control_policy { + annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID] = policy.uid.clone().into(); + annotations[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION] = + policy.generation.to_string().into(); + } patch_dynamic_object_with_resource_version_retry(&sandbox_api.api, cr_name, |version| { serde_json::json!({ - "metadata": { - "resourceVersion": version, - "annotations": binding_annotations.clone(), - } + "metadata": {"resourceVersion": version, "annotations": annotations.clone()} }) }) .await?; @@ -3262,8 +3085,8 @@ impl KubernetesComputeDriver { })?, proxy_ca.certificate_pem.into_bytes(), proxy_ca.private_key_pem.into_bytes(), - upstream_proxy_ca_bundle.clone(), - client_tls, + staged.upstream_proxy_ca_bundle, + staged.client_tls, supervisor_owner.clone(), ); let secrets = Api::::namespaced(self.client.clone(), namespace); @@ -3441,11 +3264,7 @@ impl KubernetesComputeDriver { encoded_authentication: &[u8], encoded_expected_runtime_identity: &str, ) -> Result { - // Read once per call so the whole generation stages, mounts, and - // references the same bytes, and so a bundle that became unreadable - // fails before any Service, Pod, or Secret is created. - let upstream_proxy_ca_bundle = - read_staged_upstream_proxy_ca_bundle(self.config.proxy_ca_bundle.as_deref()).await?; + let staged = self.read_staged_bootstrap_material().await?; let generation = openshell_core::sandbox_generation::SandboxGenerationId::parse( encoded_generation.to_string(), ) @@ -3598,55 +3417,25 @@ impl KubernetesComputeDriver { agent_gid, ), )?; - let supervisor = pods - .create( - &PostParams::default(), - &supervisor_pod( - &namespace, - &names, - sandbox_id, - &sandbox_name, - &self.config.gateway_id, - &self.config.supervisor_image, - self.config.supervisor_image_pull_policy, - &self.config.service_account_name, - agent_uid, - agent_gid, - &self.generation_image_pull_secret_names(&names), - &self.config.grpc_endpoint, - self.config.supervisor_client_tls(), - &main_process_spec, - &log_level, - self.config.effective_sa_token_ttl_secs(), - self.config.https_proxy.as_deref(), - self.config.no_proxy.as_deref(), - self.config - .proxy_auth_secret_name - .as_deref() - .zip(self.config.proxy_auth_secret_key.as_deref()), - self.config.proxy_auth_allow_insecure == Some(true), - self.config.proxy_connect_by_hostname == Some(true), - upstream_proxy_ca_bundle.is_some(), - self.config.provider_spiffe_enabled().then_some( - self.config - .provider_spiffe_workload_api_socket_path - .as_str(), - ), - sandbox_runtime_sandbox_owner_reference( - cr_name, - cr_uid, - &sandbox_api.resource.api_version, - false, - ), - &self.supervisor_otlp_environment(), - ) - .map_err(KubernetesDriverError::Message)?, + let supervisor_uid = self + .create_sandbox_runtime_supervisor( + &namespace, + &names, + sandbox_id, + &sandbox_name, + sandbox_runtime_sandbox_owner_reference( + cr_name, + cr_uid, + &sandbox_api.resource.api_version, + false, + ), + agent_uid, + agent_gid, + &main_process_spec, + &log_level, + staged.upstream_proxy_ca_bundle.is_some(), ) - .await - .map_err(KubernetesDriverError::from_kube)?; - let supervisor_uid = supervisor.metadata.uid.ok_or_else(|| { - KubernetesDriverError::Message("replacement supervisor Pod has no UID".to_string()) - })?; + .await?; let mut volumes = object .data @@ -3706,13 +3495,13 @@ impl KubernetesComputeDriver { sandbox_id, cr_uid, &names, - generation.as_str(), control_connection, &supervisor_uid, agent_uid, agent_gid, child_env, &launch_authentication, + staged, ) .await } @@ -4235,8 +4024,14 @@ impl KubernetesComputeDriver { continue; } if workload_may_run { + let require_service = !sandbox_runtime_bootstrap_in_progress(&object); match self - .verify_sandbox_runtime_control_path(&object, &sandbox_id, namespace) + .verify_sandbox_runtime_control_path( + &object, + &sandbox_id, + namespace, + require_service, + ) .await { Ok(()) => {} @@ -4655,24 +4450,8 @@ impl KubernetesComputeDriver { .delete_sandbox_runtime_supervisor(sandbox_id, namespace, generations) .await; } + // The boundary Service was checked by verify_sandbox_runtime_control_path. let names = SandboxRuntimeNames::new(sandbox_id); - let services = Api::::namespaced(self.client.clone(), namespace); - let service_exists = - tokio::time::timeout(KUBE_API_TIMEOUT, services.get_opt(&names.boundary_service)) - .await - .map_err(|_| { - KubernetesDriverError::Message( - "timed out reading sandbox-runtime boundary Service".to_string(), - ) - })? - .map_err(KubernetesDriverError::from_kube)? - .is_some(); - if !service_exists { - return Err(KubernetesDriverError::Precondition(format!( - "sandbox-runtime boundary Service {} is missing and its allocated address cannot be safely reconstructed", - names.boundary_service - ))); - } let pods = Api::::namespaced(self.client.clone(), namespace); tokio::time::timeout( KUBE_API_TIMEOUT, @@ -7424,32 +7203,28 @@ fn sandbox_runtime_namespace_fence_generation_matches( policy: &NetworkPolicy, sandbox: &DynamicObject, ) -> bool { - let Some(annotations) = sandbox.metadata.annotations.as_ref() else { - return false; - }; - policy.metadata.uid.as_deref() - == annotations - .get(ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID) - .map(String::as_str) - && annotations - .get(ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION) - .and_then(|generation| generation.parse::().ok()) - == policy.metadata.generation + sandbox_runtime_policy_generation_matches( + policy, + sandbox, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID, + ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION, + ) } -fn sandbox_runtime_control_policy_generation_matches( +/// Whether `policy` is the exact object and generation recorded on the +/// sandbox under the given annotation keys. +fn sandbox_runtime_policy_generation_matches( policy: &NetworkPolicy, sandbox: &DynamicObject, + uid_annotation: &str, + generation_annotation: &str, ) -> bool { let Some(annotations) = sandbox.metadata.annotations.as_ref() else { return false; }; - policy.metadata.uid.as_deref() - == annotations - .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID) - .map(String::as_str) + policy.metadata.uid.as_deref() == annotations.get(uid_annotation).map(String::as_str) && annotations - .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION) + .get(generation_annotation) .and_then(|generation| generation.parse::().ok()) == policy.metadata.generation } @@ -9299,30 +9074,6 @@ mod tests { )); } - #[test] - fn binding_annotations_add_control_policy_only_for_runtime_initiated() { - let legacy = sandbox_runtime_binding_annotations("w", "s", "f", 3, None); - assert!( - legacy - .get(ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID) - .is_none() - ); - let policy = VerifiedControlPolicy::from_policy(&stored_control_policy(4)).unwrap(); - let runtime = sandbox_runtime_binding_annotations("w", "s", "f", 3, Some(&policy)); - assert_eq!( - runtime[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID], - "control-uid" - ); - assert_eq!( - runtime[ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION], - "4" - ); - assert_eq!( - runtime[ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION], - "3" - ); - } - #[tokio::test] async fn reconcile_accepts_each_mode_with_its_own_endpoint() { let (driver, steps) = scripted_control_driver(vec![( @@ -9335,6 +9086,7 @@ mod tests { &control_mode_object(&[]), "sandbox-1", "openshell", + true, ) .await .expect("legacy sandbox with workload Service"); @@ -9367,6 +9119,7 @@ mod tests { ]), "sandbox-1", "openshell", + true, ) .await .expect("runtime-initiated sandbox with bound control policy"); @@ -9424,6 +9177,15 @@ mod tests { ), )], ), + // A running sandbox whose Service disappeared. + ( + &[], + vec![( + http::Method::GET, + SERVICE_PATH, + kube_test_not_found("services", "os-boundary-sandbox-1"), + )], + ), // The control policy was deleted. ( &runtime_annotations, @@ -9487,6 +9249,23 @@ mod tests { vec![], ), ]; + // A bootstrapping generation tolerates a missing Service (availability + // and reaping handle it) but still rejects drift in what exists. + let (driver, remaining) = scripted_control_driver(vec![( + http::Method::GET, + SERVICE_PATH, + kube_test_not_found("services", "os-boundary-sandbox-1"), + )]); + driver + .verify_sandbox_runtime_control_path( + &control_mode_object(&[]), + "sandbox-1", + "openshell", + false, + ) + .await + .expect("bootstrap without a Service is left to availability"); + assert!(remaining.lock().unwrap().is_empty()); for (index, (annotations, steps)) in cases.into_iter().enumerate() { let (driver, remaining) = scripted_control_driver(steps); let result = driver @@ -9494,6 +9273,7 @@ mod tests { &control_mode_object(annotations), "sandbox-1", "openshell", + true, ) .await; assert!( diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index ed095f4e56..cfb4c78cfc 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -41,31 +41,24 @@ struct KubernetesOuterFenceEvidence<'a> { /// Present only for runtime-initiated control. Omitted otherwise so the /// supervisor-initiated evidence encoding and digest are unchanged. #[serde(skip_serializing_if = "Option::is_none")] - runtime_control_egress: Option>, + runtime_control_policy: Option>, } -/// Facts about the per-sandbox policy that admits only the runtime's dial to -/// its own supervisor. The driver records them only after verifying the -/// stored policy matches [`runtime_control_policy`]. +/// The stored per-pair control policy bound into this generation. The driver +/// supplies it only after verifying the stored object equals +/// [`runtime_control_policy`] for this pair and port, so its only added path +/// is the runtime's connection to its own supervisor. #[derive(Serialize)] -struct KubernetesControlEgressEvidence<'a> { +struct KubernetesControlPolicyEvidence<'a> { network_policy_uid: &'a str, network_policy_resource_version: &'a str, - /// Egress rules in the policy; the only one targets the paired supervisor. - egress_rule_count: u32, - /// Ingress rules in the policy; the only one admits the paired workload. - ingress_rule_count: u32, - paired_supervisor_only: bool, port: u16, } -impl KubernetesControlEgressEvidence<'_> { - fn is_managed_control_path(&self) -> bool { +impl KubernetesControlPolicyEvidence<'_> { + fn is_bound(&self) -> bool { !self.network_policy_uid.is_empty() && !self.network_policy_resource_version.is_empty() - && self.egress_rule_count == 1 - && self.ingress_rule_count == 1 - && self.paired_supervisor_only && self.port != 0 } } @@ -82,7 +75,7 @@ impl KubernetesOuterFenceEvidence<'_> { "Kubernetes provisioning evidence is incomplete".to_string(), )); } - if self.runtime_control_egress.as_ref().is_some_and(|control| !control.is_managed_control_path()) { + if self.runtime_control_policy.as_ref().is_some_and(|control| !control.is_bound()) { return Err(BackendError::Descriptor("Kubernetes control policy evidence is incomplete".into())); } let encoded = serde_json::to_vec(self).map_err(|error| { @@ -422,13 +415,10 @@ impl KubernetesSandboxRuntimeBoundarySpec { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: control_port.map(|port| KubernetesControlEgressEvidence { + runtime_control_policy: control_port.map(|port| KubernetesControlPolicyEvidence { network_policy_uid: &resource_claims["kubernetes.control_policy_uid"], network_policy_resource_version: &resource_claims ["kubernetes.control_policy_resource_version"], - egress_rule_count: 1, - ingress_rule_count: 1, - paired_supervisor_only: true, port, }), } @@ -480,7 +470,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: None, + runtime_control_policy: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -488,7 +478,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: None, + runtime_control_policy: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -496,7 +486,7 @@ mod tests { ingress_isolated: false, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: None, + runtime_control_policy: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -504,7 +494,7 @@ mod tests { ingress_isolated: true, egress_isolated: false, egress_rule_count: 0, - runtime_control_egress: None, + runtime_control_policy: None, }, KubernetesOuterFenceEvidence { network_policy_uid: "uid", @@ -512,7 +502,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 1, - runtime_control_egress: None, + runtime_control_policy: None, }, ] { assert!(evidence.project("generation-1").is_err()); @@ -643,7 +633,7 @@ mod tests { ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: None, + runtime_control_policy: None, }; assert_eq!( serde_json::to_string(&evidence).unwrap(), @@ -703,27 +693,19 @@ mod tests { } #[test] - fn runtime_control_evidence_rejects_any_unmanaged_egress() { - let control = |uid, rules, paired| KubernetesControlEgressEvidence { - network_policy_uid: uid, - network_policy_resource_version: "1", - egress_rule_count: rules, - ingress_rule_count: 1, - paired_supervisor_only: paired, - port: 5500, - }; - for runtime_control_egress in [ - control("", 1, true), - control("uid", 2, true), - control("uid", 1, false), - ] { + fn runtime_control_evidence_requires_bound_policy_coordinates() { + for (uid, resource_version, port) in [("", "1", 5500), ("uid", "", 5500), ("uid", "1", 0)] { let evidence = KubernetesOuterFenceEvidence { network_policy_uid: "uid", network_policy_resource_version: "1", ingress_isolated: true, egress_isolated: true, egress_rule_count: 0, - runtime_control_egress: Some(runtime_control_egress), + runtime_control_policy: Some(KubernetesControlPolicyEvidence { + network_policy_uid: uid, + network_policy_resource_version: resource_version, + port, + }), }; assert!(evidence.project("generation-1").is_err()); } diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index caeb671249..50102fdbc3 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -102,25 +102,20 @@ retain generation, session, resource, identity, and provisioning evidence checks ## Boundary transport establishment -Backends may supply a `BoundaryTransportConnector` for raw stream establishment. -The shared client retains TLS peer verification, session bearer authentication, -generation checks, and reconnection handling. The connector is reused for policy -discovery and attachment; it does not define platform launch configuration or -replace the isolation interface. - -### Runtime-initiated transport - -`SandboxTransport::RuntimeInitiatedTcp` and -`BoundaryListener::RuntimeInitiatedTlsTcp` reverse only the TCP dial: the -sandbox runtime connects to the supervisor, then remains the TLS server and gRPC -server. The supervisor binds one `RuntimeInitiatedListener` per sandbox session -and shares it between policy discovery and attachment. The listener completes -TLS with the descriptor's pinned identity before it queues a stream, bounds -concurrent unauthenticated handshakes and their duration, and keeps at most two -unclaimed authenticated streams. The client wraps each stream in the same -abortable transport as dialed connections, so cancellation closes the socket. -The runtime keeps one unclaimed spare connection open so reconnects and -credential rotation never wait for a new dial. +The supervisor chooses one `BoundaryTransport` per runtime descriptor and uses +clones of it for policy discovery and attachment. It dials the descriptor's +endpoint, uses a backend-supplied `BoundaryTransportConnector` for raw stream +establishment, or, for `SandboxTransport::RuntimeInitiatedTcp`, accepts the +connections the sandbox runtime opens. Every stream from every source completes +the same pinned TLS client handshake before it carries a request; the shared +client also keeps session bearer authentication, generation checks, and +reconnection handling. + +For runtime-initiated connections, the supervisor authenticates each accepted +socket before queueing it. Concurrent unauthenticated handshakes are bounded in +number and duration, and at most two authenticated spares wait for a client, so +a new connection cannot displace a live channel. The runtime keeps one spare +open, so reconnects and credential rotation never wait for a new dial. The new variants leave existing encodings unchanged. Binaries that predate them reject runtime-initiated descriptors instead of falling back to another mode. diff --git a/crates/openshell-sandbox-backend/src/lib.rs b/crates/openshell-sandbox-backend/src/lib.rs index 0baad01772..e38e216d02 100644 --- a/crates/openshell-sandbox-backend/src/lib.rs +++ b/crates/openshell-sandbox-backend/src/lib.rs @@ -27,8 +27,7 @@ pub use runtime::{ ConfirmedBoundary, ExecSession, NetworkMediationSource, OpenShellRuntimeBackend, ProcessAttachment, ReadyBoundary, RunningBoundary, }; -pub use runtime::BoundaryTransportConnector; -pub use runtime_initiated::RuntimeInitiatedListener; +pub use runtime::{BoundaryTransport, BoundaryTransportConnector}; /// Stable isolation backend name implemented by `openshell-sandbox`. pub const BACKEND_NAME: &str = openshell_core::isolation_registration::BUILTIN_BACKEND_NAME; diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 4d5354fc72..d08f6ec561 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -79,11 +79,79 @@ pub trait BoundaryTransportConnector: std::fmt::Debug + Send + Sync { ) -> Result; } -/// Concrete RPC client used for both in-tree and endpoint-registered backends. +/// How the supervisor obtains each physical Sandbox Protocol connection. +/// +/// Choose it once per runtime descriptor and share clones between policy +/// discovery and attachment. Every stream, whatever its source, completes the +/// descriptor's pinned TLS handshake in [`authenticate_boundary_stream`] +/// before it carries a request. +#[derive(Clone, Debug, Default)] +pub struct BoundaryTransport(TransportSource); + +#[derive(Clone, Debug, Default)] +enum TransportSource { + /// Dial the descriptor's endpoint. + #[default] + Dial, + /// Obtain raw streams from a backend-owned connector. + Connector(Arc), + /// Accept streams the sandbox runtime opens to this supervisor. + RuntimeInitiated(Arc), +} + +impl BoundaryTransport { + /// Select the transport the descriptor requires. A runtime-initiated + /// descriptor binds its control listener, so call this inside the + /// supervisor's Tokio runtime. + pub fn for_descriptor(descriptor: &SandboxRuntimeDescriptor) -> Result { + Ok(Self(match descriptor.transport { + SandboxTransport::RuntimeInitiatedTcp { .. } => TransportSource::RuntimeInitiated( + crate::runtime_initiated::RuntimeInitiatedListener::bind(descriptor)?, + ), + _ => TransportSource::Dial, + })) + } + + /// Establish raw streams through a backend-owned connector. + #[must_use] + pub fn connector(connector: Arc) -> Self { + Self(TransportSource::Connector(connector)) + } + + /// Open one authenticated physical connection, retrying within the + /// connect window while the remote boundary starts. + async fn connect( + &self, + descriptor: &SandboxRuntimeDescriptor, + ) -> Result { + let connector = match &self.0 { + TransportSource::Dial => None, + TransportSource::Connector(connector) => Some(connector.as_ref()), + TransportSource::RuntimeInitiated(listener) => { + return tokio::time::timeout(CONNECT_RETRY_TIMEOUT, listener.accept(descriptor)) + .await + .map_err(|_| { + BackendError::Unavailable( + "timed out waiting for the sandbox runtime to connect".to_string(), + ) + })?; + } + }; + let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; + loop { + match connect_boundary_using_connector(descriptor, connector).await { + Ok(stream) => return Ok(stream), + Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), + Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, + } + } + } +} + +/// Host-side `OpenShell` Sandbox Protocol implementation registered with the supervisor. #[derive(Debug)] pub struct OpenShellRuntimeBackend { - connector: Option>, - runtime_initiated: Option>, + transport: BoundaryTransport, ca_file_paths: Arc>>, provider_credentials: openshell_core::provider_credentials::ProviderCredentialState, sandbox_bearer: openshell_core::jwt::SessionBearerTokenSlot, @@ -111,35 +179,10 @@ impl OpenShellRuntimeBackend { pub async fn discover_policy( descriptor: SandboxRuntimeDescriptor, bearer: openshell_core::jwt::SessionBearerTokenSlot, - ) -> Result<(Option, bool), BackendError> { - Self::discover_policy_with_connector(descriptor, bearer, None).await - } - - pub async fn discover_policy_with_connector( - descriptor: SandboxRuntimeDescriptor, - bearer: openshell_core::jwt::SessionBearerTokenSlot, - connector: Option>, + transport: BoundaryTransport, ) -> Result<(Option, bool), BackendError> { let mut client = BoundaryClient::new(descriptor, bearer); - client.connector = connector; - Self::discover_policy_with_client(client).await - } - - /// Read image policy over a runtime-initiated connection accepted by - /// `listener`. Share the same listener with the attached backend. - pub async fn discover_policy_with_runtime_initiated_listener( - descriptor: SandboxRuntimeDescriptor, - bearer: openshell_core::jwt::SessionBearerTokenSlot, - listener: Arc, - ) -> Result<(Option, bool), BackendError> { - let mut client = BoundaryClient::new(descriptor, bearer); - client.runtime_initiated = Some(listener); - Self::discover_policy_with_client(client).await - } - - async fn discover_policy_with_client( - client: BoundaryClient, - ) -> Result<(Option, bool), BackendError> { + client.transport = transport; match client.call_idempotent(Request::DiscoverPolicy).await? { Response::ImagePolicy { yaml, invalid } => Ok((yaml, invalid)), _ => Err(BackendError::Descriptor( @@ -154,8 +197,7 @@ impl OpenShellRuntimeBackend { sandbox_bearer: openshell_core::jwt::SessionBearerTokenSlot, ) -> Self { Self { - connector: None, - runtime_initiated: None, + transport: BoundaryTransport::default(), ca_file_paths, provider_credentials, sandbox_bearer, @@ -197,17 +239,12 @@ impl OpenShellRuntimeBackend { client.discover_policy().await } - /// Accept runtime-initiated connections through `listener`, which must be - /// bound for the same descriptor this backend will attach. + /// Use the transport selected for the descriptor this backend will attach. #[must_use] - pub fn with_runtime_initiated_listener( - mut self, - listener: Arc, - ) -> Self { - self.runtime_initiated = Some(listener); + pub fn with_transport(mut self, transport: BoundaryTransport) -> Self { + self.transport = transport; self } - } pub fn backend_name(&self) -> &str { self.delegated @@ -249,8 +286,7 @@ impl OpenShellRuntimeBackend { self.backend_name().to_string(), descriptor.payload.clone(), ); - client.connector = self.connector.clone(); - client.runtime_initiated = self.runtime_initiated.clone(); + client.transport = self.transport.clone(); let client = Arc::new(client); let response = client .call_idempotent(Request::Attach { @@ -340,7 +376,7 @@ pub fn validate_runtime_coordinates( Ok(()) } -fn validate_client_tls(tls: &SandboxTlsClientConfig) -> Result<(), BackendError> { +pub fn validate_client_tls(tls: &SandboxTlsClientConfig) -> Result<(), BackendError> { rustls::pki_types::ServerName::try_from(tls.server_name.clone()).map_err(|error| { BackendError::Descriptor(format!( "boundary TLS server name {:?} is invalid: {error}", @@ -350,9 +386,7 @@ fn validate_client_tls(tls: &SandboxTlsClientConfig) -> Result<(), BackendError> tls_client_config(tls).map(|_| ()) } -pub fn tls_client_config( - tls: &SandboxTlsClientConfig, -) -> Result { +fn tls_client_config(tls: &SandboxTlsClientConfig) -> Result { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let certificates = rustls_pemfile::certs(&mut tls.trust_anchor_pem.as_bytes()) .collect::, _>>() @@ -1106,8 +1140,7 @@ async fn dispatch_client_mediation_frame( } struct BoundaryClient { - connector: Option>, - runtime_initiated: Option>, + transport: BoundaryTransport, runtime_descriptor: SandboxRuntimeDescriptor, backend_name: String, driver_descriptor: Vec, @@ -1299,8 +1332,7 @@ impl BoundaryClient { driver_descriptor: Vec, ) -> Self { Self { - connector: None, - runtime_initiated: None, + transport: BoundaryTransport::default(), runtime_descriptor, backend_name, driver_descriptor, @@ -1930,8 +1962,7 @@ impl BoundaryClient { credential: &BoundaryCredential, ) -> Result<(tonic::transport::Channel, Arc), BackendError> { let runtime_descriptor = self.runtime_descriptor.clone(); - let connector = self.connector.clone(); - let runtime_initiated = self.runtime_initiated.clone(); + let source = self.transport.clone(); let transport = Arc::new(TransportAbort::default()); let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { @@ -1953,8 +1984,7 @@ impl BoundaryClient { let channel = endpoint .connect_with_connector(tower::service_fn(move |_: tonic::transport::Uri| { let runtime_descriptor = runtime_descriptor.clone(); - let connector = connector.clone(); - let runtime_initiated = runtime_initiated.clone(); + let source = source.clone(); let abort = connector_transport.clone(); async move { // A channel is one authenticated connection. Never let it @@ -1962,11 +1992,8 @@ impl BoundaryClient { if abort.is_aborted() || abort.connected.swap(true, Ordering::AcqRel) { return Err(AbortableTransport::aborted_error()); } - let source = runtime_initiated.as_deref().map_or( - TransportSource::Dial(connector.as_deref()), - TransportSource::RuntimeInitiated, - ); - connect_boundary_with_retry(&runtime_descriptor, source) + source + .connect(&runtime_descriptor) .await .map(|inner| TokioIo::new(AbortableTransport { inner, abort })) .map_err(|error| std::io::Error::other(error.to_string())) @@ -2060,35 +2087,6 @@ impl BoundaryClient { } } -/// Where a client obtains each physical connection. -#[derive(Clone, Copy)] -enum TransportSource<'a> { - /// Dial the descriptor's endpoint, optionally through a backend connector. - Dial(Option<&'a dyn BoundaryTransportConnector>), - /// Take a connection the runtime opened to the supervisor's listener. - RuntimeInitiated(&'a crate::RuntimeInitiatedListener), -} - -async fn connect_boundary_with_retry( - runtime_descriptor: &SandboxRuntimeDescriptor, - source: TransportSource<'_>, -) -> Result { - let connector = match source { - TransportSource::Dial(connector) => connector, - TransportSource::RuntimeInitiated(listener) => { - return connect_runtime_initiated(runtime_descriptor, listener).await; - } - }; - let deadline = tokio::time::Instant::now() + CONNECT_RETRY_TIMEOUT; - loop { - match connect_boundary_using_connector(runtime_descriptor, connector).await { - Ok(stream) => return Ok(stream), - Err(error) if tokio::time::Instant::now() >= deadline => return Err(error), - Err(_) => tokio::time::sleep(Duration::from_millis(25)).await, - } - } -} - #[cfg(test)] async fn connect_boundary_once( runtime_descriptor: &SandboxRuntimeDescriptor, @@ -2096,27 +2094,6 @@ async fn connect_boundary_once( connect_boundary_using_connector(runtime_descriptor, None).await } -/// The listener completes TLS with the descriptor's pinned identity before a -/// stream is queued, so the shared TLS step below is not applied twice. -async fn connect_runtime_initiated( - runtime_descriptor: &SandboxRuntimeDescriptor, - listener: &crate::RuntimeInitiatedListener, -) -> Result { - let SandboxTransport::RuntimeInitiatedTcp { authority, .. } = &runtime_descriptor.transport - else { - return Err(BackendError::Descriptor( - "runtime-initiated listener supplied for a supervisor-initiated transport".to_string(), - )); - }; - tokio::time::timeout(CONNECT_RETRY_TIMEOUT, listener.accept(runtime_descriptor)) - .await - .map_err(|_| { - BackendError::Unavailable(format!( - "timed out waiting for the sandbox runtime to connect through {authority}" - )) - })? -} - async fn connect_boundary_using_connector( runtime_descriptor: &SandboxRuntimeDescriptor, connector: Option<&dyn BoundaryTransportConnector>, @@ -2158,13 +2135,20 @@ async fn connect_boundary_using_connector( SandboxTransport::Vsock { guest_cid, port } => connect_host_vsock(*guest_cid, *port)?, SandboxTransport::RuntimeInitiatedTcp { .. } => { return Err(BackendError::Descriptor( - "runtime-initiated transport requires a supervisor control listener" - .to_string(), + "a runtime-initiated transport cannot be dialed".to_string(), )); } } }; - let tls = &runtime_descriptor.tls; + authenticate_boundary_stream(&runtime_descriptor.tls, stream).await +} + +/// The only Sandbox Protocol client handshake: verify the generation-pinned +/// server identity before a stream from any source can carry a request. +pub async fn authenticate_boundary_stream( + tls: &SandboxTlsClientConfig, + stream: BoundaryDuplexStream, +) -> Result { let server_name = rustls::pki_types::ServerName::try_from(tls.server_name.clone()).map_err(|error| { BackendError::Descriptor(format!( @@ -2477,6 +2461,7 @@ mod tests { mod flow_control; mod network_recovery; mod protocol_errors; + mod runtime_initiated; use crate::isolation_protocol::exchange_frame; use std::path::PathBuf; @@ -3672,7 +3657,7 @@ mod tests { tls_runtime_descriptor(address, certificate.client_tls), test_bearer(&"a".repeat(32)), ); - client.connector = Some(connector); + client.transport = BoundaryTransport::connector(connector); let response = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) .await @@ -3713,190 +3698,6 @@ mod tests { server.await.unwrap(); } - /// Records when the runtime side of a connection observes EOF or failure. - struct ClosureProbe { - inner: BoundaryDuplexStream, - closed: Arc, - } - - impl tokio::io::AsyncRead for ClosureProbe { - fn poll_read( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &mut tokio::io::ReadBuf<'_>, - ) -> Poll> { - let before = buffer.filled().len(); - let result = Pin::new(&mut self.inner).poll_read(context, buffer); - match &result { - Poll::Ready(Ok(())) if buffer.filled().len() == before => { - self.closed.store(true, Ordering::Release); - } - Poll::Ready(Err(_)) => self.closed.store(true, Ordering::Release), - _ => {} - } - result - } - } - - impl tokio::io::AsyncWrite for ClosureProbe { - fn poll_write( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &[u8], - ) -> Poll> { - Pin::new(&mut self.inner).poll_write(context, buffer) - } - - fn poll_flush( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_flush(context) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_shutdown(context) - } - } - - /// Dial the supervisor listener as `openshell-sandbox` does and serve the - /// protocol over the outbound connection. - async fn serve_as_runtime_initiated_boundary( - address: std::net::SocketAddr, - server_config: Arc, - expected_token: String, - closed: Arc, - ) { - let stream = tokio::net::TcpStream::connect(address).await.unwrap(); - let stream = tokio_rustls::TlsAcceptor::from(server_config) - .accept(stream) - .await - .unwrap(); - serve_test_grpc( - Box::new(ClosureProbe { - inner: Box::new(stream), - closed, - }), - expected_token, - ) - .await; - } - - fn runtime_initiated_descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { - let mut descriptor = tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), tls); - descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), - listen_address: "127.0.0.1:5500".parse().unwrap(), - }; - descriptor - } - - #[tokio::test] - async fn runtime_initiated_transport_preserves_tls_bearer_and_physical_abort() { - let certificate = test_certificate(); - let descriptor = runtime_initiated_descriptor(certificate.client_tls); - let listener = crate::RuntimeInitiatedListener::bind_address( - "127.0.0.1:0".parse().unwrap(), - &descriptor.tls, - ) - .unwrap(); - let address = listener.local_address(); - let first_closed = Arc::new(AtomicBool::new(false)); - let second_closed = Arc::new(AtomicBool::new(false)); - let first = tokio::spawn(serve_as_runtime_initiated_boundary( - address, - certificate.server_config.clone(), - "a".repeat(32), - first_closed.clone(), - )); - let mut client = BoundaryClient::new(descriptor, test_bearer(&"a".repeat(32))); - client.runtime_initiated = Some(listener); - let response = - tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap() - .unwrap(); - assert!(matches!(response, Response::Confirmed { .. })); - - // Aborting the cached channel closes the runtime's physical socket, - // and the next request uses a fresh runtime-initiated connection. - let second = tokio::spawn(serve_as_runtime_initiated_boundary( - address, - certificate.server_config, - "a".repeat(32), - second_closed.clone(), - )); - client - .grpc_channel - .lock() - .await - .take() - .unwrap() - .transport - .abort(); - tokio::time::timeout(Duration::from_secs(5), async { - while !first_closed.load(Ordering::Acquire) { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("aborting the channel must close the runtime-initiated socket"); - let response = - tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap() - .unwrap(); - assert!(matches!(response, Response::Confirmed { .. })); - assert!(!second_closed.load(Ordering::Acquire)); - first.abort(); - second.abort(); - } - - #[tokio::test] - async fn runtime_initiated_transport_rejects_a_wrong_bearer() { - let certificate = test_certificate(); - let descriptor = runtime_initiated_descriptor(certificate.client_tls); - let listener = crate::RuntimeInitiatedListener::bind_address( - "127.0.0.1:0".parse().unwrap(), - &descriptor.tls, - ) - .unwrap(); - let runtime = tokio::spawn(serve_as_runtime_initiated_boundary( - listener.local_address(), - certificate.server_config, - "a".repeat(32), - Arc::new(AtomicBool::new(false)), - )); - let mut client = BoundaryClient::new(descriptor, test_bearer(&"b".repeat(32))); - client.runtime_initiated = Some(listener); - let result = - tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap(); - assert!(result.is_err()); - runtime.abort(); - } - - #[test] - fn runtime_initiated_descriptor_requires_a_listen_port_and_authority() { - let certificate = test_certificate(); - let mut descriptor = runtime_initiated_descriptor(certificate.client_tls); - assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_ok()); - descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { - authority: String::new(), - listen_address: "0.0.0.0:5500".parse().unwrap(), - }; - assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); - descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), - listen_address: "0.0.0.0:0".parse().unwrap(), - }; - assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); - } - #[tokio::test] async fn tls_tcp_rejects_tls12_only_server() { let certificate = test_certificate_with_protocol_versions(&[&rustls::version::TLS12]); diff --git a/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs new file mode 100644 index 0000000000..7693411d28 --- /dev/null +++ b/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs @@ -0,0 +1,337 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Runtime-initiated transport: admission at the supervisor listener and the +//! client path that shares one listener between discovery and attachment. + +use super::*; +use crate::runtime_initiated::RuntimeInitiatedListener; + +/// Records when the runtime side of a connection observes EOF or failure. +struct ClosureProbe { + inner: BoundaryDuplexStream, + closed: Arc, +} + +impl tokio::io::AsyncRead for ClosureProbe { + fn poll_read( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &mut tokio::io::ReadBuf<'_>, + ) -> Poll> { + let before = buffer.filled().len(); + let result = Pin::new(&mut self.inner).poll_read(context, buffer); + if matches!(&result, Poll::Ready(Err(_))) + || (matches!(&result, Poll::Ready(Ok(()))) && buffer.filled().len() == before) + { + self.closed.store(true, Ordering::Release); + } + result + } +} + +impl tokio::io::AsyncWrite for ClosureProbe { + fn poll_write( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + buffer: &[u8], + ) -> Poll> { + Pin::new(&mut self.inner).poll_write(context, buffer) + } + + fn poll_flush( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_flush(context) + } + + fn poll_shutdown( + mut self: Pin<&mut Self>, + context: &mut Context<'_>, + ) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(context) + } +} + +fn runtime_initiated_descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { + let mut descriptor = tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), tls); + descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { + authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), + listen_address: "127.0.0.1:5500".parse().unwrap(), + }; + descriptor +} + +/// Production descriptors name a fixed port; tests bind an ephemeral one. +fn bind_ephemeral(tls: &SandboxTlsClientConfig) -> Arc { + RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), tls).unwrap() +} + +/// Dial the listener as `openshell-sandbox` does and complete TLS as the server. +async fn dial_as_runtime( + address: std::net::SocketAddr, + server: Arc, +) -> std::io::Result> { + let stream = tokio::net::TcpStream::connect(address).await?; + tokio_rustls::TlsAcceptor::from(server).accept(stream).await +} + +/// Dial as the runtime and serve the protocol over the outbound connection. +fn spawn_runtime_connection( + address: std::net::SocketAddr, + server: Arc, + expected_token: &str, +) -> (Arc, tokio::task::JoinHandle<()>) { + let closed = Arc::new(AtomicBool::new(false)); + let probe = closed.clone(); + let expected_token = expected_token.to_string(); + let task = tokio::spawn(async move { + let stream = dial_as_runtime(address, server).await.unwrap(); + let stream = ClosureProbe { + inner: Box::new(stream), + closed: probe, + }; + serve_test_grpc(Box::new(stream), expected_token).await; + }); + (closed, task) +} + +fn queued(listener: &RuntimeInitiatedListener) -> usize { + listener.queued_spares() +} + +async fn wait_until(condition: impl Fn() -> bool + Send + Sync, failure: &str) { + tokio::time::timeout(Duration::from_secs(5), async { + while !condition() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect(failure); +} + +async fn assert_closed(stream: &mut (impl tokio::io::AsyncRead + Unpin)) { + let mut byte = [0; 1]; + let read = tokio::time::timeout(Duration::from_secs(2), stream.read(&mut byte)) + .await + .unwrap(); + assert!(matches!(read, Ok(0) | Err(_))); +} + +#[tokio::test] +async fn discovery_and_attach_share_one_listener_and_abort_closes_the_socket() { + let certificate = test_certificate(); + let descriptor = runtime_initiated_descriptor(certificate.client_tls); + let listener = bind_ephemeral(&descriptor.tls); + let address = listener.local_address(); + let transport = BoundaryTransport(TransportSource::RuntimeInitiated(listener)); + let token = "a".repeat(32); + + // Discovery and attachment are separate clients over clones of one + // transport, as in the supervisor; each takes its own runtime connection. + let (_, discovery_runtime) = + spawn_runtime_connection(address, certificate.server_config.clone(), &token); + let mut discovery = BoundaryClient::new(descriptor.clone(), test_bearer(&token)); + discovery.transport = transport.clone(); + let response = + tokio::time::timeout(Duration::from_secs(5), discovery.exchange(Request::Confirm)) + .await + .unwrap() + .unwrap(); + assert!(matches!(response, Response::Confirmed { .. })); + + let (attach_closed, attach_runtime) = + spawn_runtime_connection(address, certificate.server_config.clone(), &token); + let mut client = BoundaryClient::new(descriptor, test_bearer(&token)); + client.transport = transport; + let response = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap() + .unwrap(); + assert!(matches!(response, Response::Confirmed { .. })); + + // Aborting the cached channel closes the runtime's physical socket, and + // the next request uses a fresh runtime-initiated connection. + let (replacement_closed, replacement_runtime) = + spawn_runtime_connection(address, certificate.server_config, &token); + client + .grpc_channel + .lock() + .await + .take() + .unwrap() + .transport + .abort(); + wait_until( + || attach_closed.load(Ordering::Acquire), + "aborting the channel must close the runtime-initiated socket", + ) + .await; + let response = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap() + .unwrap(); + assert!(matches!(response, Response::Confirmed { .. })); + assert!(!replacement_closed.load(Ordering::Acquire)); + for task in [discovery_runtime, attach_runtime, replacement_runtime] { + task.abort(); + } +} + +#[tokio::test] +async fn runtime_initiated_transport_rejects_a_wrong_bearer() { + let certificate = test_certificate(); + let descriptor = runtime_initiated_descriptor(certificate.client_tls); + let listener = bind_ephemeral(&descriptor.tls); + let (_, runtime) = spawn_runtime_connection( + listener.local_address(), + certificate.server_config, + &"a".repeat(32), + ); + let mut client = BoundaryClient::new(descriptor, test_bearer(&"b".repeat(32))); + client.transport = BoundaryTransport(TransportSource::RuntimeInitiated(listener)); + let result = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) + .await + .unwrap(); + assert!(result.is_err()); + runtime.abort(); +} + +#[test] +fn runtime_initiated_descriptor_requires_a_fixed_listen_port_and_authority() { + let certificate = test_certificate(); + let mut descriptor = runtime_initiated_descriptor(certificate.client_tls); + assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_ok()); + for (authority, listen_address) in [("", "0.0.0.0:5500"), ("authority:5500", "0.0.0.0:0")] { + descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { + authority: authority.to_string(), + listen_address: listen_address.parse().unwrap(), + }; + assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); + } + // The listener independently refuses an ephemeral production port. + assert!(matches!( + RuntimeInitiatedListener::bind(&descriptor), + Err(BackendError::Descriptor(_)) + )); +} + +#[tokio::test] +async fn untrusted_and_silent_peers_cannot_displace_the_genuine_runtime() { + let genuine = test_certificate(); + let impostor = test_certificate(); + let descriptor = runtime_initiated_descriptor(genuine.client_tls); + let listener = bind_ephemeral(&descriptor.tls); + let address = listener.local_address(); + + // Silent sockets occupy handshake slots but never reach the queue. + let mut silent = Vec::new(); + for _ in 0..6 { + silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + // A peer presenting another session's identity fails verification. TLS + // 1.3 servers can finish before the client's alert arrives, so the + // connection must then be closed without being queued. + if let Ok(mut rejected) = dial_as_runtime(address, impostor.server_config).await { + assert_closed(&mut rejected).await; + } + assert_eq!(queued(&listener), 0); + + let runtime = tokio::spawn(dial_as_runtime(address, genuine.server_config)); + let accepted = tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)).await; + assert!(accepted.is_ok_and(|stream| stream.is_ok())); + assert!(runtime.await.unwrap().is_ok()); + drop(silent); +} + +#[tokio::test] +async fn handshake_slots_bound_unauthenticated_sockets() { + let genuine = test_certificate(); + let descriptor = runtime_initiated_descriptor(genuine.client_tls); + let listener = bind_ephemeral(&descriptor.tls); + let address = listener.local_address(); + let mut silent = Vec::new(); + for _ in 0..8 { + silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); + } + // Let the listener assign every slot before the excess socket arrives. + tokio::time::sleep(Duration::from_millis(200)).await; + let mut excess = tokio::net::TcpStream::connect(address).await.unwrap(); + assert_closed(&mut excess).await; + + // Freed slots admit the genuine runtime again. + drop(silent); + let runtime = tokio::spawn(dial_as_runtime(address, genuine.server_config)); + assert!( + tokio::time::timeout(Duration::from_secs(10), listener.accept(&descriptor)) + .await + .is_ok() + ); + assert!(runtime.await.unwrap().is_ok()); +} + +#[tokio::test] +async fn spare_queue_is_bounded_and_closes_the_excess_spare() { + let genuine = test_certificate(); + let descriptor = runtime_initiated_descriptor(genuine.client_tls); + let listener = bind_ephemeral(&descriptor.tls); + let mut runtimes = Vec::new(); + for count in 1..=2 { + runtimes.push( + dial_as_runtime(listener.local_address(), genuine.server_config.clone()) + .await + .unwrap(), + ); + wait_until( + || queued(&listener) == count, + "authenticated spare was not queued", + ) + .await; + } + let mut excess = dial_as_runtime(listener.local_address(), genuine.server_config) + .await + .unwrap(); + assert_closed(&mut excess).await; + assert_eq!(queued(&listener), 2); + drop(runtimes); +} + +#[tokio::test] +async fn listener_rejects_an_invalid_pinned_identity_before_binding() { + let mut tls = test_certificate().client_tls; + tls.trust_anchor_pem = "not a certificate".to_string(); + assert!(matches!( + RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), &tls), + Err(BackendError::Descriptor(_)) + )); +} + +#[tokio::test] +async fn listener_refuses_a_descriptor_for_another_session() { + let genuine = test_certificate(); + let other = test_certificate(); + let listener = bind_ephemeral(&genuine.client_tls); + assert!(matches!( + listener + .accept(&runtime_initiated_descriptor(other.client_tls)) + .await, + Err(BackendError::Descriptor(_)) + )); +} + +#[tokio::test] +async fn dropping_the_listener_closes_queued_spares() { + let genuine = test_certificate(); + let listener = bind_ephemeral(&genuine.client_tls); + let mut runtime = dial_as_runtime(listener.local_address(), genuine.server_config) + .await + .unwrap(); + wait_until( + || queued(&listener) == 1, + "authenticated spare was not queued", + ) + .await; + drop(listener); + assert_closed(&mut runtime).await; +} diff --git a/crates/openshell-sandbox-backend/src/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime_initiated.rs index ecdf911a2c..574c3e4cd8 100644 --- a/crates/openshell-sandbox-backend/src/runtime_initiated.rs +++ b/crates/openshell-sandbox-backend/src/runtime_initiated.rs @@ -3,18 +3,16 @@ //! Supervisor listener for runtime-initiated Sandbox Protocol connections. //! -//! Only the TCP dial direction is reversed. The supervisor remains the TLS -//! client and gRPC client: it verifies the runtime's generation-pinned server -//! identity on every accepted socket before the socket can carry a request, and -//! every request still carries the Sandbox Protocol bearer token. A peer -//! address is never trusted; it is only the source of a socket to authenticate. +//! Only the TCP dial direction is reversed. Each accepted socket completes the +//! same pinned TLS client handshake as a dialed connection before it is queued, +//! and every request still carries the Sandbox Protocol bearer token. A peer +//! address is never trusted. -use std::collections::VecDeque; use std::net::SocketAddr; -use std::sync::{Arc, Mutex}; use std::time::Duration; use openshell_isolation_interface::contract::{BackendError, BoundaryDuplexStream}; +use tokio::sync::mpsc; use crate::boundary_protocol::{ SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, @@ -31,59 +29,45 @@ const MAX_QUEUED_CONNECTIONS: usize = 2; /// Accepts runtime-initiated connections for exactly one sandbox session. /// -/// Live channels own their streams; this listener only holds unclaimed, -/// already-authenticated spares, so no new connection can evict a live owner. -/// Dropping the listener closes the socket, every pending handshake, and every -/// queued spare. +/// Live channels own their streams; the queue holds only unclaimed, +/// authenticated spares, so no new connection can displace a live owner. +/// Dropping the listener closes the socket, pending handshakes, and spares. +#[derive(Debug)] pub struct RuntimeInitiatedListener { tls: SandboxTlsClientConfig, + #[cfg(test)] local_address: SocketAddr, - queue: Arc, + spares: tokio::sync::Mutex>, accept_task: tokio::task::AbortHandle, } -impl std::fmt::Debug for RuntimeInitiatedListener { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter - .debug_struct("RuntimeInitiatedListener") - .field("local_address", &self.local_address) - .field("server_name", &self.tls.server_name) - .finish_non_exhaustive() - } -} - impl RuntimeInitiatedListener { /// Bind the descriptor's listen address and start authenticating peers. - /// - /// Must be called inside a Tokio runtime. - pub fn bind(descriptor: &SandboxRuntimeDescriptor) -> Result, BackendError> { - let SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } = &descriptor.transport - else { - return Err(BackendError::Descriptor( - "runtime-initiated listener requires a runtime-initiated transport".to_string(), - )); - }; - if listen_address.port() == 0 { - return Err(BackendError::Descriptor( - "runtime-initiated listen address must have a nonzero port".to_string(), - )); + pub fn bind( + descriptor: &SandboxRuntimeDescriptor, + ) -> Result, BackendError> { + match descriptor.transport { + SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } + if listen_address.port() != 0 => + { + Self::bind_address(listen_address, &descriptor.tls) + } + _ => Err(BackendError::Descriptor( + "runtime-initiated listener requires a nonzero listen port".to_string(), + )), } - Self::bind_address(*listen_address, &descriptor.tls) } pub(crate) fn bind_address( address: SocketAddr, tls: &SandboxTlsClientConfig, - ) -> Result, BackendError> { - let server_name = rustls::pki_types::ServerName::try_from(tls.server_name.clone()) - .map_err(|error| { - BackendError::Descriptor(format!( - "boundary TLS server name {:?} is invalid: {error}", - tls.server_name - )) - })?; - let connector = - tokio_rustls::TlsConnector::from(Arc::new(crate::runtime::tls_client_config(tls)?)); + ) -> Result, BackendError> { + let runtime = tokio::runtime::Handle::try_current().map_err(|_| { + BackendError::Unavailable("runtime-initiated listener requires a Tokio runtime".into()) + })?; + // Fail before opening a socket that could never authenticate a peer. + crate::runtime::validate_client_tls(tls)?; + let _entered = runtime.enter(); let listener = bind_tcp(address).map_err(|error| { BackendError::Unavailable(format!( "bind runtime-initiated control listener {address}: {error}" @@ -92,29 +76,34 @@ impl RuntimeInitiatedListener { let local_address = listener.local_addr().map_err(|error| { BackendError::Unavailable(format!("read control listener address: {error}")) })?; - let queue = Arc::new(AdmissionQueue::default()); - let accept_task = - tokio::spawn(accept_loop(listener, connector, server_name, queue.clone())) - .abort_handle(); + let (sender, spares) = mpsc::channel(MAX_QUEUED_CONNECTIONS); + let accept_task = runtime + .spawn(accept_loop(listener, tls.clone(), sender)) + .abort_handle(); tracing::info!(%local_address, "Runtime-initiated Sandbox Protocol listener ready"); - Ok(Arc::new(Self { + Ok(std::sync::Arc::new(Self { tls: tls.clone(), + #[cfg(test)] local_address, - queue, + spares: tokio::sync::Mutex::new(spares), accept_task, })) } /// Address actually bound, including an OS-assigned port. - #[must_use] + #[cfg(test)] pub fn local_address(&self) -> SocketAddr { self.local_address } - /// Wait for one authenticated runtime connection for `descriptor`. - /// - /// The returned stream has completed TLS with the descriptor's pinned - /// identity. A listener bound for another session is refused. + /// Authenticated spares waiting for a client. + #[cfg(test)] + pub fn queued_spares(&self) -> usize { + self.spares.try_lock().map_or(0, |spares| spares.len()) + } + + /// Wait for one authenticated runtime connection for `descriptor`. A + /// listener bound for another session is refused. pub(crate) async fn accept( &self, descriptor: &SandboxRuntimeDescriptor, @@ -124,7 +113,9 @@ impl RuntimeInitiatedListener { "runtime-initiated listener is bound to a different sandbox session".to_string(), )); } - Ok(self.queue.pop().await) + self.spares.lock().await.recv().await.ok_or_else(|| { + BackendError::Unavailable("runtime-initiated control listener stopped".to_string()) + }) } } @@ -147,64 +138,15 @@ fn bind_tcp(address: SocketAddr) -> std::io::Result { tokio::net::TcpListener::from_std(socket.into()) } -#[derive(Default)] -struct AdmissionQueue { - streams: Mutex>, - ready: tokio::sync::Notify, -} - -impl AdmissionQueue { - fn push(&self, stream: BoundaryDuplexStream) { - let displaced = { - let mut streams = self - .streams - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - let displaced = (streams.len() >= MAX_QUEUED_CONNECTIONS) - .then(|| streams.pop_front()) - .flatten(); - streams.push_back(stream); - displaced - }; - if displaced.is_some() { - tracing::debug!("discarding an unclaimed runtime-initiated spare connection"); - } - self.ready.notify_one(); - } - - async fn pop(&self) -> BoundaryDuplexStream { - loop { - let notified = self.ready.notified(); - tokio::pin!(notified); - notified.as_mut().enable(); - let stream = self - .streams - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .pop_front(); - if let Some(stream) = stream { - return stream; - } - notified.await; - } - } - - #[cfg(test)] - fn len(&self) -> usize { - self.streams - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner) - .len() - } -} - +/// Authenticate peers concurrently, bounded in count and time, and queue only +/// streams that completed the pinned handshake. A full queue closes the new +/// spare; the runtime dials again once a spare is used or lost. async fn accept_loop( listener: tokio::net::TcpListener, - connector: tokio_rustls::TlsConnector, - server_name: rustls::pki_types::ServerName<'static>, - queue: Arc, + tls: SandboxTlsClientConfig, + spares: mpsc::Sender, ) { - let pending = Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); + let pending = std::sync::Arc::new(tokio::sync::Semaphore::new(MAX_PENDING_HANDSHAKES)); // Owned here so aborting this task also cancels every pending handshake. let mut handshakes = tokio::task::JoinSet::new(); loop { @@ -225,24 +167,25 @@ async fn accept_loop( }; openshell_core::net::set_tcp_nodelay_best_effort(&stream); crate::runtime::enable_boundary_tcp_keepalive(&stream); - let connector = connector.clone(); - let server_name = server_name.clone(); - let queue = queue.clone(); + let tls = tls.clone(); + let spares = spares.clone(); handshakes.spawn(async move { let _permit = permit; - match tokio::time::timeout( + let authenticated = tokio::time::timeout( HANDSHAKE_TIMEOUT, - connector.connect(server_name, stream), + crate::runtime::authenticate_boundary_stream(&tls, Box::new(stream)), ) - .await - { - Ok(Ok(stream)) => queue.push(Box::new(stream)), + .await; + match authenticated { + Ok(Ok(stream)) => { + if spares.try_send(stream).is_err() { + tracing::debug!(%peer, "runtime-initiated spare queue full; closing connection"); + } + } Ok(Err(error)) => { tracing::debug!(%peer, %error, "rejected unauthenticated runtime-initiated connection"); } - Err(_) => { - tracing::debug!(%peer, "runtime-initiated TLS handshake timed out"); - } + Err(_) => tracing::debug!(%peer, "runtime-initiated TLS handshake timed out"), } }); } @@ -250,247 +193,3 @@ async fn accept_loop( } } } - -#[cfg(test)] -mod tests { - use super::*; - use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; - - struct TestIdentity { - client: SandboxTlsClientConfig, - server: Arc, - } - - fn identity() -> TestIdentity { - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let material = crate::boundary_protocol::generate_sandbox_tls_material( - openshell_core::SandboxSessionId::new(), - ) - .unwrap(); - let certificates = rustls_pemfile::certs(&mut material.certificate_chain_pem.as_bytes()) - .collect::, _>>() - .unwrap(); - let key = rustls_pemfile::private_key(&mut material.private_key_pem.as_bytes()) - .unwrap() - .unwrap(); - let mut server = - rustls::ServerConfig::builder_with_protocol_versions(&[&rustls::version::TLS13]) - .with_no_client_auth() - .with_single_cert(certificates, key) - .unwrap(); - server.alpn_protocols = vec![b"h2".to_vec()]; - TestIdentity { - client: SandboxTlsClientConfig { - server_name: material.server_name, - trust_anchor_pem: material.trust_anchor_pem, - }, - server: Arc::new(server), - } - } - - fn descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { - SandboxRuntimeDescriptor { - boundary_id: "sandbox-1".to_string(), - generation: "generation-1".to_string(), - session_id: openshell_core::SandboxSessionId::new(), - workload_identity: - openshell_isolation_interface::contract::ResolvedWorkloadIdentity::new( - 1000, - 1000, - Vec::new(), - "test".to_string(), - "test".to_string(), - ) - .unwrap(), - transport: SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), - listen_address: "127.0.0.1:0".parse().unwrap(), - }, - tls, - host_gateway_ip: None, - resource_claims: std::collections::BTreeMap::new(), - outer_fence: openshell_isolation_interface::contract::OuterFenceGuarantees::from_enforcement_evidence( - "generation-1", - [], - b"test", - ) - .unwrap(), - } - } - - /// Production descriptors require a fixed port; tests bind an ephemeral one. - fn bind_ephemeral( - descriptor: &SandboxRuntimeDescriptor, - ) -> Result, BackendError> { - RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), &descriptor.tls) - } - - #[test] - fn production_bind_requires_a_fixed_port() { - let _runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - let identity = identity(); - assert!(matches!( - RuntimeInitiatedListener::bind(&descriptor(identity.client)), - Err(BackendError::Descriptor(_)) - )); - } - - /// Dial the listener as the runtime does and serve TLS on the outbound socket. - async fn dial_as_runtime( - address: SocketAddr, - server: Arc, - ) -> std::io::Result> { - let stream = tokio::net::TcpStream::connect(address).await?; - tokio_rustls::TlsAcceptor::from(server).accept(stream).await - } - - async fn queued_within(listener: &RuntimeInitiatedListener, count: usize) { - tokio::time::timeout(Duration::from_secs(5), async { - while listener.queue.len() < count { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect("authenticated connection was not queued"); - } - - #[tokio::test] - async fn authenticated_runtime_connection_is_accepted_and_carries_data() { - let identity = identity(); - let descriptor = descriptor(identity.client); - let listener = bind_ephemeral(&descriptor).unwrap(); - let runtime = tokio::spawn(dial_as_runtime(listener.local_address(), identity.server)); - - let mut supervisor_side = - tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)) - .await - .unwrap() - .unwrap(); - let mut runtime_side = runtime.await.unwrap().unwrap(); - supervisor_side.write_all(b"ping").await.unwrap(); - supervisor_side.flush().await.unwrap(); - let mut received = [0; 4]; - runtime_side.read_exact(&mut received).await.unwrap(); - assert_eq!(&received, b"ping"); - } - - #[tokio::test] - async fn untrusted_and_silent_peers_cannot_displace_the_genuine_runtime() { - let genuine = identity(); - let impostor = identity(); - let descriptor = descriptor(genuine.client); - let listener = bind_ephemeral(&descriptor).unwrap(); - let address = listener.local_address(); - - // Silent sockets occupy handshake slots but never reach the queue. - let mut silent = Vec::new(); - for _ in 0..(MAX_PENDING_HANDSHAKES - 2) { - silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); - } - // A peer presenting another session's identity fails TLS verification. - let rejected = dial_as_runtime(address, impostor.server).await; - assert!( - rejected.is_err() || { - // TLS 1.3 servers can finish before the client's alert arrives; - // the connection must then be closed without being queued. - let mut stream = rejected.unwrap(); - let mut byte = [0; 1]; - stream.read(&mut byte).await.map_or(true, |read| read == 0) - } - ); - assert_eq!(listener.queue.len(), 0); - - let runtime = tokio::spawn(dial_as_runtime(address, genuine.server)); - let accepted = - tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)).await; - assert!(accepted.is_ok_and(|stream| stream.is_ok())); - assert!(runtime.await.unwrap().is_ok()); - drop(silent); - } - - #[tokio::test] - async fn handshake_slots_bound_unauthenticated_sockets() { - let genuine = identity(); - let descriptor = descriptor(genuine.client); - let listener = bind_ephemeral(&descriptor).unwrap(); - let address = listener.local_address(); - let mut silent = Vec::new(); - for _ in 0..MAX_PENDING_HANDSHAKES { - silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); - } - // Let the listener assign every slot before the excess socket arrives. - tokio::time::sleep(Duration::from_millis(200)).await; - let mut excess = tokio::net::TcpStream::connect(address).await.unwrap(); - let mut byte = [0; 1]; - let closed = tokio::time::timeout(Duration::from_secs(2), excess.read(&mut byte)).await; - assert!( - matches!(closed, Ok(Ok(0) | Err(_))), - "a socket beyond the handshake limit must be closed" - ); - - // Freed slots admit the genuine runtime again after the timeout. - drop(silent); - let runtime = tokio::spawn(dial_as_runtime(address, genuine.server)); - assert!( - tokio::time::timeout(HANDSHAKE_TIMEOUT * 2, listener.accept(&descriptor)) - .await - .is_ok() - ); - assert!(runtime.await.unwrap().is_ok()); - } - - #[tokio::test] - async fn spare_queue_is_bounded_and_never_holds_more_than_the_limit() { - let genuine = identity(); - let descriptor = descriptor(genuine.client); - let listener = bind_ephemeral(&descriptor).unwrap(); - let mut runtimes = Vec::new(); - for _ in 0..=MAX_QUEUED_CONNECTIONS { - runtimes.push( - dial_as_runtime(listener.local_address(), genuine.server.clone()) - .await - .unwrap(), - ); - } - queued_within(&listener, MAX_QUEUED_CONNECTIONS).await; - tokio::time::sleep(Duration::from_millis(100)).await; - assert_eq!(listener.queue.len(), MAX_QUEUED_CONNECTIONS); - // The oldest spare was displaced and closed. - let mut byte = [0; 1]; - let oldest = tokio::time::timeout(Duration::from_secs(2), runtimes[0].read(&mut byte)) - .await - .unwrap(); - assert!(matches!(oldest, Ok(0) | Err(_))); - } - - #[tokio::test] - async fn listener_refuses_a_descriptor_for_another_session() { - let genuine = identity(); - let other = identity(); - let listener = bind_ephemeral(&descriptor(genuine.client)).unwrap(); - assert!(matches!( - listener.accept(&descriptor(other.client)).await, - Err(BackendError::Descriptor(_)) - )); - } - - #[tokio::test] - async fn dropping_the_listener_closes_queued_connections() { - let genuine = identity(); - let descriptor = descriptor(genuine.client); - let listener = bind_ephemeral(&descriptor).unwrap(); - let mut runtime = dial_as_runtime(listener.local_address(), genuine.server) - .await - .unwrap(); - queued_within(&listener, 1).await; - drop(listener); - let mut byte = [0; 1]; - let read = tokio::time::timeout(Duration::from_secs(2), runtime.read(&mut byte)) - .await - .unwrap(); - assert!(matches!(read, Ok(0) | Err(_))); - } -} diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index c4a5e7e0db..bb6b964c13 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -650,8 +650,10 @@ mod linux { /// /// The supervisor claims a spare by starting HTTP/2 on it; then the next /// spare is dialed so credential rotation and recovery, which open a new - /// connection before retiring the old one, never wait on this loop. Live - /// connections keep the existing disconnect, freeze, and termination path. + /// connection before retiring the old one, never wait on this loop. TLS + /// failures back off like dial failures. Established spares are served + /// like accepted connections: same identity, slot limit, and disconnect, + /// freeze, and termination handling. async fn run_runtime_initiated_control( addresses: Vec, server_config: Arc, @@ -659,59 +661,51 @@ mod linux { active_connections: Arc, ) { let mut backoff = RUNTIME_DIAL_BACKOFF_MIN; - let mut first_failure: Option = None; - let mut last_warning: Option = None; + let mut warn_at: Option = None; loop { - let Some(slot) = acquire_control_connection_slot(&active_connections) else { - tokio::time::sleep(RUNTIME_DIAL_BACKOFF_MAX).await; - continue; + let established = match dial_supervisor(&addresses).await { + Ok(stream) => ControlStream::PendingTls { + stream: PlainControlStream::Tcp(stream), + server_config: server_config.clone(), + } + .establish_async(&runtime.process_runtime) + .await + .map_err(|error| format!("authenticate boundary transport: {error}")), + Err(error) => Err(format!("dial supervisor: {error}")), }; - let dialed_at = tokio::time::Instant::now(); - let stream = match dial_runtime_initiated(&addresses, server_config.clone()).await { + let healthy = match established { Ok(stream) => { - first_failure = None; - stream + warn_at = None; + let established_at = tokio::time::Instant::now(); + let (claimed, claim) = tokio::sync::oneshot::channel(); + let served = serve_authenticated_connection( + stream, + runtime.clone(), + active_connections.clone(), + Some(claimed), + ); + runtime.process_runtime.spawn(async move { + if let Err(error) = served.await { + tracing::debug!(%error, "Boundary control connection ended"); + } + }); + // A used spare, or one that stayed open, needs no backoff. + claim.await.is_ok() || established_at.elapsed() >= RUNTIME_SPARE_MIN_LIFETIME } Err(error) => { + // Failures are expected while the supervisor starts; report + // a persistent one at most once per interval. let now = tokio::time::Instant::now(); - let failing_since = *first_failure.get_or_insert(now); - if now.duration_since(failing_since) >= RUNTIME_DIAL_WARNING_INTERVAL - && last_warning.is_none_or(|last| { - now.duration_since(last) >= RUNTIME_DIAL_WARNING_INTERVAL - }) - { - last_warning = Some(now); - tracing::warn!( - %error, - failing_for_secs = now.duration_since(failing_since).as_secs(), - "Runtime-initiated control cannot reach the supervisor; retrying" - ); + if now >= *warn_at.get_or_insert(now + RUNTIME_DIAL_WARNING_INTERVAL) { + warn_at = Some(now + RUNTIME_DIAL_WARNING_INTERVAL); + tracing::warn!(%error, "Runtime-initiated control cannot reach the supervisor; retrying"); } else { tracing::debug!(%error, "Runtime-initiated control dial failed; retrying"); } - drop(slot); - tokio::time::sleep(backoff).await; - backoff = (backoff * 2).min(RUNTIME_DIAL_BACKOFF_MAX); - continue; + false } }; - let (claimed, claim) = tokio::sync::oneshot::channel(); - runtime.process_runtime.spawn({ - let runtime = runtime.clone(); - async move { - let _slot = slot; - let stream = Box::new(ClaimOnFirstRead { - inner: stream, - claimed: Some(claimed), - }); - if let Err(error) = - serve_grpc(stream, runtime, SandboxConnectionId::new()).await - { - tracing::debug!(%error, "Boundary control connection ended"); - } - } - }); - if claim.await.is_ok() || dialed_at.elapsed() >= RUNTIME_SPARE_MIN_LIFETIME { + if healthy { backoff = RUNTIME_DIAL_BACKOFF_MIN; } else { tokio::time::sleep(backoff).await; @@ -720,10 +714,10 @@ mod linux { } } - async fn dial_runtime_initiated( + /// Dial only driver-provided supervisor addresses from the bootstrap. + async fn dial_supervisor( addresses: &[std::net::SocketAddr], - server_config: Arc, - ) -> io::Result { + ) -> io::Result { let stream = tokio::time::timeout( RUNTIME_DIAL_TIMEOUT, openshell_core::net::connect_tcp_nodelay_best_effort(addresses), @@ -737,14 +731,7 @@ mod linux { .with_interval(Duration::from_secs(5)) .with_retries(3); let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); - let stream = tokio::time::timeout( - CONTROL_HANDSHAKE_TIMEOUT, - tokio_rustls::TlsAcceptor::from(server_config).accept(stream), - ) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "boundary TLS handshake timed out"))? - .map_err(|error| io::Error::new(io::ErrorKind::InvalidData, error))?; - Ok(Box::new(stream)) + stream.into_std() } /// Signals once the supervisor sends its first application bytes. @@ -811,10 +798,30 @@ mod linux { .await .map_err(|error| format!("authenticate boundary transport: {error}"))?; drop(pending); + serve_authenticated_connection(stream, runtime, active_connections, None).await + } + + /// Serve one TLS-established connection, accepted or runtime-initiated. + /// `claimed`, used by runtime-initiated spares, fires when the supervisor + /// first sends data. + async fn serve_authenticated_connection( + stream: ControlStream, + runtime: Arc, + active_connections: Arc, + claimed: Option>, + ) -> Result<(), String> { let Some(_slot) = acquire_control_connection_slot(&active_connections) else { return Err("authenticated control connection limit reached".to_string()); }; - serve_grpc(stream.into_tokio()?, runtime, SandboxConnectionId::new()).await + let stream = stream.into_tokio()?; + let stream = match claimed { + Some(claimed) => Box::new(ClaimOnFirstRead { + inner: stream, + claimed: Some(claimed), + }), + None => stream, + }; + serve_grpc(stream, runtime, SandboxConnectionId::new()).await } async fn serve_grpc( diff --git a/crates/openshell-supervisor/src/backend_setup.rs b/crates/openshell-supervisor/src/backend_setup.rs index 3e40416fb4..fe11068b52 100644 --- a/crates/openshell-supervisor/src/backend_setup.rs +++ b/crates/openshell-supervisor/src/backend_setup.rs @@ -231,47 +231,23 @@ impl BackendSetup for OpenShellBackendSetup { gid: descriptor.workload_identity.gid, }), }; + let transport = openshell_sandbox_backend::BoundaryTransport::for_descriptor(&descriptor)?; Ok(( identity, Box::new(OpenShellLaunch { descriptor, - runtime_initiated: std::sync::OnceLock::new(), + transport, }), )) } } +/// Launch data plus the transport chosen for it. Discovery and the attached +/// backend use clones of one transport, so a runtime-initiated descriptor binds +/// its control listener once for the supervisor's lifetime. struct OpenShellLaunch { descriptor: openshell_sandbox_backend::boundary_protocol::SandboxRuntimeDescriptor, - /// Bound once and shared by discovery and the attached backend, so a - /// runtime spare accepted during discovery remains usable for attach. - runtime_initiated: - std::sync::OnceLock>, -} - -impl OpenShellLaunch { - /// The listener exists only for a runtime-initiated transport. Binding - /// requires the supervisor's Tokio runtime and happens before attach. - fn runtime_initiated_listener( - &self, - ) -> std::result::Result< - Option>, - BackendError, - > { - if !matches!( - self.descriptor.transport, - openshell_sandbox_backend::boundary_protocol::SandboxTransport::RuntimeInitiatedTcp { .. } - ) { - return Ok(None); - } - if let Some(listener) = self.runtime_initiated.get() { - return Ok(Some(listener.clone())); - } - let listener = openshell_sandbox_backend::RuntimeInitiatedListener::bind(&self.descriptor)?; - Ok(Some( - self.runtime_initiated.get_or_init(|| listener).clone(), - )) - } + transport: openshell_sandbox_backend::BoundaryTransport, } #[tonic::async_trait] @@ -280,23 +256,12 @@ impl PreparedBackend for OpenShellLaunch { &self, bearer: SessionBearerTokenSlot, ) -> std::result::Result<(Option, bool), BackendError> { - match self.runtime_initiated_listener()? { - Some(listener) => { - openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy_with_runtime_initiated_listener( - self.descriptor.clone(), - bearer, - listener, - ) - .await - } - None => { - openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( - self.descriptor.clone(), - bearer, - ) - .await - } - } + openshell_sandbox_backend::OpenShellRuntimeBackend::discover_policy( + self.descriptor.clone(), + bearer, + self.transport.clone(), + ) + .await } fn build( @@ -304,18 +269,17 @@ impl PreparedBackend for OpenShellLaunch { services: BackendServices, ) -> std::result::Result { let payload = self.descriptor.backend_descriptor()?.payload; - let mut backend = openshell_sandbox_backend::OpenShellRuntimeBackend::new( - services.ca_file_paths, - services.provider_credentials, - services.sandbox_bearer, - ); - if let Some(listener) = self.runtime_initiated_listener()? { - backend = backend.with_runtime_initiated_listener(listener); - } Ok(BuiltBackend { payload, proxy_listener: None, - backend: Arc::new(backend), + backend: Arc::new( + openshell_sandbox_backend::OpenShellRuntimeBackend::new( + services.ca_file_paths, + services.provider_credentials, + services.sandbox_bearer, + ) + .with_transport(self.transport), + ), }) } } diff --git a/crates/openshell-supervisor/src/backend_setup/tests.rs b/crates/openshell-supervisor/src/backend_setup/tests.rs index 0b5e0bf38c..1025860f7e 100644 --- a/crates/openshell-supervisor/src/backend_setup/tests.rs +++ b/crates/openshell-supervisor/src/backend_setup/tests.rs @@ -813,58 +813,3 @@ async fn shared_startup_uses_selected_backend_through_readiness_and_shutdown() { .is_err() ); } - -#[tokio::test] -async fn runtime_initiated_launch_shares_one_listener_between_discovery_and_attach() { - use openshell_sandbox_backend::boundary_protocol::{ - SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, - generate_sandbox_tls_material, - }; - - let material = generate_sandbox_tls_material(SandboxSessionId::new()).unwrap(); - // Production descriptors name a fixed port; reserve a free one for the test. - let port = std::net::TcpListener::bind("127.0.0.1:0") - .unwrap() - .local_addr() - .unwrap() - .port(); - let mut descriptor = SandboxRuntimeDescriptor { - boundary_id: "sandbox-1".into(), - generation: "generation-1".into(), - session_id: SandboxSessionId::new(), - workload_identity: identity(), - transport: SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox-1.ns.svc:5500".into(), - listen_address: std::net::SocketAddr::from(([127, 0, 0, 1], port)), - }, - tls: SandboxTlsClientConfig { - server_name: material.server_name, - trust_anchor_pem: material.trust_anchor_pem, - }, - host_gateway_ip: None, - resource_claims: BTreeMap::new(), - outer_fence: OuterFenceGuarantees::from_enforcement_evidence( - "generation-1", - [], - b"test-evidence", - ) - .unwrap(), - }; - let launch = OpenShellLaunch { - descriptor: descriptor.clone(), - runtime_initiated: std::sync::OnceLock::new(), - }; - let first = launch.runtime_initiated_listener().unwrap().unwrap(); - let second = launch.runtime_initiated_listener().unwrap().unwrap(); - assert!(Arc::ptr_eq(&first, &second)); - assert_eq!(first.local_address().port(), port); - - descriptor.transport = SandboxTransport::Unix { - socket_path: "/unused-test-runtime".into(), - }; - let legacy = OpenShellLaunch { - descriptor, - runtime_initiated: std::sync::OnceLock::new(), - }; - assert!(legacy.runtime_initiated_listener().unwrap().is_none()); -} From e1a8bee6ec65914f4f40db7f3a7321888a2c5d5e Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 12:46:33 -0700 Subject: [PATCH 3/9] test(sandbox): remove prototype-specific unit tests Signed-off-by: Drew Newberry --- .../openshell-driver-kubernetes/src/config.rs | 19 - .../openshell-driver-kubernetes/src/driver.rs | 301 ---------------- .../src/isolation.rs | 139 -------- .../src/sandbox_runtime.rs | 60 +--- .../src/boundary_protocol.rs | 61 ---- .../openshell-sandbox-backend/src/runtime.rs | 1 - .../src/runtime/tests/runtime_initiated.rs | 337 ------------------ .../src/runtime_initiated.rs | 16 - .../openshell-sandbox/src/boundary_server.rs | 90 ----- 9 files changed, 12 insertions(+), 1012 deletions(-) delete mode 100644 crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index a0b5f8354a..040cccfacd 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -997,25 +997,6 @@ fn validate_provider_spiffe_workload_api_socket_path_value( #[cfg(test)] mod tests { use super::*; - - #[test] - fn control_connection_creation_default_is_runtime_initiated_and_overridable() { - assert_eq!( - KubernetesSandboxRuntimeConfig::default().control_connection, - KubernetesControlConnection::RuntimeInitiated - ); - let parsed: KubernetesSandboxRuntimeConfig = - toml::from_str("control_connection = \"supervisor-initiated\"").unwrap(); - assert_eq!( - parsed.control_connection, - KubernetesControlConnection::SupervisorInitiated - ); - assert_eq!(parsed.boundary_port, 5500); - assert!( - toml::from_str::("control_connection = \"both\"") - .is_err() - ); - } use std::collections::BTreeMap as HashMap; #[test] diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 496498bd29..07d8f96e45 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -8984,307 +8984,6 @@ mod tests { assert!(steps.lock().unwrap().is_empty()); } - fn scripted_control_driver( - steps: Vec, - ) -> ( - KubernetesComputeDriver, - Arc>>, - ) { - let (driver, steps, _) = scripted_driver(KubernetesComputeConfig::default(), steps); - (driver, steps) - } - - fn control_mode_object(annotations: &[(&str, &str)]) -> DynamicObject { - let mut object = sandbox_object_for_test("resource-uid", "sandbox-1"); - object.metadata.annotations = Some( - annotations - .iter() - .map(|(key, value)| ((*key).to_string(), (*value).to_string())) - .collect(), - ); - object - } - - fn boundary_service_json(role: &str, publish_not_ready: bool) -> serde_json::Value { - let mut service = serde_json::json!({ - "apiVersion": "v1", - "kind": "Service", - "metadata": {"name": "os-boundary-sandbox-1", "namespace": "openshell"}, - "spec": { - "clusterIP": "10.43.0.7", - "selector": {BOUNDARY_PAIR_LABEL: "sandbox-1", BOUNDARY_ROLE_LABEL: role} - } - }); - if publish_not_ready { - service["spec"]["publishNotReadyAddresses"] = true.into(); - } - service - } - - /// The Service each mode renders: runtime-initiated publishes the - /// supervisor before it is Ready. - fn boundary_service_response(role: &str) -> http::Response { - kube_test_response( - http::StatusCode::OK, - boundary_service_json(role, role == "supervisor"), - ) - } - - fn stored_control_policy(generation: i64) -> NetworkPolicy { - let names = SandboxRuntimeNames::new("sandbox-1"); - let mut policy = control_policy( - "openshell", - &names, - "sandbox-1", - 5500, - OwnerReference::default(), - ); - policy.metadata.uid = Some("control-uid".to_string()); - policy.metadata.generation = Some(generation); - policy.metadata.resource_version = Some("77".to_string()); - policy - } - - type Annotations<'a> = &'a [(&'a str, &'a str)]; - - const SERVICE_PATH: &str = "/api/v1/namespaces/openshell/services/os-boundary-sandbox-1"; - const CONTROL_POLICY_PATH: &str = - "/apis/networking.k8s.io/v1/namespaces/openshell/networkpolicies/os-control-sandbox-1"; - - #[test] - fn control_connection_is_read_from_the_sandbox_and_defaults_to_legacy() { - assert_eq!( - sandbox_runtime_control_connection(&control_mode_object(&[])).unwrap(), - KubernetesControlConnection::SupervisorInitiated - ); - assert_eq!( - sandbox_runtime_control_connection(&control_mode_object(&[( - ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, - "runtime-initiated" - )])) - .unwrap(), - KubernetesControlConnection::RuntimeInitiated - ); - assert!(matches!( - sandbox_runtime_control_connection(&control_mode_object(&[( - ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, - "Runtime-Initiated" - )])), - Err(KubernetesDriverError::Precondition(_)) - )); - } - - #[tokio::test] - async fn reconcile_accepts_each_mode_with_its_own_endpoint() { - let (driver, steps) = scripted_control_driver(vec![( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("workload"), - )]); - driver - .verify_sandbox_runtime_control_path( - &control_mode_object(&[]), - "sandbox-1", - "openshell", - true, - ) - .await - .expect("legacy sandbox with workload Service"); - assert!(steps.lock().unwrap().is_empty()); - - let (driver, steps) = scripted_control_driver(vec![ - ( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("supervisor"), - ), - ( - http::Method::GET, - CONTROL_POLICY_PATH, - kube_test_response( - http::StatusCode::OK, - serde_json::to_value(stored_control_policy(4)).unwrap(), - ), - ), - ]); - driver - .verify_sandbox_runtime_control_path( - &control_mode_object(&[ - ( - ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, - "runtime-initiated", - ), - (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, "control-uid"), - (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, "4"), - ]), - "sandbox-1", - "openshell", - true, - ) - .await - .expect("runtime-initiated sandbox with bound control policy"); - assert!(steps.lock().unwrap().is_empty()); - } - - #[tokio::test] - async fn reconcile_fails_closed_on_service_or_control_policy_drift() { - let runtime_annotations = [ - ( - ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, - "runtime-initiated", - ), - (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID, "control-uid"), - (ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_GENERATION, "4"), - ]; - let mut broadened = stored_control_policy(4); - broadened - .spec - .as_mut() - .unwrap() - .egress - .as_mut() - .unwrap() - .push(k8s_openapi::api::networking::v1::NetworkPolicyEgressRule::default()); - let cases: Vec<(Annotations<'_>, Vec)> = vec![ - // A legacy sandbox's Service must keep selecting its workload. - ( - &[], - vec![( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("supervisor"), - )], - ), - // A runtime-initiated Service must keep selecting its supervisor. - ( - &runtime_annotations, - vec![( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("workload"), - )], - ), - // A runtime-initiated Service that withholds the not-Ready - // supervisor would deadlock bootstrap and recovery. - ( - &runtime_annotations, - vec![( - http::Method::GET, - SERVICE_PATH, - kube_test_response( - http::StatusCode::OK, - boundary_service_json("supervisor", false), - ), - )], - ), - // A running sandbox whose Service disappeared. - ( - &[], - vec![( - http::Method::GET, - SERVICE_PATH, - kube_test_not_found("services", "os-boundary-sandbox-1"), - )], - ), - // The control policy was deleted. - ( - &runtime_annotations, - vec![ - ( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("supervisor"), - ), - ( - http::Method::GET, - CONTROL_POLICY_PATH, - kube_test_not_found("networkpolicies", "os-control-sandbox-1"), - ), - ], - ), - // The control policy was broadened in place. - ( - &runtime_annotations, - vec![ - ( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("supervisor"), - ), - ( - http::Method::GET, - CONTROL_POLICY_PATH, - kube_test_response( - http::StatusCode::OK, - serde_json::to_value(&broadened).unwrap(), - ), - ), - ], - ), - // The control policy changed after it was bound. - ( - &runtime_annotations, - vec![ - ( - http::Method::GET, - SERVICE_PATH, - boundary_service_response("supervisor"), - ), - ( - http::Method::GET, - CONTROL_POLICY_PATH, - kube_test_response( - http::StatusCode::OK, - serde_json::to_value(stored_control_policy(5)).unwrap(), - ), - ), - ], - ), - // An unknown recorded mode is never reinterpreted. - ( - &[( - ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION, - "bidirectional", - )], - vec![], - ), - ]; - // A bootstrapping generation tolerates a missing Service (availability - // and reaping handle it) but still rejects drift in what exists. - let (driver, remaining) = scripted_control_driver(vec![( - http::Method::GET, - SERVICE_PATH, - kube_test_not_found("services", "os-boundary-sandbox-1"), - )]); - driver - .verify_sandbox_runtime_control_path( - &control_mode_object(&[]), - "sandbox-1", - "openshell", - false, - ) - .await - .expect("bootstrap without a Service is left to availability"); - assert!(remaining.lock().unwrap().is_empty()); - for (index, (annotations, steps)) in cases.into_iter().enumerate() { - let (driver, remaining) = scripted_control_driver(steps); - let result = driver - .verify_sandbox_runtime_control_path( - &control_mode_object(annotations), - "sandbox-1", - "openshell", - true, - ) - .await; - assert!( - matches!(result, Err(KubernetesDriverError::Precondition(_))), - "case {index}: {:?}", - result.err() - ); - assert!(remaining.lock().unwrap().is_empty(), "case {index}"); - } - } - #[test] fn bootstrap_phase_round_trips_releasing() { let phase = SandboxRuntimeBootstrapPhase::Releasing; diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index cfb4c78cfc..97e3676f4f 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -613,145 +613,6 @@ mod tests { ); } - fn runtime_initiated_spec() -> KubernetesSandboxRuntimeBoundarySpec { - let mut spec = spec(); - spec.control_path = KubernetesControlPath::RuntimeInitiated { - control_authority: "os-boundary-sandbox.default.svc:5500".to_string(), - supervisor_address: "10.43.0.7:5500".parse().expect("valid target"), - supervisor_listen: "0.0.0.0:5500".parse().expect("valid listener"), - control_policy_uid: "control-policy-uid".to_string(), - control_policy_resource_version: "2001".to_string(), - }; - spec - } - - #[test] - fn supervisor_initiated_evidence_encoding_is_unchanged() { - let evidence = KubernetesOuterFenceEvidence { - network_policy_uid: "uid", - network_policy_resource_version: "1", - ingress_isolated: true, - egress_isolated: true, - egress_rule_count: 0, - runtime_control_policy: None, - }; - assert_eq!( - serde_json::to_string(&evidence).unwrap(), - r#"{"network_policy_uid":"uid","network_policy_resource_version":"1","ingress_isolated":true,"egress_isolated":true,"egress_rule_count":0}"# - ); - } - - #[test] - fn runtime_initiated_provisioning_reverses_only_the_dial_direction() { - let provisioned = runtime_initiated_spec().provision().unwrap(); - assert_eq!( - provisioned.boundary_config.listener, - BoundaryListener::RuntimeInitiatedTlsTcp { - authority: "os-boundary-sandbox.default.svc:5500".to_string(), - addresses: vec!["10.43.0.7:5500".parse().expect("valid target")], - tls: SandboxTlsServerConfig { - certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"), - private_key_path: PathBuf::from("/run/boundary/tls.key"), - }, - } - ); - assert_eq!( - provisioned.runtime_descriptor.transport, - SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox.default.svc:5500".to_string(), - listen_address: "0.0.0.0:5500".parse().expect("valid listener"), - } - ); - // TLS identity and every legacy binding are unchanged; the control - // policy is bound in addition. - let legacy = spec().provision().unwrap(); - assert_eq!( - provisioned.runtime_descriptor.tls, - legacy.runtime_descriptor.tls - ); - let claims = &provisioned.runtime_descriptor.resource_claims; - assert_eq!(claims, &provisioned.boundary_config.resource_claims); - for (key, value) in &legacy.runtime_descriptor.resource_claims { - assert_eq!(claims.get(key), Some(value)); - } - assert_eq!( - claims["kubernetes.control_policy_uid"], - "control-policy-uid" - ); - assert_eq!(claims["kubernetes.control_policy_resource_version"], "2001"); - assert!( - provisioned - .runtime_descriptor - .outer_fence - .validate("generation-1") - .is_ok() - ); - assert_ne!( - provisioned.runtime_descriptor.outer_fence.evidence_digest, - legacy.runtime_descriptor.outer_fence.evidence_digest - ); - } - - #[test] - fn runtime_control_evidence_requires_bound_policy_coordinates() { - for (uid, resource_version, port) in [("", "1", 5500), ("uid", "", 5500), ("uid", "1", 0)] { - let evidence = KubernetesOuterFenceEvidence { - network_policy_uid: "uid", - network_policy_resource_version: "1", - ingress_isolated: true, - egress_isolated: true, - egress_rule_count: 0, - runtime_control_policy: Some(KubernetesControlPolicyEvidence { - network_policy_uid: uid, - network_policy_resource_version: resource_version, - port, - }), - }; - assert!(evidence.project("generation-1").is_err()); - } - } - - #[test] - fn runtime_control_policy_admits_only_the_paired_supervisor_port() { - let policy = runtime_control_policy("sandbox-ns", "os-control-pair", "pair", 5500); - let spec = policy.spec.expect("policy has a spec"); - // Supervisors may already be ingress-isolated (managed SSH policy), - // so the one flow is admitted at both enforcement points. - assert_eq!( - spec.policy_types, - Some(vec!["Ingress".to_string(), "Egress".to_string()]) - ); - assert_eq!( - spec.pod_selector.match_labels, - Some(BTreeMap::from([( - BOUNDARY_PAIR_LABEL.to_string(), - "pair".to_string() - )])) - ); - let only_peer_and_port = |peers: &Option>, - ports: &Option>, - role: &str| { - let peers = peers.as_ref().expect("rule has peers"); - assert_eq!(peers.len(), 1); - assert!(peers[0].ip_block.is_none()); - assert!(peers[0].namespace_selector.is_none()); - assert_eq!( - peers[0].pod_selector.as_ref().unwrap().match_labels, - Some(pair_role_labels("pair", role)) - ); - let ports = ports.as_ref().expect("rule has ports"); - assert_eq!(ports.len(), 1); - assert_eq!(ports[0].protocol.as_deref(), Some("TCP")); - assert_eq!(ports[0].port, Some(IntOrString::Int(5500))); - }; - let egress = spec.egress.expect("policy has egress rules"); - assert_eq!(egress.len(), 1); - only_peer_and_port(&egress[0].to, &egress[0].ports, SUPERVISOR_ROLE); - let ingress = spec.ingress.expect("policy has ingress rules"); - assert_eq!(ingress.len(), 1); - only_peer_and_port(&ingress[0].from, &ingress[0].ports, WORKLOAD_ROLE); - } - #[test] fn network_fence_denies_all_workload_initiated_egress() { let fence = KubernetesSandboxRuntimeNetworkFenceSpec { diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index d4295253ef..0759ee5c1a 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -23,7 +23,7 @@ use k8s_openapi::api::networking::v1::NetworkPolicy; use crate::config::KubernetesControlConnection; #[cfg(test)] -use crate::isolation::{BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL}; +use crate::isolation::BOUNDARY_ROLE_LABEL; use crate::isolation::{ KubernetesSandboxRuntimeNetworkFence, KubernetesSandboxRuntimeNetworkFenceSpec, boundary_service_role, runtime_control_policy, @@ -779,55 +779,19 @@ mod tests { } #[test] - fn service_selects_the_listener_of_the_recorded_mode() { + fn service_selects_only_the_workload_boundary() { let names = SandboxRuntimeNames::new("4b67c0d0-1111-2222-3333-444444444444"); - for (mode, role, other) in [ - ( - KubernetesControlConnection::SupervisorInitiated, - "workload", - KubernetesControlConnection::RuntimeInitiated, - ), - ( - KubernetesControlConnection::RuntimeInitiated, - "supervisor", - KubernetesControlConnection::SupervisorInitiated, - ), - ] { - let service = boundary_service("sandbox", &names, "Pair", 5500, mode, owner()); - let selector = service.spec.as_ref().unwrap().selector.clone().unwrap(); - assert_eq!(selector[BOUNDARY_ROLE_LABEL], role); - // The runtime must reach a not-yet-Ready supervisor; the legacy - // Service encoding is unchanged. - let serialized = serde_json::to_value(&service).unwrap(); - assert_eq!( - serialized["spec"].get("publishNotReadyAddresses"), - (mode == KubernetesControlConnection::RuntimeInitiated) - .then_some(&serde_json::Value::Bool(true)) - ); - assert_eq!(selector[BOUNDARY_PAIR_LABEL], "pair"); - assert!(boundary_service_matches_mode(&service, "Pair", mode)); - assert!(!boundary_service_matches_mode(&service, "Pair", other)); - assert!(!boundary_service_matches_mode(&service, "other", mode)); - let mut flipped = service.clone(); - let spec = flipped.spec.as_mut().unwrap(); - spec.publish_not_ready_addresses = match spec.publish_not_ready_addresses { - Some(true) => None, - _ => Some(true), - }; - assert!(!boundary_service_matches_mode(&flipped, "Pair", mode)); - } - } - - #[test] - fn control_policy_is_owned_and_scoped_to_one_pair() { - let names = SandboxRuntimeNames::for_generation("Pair", "generation"); - let policy = control_policy("sandbox", &names, "Pair", 5500, owner()); - assert_eq!(policy.metadata.name.as_deref(), Some("os-control-pair")); - assert_eq!(policy.metadata.owner_references, Some(vec![owner()])); - let spec = policy.spec.unwrap(); + let service = boundary_service( + "sandbox", + &names, + "pair", + 5500, + KubernetesControlConnection::SupervisorInitiated, + owner(), + ); assert_eq!( - spec.pod_selector.match_labels.unwrap()[BOUNDARY_PAIR_LABEL], - "pair" + service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL], + "workload" ); } diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index 2cf6431c60..fc0b0b9ada 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -1774,67 +1774,6 @@ mod tests { assert_eq!(decoded, transport); } - #[test] - fn legacy_control_encodings_are_unchanged_by_runtime_initiated_variants() { - // Binaries from before runtime-initiated control decode exactly these - // shapes; the new variants must not alter them. - let transport = SandboxTransport::Tcp { - authority: "os-boundary-a.ns.svc:5500".to_string(), - addresses: vec!["10.43.0.9:5500".parse().expect("test address")], - }; - assert_eq!( - serde_json::to_string(&transport).unwrap(), - r#"{"kind":"tcp","authority":"os-boundary-a.ns.svc:5500","addresses":["10.43.0.9:5500"]}"# - ); - let listener = BoundaryListener::TlsTcp { - address: "0.0.0.0:5500".parse().expect("test address"), - tls: SandboxTlsServerConfig { - certificate_chain_path: PathBuf::from("/run/tls.crt"), - private_key_path: PathBuf::from("/run/tls.key"), - }, - }; - assert_eq!( - serde_json::to_string(&listener).unwrap(), - r#"{"kind":"tls-tcp","address":"0.0.0.0:5500","tls":{"certificate_chain_path":"/run/tls.crt","private_key_path":"/run/tls.key"}}"# - ); - } - - #[test] - fn runtime_initiated_control_variants_round_trip_and_reject_unknown_fields() { - let transport = SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-a.ns.svc:5500".to_string(), - listen_address: "0.0.0.0:5500".parse().expect("test address"), - }; - let encoded = serde_json::to_string(&transport).unwrap(); - assert_eq!( - encoded, - r#"{"kind":"runtime-initiated-tcp","authority":"os-boundary-a.ns.svc:5500","listen_address":"0.0.0.0:5500"}"# - ); - assert_eq!( - serde_json::from_str::(&encoded).unwrap(), - transport - ); - let listener = BoundaryListener::RuntimeInitiatedTlsTcp { - authority: "os-boundary-a.ns.svc:5500".to_string(), - addresses: vec!["10.43.0.9:5500".parse().expect("test address")], - tls: SandboxTlsServerConfig { - certificate_chain_path: PathBuf::from("/run/tls.crt"), - private_key_path: PathBuf::from("/run/tls.key"), - }, - }; - let encoded = serde_json::to_string(&listener).unwrap(); - assert_eq!( - serde_json::from_str::(&encoded).unwrap(), - listener - ); - assert!( - serde_json::from_str::( - r#"{"kind":"runtime-initiated-tcp","authority":"a","listen_address":"0.0.0.0:5500","target":"10.0.0.1:1"}"# - ) - .is_err() - ); - } - #[test] fn exec_wire_accepts_legacy_direct_exec_without_shell_intent() { let wire: ExecSpecWire = serde_json::from_str( diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index d08f6ec561..246f85de72 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -2461,7 +2461,6 @@ mod tests { mod flow_control; mod network_recovery; mod protocol_errors; - mod runtime_initiated; use crate::isolation_protocol::exchange_frame; use std::path::PathBuf; diff --git a/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs deleted file mode 100644 index 7693411d28..0000000000 --- a/crates/openshell-sandbox-backend/src/runtime/tests/runtime_initiated.rs +++ /dev/null @@ -1,337 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Runtime-initiated transport: admission at the supervisor listener and the -//! client path that shares one listener between discovery and attachment. - -use super::*; -use crate::runtime_initiated::RuntimeInitiatedListener; - -/// Records when the runtime side of a connection observes EOF or failure. -struct ClosureProbe { - inner: BoundaryDuplexStream, - closed: Arc, -} - -impl tokio::io::AsyncRead for ClosureProbe { - fn poll_read( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &mut tokio::io::ReadBuf<'_>, - ) -> Poll> { - let before = buffer.filled().len(); - let result = Pin::new(&mut self.inner).poll_read(context, buffer); - if matches!(&result, Poll::Ready(Err(_))) - || (matches!(&result, Poll::Ready(Ok(()))) && buffer.filled().len() == before) - { - self.closed.store(true, Ordering::Release); - } - result - } -} - -impl tokio::io::AsyncWrite for ClosureProbe { - fn poll_write( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - buffer: &[u8], - ) -> Poll> { - Pin::new(&mut self.inner).poll_write(context, buffer) - } - - fn poll_flush( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_flush(context) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - context: &mut Context<'_>, - ) -> Poll> { - Pin::new(&mut self.inner).poll_shutdown(context) - } -} - -fn runtime_initiated_descriptor(tls: SandboxTlsClientConfig) -> SandboxRuntimeDescriptor { - let mut descriptor = tls_runtime_descriptor("127.0.0.1:1".parse().unwrap(), tls); - descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { - authority: "os-boundary-sandbox-1.ns.svc:5500".to_string(), - listen_address: "127.0.0.1:5500".parse().unwrap(), - }; - descriptor -} - -/// Production descriptors name a fixed port; tests bind an ephemeral one. -fn bind_ephemeral(tls: &SandboxTlsClientConfig) -> Arc { - RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), tls).unwrap() -} - -/// Dial the listener as `openshell-sandbox` does and complete TLS as the server. -async fn dial_as_runtime( - address: std::net::SocketAddr, - server: Arc, -) -> std::io::Result> { - let stream = tokio::net::TcpStream::connect(address).await?; - tokio_rustls::TlsAcceptor::from(server).accept(stream).await -} - -/// Dial as the runtime and serve the protocol over the outbound connection. -fn spawn_runtime_connection( - address: std::net::SocketAddr, - server: Arc, - expected_token: &str, -) -> (Arc, tokio::task::JoinHandle<()>) { - let closed = Arc::new(AtomicBool::new(false)); - let probe = closed.clone(); - let expected_token = expected_token.to_string(); - let task = tokio::spawn(async move { - let stream = dial_as_runtime(address, server).await.unwrap(); - let stream = ClosureProbe { - inner: Box::new(stream), - closed: probe, - }; - serve_test_grpc(Box::new(stream), expected_token).await; - }); - (closed, task) -} - -fn queued(listener: &RuntimeInitiatedListener) -> usize { - listener.queued_spares() -} - -async fn wait_until(condition: impl Fn() -> bool + Send + Sync, failure: &str) { - tokio::time::timeout(Duration::from_secs(5), async { - while !condition() { - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .expect(failure); -} - -async fn assert_closed(stream: &mut (impl tokio::io::AsyncRead + Unpin)) { - let mut byte = [0; 1]; - let read = tokio::time::timeout(Duration::from_secs(2), stream.read(&mut byte)) - .await - .unwrap(); - assert!(matches!(read, Ok(0) | Err(_))); -} - -#[tokio::test] -async fn discovery_and_attach_share_one_listener_and_abort_closes_the_socket() { - let certificate = test_certificate(); - let descriptor = runtime_initiated_descriptor(certificate.client_tls); - let listener = bind_ephemeral(&descriptor.tls); - let address = listener.local_address(); - let transport = BoundaryTransport(TransportSource::RuntimeInitiated(listener)); - let token = "a".repeat(32); - - // Discovery and attachment are separate clients over clones of one - // transport, as in the supervisor; each takes its own runtime connection. - let (_, discovery_runtime) = - spawn_runtime_connection(address, certificate.server_config.clone(), &token); - let mut discovery = BoundaryClient::new(descriptor.clone(), test_bearer(&token)); - discovery.transport = transport.clone(); - let response = - tokio::time::timeout(Duration::from_secs(5), discovery.exchange(Request::Confirm)) - .await - .unwrap() - .unwrap(); - assert!(matches!(response, Response::Confirmed { .. })); - - let (attach_closed, attach_runtime) = - spawn_runtime_connection(address, certificate.server_config.clone(), &token); - let mut client = BoundaryClient::new(descriptor, test_bearer(&token)); - client.transport = transport; - let response = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap() - .unwrap(); - assert!(matches!(response, Response::Confirmed { .. })); - - // Aborting the cached channel closes the runtime's physical socket, and - // the next request uses a fresh runtime-initiated connection. - let (replacement_closed, replacement_runtime) = - spawn_runtime_connection(address, certificate.server_config, &token); - client - .grpc_channel - .lock() - .await - .take() - .unwrap() - .transport - .abort(); - wait_until( - || attach_closed.load(Ordering::Acquire), - "aborting the channel must close the runtime-initiated socket", - ) - .await; - let response = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap() - .unwrap(); - assert!(matches!(response, Response::Confirmed { .. })); - assert!(!replacement_closed.load(Ordering::Acquire)); - for task in [discovery_runtime, attach_runtime, replacement_runtime] { - task.abort(); - } -} - -#[tokio::test] -async fn runtime_initiated_transport_rejects_a_wrong_bearer() { - let certificate = test_certificate(); - let descriptor = runtime_initiated_descriptor(certificate.client_tls); - let listener = bind_ephemeral(&descriptor.tls); - let (_, runtime) = spawn_runtime_connection( - listener.local_address(), - certificate.server_config, - &"a".repeat(32), - ); - let mut client = BoundaryClient::new(descriptor, test_bearer(&"b".repeat(32))); - client.transport = BoundaryTransport(TransportSource::RuntimeInitiated(listener)); - let result = tokio::time::timeout(Duration::from_secs(5), client.exchange(Request::Confirm)) - .await - .unwrap(); - assert!(result.is_err()); - runtime.abort(); -} - -#[test] -fn runtime_initiated_descriptor_requires_a_fixed_listen_port_and_authority() { - let certificate = test_certificate(); - let mut descriptor = runtime_initiated_descriptor(certificate.client_tls); - assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_ok()); - for (authority, listen_address) in [("", "0.0.0.0:5500"), ("authority:5500", "0.0.0.0:0")] { - descriptor.transport = SandboxTransport::RuntimeInitiatedTcp { - authority: authority.to_string(), - listen_address: listen_address.parse().unwrap(), - }; - assert!(validate_runtime_descriptor(&descriptor, &sandbox()).is_err()); - } - // The listener independently refuses an ephemeral production port. - assert!(matches!( - RuntimeInitiatedListener::bind(&descriptor), - Err(BackendError::Descriptor(_)) - )); -} - -#[tokio::test] -async fn untrusted_and_silent_peers_cannot_displace_the_genuine_runtime() { - let genuine = test_certificate(); - let impostor = test_certificate(); - let descriptor = runtime_initiated_descriptor(genuine.client_tls); - let listener = bind_ephemeral(&descriptor.tls); - let address = listener.local_address(); - - // Silent sockets occupy handshake slots but never reach the queue. - let mut silent = Vec::new(); - for _ in 0..6 { - silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); - } - // A peer presenting another session's identity fails verification. TLS - // 1.3 servers can finish before the client's alert arrives, so the - // connection must then be closed without being queued. - if let Ok(mut rejected) = dial_as_runtime(address, impostor.server_config).await { - assert_closed(&mut rejected).await; - } - assert_eq!(queued(&listener), 0); - - let runtime = tokio::spawn(dial_as_runtime(address, genuine.server_config)); - let accepted = tokio::time::timeout(Duration::from_secs(5), listener.accept(&descriptor)).await; - assert!(accepted.is_ok_and(|stream| stream.is_ok())); - assert!(runtime.await.unwrap().is_ok()); - drop(silent); -} - -#[tokio::test] -async fn handshake_slots_bound_unauthenticated_sockets() { - let genuine = test_certificate(); - let descriptor = runtime_initiated_descriptor(genuine.client_tls); - let listener = bind_ephemeral(&descriptor.tls); - let address = listener.local_address(); - let mut silent = Vec::new(); - for _ in 0..8 { - silent.push(tokio::net::TcpStream::connect(address).await.unwrap()); - } - // Let the listener assign every slot before the excess socket arrives. - tokio::time::sleep(Duration::from_millis(200)).await; - let mut excess = tokio::net::TcpStream::connect(address).await.unwrap(); - assert_closed(&mut excess).await; - - // Freed slots admit the genuine runtime again. - drop(silent); - let runtime = tokio::spawn(dial_as_runtime(address, genuine.server_config)); - assert!( - tokio::time::timeout(Duration::from_secs(10), listener.accept(&descriptor)) - .await - .is_ok() - ); - assert!(runtime.await.unwrap().is_ok()); -} - -#[tokio::test] -async fn spare_queue_is_bounded_and_closes_the_excess_spare() { - let genuine = test_certificate(); - let descriptor = runtime_initiated_descriptor(genuine.client_tls); - let listener = bind_ephemeral(&descriptor.tls); - let mut runtimes = Vec::new(); - for count in 1..=2 { - runtimes.push( - dial_as_runtime(listener.local_address(), genuine.server_config.clone()) - .await - .unwrap(), - ); - wait_until( - || queued(&listener) == count, - "authenticated spare was not queued", - ) - .await; - } - let mut excess = dial_as_runtime(listener.local_address(), genuine.server_config) - .await - .unwrap(); - assert_closed(&mut excess).await; - assert_eq!(queued(&listener), 2); - drop(runtimes); -} - -#[tokio::test] -async fn listener_rejects_an_invalid_pinned_identity_before_binding() { - let mut tls = test_certificate().client_tls; - tls.trust_anchor_pem = "not a certificate".to_string(); - assert!(matches!( - RuntimeInitiatedListener::bind_address("127.0.0.1:0".parse().unwrap(), &tls), - Err(BackendError::Descriptor(_)) - )); -} - -#[tokio::test] -async fn listener_refuses_a_descriptor_for_another_session() { - let genuine = test_certificate(); - let other = test_certificate(); - let listener = bind_ephemeral(&genuine.client_tls); - assert!(matches!( - listener - .accept(&runtime_initiated_descriptor(other.client_tls)) - .await, - Err(BackendError::Descriptor(_)) - )); -} - -#[tokio::test] -async fn dropping_the_listener_closes_queued_spares() { - let genuine = test_certificate(); - let listener = bind_ephemeral(&genuine.client_tls); - let mut runtime = dial_as_runtime(listener.local_address(), genuine.server_config) - .await - .unwrap(); - wait_until( - || queued(&listener) == 1, - "authenticated spare was not queued", - ) - .await; - drop(listener); - assert_closed(&mut runtime).await; -} diff --git a/crates/openshell-sandbox-backend/src/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime_initiated.rs index 574c3e4cd8..e867e580fb 100644 --- a/crates/openshell-sandbox-backend/src/runtime_initiated.rs +++ b/crates/openshell-sandbox-backend/src/runtime_initiated.rs @@ -35,8 +35,6 @@ const MAX_QUEUED_CONNECTIONS: usize = 2; #[derive(Debug)] pub struct RuntimeInitiatedListener { tls: SandboxTlsClientConfig, - #[cfg(test)] - local_address: SocketAddr, spares: tokio::sync::Mutex>, accept_task: tokio::task::AbortHandle, } @@ -83,25 +81,11 @@ impl RuntimeInitiatedListener { tracing::info!(%local_address, "Runtime-initiated Sandbox Protocol listener ready"); Ok(std::sync::Arc::new(Self { tls: tls.clone(), - #[cfg(test)] - local_address, spares: tokio::sync::Mutex::new(spares), accept_task, })) } - /// Address actually bound, including an OS-assigned port. - #[cfg(test)] - pub fn local_address(&self) -> SocketAddr { - self.local_address - } - - /// Authenticated spares waiting for a client. - #[cfg(test)] - pub fn queued_spares(&self) -> usize { - self.spares.try_lock().map_or(0, |spares| spares.len()) - } - /// Wait for one authenticated runtime connection for `descriptor`. A /// listener bound for another session is refused. pub(crate) async fn accept( diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index bb6b964c13..34a95c06e9 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -5767,96 +5767,6 @@ mod linux { server.abort(); } - #[test] - fn runtime_initiated_targets_must_be_concrete_remote_and_single_port() { - let tls = placeholder_server_tls(); - let address = |value: &str| value.parse::().unwrap(); - assert!( - validate_runtime_initiated_targets( - "os-boundary-a.ns.svc:5500", - &[address("10.43.0.9:5500"), address("[fd00::9]:5500")], - &tls - ) - .is_ok() - ); - for addresses in [ - vec![], - vec![address("127.0.0.1:5500")], - vec![address("[::ffff:127.0.0.1]:5500")], - vec![address("0.0.0.0:5500")], - vec![address("10.43.0.9:0")], - vec![address("10.43.0.9:5500"), address("10.43.0.10:5501")], - ] { - assert!( - validate_runtime_initiated_targets( - "os-boundary-a.ns.svc:5500", - &addresses, - &tls - ) - .is_err(), - "{addresses:?}" - ); - } - assert!( - validate_runtime_initiated_targets("", &[address("10.43.0.9:5500")], &tls).is_err() - ); - } - - #[tokio::test(flavor = "multi_thread")] - async fn runtime_initiated_control_keeps_one_spare_and_redials_after_loss() { - let (runtime, _token) = availability_test_runtime(); - let directory = tempfile::tempdir().expect("temporary directory"); - let (server_tls, client_tls) = stage_test_tls(directory.path(), "dial"); - let server_config = - Arc::new(load_tls_server_config(&server_tls).expect("load TLS identity")); - let supervisor = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let dialer = tokio::spawn(run_runtime_initiated_control( - vec![supervisor.local_addr().unwrap()], - server_config, - runtime, - Arc::new(AtomicUsize::new(0)), - )); - let connector = - tokio_rustls::TlsConnector::from(Arc::new(test_client_config(&client_tls))); - let server_name = - rustls::pki_types::ServerName::try_from(client_tls.server_name.clone()).unwrap(); - let accept_authenticated = || async { - let (stream, _) = tokio::time::timeout(Duration::from_secs(5), supervisor.accept()) - .await - .expect("runtime dials the supervisor") - .unwrap(); - connector - .connect(server_name.clone(), stream) - .await - .expect("runtime presents its pinned identity") - }; - let no_new_dial = || async { - tokio::time::timeout(Duration::from_millis(400), supervisor.accept()) - .await - .is_err() - }; - - let mut first = accept_authenticated().await; - assert!(no_new_dial().await, "an unclaimed spare suppresses dialing"); - - // Starting HTTP/2 claims the spare; the runtime dials a new one. - first - .write_all(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") - .await - .unwrap(); - first.flush().await.unwrap(); - let spare = accept_authenticated().await; - assert!(no_new_dial().await, "only one spare is kept"); - - // Losing the claimed connection does not add spares; losing the - // spare is repaired by a redial. - drop(first); - assert!(no_new_dial().await); - drop(spare); - let _replacement = accept_authenticated().await; - dialer.abort(); - } - #[test] fn pod_control_listener_rejects_loopback_peers_before_tls() { let directory = tempfile::tempdir().expect("temporary directory"); From 7c58076757ba18e5808f9b1671314f3e1695cce9 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 13:01:25 -0700 Subject: [PATCH 4/9] feat(kubernetes): migrate legacy control flow on restart Signed-off-by: Drew Newberry --- .agents/skills/helm-dev-environment/SKILL.md | 11 +-- crates/openshell-driver-kubernetes/README.md | 16 +++-- .../openshell-driver-kubernetes/src/config.rs | 10 +-- .../openshell-driver-kubernetes/src/driver.rs | 71 +++++++++++++++---- .../openshell-driver-kubernetes/src/main.rs | 14 +--- .../src/sandbox_runtime.rs | 4 +- .../helm/openshell/templates/clusterrole.yaml | 3 +- deploy/helm/openshell/templates/role.yaml | 3 +- .../openshell/tests/clusterrole_test.yaml | 4 +- .../tests/sandbox_namespace_test.yaml | 2 +- docs/how-it-works/gateways/configuration.mdx | 20 +++--- docs/how-it-works/sandboxes/runtimes.mdx | 2 +- skills/debug-openshell-cluster/SKILL.md | 19 +++-- 13 files changed, 107 insertions(+), 72 deletions(-) diff --git a/.agents/skills/helm-dev-environment/SKILL.md b/.agents/skills/helm-dev-environment/SKILL.md index 64881ea44f..03c46b4f46 100644 --- a/.agents/skills/helm-dev-environment/SKILL.md +++ b/.agents/skills/helm-dev-environment/SKILL.md @@ -81,12 +81,13 @@ The Skaffold flow builds distinct `gateway`, `sandbox`, and `supervisor` images and deploys the OpenShell Helm chart. The Kubernetes driver creates a capability-free workload Pod and a directly managed capability-free supervisor Pod. One namespace-wide NetworkPolicy denies direct egress from every OpenShell -workload Pod. New sandboxes default to `runtime-initiated` control: the sandbox +workload Pod. New sandbox generations use `runtime-initiated` control: the sandbox runtime dials its supervisor Pod, and the driver adds a per-sandbox -`os-control-` NetworkPolicy for that path only. Set -`gatewayConfig."openshell.drivers.kubernetes".sandbox_runtime.control_connection` -to `supervisor-initiated` when testing with an older supervisor or sandbox -runtime image. Existing sandboxes keep the mode recorded at creation. The +`os-control-` NetworkPolicy for that path only. Existing running generations +keep their connection until replaced; stop/start and gateway recovery that +recreates both Pods upgrade legacy generations automatically. Use compatible +gateway, supervisor, and sandbox runtime images together; there is no legacy +rollout flag. The `pkiInitJob` hook (a pre-install Job that runs `openshell-gateway generate-certs`) generates gateway and CLI TLS secrets on first install. Supervisor Pods project only `ca.crt` and authenticate gateway RPCs with sandbox bearer tokens. User diff --git a/crates/openshell-driver-kubernetes/README.md b/crates/openshell-driver-kubernetes/README.md index a3328315ac..612eca70d8 100644 --- a/crates/openshell-driver-kubernetes/README.md +++ b/crates/openshell-driver-kubernetes/README.md @@ -105,13 +105,15 @@ enforcement, so operators must verify CNI support before running sandboxes. ### Control connection direction -Each sandbox records which side opens its Sandbox Protocol TCP connection in -the Sandbox annotation `openshell.ai/sandbox-runtime-control-connection`, -written once at creation from `sandbox_runtime.control_connection` (default -`runtime-initiated`). A sandbox without the annotation was created before the -mode existed and is `supervisor-initiated`. Start, gateway-restart recovery, -and image upgrades read the annotation and never apply the current default, so -both modes run side by side in one namespace. An unknown value fails closed. +Every new runtime generation uses `runtime-initiated` control. The Sandbox +annotation `openshell.ai/sandbox-runtime-control-connection` records the current +generation's mode. A missing annotation identifies a legacy, +`supervisor-initiated` generation, which keeps working until replaced. Stop/start +upgrades it automatically: with both old Pods removed, the driver changes the +Service selector in place, preserves its ClusterIP, and records the new mode +before releasing the replacement workload. Gateway recovery that recreates both +Pods also upgrades the generation. Reconciliation never migrates a running +generation. An unknown annotation value fails closed; there is no rollout flag. Only the dial direction differs. In both modes the sandbox runtime presents the generation-pinned TLS server identity and serves gRPC, and the supervisor diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 040cccfacd..14adb9ee01 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -76,9 +76,9 @@ pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi"; /// Which side opens the Sandbox Protocol TCP connection. /// -/// Recorded on each Sandbox resource at creation. Existing sandboxes keep their -/// recorded mode across stop/start, restarts, and upgrades; a sandbox created -/// before this setting existed is `SupervisorInitiated`. +/// Internal state for the current runtime generation. A missing Sandbox +/// annotation identifies a legacy generation; every new generation uses +/// `RuntimeInitiated`. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum KubernetesControlConnection { @@ -121,16 +121,12 @@ pub struct KubernetesSandboxRuntimeConfig { /// TCP port of the Sandbox Protocol listener: the workload boundary for /// supervisor-initiated sandboxes, the supervisor for runtime-initiated ones. pub boundary_port: u16, - /// Connection mode recorded on sandboxes created from now on. Changing it - /// never migrates an existing sandbox. - pub control_connection: KubernetesControlConnection, } impl Default for KubernetesSandboxRuntimeConfig { fn default() -> Self { Self { boundary_port: 5500, - control_connection: KubernetesControlConnection::default(), } } } diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 07d8f96e45..32ff984029 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -107,8 +107,8 @@ const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_UID: &str = "openshell.ai/sandbox-runtime-network-policy-uid"; const ANNOTATION_SANDBOX_RUNTIME_NETWORK_POLICY_GENERATION: &str = "openshell.ai/sandbox-runtime-network-policy-generation"; -/// Durable, creation-time Sandbox Protocol connection mode. Absent on -/// sandboxes created before the mode existed, which are supervisor-initiated. +/// Connection mode of the current runtime generation. Absent on legacy +/// generations, which remain supervisor-initiated until they are replaced. const ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION: &str = "openshell.ai/sandbox-runtime-control-connection"; const ANNOTATION_SANDBOX_RUNTIME_CONTROL_POLICY_UID: &str = @@ -156,8 +156,7 @@ fn boundary_service_authority( ) } -/// Read the connection mode recorded when the sandbox was created. Starting a -/// sandbox never consults the current creation default. +/// Read the current generation's mode without migrating a running workload. fn sandbox_runtime_control_connection( object: &DynamicObject, ) -> Result { @@ -2180,7 +2179,7 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL.to_string(), log_level.clone(), ); - let control_connection = self.config.sandbox_runtime.control_connection; + let control_connection = KubernetesControlConnection::RuntimeInitiated; annotations.insert( ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION.to_string(), control_connection.as_str().to_string(), @@ -2333,7 +2332,7 @@ impl KubernetesComputeDriver { } /// Fail closed when a sandbox's boundary Service or control policy no - /// longer matches the mode recorded at creation. This is the reconcile + /// longer matches the mode recorded for its generation. This is the reconcile /// pass's only Service read; a missing Service is drift once the sandbox /// should be running. Returns `Precondition` for drift that must suspend. async fn verify_sandbox_runtime_control_path( @@ -2904,8 +2903,7 @@ impl KubernetesComputeDriver { .get(&names.boundary_service) .await .map_err(KubernetesDriverError::from_kube)?; - // The Service and its selected role were fixed when the sandbox was - // created; restart must not reinterpret them under another mode. + // Running generations must retain the Service role prepared for them. if !boundary_service_matches_mode(&service, sandbox_id, control_connection) { return Err(KubernetesDriverError::Precondition(format!( "boundary Service {} does not select the {} listener recorded for this sandbox", @@ -3360,16 +3358,17 @@ impl KubernetesComputeDriver { &namespace, ); let pods = Api::::namespaced(self.client.clone(), &namespace); - if pods + let workload_exists = pods .get_opt(cr_name) .await .map_err(KubernetesDriverError::from_kube)? - .is_some() - { + .is_some(); + if workload_exists || sandbox_runtime_should_run(&object) { if !encoded_authentication.is_empty() { // Gateway restart creates a fresh in-memory launch session. - // Recreate both Pods so neither side retains credentials for - // the gateway instance that was replaced. + // Suspend the controller and recreate both Pods so neither + // side retains credentials for the replaced gateway. Suspend + // even if the old workload Pod disappeared before this call. self.stop_sandbox_inner(sandbox_id).await?; return Box::pin(self.start_sandbox_runtime_generation( sandbox_id, @@ -3384,7 +3383,8 @@ impl KubernetesComputeDriver { )); } - let control_connection = sandbox_runtime_control_connection(&object)?; + let previous_control_connection = sandbox_runtime_control_connection(&object)?; + let control_connection = KubernetesControlConnection::RuntimeInitiated; let names = SandboxRuntimeNames::for_generation(sandbox_id, generation.as_str()); self.create_sandbox_runtime_fence(&namespace, &names) .await?; @@ -3392,6 +3392,48 @@ impl KubernetesComputeDriver { stale_generations.push(generation.as_str()); self.delete_sandbox_runtime_supervisor(sandbox_id, &namespace, &stale_generations) .await?; + // Both old Pods are gone and the lifecycle gate excludes reconcile. + // Upgrade the endpoint in place, preserving its stable ClusterIP. A + // retry may find this update completed before the new mode was saved. + let services = Api::::namespaced(self.client.clone(), &namespace); + let service = services + .get(&names.boundary_service) + .await + .map_err(KubernetesDriverError::from_kube)?; + if !boundary_service_matches_mode(&service, sandbox_id, control_connection) { + if !boundary_service_matches_mode(&service, sandbox_id, previous_control_connection) { + return Err(KubernetesDriverError::Precondition(format!( + "boundary Service {} no longer matches the recorded control connection", + names.boundary_service + ))); + } + let version = service + .metadata + .resource_version + .as_deref() + .ok_or_else(|| { + KubernetesDriverError::Message( + "boundary Service has no resourceVersion".to_string(), + ) + })?; + services + .patch( + &names.boundary_service, + &PatchParams::default(), + &Patch::Merge(serde_json::json!({ + "metadata": {"resourceVersion": version}, + "spec": { + "selector": crate::isolation::pair_role_labels( + &crate::sandbox_runtime::pair_label_value(sandbox_id), + crate::isolation::boundary_service_role(control_connection), + ), + "publishNotReadyAddresses": true, + }, + })), + ) + .await + .map_err(KubernetesDriverError::from_kube)?; + } let main_process_spec = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_MAIN_PROCESS_SPEC)?; let log_level = required_sandbox_annotation(&object, ANNOTATION_SANDBOX_RUNTIME_LOG_LEVEL)?; @@ -3476,6 +3518,7 @@ impl KubernetesComputeDriver { ANNOTATION_SANDBOX_RUNTIME_BOOTSTRAP_SESSION: launch_authentication.supervisor.session_id.to_string(), ANNOTATION_SANDBOX_RUNTIME_GENERATION: generation.as_str(), ANNOTATION_SANDBOX_RUNTIME_SUPERVISOR_UID: supervisor_uid.clone(), + ANNOTATION_SANDBOX_RUNTIME_CONTROL_CONNECTION: control_connection.as_str(), }); running_patch["spec"]["podTemplate"]["spec"] = restart_pod_template_spec( &volumes, diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 9be1e6cccc..ab6e0bf1b8 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -12,9 +12,8 @@ use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; use openshell_driver_kubernetes::{ ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesControlConnection, - KubernetesImagePullPolicy, KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, - WorkspaceMode, + KubernetesComputeConfig, KubernetesComputeDriver, KubernetesImagePullPolicy, + KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode, }; #[derive(Parser, Debug)] @@ -158,14 +157,6 @@ struct Args { )] sandbox_runtime_boundary_port: u16, - /// Sandbox Protocol connection mode recorded on newly created sandboxes. - #[arg( - long, - env = "OPENSHELL_K8S_SANDBOX_RUNTIME_CONTROL_CONNECTION", - default_value = "runtime-initiated" - )] - sandbox_runtime_control_connection: KubernetesControlConnection, - /// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress. #[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")] https_proxy: Option, @@ -295,7 +286,6 @@ async fn main() -> Result<()> { supervisor_image_pull_policy: args.supervisor_image_pull_policy, sandbox_runtime: KubernetesSandboxRuntimeConfig { boundary_port: args.sandbox_runtime_boundary_port, - control_connection: args.sandbox_runtime_control_connection, }, https_proxy: args.https_proxy, no_proxy: args.no_proxy, diff --git a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs index 0759ee5c1a..504eb62121 100644 --- a/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs +++ b/crates/openshell-driver-kubernetes/src/sandbox_runtime.rs @@ -185,8 +185,8 @@ pub fn workload_fence( } /// Per-sandbox Sandbox Protocol endpoint. Its stable `ClusterIP` is allocated -/// before either Pod runs; the selected role follows the sandbox's recorded -/// connection mode and never changes for the sandbox's lifetime. +/// before either Pod runs; the selected role follows the current generation's +/// connection mode. A legacy endpoint is upgraded when both Pods are replaced. #[must_use] pub fn boundary_service( namespace: &str, diff --git a/deploy/helm/openshell/templates/clusterrole.yaml b/deploy/helm/openshell/templates/clusterrole.yaml index a8f2e6f57c..789e53ee79 100644 --- a/deploy/helm/openshell/templates/clusterrole.yaml +++ b/deploy/helm/openshell/templates/clusterrole.yaml @@ -101,7 +101,8 @@ rules: {{- if ne $workspaceMode "shared" }} - apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + # Restart upgrades a legacy control endpoint without replacing its ClusterIP. + verbs: ["create", "get", "patch"] - apiGroups: ["networking.k8s.io"] resources: ["networkpolicies"] verbs: ["create", "get"] diff --git a/deploy/helm/openshell/templates/role.yaml b/deploy/helm/openshell/templates/role.yaml index 4097976ce1..f79c05a969 100644 --- a/deploy/helm/openshell/templates/role.yaml +++ b/deploy/helm/openshell/templates/role.yaml @@ -58,7 +58,8 @@ rules: - watch - apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + # Restart upgrades a legacy control endpoint without replacing its ClusterIP. + verbs: ["create", "get", "patch"] - apiGroups: [""] resources: ["secrets"] # Bootstrap Secrets are generation-scoped. Recovery deletes them by exact diff --git a/deploy/helm/openshell/tests/clusterrole_test.yaml b/deploy/helm/openshell/tests/clusterrole_test.yaml index 309b10fc0e..a02219a986 100644 --- a/deploy/helm/openshell/tests/clusterrole_test.yaml +++ b/deploy/helm/openshell/tests/clusterrole_test.yaml @@ -156,7 +156,7 @@ tests: content: apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + verbs: ["create", "get", "patch"] - contains: path: rules content: @@ -181,7 +181,7 @@ tests: content: apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + verbs: ["create", "get", "patch"] - it: omits the ClusterRole when cluster-scoped RBAC is disabled set: rbac.clusterScoped.create: false diff --git a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml index 596f29b307..1a8a6306d2 100644 --- a/deploy/helm/openshell/tests/sandbox_namespace_test.yaml +++ b/deploy/helm/openshell/tests/sandbox_namespace_test.yaml @@ -67,7 +67,7 @@ tests: content: apiGroups: [""] resources: ["services"] - verbs: ["create", "get"] + verbs: ["create", "get", "patch"] - contains: path: rules content: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index a5f055c461..1a552f04d3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -991,9 +991,6 @@ gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes [openshell.drivers.kubernetes.sandbox_runtime] # Port of the TLS-protected Sandbox Protocol listener on the paired Pod. boundary_port = 5500 -# Which side opens the Sandbox Protocol connection for sandboxes created from -# now on: "runtime-initiated" (default) or "supervisor-initiated". -control_connection = "runtime-initiated" ``` The Kubernetes driver always creates the workload `NetworkPolicy` before @@ -1003,11 +1000,11 @@ enforcement. #### Sandbox Control Connection -`control_connection` applies only when a sandbox is created. The driver records -the mode on the sandbox, and stop/start, gateway restarts, and upgrades keep -it. Sandboxes created before this setting existed stay `supervisor-initiated`. -With `runtime-initiated`, the sandbox runtime dials its own supervisor Pod -through the sandbox's boundary Service, and the driver adds one per-sandbox +New Kubernetes sandboxes use `runtime-initiated` control automatically. Existing +running sandboxes keep their current connection. Stop/start upgrades a legacy +sandbox to `runtime-initiated`; gateway recovery that replaces both Pods does +the same. There is no configuration flag. The sandbox runtime dials its +supervisor Pod through the sandbox's boundary Service, and the driver adds one per-sandbox `NetworkPolicy` that allows only that connection, as egress from the workload Pod and ingress to the supervisor Pod. Authentication is the same in both modes. @@ -1019,10 +1016,9 @@ it already must for the default `networkPolicy.enabled` SSH policy. Do not loosen `os-control-`; add any extra supervisor ingress your platform needs, such as monitoring, as a separate policy. -Use `runtime-initiated` only when the gateway, `supervisor_image`, and -`sandbox_runtime_image` all support it. If you pin an older supervisor or -sandbox runtime image, set `control_connection = "supervisor-initiated"` until -you upgrade both images. +Deploy compatible gateway, `supervisor_image`, and `sandbox_runtime_image` +versions together. Newly created or restarted sandboxes require all three to +support runtime-initiated control; pinned older images cannot start them. Rolling back is not supported for `runtime-initiated` sandboxes. Older gateways, supervisors, and sandbox runtimes cannot start or restart them: an older driver diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index 93f953c91d..a9b12679dd 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -299,7 +299,7 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `proxy_auth_secret_name` / `proxy_auth_secret_key` | `upstreamProxy.authSecret.name` / `.key` | Secret holding the proxy `user:pass` credential. | | `proxy_ca_bundle` | `upstreamProxy.caBundle.configMapName` / `.key` | ConfigMap with the proxy's CA certificate. | -New Kubernetes sandboxes default to a runtime-initiated control connection, which needs a supervisor image and sandbox runtime image that support it. If you override either image with an older version, first set `gatewayConfig["openshell.drivers.kubernetes"].sandbox_runtime.control_connection` to `supervisor-initiated`. Existing sandboxes keep their mode. See [Sandbox Control Connection](/how-it-works/gateways/configuration#sandbox-control-connection). +New and restarted Kubernetes sandboxes use a runtime-initiated control connection automatically. Existing running generations keep their current connection until replaced. Deploy compatible gateway, supervisor, and sandbox runtime images together. See [Sandbox Control Connection](/how-it-works/gateways/configuration#sandbox-control-connection). For the full list, see the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 9e10e77dac..760bcfc641 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -1018,16 +1018,17 @@ provider credentials, DNS, and mediated upstream connections. Each sandbox records which side opens that connection in the Sandbox annotation `openshell.ai/sandbox-runtime-control-connection`. A missing annotation means `supervisor-initiated`: the `os-boundary-` Service selects the workload Pod -and the supervisor dials it. With `runtime-initiated` (the default for new -sandboxes), the Service selects the supervisor Pod, the sandbox runtime dials +and the supervisor dials it. With `runtime-initiated` (used for new +generations), the Service selects the supervisor Pod, the sandbox runtime dials it, and a per-sandbox NetworkPolicy `os-control-` admits only that connection at both the workload (egress) and the supervisor (ingress). The Service publishes the supervisor before it is Ready, because readiness follows the runtime's attach. If the runtime logs `cannot reach the supervisor`, check that the CNI applied both directions of `os-control-`. -Stop/start and gateway restarts keep the recorded mode. Check the mode first -when a sandbox never becomes ready, especially after changing images or -`sandbox_runtime.control_connection`: +Stop/start upgrades legacy sandboxes to runtime-initiated control, as does +gateway recovery that recreates both Pods. Existing running generations keep +their current connection until replaced. Check the mode first when a sandbox +never becomes ready, especially after changing images: ```bash kubectl -n get sandbox \ @@ -1041,8 +1042,12 @@ runtime image that all support the mode. An older supervisor or runtime fails to decode its launch descriptor, so the sandbox cannot start or restart. Rolling the gateway or images back is unsupported for `runtime-initiated` sandboxes; running Pods keep their connection, but their next start fails. -Upgrade the images, or set `control_connection = "supervisor-initiated"` for -new sandboxes. +Upgrade the images together. There is no legacy rollout flag. +Migrating a legacy endpoint on restart also requires `patch` on Services in the +sandbox namespace. Helm grants it through the shared-mode Role or the managed +and operator-mode ClusterRole. For a `Forbidden` restart error, check the +gateway ServiceAccount with `kubectl auth can-i patch services --namespace + --as system:serviceaccount::`. Do not edit the annotation, Service selector, or `os-control-` policy: reconciliation suspends a sandbox whose Service or control policy no longer matches its recorded mode. From 2650b4fa2cfc5d25fd8e0d1c43ac7b7627ed405c Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 13:02:14 -0700 Subject: [PATCH 5/9] docs: remove prototype changes from published docs Signed-off-by: Drew Newberry --- docs/how-it-works/gateways/configuration.mdx | 33 +------------------- docs/how-it-works/sandboxes/runtimes.mdx | 2 -- 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 1a552f04d3..aab1204eb5 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -989,7 +989,7 @@ gateway_namespace = "openshell" gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" } [openshell.drivers.kubernetes.sandbox_runtime] -# Port of the TLS-protected Sandbox Protocol listener on the paired Pod. +# TLS-protected boundary listener reached only by the paired control pod. boundary_port = 5500 ``` @@ -998,37 +998,6 @@ releasing a sandbox Pod. Your cluster CNI must enforce ingress and egress `NetworkPolicy` in every sandbox namespace; the Kubernetes API does not verify enforcement. -#### Sandbox Control Connection - -New Kubernetes sandboxes use `runtime-initiated` control automatically. Existing -running sandboxes keep their current connection. Stop/start upgrades a legacy -sandbox to `runtime-initiated`; gateway recovery that replaces both Pods does -the same. There is no configuration flag. The sandbox runtime dials its -supervisor Pod through the sandbox's boundary Service, and the driver adds one per-sandbox -`NetworkPolicy` that allows only that connection, as egress from the workload -Pod and ingress to the supervisor Pod. Authentication is the same in both modes. - -That policy makes the supervisor Pod ingress-isolated even when Helm -`networkPolicy.enabled` is `false`. The sandbox needs no other ingress to its -supervisor: exec, file transfer, and logs use connections the supervisor opens. -Your CNI must still allow kubelet readiness probes to ingress-isolated Pods, as -it already must for the default `networkPolicy.enabled` SSH policy. Do not -loosen `os-control-`; add any extra supervisor ingress your platform needs, -such as monitoring, as a separate policy. - -Deploy compatible gateway, `supervisor_image`, and `sandbox_runtime_image` -versions together. Newly created or restarted sandboxes require all three to -support runtime-initiated control; pinned older images cannot start them. - -Rolling back is not supported for `runtime-initiated` sandboxes. Older gateways, -supervisors, and sandbox runtimes cannot start or restart them: an older driver -does not read the recorded mode, the sandbox's Service selects the supervisor -rather than the workload, and older supervisors and runtimes reject the new -launch descriptors. Running supervisor and sandbox processes can keep their -established connection, but starting a new generation with incompatible -binaries fails closed. Delete those sandboxes before rolling back, or roll -forward. - In managed workspace mode, the Kubernetes driver copies each explicitly named `image_pull_secrets` Secret from `namespace` into the managed workspace namespace on sandbox creation. Shared and operator modes require the Secret to diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index a9b12679dd..89ec2c8d44 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -299,8 +299,6 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `proxy_auth_secret_name` / `proxy_auth_secret_key` | `upstreamProxy.authSecret.name` / `.key` | Secret holding the proxy `user:pass` credential. | | `proxy_ca_bundle` | `upstreamProxy.caBundle.configMapName` / `.key` | ConfigMap with the proxy's CA certificate. | -New and restarted Kubernetes sandboxes use a runtime-initiated control connection automatically. Existing running generations keep their current connection until replaced. Deploy compatible gateway, supervisor, and sandbox runtime images together. See [Sandbox Control Connection](/how-it-works/gateways/configuration#sandbox-control-connection). - For the full list, see the [Gateway Configuration File](/how-it-works/gateways/configuration) reference. On OpenShift, copy the cluster proxy's CA into the gateway namespace and set `upstreamProxy.caBundle.configMapName` to `corporate-proxy-ca`: From 758a70e4b813b3f7ec502cc2cc2e107bccaf2a9d Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Fri, 16 Oct 2026 14:35:46 -0700 Subject: [PATCH 6/9] fix(sandbox): adapt reverse control to published backend protocol Signed-off-by: Drew Newberry --- crates/openshell-core/src/isolation_registration.rs | 13 ++++++++++--- crates/openshell-driver-kubernetes/src/isolation.rs | 10 ++++++++-- crates/openshell-sandbox-backend/src/runtime.rs | 12 +++++------- .../src/runtime/test_support.rs | 4 ++-- 4 files changed, 25 insertions(+), 14 deletions(-) diff --git a/crates/openshell-core/src/isolation_registration.rs b/crates/openshell-core/src/isolation_registration.rs index 873c39f265..6ef2032462 100644 --- a/crates/openshell-core/src/isolation_registration.rs +++ b/crates/openshell-core/src/isolation_registration.rs @@ -51,9 +51,10 @@ impl SandboxTransport { .iter() .all(|address| address.port() != 0 && !address.ip().is_unspecified()) } - Self::RuntimeInitiatedTcp { authority, listen_address } => { - !authority.is_empty() && listen_address.port() != 0 - } + Self::RuntimeInitiatedTcp { + authority, + listen_address, + } => !authority.is_empty() && listen_address.port() != 0, Self::Vsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } @@ -91,6 +92,12 @@ pub fn validate_registrations(registrations: &[BackendRegistration]) -> Result<( let mut names = BTreeSet::new(); for registration in registrations { validate_backend_name(®istration.name)?; + if matches!( + registration.endpoint, + SandboxTransport::RuntimeInitiatedTcp { .. } + ) { + return Err("runtime-initiated control is not a registered backend endpoint"); + } if !names.insert(®istration.name) { return Err("duplicate isolation backend registration"); } diff --git a/crates/openshell-driver-kubernetes/src/isolation.rs b/crates/openshell-driver-kubernetes/src/isolation.rs index 97e3676f4f..c9da7f3e3a 100644 --- a/crates/openshell-driver-kubernetes/src/isolation.rs +++ b/crates/openshell-driver-kubernetes/src/isolation.rs @@ -75,8 +75,14 @@ impl KubernetesOuterFenceEvidence<'_> { "Kubernetes provisioning evidence is incomplete".to_string(), )); } - if self.runtime_control_policy.as_ref().is_some_and(|control| !control.is_bound()) { - return Err(BackendError::Descriptor("Kubernetes control policy evidence is incomplete".into())); + if self + .runtime_control_policy + .as_ref() + .is_some_and(|control| !control.is_bound()) + { + return Err(BackendError::Descriptor( + "Kubernetes control policy evidence is incomplete".into(), + )); } let encoded = serde_json::to_vec(self).map_err(|error| { BackendError::Descriptor(format!("encode Kubernetes outer fence evidence: {error}")) diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 246f85de72..03ffdbd29d 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -183,12 +183,7 @@ impl OpenShellRuntimeBackend { ) -> Result<(Option, bool), BackendError> { let mut client = BoundaryClient::new(descriptor, bearer); client.transport = transport; - match client.call_idempotent(Request::DiscoverPolicy).await? { - Response::ImagePolicy { yaml, invalid } => Ok((yaml, invalid)), - _ => Err(BackendError::Descriptor( - "expected image policy discovery response".to_string(), - )), - } + client.discover_policy().await } pub fn new( @@ -1966,7 +1961,8 @@ impl BoundaryClient { let transport = Arc::new(TransportAbort::default()); let connector_transport = transport.clone(); let authority = match &runtime_descriptor.transport { - SandboxTransport::Tcp { authority, .. } => authority.as_str(), + SandboxTransport::Tcp { authority, .. } + | SandboxTransport::RuntimeInitiatedTcp { authority, .. } => authority.as_str(), SandboxTransport::Unix { .. } | SandboxTransport::Vsock { .. } => { "boundary.openshell.internal" } @@ -2610,6 +2606,7 @@ mod tests { let result = OpenShellRuntimeBackend::discover_policy( tls_runtime_descriptor(address, client_tls), test_bearer(&"a".repeat(32)), + BoundaryTransport::default(), ) .await; assert!( @@ -2652,6 +2649,7 @@ mod tests { let result = OpenShellRuntimeBackend::discover_policy( tls_runtime_descriptor(address, client_tls), test_bearer(&"a".repeat(32)), + BoundaryTransport::default(), ) .await; assert!( diff --git a/crates/openshell-sandbox-backend/src/runtime/test_support.rs b/crates/openshell-sandbox-backend/src/runtime/test_support.rs index 464bcc83bf..8d29407300 100644 --- a/crates/openshell-sandbox-backend/src/runtime/test_support.rs +++ b/crates/openshell-sandbox-backend/src/runtime/test_support.rs @@ -723,7 +723,7 @@ fn client(peer: Peer) -> Arc { .unwrap(); let mut client = BoundaryClient::with_backend(descriptor, bearer, crate::BACKEND_NAME.into(), vec![]); - client.connector = Some(Arc::new(Connector { + client.transport = crate::BoundaryTransport::connector(Arc::new(Connector { peer, address, task, @@ -861,7 +861,7 @@ pub fn backend( ) .unwrap(); let mut backend = crate::OpenShellRuntimeBackend::new(ca, credentials, bearer); - backend.connector.clone_from(&peer.connector); + backend.transport.clone_from(&peer.transport); backend.delegated = Some(crate::runtime::DelegatedRuntime { backend_name: name, descriptor, From 32ced96aee43fbc33ef32a17a952c0301ef95aec Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 12:08:27 -0700 Subject: [PATCH 7/9] feat(sandbox): initiate supervisor connections across compute drivers Signed-off-by: Drew Newberry --- .../src/isolation_registration.rs | 19 +- crates/openshell-driver-docker/README.md | 8 +- .../openshell-driver-docker/src/isolation.rs | 4 +- crates/openshell-driver-docker/src/lib.rs | 119 +++++++++-- crates/openshell-driver-docker/src/tests.rs | 2 +- crates/openshell-driver-podman/README.md | 7 + .../openshell-driver-podman/src/container.rs | 13 +- crates/openshell-driver-podman/src/driver.rs | 60 +++++- crates/openshell-driver-podman/src/grpc.rs | 3 +- .../openshell-driver-podman/src/isolation.rs | 51 ++++- crates/openshell-driver-vm/README.md | 8 + crates/openshell-driver-vm/src/driver.rs | 30 +-- .../openshell-driver-vm/src/isolation/mod.rs | 20 +- crates/openshell-driver-vm/src/main.rs | 5 +- crates/openshell-driver-vm/src/runtime.rs | 4 +- crates/openshell-sandbox-backend/Cargo.toml | 2 - crates/openshell-sandbox-backend/README.md | 5 + .../src/boundary_protocol.rs | 10 + .../openshell-sandbox-backend/src/runtime.rs | 15 +- .../src/runtime_initiated.rs | 196 +++++++++++++++--- .../openshell-sandbox/src/boundary_server.rs | 118 ++++++++--- skills/debug-openshell-cluster/SKILL.md | 7 + 22 files changed, 576 insertions(+), 130 deletions(-) diff --git a/crates/openshell-core/src/isolation_registration.rs b/crates/openshell-core/src/isolation_registration.rs index 6ef2032462..25b364e1af 100644 --- a/crates/openshell-core/src/isolation_registration.rs +++ b/crates/openshell-core/src/isolation_registration.rs @@ -35,12 +35,21 @@ pub enum SandboxTransport { authority: String, listen_address: SocketAddr, }, + RuntimeInitiatedUnix { + socket_path: PathBuf, + }, + RuntimeInitiatedVsock { + guest_cid: u32, + port: u32, + }, } impl SandboxTransport { pub fn validate(&self) -> Result<(), &'static str> { let valid = match self { - Self::Unix { socket_path } => socket_path.is_absolute(), + Self::Unix { socket_path } | Self::RuntimeInitiatedUnix { socket_path } => { + socket_path.is_absolute() + } Self::Tcp { authority, addresses, @@ -55,7 +64,7 @@ impl SandboxTransport { authority, listen_address, } => !authority.is_empty() && listen_address.port() != 0, - Self::Vsock { guest_cid, port } => { + Self::Vsock { guest_cid, port } | Self::RuntimeInitiatedVsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } }; @@ -95,6 +104,8 @@ pub fn validate_registrations(registrations: &[BackendRegistration]) -> Result<( if matches!( registration.endpoint, SandboxTransport::RuntimeInitiatedTcp { .. } + | SandboxTransport::RuntimeInitiatedUnix { .. } + | SandboxTransport::RuntimeInitiatedVsock { .. } ) { return Err("runtime-initiated control is not a registered backend endpoint"); } @@ -117,7 +128,9 @@ impl TryFrom<&BackendRegistration> for crate::proto::compute::v1::IsolationBacke validate_backend_name(®istration.name)?; registration.endpoint.validate()?; let endpoint = match ®istration.endpoint { - SandboxTransport::RuntimeInitiatedTcp { .. } => { + SandboxTransport::RuntimeInitiatedTcp { .. } + | SandboxTransport::RuntimeInitiatedUnix { .. } + | SandboxTransport::RuntimeInitiatedVsock { .. } => { return Err("runtime-initiated control is not a registered backend endpoint"); } SandboxTransport::Unix { socket_path } => Endpoint::UnixSocketPath( diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index ee77408fdc..94c2b1be79 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -108,10 +108,16 @@ LSM decisions remain authoritative. | `network_mode = host` on the supervisor | Lets the trusted supervisor reach the gateway's primary loopback listener and originate approved upstream connections. | | `restart_policy = no` | Keeps canonical main-process exit terminal. | | `PidsLimit` | Applies the configured sandbox PID budget. Omit `sandbox_pids_limit` to use OpenShell's default. Explicit zero is invalid. | -| Private named volumes | One carries the authenticated sandbox/supervisor channel. The other is mounted only into the supervisor and contains its JWT and private gateway credentials. | +| Private named volumes | Separate volumes carry sandbox bootstrap keys, the runtime-initiated Unix socket, and supervisor-only credentials. The supervisor mounts bootstrap keys read-only and only the socket volume writable. | | In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | +New containers dial the supervisor at `/.openshell/channel/control/control.sock`. +The runtime remains the TLS/gRPC server; the supervisor verifies its pinned +identity and supplies the existing bearer credential. Saved descriptors retain +the legacy Unix listener for existing containers, including across restart. +New generations require matching supervisor and sandbox runtime binaries. + ## Stop, Start, and Delete Stop terminates the supervisor companion and stops the workload container diff --git a/crates/openshell-driver-docker/src/isolation.rs b/crates/openshell-driver-docker/src/isolation.rs index a817ec484b..9935ff0cbe 100644 --- a/crates/openshell-driver-docker/src/isolation.rs +++ b/crates/openshell-driver-docker/src/isolation.rs @@ -109,7 +109,7 @@ impl DockerBoundarySpec { auth_epoch: self.auth_epoch, gateway_id: self.gateway_id, verification_keys: self.verification_keys, - listener: BoundaryListener::Unix { + listener: BoundaryListener::RuntimeInitiatedUnix { socket_path: self.listener_socket, tls: self.sandbox_tls, }, @@ -124,7 +124,7 @@ impl DockerBoundarySpec { generation: self.generation, session_id: self.session_id, workload_identity: self.workload_identity, - transport: SandboxTransport::Unix { + transport: SandboxTransport::RuntimeInitiatedUnix { socket_path: self.control_socket, }, tls: self.supervisor_tls, diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 059ba1535c..8c1a2e3aa2 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -107,7 +107,8 @@ const SUPERVISOR_UID: u32 = 65_534; const SUPERVISOR_GID: u32 = 65_534; const BOUNDARY_MOUNT_PATH: &str = "/.openshell/channel"; const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; -const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/sandbox/control.sock"; +const BOUNDARY_CONTROL_MOUNT_PATH: &str = "/.openshell/channel/control"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/control/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; @@ -1309,7 +1310,9 @@ impl DockerComputeDriver { Status::unavailable("volume admission lookup failed") } })?; - if name == docker_channel_volume_name_by_id(sandbox_id, &self.config) { + if name == docker_channel_volume_name_by_id(sandbox_id, &self.config) + || name == docker_control_volume_name_by_id(sandbox_id, &self.config) + { if volume.driver != "local" || !volume.options.is_empty() || volume.labels.get(LABEL_SANDBOX_ID) != Some(sandbox_id) @@ -4126,28 +4129,36 @@ fn docker_supervisor_volume_name_by_id( format!("openshell-supervisor-{}", &digest[..32]) } +fn docker_control_volume_name_by_id( + sandbox_id: &str, + config: &DockerDriverRuntimeConfig, +) -> String { + docker_channel_volume_name_by_id(sandbox_id, config).replacen( + "openshell-channel-", + "openshell-control-", + 1, + ) +} + async fn create_docker_channel_volume( docker: &Docker, sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig, ) -> Result<(), Status> { - create_docker_runtime_volume( - docker, - sandbox, - config, + let names = [ docker_channel_volume_name(sandbox, config), - ) - .await?; - if let Err(error) = create_docker_runtime_volume( - docker, - sandbox, - config, docker_supervisor_volume_name(sandbox, config), - ) - .await - { - let _ = remove_docker_volume(docker, &docker_channel_volume_name(sandbox, config)).await; - return Err(error); + docker_control_volume_name_by_id(&sandbox.id, config), + ]; + for (index, name) in names.iter().enumerate() { + if let Err(error) = + create_docker_runtime_volume(docker, sandbox, config, name.clone()).await + { + for created in &names[..index] { + let _ = remove_docker_volume(docker, created).await; + } + return Err(error); + } } Ok(()) } @@ -4213,6 +4224,11 @@ async fn remove_docker_channel_volume_by_id( docker, &docker_supervisor_volume_name_by_id(sandbox_id, config), ) + .await?; + remove_docker_volume( + docker, + &docker_control_volume_name_by_id(sandbox_id, config), + ) .await } @@ -4472,6 +4488,15 @@ fn docker_sandbox_bundle_archive( identity.uid, identity.gid, )?; + // The supervisor writes only the separate socket volume; its bootstrap + // and runtime key mounts stay read-only. + append_docker_archive_directory( + &mut archive, + ".openshell/channel/control", + 0o711, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; append_docker_archive_file( &mut archive, ".openshell/runtime/openshell-sandbox", @@ -4551,7 +4576,37 @@ async fn stage_docker_sandbox_bundle( bollard::body_full(Bytes::from(archive)), ) .await - .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}"))) + .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}")))?; + let bootstrap: BoundaryConfig = serde_json::from_slice(boundary_config) + .map_err(|error| Status::internal(format!("decode Docker sandbox bootstrap: {error}")))?; + if matches!( + bootstrap.listener, + openshell_sandbox_backend::boundary_protocol::BoundaryListener::RuntimeInitiatedUnix { .. } + ) { + // Docker resolves an archive destination to one volume. Initialize the + // nested socket volume directly instead of writing into its shadowed + // mountpoint in the read-only bootstrap volume. + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory(&mut archive, ".", 0o711, SUPERVISOR_UID, SUPERVISOR_GID)?; + let bytes = archive + .into_inner() + .map_err(|error| Status::internal(format!("finish control volume archive: {error}")))?; + let options = UploadToContainerOptionsBuilder::default() + .path(BOUNDARY_CONTROL_MOUNT_PATH) + .copy_uidgid("true") + .build(); + docker + .upload_to_container( + container_id, + Some(options), + bollard::body_full(Bytes::from(bytes)), + ) + .await + .map_err(|error| { + Status::internal(format!("stage Docker control socket volume: {error}")) + })?; + } + Ok(()) } fn decode_docker_launch_authentication( @@ -5245,6 +5300,8 @@ async fn spawn_docker_control_process( &config.upstream_proxy, config.proxy_ca_bundle.is_some(), )); + let runtime_initiated = read_docker_runtime_descriptor(&sandbox.id, config).await? + .is_some_and(|descriptor| matches!(descriptor.transport, openshell_sandbox_backend::boundary_protocol::SandboxTransport::RuntimeInitiatedUnix { .. })); let mut supervisor_mounts = vec![ Mount { target: Some(BOUNDARY_MOUNT_PATH.to_string()), @@ -5269,6 +5326,20 @@ async fn spawn_docker_control_process( ..Default::default() }, ]; + if runtime_initiated { + supervisor_mounts.push(Mount { + target: Some(BOUNDARY_CONTROL_MOUNT_PATH.to_string()), + source: Some(docker_control_volume_name_by_id(&sandbox.id, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }); + } + if let Some(socket) = config.provider_spiffe_workload_api_socket.as_ref() { let parent = socket.parent().ok_or_else(|| { Status::failed_precondition("provider SPIFFE socket has no parent directory") @@ -5931,6 +6002,18 @@ fn build_container_create_body_for_image( }), ..Default::default() }); + user_mounts.push(Mount { + target: Some(BOUNDARY_CONTROL_MOUNT_PATH.to_string()), + source: Some(docker_control_volume_name_by_id(&sandbox.id, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }); + let user_bind_strings = docker_driver_bind_strings(driver_config)?; let device_requests = gpu_device_ids.map(|device_ids| { vec![DeviceRequest { diff --git a/crates/openshell-driver-docker/src/tests.rs b/crates/openshell-driver-docker/src/tests.rs index 0159ecd99f..b679a7f5a6 100644 --- a/crates/openshell-driver-docker/src/tests.rs +++ b/crates/openshell-driver-docker/src/tests.rs @@ -2056,7 +2056,7 @@ fn build_container_create_body_includes_driver_config_mounts() { .mounts .expect("driver config mounts should be set"); - assert_eq!(mounts.len(), 3); + assert_eq!(mounts.len(), 4); assert_eq!(mounts[0].typ, Some(MountTypeEnum::VOLUME)); assert_eq!(mounts[0].source.as_deref(), Some("work-nfs")); assert_eq!(mounts[0].target.as_deref(), Some("/sandbox/work")); diff --git a/crates/openshell-driver-podman/README.md b/crates/openshell-driver-podman/README.md index f7ceed653b..f582cb03b3 100644 --- a/crates/openshell-driver-podman/README.md +++ b/crates/openshell-driver-podman/README.md @@ -7,6 +7,13 @@ REST API over a Unix socket. Each sandbox has two independent containers: - `openshell-supervisor` evaluates policy, holds gateway credentials, and proxies approved egress in a separate companion container. +New containers dial the supervisor over a separate private Unix socket volume +at `/.openshell/channel/control/control.sock`; workload networking remains +`none`. The supervisor's bootstrap-key mount stays read-only. Restart metadata +records the direction, with missing metadata retaining the legacy runtime +listener. Deploy matching supervisor and sandbox runtime binaries for new +containers. + The driver provisions placement, identity, credentials, transport, and lifecycle. The shared isolation interface supplies exec, attach, signal, terminate, binary identity, DNS, TCP, and loopback-forwarding semantics. diff --git a/crates/openshell-driver-podman/src/container.rs b/crates/openshell-driver-podman/src/container.rs index 9c3841d000..8e7a9f4228 100644 --- a/crates/openshell-driver-podman/src/container.rs +++ b/crates/openshell-driver-podman/src/container.rs @@ -1677,6 +1677,12 @@ pub fn build_isolation_specs( options: vec!["rw".into(), "z".into()], }); + workload.volumes.push(NamedVolume { + name: crate::isolation::control_volume_name(&input.sandbox.id), + dest: crate::isolation::CONTROL_ROOT.into(), + options: vec!["rw".into(), "z".into()], + }); + supervisor.name = crate::isolation::supervisor_name(&input.sandbox.id); supervisor .labels @@ -1732,6 +1738,11 @@ pub fn build_isolation_specs( dest: crate::isolation::CHANNEL_ROOT.into(), options: vec!["ro".into(), "z".into()], }]; + supervisor.volumes.push(NamedVolume { + name: crate::isolation::control_volume_name(&input.sandbox.id), + dest: crate::isolation::CONTROL_ROOT.into(), + options: vec!["rw".into(), "z".into()], + }); supervisor.mounts.retain(|mount| { trusted_mount(&mount.destination) && mount.destination != openshell_core::container_paths::NETNS_MOUNT_ROOT @@ -2061,7 +2072,7 @@ mod tests { assert_eq!(specs.supervisor.secrets.len(), 1); assert_eq!(specs.supervisor.secrets[0].source, "jwt"); assert_eq!(specs.supervisor.secrets[0].uid, 1000); - assert_eq!(specs.supervisor.volumes.len(), 1); + assert_eq!(specs.supervisor.volumes.len(), 2); assert!(specs.workload.volumes[0].options.contains(&"rw".into())); assert!(!specs.workload.volumes[0].options.contains(&"nocopy".into())); assert!(specs.supervisor.volumes[0].options.contains(&"ro".into())); diff --git a/crates/openshell-driver-podman/src/driver.rs b/crates/openshell-driver-podman/src/driver.rs index b6bbe6a36f..06d4d7b6fd 100644 --- a/crates/openshell-driver-podman/src/driver.rs +++ b/crates/openshell-driver-podman/src/driver.rs @@ -829,7 +829,9 @@ impl PodmanComputeDriver { return Err(missing()); } let workspace_volume = name == container::volume_name(sandbox_id); - if workspace_volume || name == crate::isolation::channel_volume_name(sandbox_id) + if workspace_volume + || name == crate::isolation::channel_volume_name(sandbox_id) + || name == crate::isolation::control_volume_name(sandbox_id) { let owned = volume.labels.as_ref().is_some_and(|labels| { labels.get(LABEL_SANDBOX_ID) == Some(sandbox_id) @@ -1064,6 +1066,7 @@ impl PodmanComputeDriver { .resolve_workload_identity(sandbox, &resolved_image.id, &resolved_image.oci_user) .await?; let channel_volume = crate::isolation::channel_volume_name(&sandbox.id); + let control_volume = crate::isolation::control_volume_name(&sandbox.id); let mut runtime_config = self.config.clone(); runtime_config.sandbox_runtime_image = self .client @@ -1157,6 +1160,7 @@ impl PodmanComputeDriver { let cleanup_created = || async { if channel_owned.load(std::sync::atomic::Ordering::Relaxed) { let _ = self.client.remove_volume(&channel_volume).await; + let _ = self.client.remove_volume(&control_volume).await; } let _ = self.client.remove_volume(&vol_name).await; cleanup_sandbox_resolver_secret(&self.client, &resolver_secret_name).await; @@ -1240,6 +1244,9 @@ impl PodmanComputeDriver { .create_owned_volume(&channel_volume, &sandbox.id, &sandbox.workspace, None) .await?; channel_owned.store(true, std::sync::atomic::Ordering::Relaxed); + self.client + .create_owned_volume(&control_volume, &sandbox.id, &sandbox.workspace, None) + .await?; let workload_id = self.client.create_typed_container(&specs.workload).await?; created_workload = Some(workload_id.clone()); self.client.verify_isolation_fence(&workload_id).await?; @@ -1260,6 +1267,7 @@ impl PodmanComputeDriver { let generation = uuid::Uuid::new_v4().to_string(); let archives = crate::isolation::bootstrap_archives( crate::isolation::BootstrapArchivesInput { + runtime_initiated: true, sandbox_id: &sandbox.id, container_id: &workload_id, generation: &generation, @@ -1280,6 +1288,15 @@ impl PodmanComputeDriver { archives.channel, ) .await?; + if let Some(control) = archives.control { + self.client + .copy_to_container( + &workload_id, + crate::isolation::CONTROL_ROOT, + control, + ) + .await?; + } if managed_workspace { self.client .copy_to_container(&workload_id, "/sandbox", archives.workspace) @@ -1611,6 +1628,7 @@ impl PodmanComputeDriver { let restart_metadata = crate::isolation::restart_metadata_from_slice(&bundle)?; let archives = crate::isolation::bootstrap_archives(crate::isolation::BootstrapArchivesInput { + runtime_initiated: restart_metadata.runtime_initiated, sandbox_id, container_id: &container_id, generation: generation.as_str(), @@ -1633,6 +1651,11 @@ impl PodmanComputeDriver { archives.channel, ) .await?; + if let Some(control) = archives.control { + self.client + .copy_to_container(&container_id, crate::isolation::CONTROL_ROOT, control) + .await?; + } self.client .copy_to_container(&supervisor, "/", archives.supervisor) .await?; @@ -1687,6 +1710,10 @@ impl PodmanComputeDriver { let Some(container_id) = self.find_container_id(sandbox_id).await? else { debug!(sandbox_id = %sandbox_id, "Sandbox container not found (already deleted)"); + let _ = self + .client + .remove_volume(&crate::isolation::control_volume_name(sandbox_id)) + .await; let vol = container::volume_name(sandbox_id); if let Err(e) = self.client.remove_volume(&vol).await { warn!(sandbox_id = %sandbox_id, volume = %vol, error = %e, "Failed to remove workspace volume"); @@ -1722,6 +1749,14 @@ impl PodmanComputeDriver { Err(e) => return Err(ComputeDriverError::from(e)), }; + if let Err(error) = self + .client + .remove_volume(&crate::isolation::control_volume_name(sandbox_id)) + .await + && !matches!(error, PodmanApiError::NotFound(_)) + { + warn!(%sandbox_id, %error, "Failed to remove private control volume"); + } // Remove workspace volume. if let Err(error) = self .client @@ -2527,6 +2562,7 @@ mod tests { uploads, [ "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel", + "/libpod/containers/workload/archive?path=%2F.openshell%2Fchannel%2Fcontrol", "/libpod/containers/workload/archive?path=%2Fsandbox", "/libpod/containers/supervisor/archive?path=%2F", ] @@ -3526,6 +3562,7 @@ mod tests { StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // control volume // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -3545,7 +3582,7 @@ mod tests { .clone(); assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[3], + requests[4], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) @@ -3662,6 +3699,7 @@ mod tests { ) .unwrap(); let bundle = serde_json::to_vec(&crate::isolation::RestartMetadata { + runtime_initiated: false, generation: "generation-1".to_string(), workload_identity: identity, child_env: HashMap::new(), @@ -3810,6 +3848,13 @@ mod tests { sandbox_id, None, )); + responses.push(StubResponse::new(StatusCode::NOT_FOUND, "")); + responses.push(StubResponse::new(StatusCode::CREATED, "{}")); + responses.push(owned_volume_response( + &crate::isolation::control_volume_name(sandbox_id), + sandbox_id, + None, + )); responses } @@ -3843,6 +3888,7 @@ mod tests { created_response("workload"), fence_response(), StubResponse::new(StatusCode::OK, "").with_archive_members(channel_archive_members()), + StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), // control volume StubResponse::new(StatusCode::OK, "").with_archive_members(&["."]), created_response("supervisor"), StubResponse::new(StatusCode::OK, ""), // supervisor archive @@ -3905,6 +3951,7 @@ mod tests { .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "create failed"), StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // control StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace StubResponse::new(StatusCode::NO_CONTENT, ""), // resolver secret StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret @@ -3950,12 +3997,13 @@ mod tests { "create-start-fail", create_setup_responses(true, sandbox_id) .into_iter() - .chain(create_launch_responses().into_iter().take(7)) + .chain(create_launch_responses().into_iter().take(8)) .chain([ StubResponse::new(StatusCode::INTERNAL_SERVER_ERROR, "supervisor start failed"), StubResponse::new(StatusCode::NO_CONTENT, ""), // supervisor StubResponse::new(StatusCode::NO_CONTENT, ""), // workload StubResponse::new(StatusCode::NO_CONTENT, ""), // channel + StubResponse::new(StatusCode::NO_CONTENT, ""), // control StubResponse::new(StatusCode::NO_CONTENT, ""), // workspace StubResponse::new(StatusCode::NO_CONTENT, ""), // resolver secret StubResponse::new(StatusCode::NO_CONTENT, ""), // proxy secret @@ -4001,7 +4049,8 @@ mod tests { vec![ StubResponse::new(StatusCode::NO_CONTENT, ""), // remove companion StubResponse::new(StatusCode::NO_CONTENT, ""), // remove channel if detached - StubResponse::new(StatusCode::OK, "[]"), // list_containers (not found) + StubResponse::new(StatusCode::OK, "[]"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // control volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove volume StubResponse::new(StatusCode::NO_CONTENT, ""), // remove token secret StubResponse::new(StatusCode::NO_CONTENT, ""), // remove resolver secret @@ -4055,6 +4104,7 @@ mod tests { StubResponse::new(StatusCode::OK, list_body), // single timed remove_container operation StubResponse::new(StatusCode::NO_CONTENT, ""), + StubResponse::new(StatusCode::NO_CONTENT, ""), // control volume // channel volume, now detached StubResponse::new(StatusCode::NO_CONTENT, ""), // remove_volume @@ -4085,7 +4135,7 @@ mod tests { ) ); assert_eq!( - requests[5], + requests[6], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/grpc.rs b/crates/openshell-driver-podman/src/grpc.rs index a6d208f707..25cfd09d7a 100644 --- a/crates/openshell-driver-podman/src/grpc.rs +++ b/crates/openshell-driver-podman/src/grpc.rs @@ -683,6 +683,7 @@ mod tests { StubResponse::new(StatusCode::NO_CONTENT, ""), // channel // list_containers returns empty (container already gone) StubResponse::new(StatusCode::OK, "[]"), + StubResponse::new(StatusCode::NO_CONTENT, ""), // control volume // remove_volume StubResponse::new(StatusCode::NO_CONTENT, ""), ], @@ -711,7 +712,7 @@ mod tests { .clone(); assert!(requests[2].contains("/libpod/containers/json")); assert_eq!( - requests[3], + requests[4], format!( "DELETE {}", api_path(&format!("/libpod/volumes/{volume_name}")) diff --git a/crates/openshell-driver-podman/src/isolation.rs b/crates/openshell-driver-podman/src/isolation.rs index 0ee226b446..6da06a5773 100644 --- a/crates/openshell-driver-podman/src/isolation.rs +++ b/crates/openshell-driver-podman/src/isolation.rs @@ -26,6 +26,8 @@ pub const BOOTSTRAP_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; pub const RUNTIME_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json"; pub const AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json"; pub const RESTART_METADATA_PATH: &str = "/.openshell/supervisor/restart-metadata.json"; +pub const CONTROL_ROOT: &str = "/.openshell/channel/control"; +const RUNTIME_INITIATED_SOCKET: &str = "/.openshell/channel/control/control.sock"; const SOCKET_PATH: &str = "/.openshell/channel/sandbox/control.sock"; #[derive(Serialize)] @@ -54,6 +56,9 @@ impl PodmanOuterFenceEvidence<'_> { pub fn supervisor_name(id: &str) -> String { format!("openshell-supervisor-{id}") } +pub fn control_volume_name(id: &str) -> String { + format!("openshell-control-{id}") +} pub fn channel_volume_name(id: &str) -> String { format!("openshell-channel-{id}") } @@ -171,6 +176,7 @@ pub fn resolve_identity( pub struct BootstrapArchives { pub channel: Vec, + pub control: Option>, pub workspace: Vec, pub supervisor: Vec, } @@ -178,11 +184,14 @@ pub struct BootstrapArchives { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct RestartMetadata { pub(crate) generation: String, + #[serde(default)] + pub(crate) runtime_initiated: bool, pub(crate) workload_identity: ResolvedWorkloadIdentity, pub(crate) child_env: HashMap, } pub struct BootstrapArchivesInput<'a> { + pub runtime_initiated: bool, pub sandbox_id: &'a str, pub container_id: &'a str, pub generation: &'a str, @@ -199,6 +208,7 @@ pub fn bootstrap_archives( input: BootstrapArchivesInput<'_>, ) -> Result { let BootstrapArchivesInput { + runtime_initiated, sandbox_id, container_id, generation, @@ -255,12 +265,22 @@ pub fn bootstrap_archives( auth_epoch: launch_authentication.supervisor.auth_epoch, gateway_id: launch_authentication.gateway_id.clone(), verification_keys, - listener: BoundaryListener::Unix { - socket_path: PathBuf::from(SOCKET_PATH), - tls: SandboxTlsServerConfig { + listener: { + let tls = SandboxTlsServerConfig { certificate_chain_path: PathBuf::from("/.openshell/channel/sandbox/server.crt"), private_key_path: PathBuf::from("/.openshell/channel/sandbox/server.key"), - }, + }; + if runtime_initiated { + BoundaryListener::RuntimeInitiatedUnix { + socket_path: RUNTIME_INITIATED_SOCKET.into(), + tls, + } + } else { + BoundaryListener::Unix { + socket_path: SOCKET_PATH.into(), + tls, + } + } }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), @@ -272,8 +292,14 @@ pub fn bootstrap_archives( boundary_id: sandbox_id.into(), generation: runtime_generation, session_id, - transport: SandboxTransport::Unix { - socket_path: PathBuf::from(SOCKET_PATH), + transport: if runtime_initiated { + SandboxTransport::RuntimeInitiatedUnix { + socket_path: RUNTIME_INITIATED_SOCKET.into(), + } + } else { + SandboxTransport::Unix { + socket_path: SOCKET_PATH.into(), + } }, tls: SandboxTlsClientConfig { server_name: tls.server_name, @@ -311,6 +337,7 @@ pub fn bootstrap_archives( &serde_json::to_vec(&launch_authentication.supervisor).map_err(invalid)?, )?; let restart_metadata = RestartMetadata { + runtime_initiated, generation: generation.to_string(), workload_identity: identity.clone(), child_env, @@ -319,7 +346,18 @@ pub fn bootstrap_archives( RESTART_METADATA_PATH, &serde_json::to_vec(&restart_metadata).map_err(invalid)?, )?; + let control = if runtime_initiated { + let mut archive = Archive::new(identity); + // The containers can use different rootless user-namespace mappings. + // Only this socket volume is writable; keys stay in the read-only + // bootstrap volume. The sticky bit protects existing socket inodes. + archive.directory(".", 0o1777, false)?; + Some(archive.finish()?) + } else { + None + }; Ok(BootstrapArchives { + control, channel, workspace: workspace.finish()?, supervisor: supervisor.finish()?, @@ -510,6 +548,7 @@ mod tests { let authentication = authentication(); let child_env = HashMap::from([("PATH".to_string(), "/agent/bin".to_string())]); let archives = bootstrap_archives(BootstrapArchivesInput { + runtime_initiated: false, sandbox_id: "sandbox", container_id: "container", generation: "generation-1", diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index fa05c48efc..deb4ca9dae 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -4,6 +4,14 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess and talks to it over the `openshell.compute.v1.ComputeDriver` Unix-socket surface. `openshell-supervisor` runs as a native host process, while `openshell-sandbox` runs as capability-free PID 1 inside each microVM and applies guest-local isolation over virtio-vsock. +New VM generations initiate the control connection to host CID 2 over vsock. +Libkrun forwards that guest connection to the supervisor's protected Unix +socket; QEMU uses a host vsock listener on a per-guest port and checks the guest +CID before TLS authentication. Existing protocol roles and authorization stay +unchanged. Running legacy generations retain their original descriptors; +newly booted generations require matching embedded runtime and supervisor +binaries. + The driver embeds libkrun, libkrunfw, the guest OCI unpacker, the portable guest sandbox, and the custom kernel runtime. Each sandbox boots from a cached immutable bootstrap ext4 root disk plus a per-sandbox writable overlay disk. When the requested sandbox image differs from the bootstrap image, the driver prepares a read-only image ext4 disk inside a bootstrap VM and mounts that unpacked rootfs as the sandbox lowerdir. ## How it fits together diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index a25ad5ab07..7bd6f39a91 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -1633,14 +1633,20 @@ impl VmDriver { trust_anchor_pem: channel_tls.trust_anchor_pem.clone(), }; let transport = if plan.backend == VmBackend::Qemu { - SandboxTransport::Vsock { - guest_cid: plan.vsock_cid.ok_or_else(|| { - Status::internal("QEMU launch plan is missing a guest vsock CID") - })?, - port: VM_CONTROL_PORT, + let guest_cid = plan + .vsock_cid + .ok_or_else(|| Status::internal("QEMU launch plan is missing a guest vsock CID"))?; + SandboxTransport::RuntimeInitiatedVsock { + guest_cid, + // Each QEMU guest connects to a distinct host port. The CID + // allocator already guarantees uniqueness among live guests. + port: VM_CONTROL_PORT + .checked_add(guest_cid) + .filter(|port| *port != u32::MAX) + .ok_or_else(|| Status::resource_exhausted("VM control port space exhausted"))?, } } else { - SandboxTransport::Unix { + SandboxTransport::RuntimeInitiatedUnix { socket_path: control_socket.clone(), } }; @@ -1730,6 +1736,7 @@ impl VmDriver { } else { let _ = tokio::fs::remove_file(&control_socket).await; command + .arg("--vm-vsock-runtime-initiated") .arg("--vm-vsock-control-port") .arg(VM_CONTROL_PORT.to_string()) .arg("--vm-vsock-control-socket") @@ -7111,13 +7118,10 @@ fn inject_guest_boundary_bundle( let tls = match &config.listener { BoundaryListener::Unix { tls, .. } | BoundaryListener::TlsTcp { tls, .. } - | BoundaryListener::Vsock { tls, .. } => tls.clone(), - // The VM driver provisions only listener transports. - BoundaryListener::RuntimeInitiatedTlsTcp { .. } => { - return Err( - "VM sandboxes do not support runtime-initiated boundary control".to_string(), - ); - } + | BoundaryListener::Vsock { tls, .. } + | BoundaryListener::RuntimeInitiatedTlsTcp { tls, .. } + | BoundaryListener::RuntimeInitiatedUnix { tls, .. } + | BoundaryListener::RuntimeInitiatedVsock { tls, .. } => tls.clone(), }; let encoded_config = config .encode() diff --git a/crates/openshell-driver-vm/src/isolation/mod.rs b/crates/openshell-driver-vm/src/isolation/mod.rs index dc7426d229..f52456b50e 100644 --- a/crates/openshell-driver-vm/src/isolation/mod.rs +++ b/crates/openshell-driver-vm/src/isolation/mod.rs @@ -100,9 +100,23 @@ impl VmBoundarySpec { auth_epoch: self.auth_epoch, gateway_id: self.gateway_id, verification_keys: self.verification_keys, - listener: BoundaryListener::Vsock { - control_port: self.control_port, - tls: self.sandbox_tls, + listener: match &self.transport { + SandboxTransport::RuntimeInitiatedVsock { port, .. } => { + BoundaryListener::RuntimeInitiatedVsock { + host_port: *port, + tls: self.sandbox_tls, + } + } + SandboxTransport::RuntimeInitiatedUnix { .. } => { + BoundaryListener::RuntimeInitiatedVsock { + host_port: self.control_port, + tls: self.sandbox_tls, + } + } + _ => BoundaryListener::Vsock { + control_port: self.control_port, + tls: self.sandbox_tls, + }, }, resource_claims: resource_claims.clone(), resource_claim_files: BTreeMap::new(), diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 796aad68d8..fb7c4accf2 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -263,6 +263,9 @@ struct Args { #[arg(long, hide = true)] vm_vsock_control_socket: Option, + #[arg(long, hide = true)] + vm_vsock_runtime_initiated: bool, + #[arg(long, hide = true)] vm_mount: Vec, } @@ -673,7 +676,7 @@ fn build_vm_launch_config(args: &Args) -> std::result::Result Some(VsockPortMap { guest_port, host_socket, - host_initiated: true, + host_initiated: !args.vm_vsock_runtime_initiated, }), (None, None) => None, _ => { diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index fbccba270a..ad300d98cf 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -345,7 +345,9 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { vm.disable_implicit_vsock()?; vm.add_vsock(0)?; if let Some(port_map) = &config.vsock_port_map { - let _ = std::fs::remove_file(&port_map.host_socket); + if port_map.host_initiated { + let _ = std::fs::remove_file(&port_map.host_socket); + } vm.add_vsock_port(port_map)?; } diff --git a/crates/openshell-sandbox-backend/Cargo.toml b/crates/openshell-sandbox-backend/Cargo.toml index 1f5908662d..15a0dd29f0 100644 --- a/crates/openshell-sandbox-backend/Cargo.toml +++ b/crates/openshell-sandbox-backend/Cargo.toml @@ -41,8 +41,6 @@ uuid = { workspace = true } [target.'cfg(unix)'.dependencies] libc = "0.2" - -[target.'cfg(target_os = "linux")'.dependencies] rustix = { workspace = true, features = ["fs", "process"] } [lints] diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index 50102fdbc3..01f89b3b7d 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -119,3 +119,8 @@ open, so reconnects and credential rotation never wait for a new dial. The new variants leave existing encodings unchanged. Binaries that predate them reject runtime-initiated descriptors instead of falling back to another mode. + +Runtime-initiated transport supports TCP, protected Unix sockets, and host +vsock listeners. These share bounded TLS handshakes, the authenticated spare +queue, and the same generation-pinned runtime identity check. The legacy +supervisor-dialed transports remain available for existing descriptors. diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index fc0b0b9ada..d4e2cf0ac7 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -363,6 +363,16 @@ pub enum BoundaryListener { addresses: Vec, tls: SandboxTlsServerConfig, }, + /// Dial the supervisor over the driver's protected shared Unix socket. + RuntimeInitiatedUnix { + socket_path: PathBuf, + tls: SandboxTlsServerConfig, + }, + /// Dial the host supervisor through the VM's vsock device. + RuntimeInitiatedVsock { + host_port: u32, + tls: SandboxTlsServerConfig, + }, } /// Public verification key staged in the sandbox's immutable auth bundle. diff --git a/crates/openshell-sandbox-backend/src/runtime.rs b/crates/openshell-sandbox-backend/src/runtime.rs index 03ffdbd29d..4bfa0c5a6f 100644 --- a/crates/openshell-sandbox-backend/src/runtime.rs +++ b/crates/openshell-sandbox-backend/src/runtime.rs @@ -105,7 +105,9 @@ impl BoundaryTransport { /// supervisor's Tokio runtime. pub fn for_descriptor(descriptor: &SandboxRuntimeDescriptor) -> Result { Ok(Self(match descriptor.transport { - SandboxTransport::RuntimeInitiatedTcp { .. } => TransportSource::RuntimeInitiated( + SandboxTransport::RuntimeInitiatedTcp { .. } + | SandboxTransport::RuntimeInitiatedUnix { .. } + | SandboxTransport::RuntimeInitiatedVsock { .. } => TransportSource::RuntimeInitiated( crate::runtime_initiated::RuntimeInitiatedListener::bind(descriptor)?, ), _ => TransportSource::Dial, @@ -1963,9 +1965,10 @@ impl BoundaryClient { let authority = match &runtime_descriptor.transport { SandboxTransport::Tcp { authority, .. } | SandboxTransport::RuntimeInitiatedTcp { authority, .. } => authority.as_str(), - SandboxTransport::Unix { .. } | SandboxTransport::Vsock { .. } => { - "boundary.openshell.internal" - } + SandboxTransport::Unix { .. } + | SandboxTransport::Vsock { .. } + | SandboxTransport::RuntimeInitiatedUnix { .. } + | SandboxTransport::RuntimeInitiatedVsock { .. } => "boundary.openshell.internal", }; let endpoint = tonic::transport::Endpoint::from_shared(format!("http://{authority}")) .map_err(|error| { @@ -2129,7 +2132,9 @@ async fn connect_boundary_using_connector( Box::new(stream) } SandboxTransport::Vsock { guest_cid, port } => connect_host_vsock(*guest_cid, *port)?, - SandboxTransport::RuntimeInitiatedTcp { .. } => { + SandboxTransport::RuntimeInitiatedTcp { .. } + | SandboxTransport::RuntimeInitiatedUnix { .. } + | SandboxTransport::RuntimeInitiatedVsock { .. } => { return Err(BackendError::Descriptor( "a runtime-initiated transport cannot be dialed".to_string(), )); diff --git a/crates/openshell-sandbox-backend/src/runtime_initiated.rs b/crates/openshell-sandbox-backend/src/runtime_initiated.rs index e867e580fb..8f2bf94e2c 100644 --- a/crates/openshell-sandbox-backend/src/runtime_initiated.rs +++ b/crates/openshell-sandbox-backend/src/runtime_initiated.rs @@ -3,7 +3,7 @@ //! Supervisor listener for runtime-initiated Sandbox Protocol connections. //! -//! Only the TCP dial direction is reversed. Each accepted socket completes the +//! Only connection establishment is reversed. Each accepted socket completes the //! same pinned TLS client handshake as a dialed connection before it is queued, //! and every request still carries the Sandbox Protocol bearer token. A peer //! address is never trusted. @@ -44,41 +44,48 @@ impl RuntimeInitiatedListener { pub fn bind( descriptor: &SandboxRuntimeDescriptor, ) -> Result, BackendError> { - match descriptor.transport { - SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } - if listen_address.port() != 0 => - { - Self::bind_address(listen_address, &descriptor.tls) + let runtime = tokio::runtime::Handle::try_current().map_err(|_| { + BackendError::Unavailable("runtime-initiated listener requires a Tokio runtime".into()) + })?; + let _entered = runtime.enter(); + crate::runtime::validate_client_tls(&descriptor.tls)?; + descriptor + .transport + .validate() + .map_err(|error| BackendError::Descriptor(error.into()))?; + let listener = match &descriptor.transport { + SandboxTransport::RuntimeInitiatedTcp { listen_address, .. } => { + Listener::Tcp(bind_tcp(*listen_address).map_err(unavailable)?) } - _ => Err(BackendError::Descriptor( - "runtime-initiated listener requires a nonzero listen port".to_string(), - )), - } + #[cfg(unix)] + SandboxTransport::RuntimeInitiatedUnix { socket_path } => { + Listener::bind_unix(socket_path).map_err(unavailable)? + } + #[cfg(target_os = "linux")] + SandboxTransport::RuntimeInitiatedVsock { guest_cid, port } => { + Listener::bind_vsock(*guest_cid, *port).map_err(unavailable)? + } + _ => { + return Err(BackendError::Descriptor( + "unsupported runtime-initiated transport on this platform".into(), + )); + } + }; + Self::start(listener, &descriptor.tls) } - pub(crate) fn bind_address( - address: SocketAddr, + fn start( + listener: Listener, tls: &SandboxTlsClientConfig, ) -> Result, BackendError> { let runtime = tokio::runtime::Handle::try_current().map_err(|_| { BackendError::Unavailable("runtime-initiated listener requires a Tokio runtime".into()) })?; - // Fail before opening a socket that could never authenticate a peer. - crate::runtime::validate_client_tls(tls)?; - let _entered = runtime.enter(); - let listener = bind_tcp(address).map_err(|error| { - BackendError::Unavailable(format!( - "bind runtime-initiated control listener {address}: {error}" - )) - })?; - let local_address = listener.local_addr().map_err(|error| { - BackendError::Unavailable(format!("read control listener address: {error}")) - })?; let (sender, spares) = mpsc::channel(MAX_QUEUED_CONNECTIONS); let accept_task = runtime .spawn(accept_loop(listener, tls.clone(), sender)) .abort_handle(); - tracing::info!(%local_address, "Runtime-initiated Sandbox Protocol listener ready"); + tracing::info!("Runtime-initiated Sandbox Protocol listener ready"); Ok(std::sync::Arc::new(Self { tls: tls.clone(), spares: tokio::sync::Mutex::new(spares), @@ -126,7 +133,7 @@ fn bind_tcp(address: SocketAddr) -> std::io::Result { /// streams that completed the pinned handshake. A full queue closes the new /// spare; the runtime dials again once a spare is used or lost. async fn accept_loop( - listener: tokio::net::TcpListener, + listener: Listener, tls: SandboxTlsClientConfig, spares: mpsc::Sender, ) { @@ -136,7 +143,7 @@ async fn accept_loop( loop { tokio::select! { accepted = listener.accept() => { - let (stream, peer) = match accepted { + let stream = match accepted { Ok(accepted) => accepted, Err(error) => { // Descriptor exhaustion and similar errors are transient. @@ -146,30 +153,28 @@ async fn accept_loop( } }; let Ok(permit) = pending.clone().try_acquire_owned() else { - tracing::debug!(%peer, "runtime-initiated handshake limit reached; closing connection"); + tracing::debug!("runtime-initiated handshake limit reached; closing connection"); continue; }; - openshell_core::net::set_tcp_nodelay_best_effort(&stream); - crate::runtime::enable_boundary_tcp_keepalive(&stream); let tls = tls.clone(); let spares = spares.clone(); handshakes.spawn(async move { let _permit = permit; let authenticated = tokio::time::timeout( HANDSHAKE_TIMEOUT, - crate::runtime::authenticate_boundary_stream(&tls, Box::new(stream)), + crate::runtime::authenticate_boundary_stream(&tls, stream), ) .await; match authenticated { Ok(Ok(stream)) => { if spares.try_send(stream).is_err() { - tracing::debug!(%peer, "runtime-initiated spare queue full; closing connection"); + tracing::debug!("runtime-initiated spare queue full; closing connection"); } } Ok(Err(error)) => { - tracing::debug!(%peer, %error, "rejected unauthenticated runtime-initiated connection"); + tracing::debug!(%error, "rejected unauthenticated runtime-initiated connection"); } - Err(_) => tracing::debug!(%peer, "runtime-initiated TLS handshake timed out"), + Err(_) => tracing::debug!("runtime-initiated TLS handshake timed out"), } }); } @@ -177,3 +182,128 @@ async fn accept_loop( } } } + +fn unavailable(error: std::io::Error) -> BackendError { + BackendError::Unavailable(format!("runtime-initiated control listener: {error}")) +} + +#[derive(Debug)] +enum Listener { + Tcp(tokio::net::TcpListener), + #[cfg(unix)] + Unix { + listener: tokio::net::UnixListener, + path: std::path::PathBuf, + inode: u64, + }, + #[cfg(target_os = "linux")] + Vsock { + listener: tokio::io::unix::AsyncFd, + guest_cid: u32, + }, +} + +impl Listener { + #[cfg(unix)] + fn bind_unix(path: &std::path::Path) -> std::io::Result { + use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; + // The driver owns the containing directory. Never replace a live socket, + // another user's inode, or a non-socket (including symlinks). + match std::fs::symlink_metadata(path) { + Ok(metadata) => { + if !metadata.file_type().is_socket() + || metadata.uid() != rustix::process::geteuid().as_raw() + { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "refusing to replace unowned control socket", + )); + } + let probe = + socket2::Socket::new(socket2::Domain::UNIX, socket2::Type::STREAM, None)?; + probe.set_nonblocking(true)?; + match probe.connect(&socket2::SockAddr::unix(path)?) { + Err(error) if error.kind() == std::io::ErrorKind::ConnectionRefused => { + std::fs::remove_file(path)?; + } + _ => { + return Err(std::io::Error::new( + std::io::ErrorKind::AddrInUse, + "control socket is still active", + )); + } + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error), + } + let listener = tokio::net::UnixListener::bind(path)?; + // Runtime and supervisor can have different UIDs. TLS pins the runtime; + // the protected mount path is inaccessible to workload code. + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o666))?; + let inode = std::fs::symlink_metadata(path)?.ino(); + Ok(Self::Unix { + listener, + path: path.into(), + inode, + }) + } + + #[cfg(target_os = "linux")] + fn bind_vsock(guest_cid: u32, port: u32) -> std::io::Result { + let socket = socket2::Socket::new(socket2::Domain::VSOCK, socket2::Type::STREAM, None)?; + socket.set_nonblocking(true)?; + socket.bind(&socket2::SockAddr::vsock(libc::VMADDR_CID_ANY, port))?; + socket.listen(64)?; + Ok(Self::Vsock { + listener: tokio::io::unix::AsyncFd::new(socket)?, + guest_cid, + }) + } + + async fn accept(&self) -> std::io::Result { + match self { + Self::Tcp(listener) => { + let (stream, _) = listener.accept().await?; + openshell_core::net::set_tcp_nodelay_best_effort(&stream); + crate::runtime::enable_boundary_tcp_keepalive(&stream); + Ok(Box::new(stream)) + } + #[cfg(unix)] + Self::Unix { listener, .. } => Ok(Box::new(listener.accept().await?.0)), + #[cfg(target_os = "linux")] + Self::Vsock { + listener, + guest_cid, + } => loop { + let mut ready = listener.readable().await?; + match ready.try_io(|fd| fd.get_ref().accept()) { + Ok(Ok((socket, address))) => { + if address.as_vsock_address().map(|(cid, _)| cid) != Some(*guest_cid) { + continue; + } + socket.set_nonblocking(true)?; + let stream = tokio::net::UnixStream::from_std( + std::os::unix::net::UnixStream::from(socket), + )?; + return Ok(Box::new(stream)); + } + Ok(Err(error)) => return Err(error), + Err(_) => {} + } + }, + } + } +} + +impl Drop for Listener { + fn drop(&mut self) { + #[cfg(unix)] + if let Self::Unix { path, inode, .. } = self { + use std::os::unix::fs::MetadataExt; + if std::fs::symlink_metadata(&*path).is_ok_and(|metadata| metadata.ino() == *inode) { + let _ = std::fs::remove_file(path); + } + } + } +} diff --git a/crates/openshell-sandbox/src/boundary_server.rs b/crates/openshell-sandbox/src/boundary_server.rs index 34a95c06e9..49e5690083 100644 --- a/crates/openshell-sandbox/src/boundary_server.rs +++ b/crates/openshell-sandbox/src/boundary_server.rs @@ -252,7 +252,10 @@ mod linux { BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { addresses.first().map(std::net::SocketAddr::port) } - BoundaryListenerConfig::Unix { .. } | BoundaryListenerConfig::Vsock { .. } => None, + BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::Vsock { .. } + | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } + | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } => None, }; let network_broker = NetworkBroker::start(listener, protected_control_port) .map_err(|error| format!("start sandbox network broker: {error}"))?; @@ -339,6 +342,7 @@ mod linux { } match &config.listener { BoundaryListenerConfig::Unix { socket_path, tls } + | BoundaryListenerConfig::RuntimeInitiatedUnix { socket_path, tls } if !socket_path.is_absolute() || !tls_paths_are_absolute(tls) => { return Err("boundary Unix socket path must be absolute".to_string()); @@ -366,12 +370,19 @@ mod linux { } => { return Err("boundary control port must be nonzero".to_string()); } + BoundaryListenerConfig::RuntimeInitiatedVsock { host_port, tls } + if *host_port == 0 || *host_port == u32::MAX || !tls_paths_are_absolute(tls) => + { + return Err("runtime-initiated vsock requires a valid host port and absolute certificate paths".into()); + } BoundaryListenerConfig::RuntimeInitiatedTlsTcp { authority, addresses, tls, } => validate_runtime_initiated_targets(authority, addresses, tls)?, BoundaryListenerConfig::Unix { .. } + | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } + | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } | BoundaryListenerConfig::TlsTcp { .. } | BoundaryListenerConfig::Vsock { .. } => {} } @@ -563,8 +574,14 @@ mod linux { } fn serve(config: &BoundaryListenerConfig, runtime: Arc) -> Result<(), String> { - if let BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, tls, .. } = config { - return serve_runtime_initiated(config, addresses.clone(), tls, runtime); + let dial_tls = match config { + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { tls, .. } + | BoundaryListenerConfig::RuntimeInitiatedUnix { tls, .. } + | BoundaryListenerConfig::RuntimeInitiatedVsock { tls, .. } => Some(tls), + _ => None, + }; + if let Some(tls) = dial_tls { + return serve_runtime_initiated(config, tls, runtime); } let listener = ControlListener::bind(config) .map_err(|error| format!("bind boundary control listener: {error}"))?; @@ -622,7 +639,6 @@ mod linux { /// TLS server and gRPC server roles are the same as for a listener. fn serve_runtime_initiated( config: &BoundaryListenerConfig, - addresses: Vec, tls: &openshell_sandbox_backend::boundary_protocol::SandboxTlsServerConfig, runtime: Arc, ) -> Result<(), String> { @@ -632,7 +648,7 @@ mod linux { ); tracing::info!(?config, "Boundary control dialer ready"); let dialer = runtime.process_runtime.spawn(run_runtime_initiated_control( - addresses, + config.clone(), server_config, runtime.clone(), Arc::new(AtomicUsize::new(0)), @@ -655,7 +671,7 @@ mod linux { /// like accepted connections: same identity, slot limit, and disconnect, /// freeze, and termination handling. async fn run_runtime_initiated_control( - addresses: Vec, + config: BoundaryListenerConfig, server_config: Arc, runtime: Arc, active_connections: Arc, @@ -663,16 +679,24 @@ mod linux { let mut backoff = RUNTIME_DIAL_BACKOFF_MIN; let mut warn_at: Option = None; loop { - let established = match dial_supervisor(&addresses).await { - Ok(stream) => ControlStream::PendingTls { - stream: PlainControlStream::Tcp(stream), - server_config: server_config.clone(), - } - .establish_async(&runtime.process_runtime) - .await - .map_err(|error| format!("authenticate boundary transport: {error}")), - Err(error) => Err(format!("dial supervisor: {error}")), - }; + let established = + match tokio::time::timeout(RUNTIME_DIAL_TIMEOUT, dial_supervisor(&config)) + .await + .unwrap_or_else(|_| { + Err(io::Error::new( + io::ErrorKind::TimedOut, + "supervisor dial timed out", + )) + }) { + Ok(stream) => ControlStream::PendingTls { + stream, + server_config: server_config.clone(), + } + .establish_async(&runtime.process_runtime) + .await + .map_err(|error| format!("authenticate boundary transport: {error}")), + Err(error) => Err(format!("dial supervisor: {error}")), + }; let healthy = match established { Ok(stream) => { warn_at = None; @@ -715,23 +739,47 @@ mod linux { } /// Dial only driver-provided supervisor addresses from the bootstrap. - async fn dial_supervisor( - addresses: &[std::net::SocketAddr], - ) -> io::Result { - let stream = tokio::time::timeout( - RUNTIME_DIAL_TIMEOUT, - openshell_core::net::connect_tcp_nodelay_best_effort(addresses), - ) - .await - .map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "supervisor dial timed out"))??; - // An idle spare must notice a silently lost supervisor well inside the - // authenticated recovery window. - let keepalive = socket2::TcpKeepalive::new() - .with_time(Duration::from_secs(10)) - .with_interval(Duration::from_secs(5)) - .with_retries(3); - let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); - stream.into_std() + async fn dial_supervisor(config: &BoundaryListenerConfig) -> io::Result { + match config { + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { addresses, .. } => { + let stream = + openshell_core::net::connect_tcp_nodelay_best_effort(addresses).await?; + let keepalive = socket2::TcpKeepalive::new() + .with_time(Duration::from_secs(10)) + .with_interval(Duration::from_secs(5)) + .with_retries(3); + let _ = socket2::SockRef::from(&stream).set_tcp_keepalive(&keepalive); + Ok(PlainControlStream::Tcp(stream.into_std()?)) + } + BoundaryListenerConfig::RuntimeInitiatedUnix { socket_path, .. } => { + Ok(PlainControlStream::Unix( + tokio::net::UnixStream::connect(socket_path) + .await? + .into_std()?, + )) + } + BoundaryListenerConfig::RuntimeInitiatedVsock { host_port, .. } => { + let socket = + socket2::Socket::new(socket2::Domain::VSOCK, socket2::Type::STREAM, None)?; + socket.set_nonblocking(true)?; + match socket.connect(&socket2::SockAddr::vsock(libc::VMADDR_CID_HOST, *host_port)) { + Ok(()) => {} + Err(error) if error.raw_os_error() == Some(libc::EINPROGRESS) => {} + Err(error) => return Err(error), + } + let stream = + tokio::net::UnixStream::from_std(std::os::unix::net::UnixStream::from(socket))?; + stream.writable().await?; + if let Some(error) = stream.take_error()? { + return Err(error); + } + Ok(PlainControlStream::Unix(stream.into_std()?)) + } + _ => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "control transport does not dial", + )), + } } /// Signals once the supervisor sends its first application bytes. @@ -3607,7 +3655,9 @@ mod linux { server_config, }) } - BoundaryListenerConfig::RuntimeInitiatedTlsTcp { .. } => Err(io::Error::new( + BoundaryListenerConfig::RuntimeInitiatedTlsTcp { .. } + | BoundaryListenerConfig::RuntimeInitiatedUnix { .. } + | BoundaryListenerConfig::RuntimeInitiatedVsock { .. } => Err(io::Error::new( io::ErrorKind::InvalidInput, "runtime-initiated control dials the supervisor and binds no listener", )), diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 760bcfc641..c848cd2018 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -335,6 +335,13 @@ Common findings: - Sandbox runtime image exits before printing `openshell-sandbox --version`: verify the configured image contains a static executable at `/openshell-sandbox`. - A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: inspect supervisor and backend logs for policy rejection, network-mediation initialization, protected-channel setup, or listener failures. A backend that cannot enforce the admitted policy must reject it before starting workload code. - Supervisor runtime validation fails: verify `supervisor_image` contains an `/openshell-supervisor` executable from the same release as the sandbox runtime, and that the dynamic loader and shared libraries it links against are available inside that image. `docker run --rm --network none --entrypoint /openshell-supervisor --version` should print that release; a `no such file or directory` error for a binary that exists means the loader or a library is missing. The supervisor runs from its own image and does not need to be static; only `/openshell-sandbox` must be. +- New Docker and Podman sandboxes dial the supervisor over a separate private + Unix socket volume; the workload keeps `network=none`. Check both containers' + mounts and socket-directory ownership if attachment times out. Existing + containers retain their saved legacy transport through restart. New VM + generations dial host CID 2 over vsock; libkrun forwards to a protected host + Unix socket, while QEMU uses a per-guest host port. Keep supervisor and runtime + binaries compatible when creating new generations. - Sandbox startup reports an `isolation control contract` failure: inspect the supervisor and sandbox-runtime image versions and update them together. An older runtime or incompatible isolation capability set fails before workload startup. See the published Sandbox Runtimes documentation for trusted image override precedence. - The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. A runtime may return `ENOSYS` for `process_vm_readv` and `process_vm_writev` while satisfying the production parent-to-workload-child task-memory probe through `/proc//mem`; only failure of both backends is fatal. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe. - A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`. From b0041e4fdcc60e3c5334b2577bd1c6d6c43221b6 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 12:12:42 -0700 Subject: [PATCH 8/9] fix(docker): stage reverse control socket for the supervisor identity Signed-off-by: Drew Newberry --- crates/openshell-driver-docker/README.md | 2 +- crates/openshell-driver-docker/src/lib.rs | 121 +++++++++++++--------- 2 files changed, 73 insertions(+), 50 deletions(-) diff --git a/crates/openshell-driver-docker/README.md b/crates/openshell-driver-docker/README.md index 94c2b1be79..2f64fd05d1 100644 --- a/crates/openshell-driver-docker/README.md +++ b/crates/openshell-driver-docker/README.md @@ -112,7 +112,7 @@ LSM decisions remain authoritative. | In-memory `/run/openshell-supervisor-ca` tmpfs | Holds only the public supervisor CA certificate and trust bundle without making all of `/run` writable. | | CDI GPU request | Assigns the exact validated CDI devices requested by driver config or count-based selection. | -New containers dial the supervisor at `/.openshell/channel/control/control.sock`. +New containers dial the supervisor at `/.openshell/channel/control/supervisor/control.sock`. The runtime remains the TLS/gRPC server; the supervisor verifies its pinned identity and supplies the existing bearer credential. Saved descriptors retain the legacy Unix listener for existing containers, including across restart. diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index 8c1a2e3aa2..b11aaa6542 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -108,7 +108,7 @@ const SUPERVISOR_GID: u32 = 65_534; const BOUNDARY_MOUNT_PATH: &str = "/.openshell/channel"; const BOUNDARY_CONFIG_MOUNT_PATH: &str = "/.openshell/channel/sandbox/bootstrap.json"; const BOUNDARY_CONTROL_MOUNT_PATH: &str = "/.openshell/channel/control"; -const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/control/control.sock"; +const BOUNDARY_SOCKET_MOUNT_PATH: &str = "/.openshell/channel/control/supervisor/control.sock"; const BOUNDARY_CERTIFICATE_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.crt"; const BOUNDARY_PRIVATE_KEY_MOUNT_PATH: &str = "/.openshell/channel/sandbox/server.key"; const SUPERVISOR_STATE_MOUNT_PATH: &str = "/.openshell/supervisor"; @@ -4577,35 +4577,6 @@ async fn stage_docker_sandbox_bundle( ) .await .map_err(|error| Status::internal(format!("stage Docker sandbox bundle: {error}")))?; - let bootstrap: BoundaryConfig = serde_json::from_slice(boundary_config) - .map_err(|error| Status::internal(format!("decode Docker sandbox bootstrap: {error}")))?; - if matches!( - bootstrap.listener, - openshell_sandbox_backend::boundary_protocol::BoundaryListener::RuntimeInitiatedUnix { .. } - ) { - // Docker resolves an archive destination to one volume. Initialize the - // nested socket volume directly instead of writing into its shadowed - // mountpoint in the read-only bootstrap volume. - let mut archive = tar::Builder::new(Vec::new()); - append_docker_archive_directory(&mut archive, ".", 0o711, SUPERVISOR_UID, SUPERVISOR_GID)?; - let bytes = archive - .into_inner() - .map_err(|error| Status::internal(format!("finish control volume archive: {error}")))?; - let options = UploadToContainerOptionsBuilder::default() - .path(BOUNDARY_CONTROL_MOUNT_PATH) - .copy_uidgid("true") - .build(); - docker - .upload_to_container( - container_id, - Some(options), - bollard::body_full(Bytes::from(bytes)), - ) - .await - .map_err(|error| { - Status::internal(format!("stage Docker control socket volume: {error}")) - })?; - } Ok(()) } @@ -5019,6 +4990,32 @@ async fn stage_docker_supervisor_bundle( Some(RemoveContainerOptionsBuilder::default().force(true).build()), ) .await; + let runtime_initiated = read_docker_runtime_descriptor(&sandbox.id, config).await? + .is_some_and(|descriptor| matches!(descriptor.transport, openshell_sandbox_backend::boundary_protocol::SandboxTransport::RuntimeInitiatedUnix { .. })); + let mut mounts = vec![Mount { + target: Some(SUPERVISOR_STATE_MOUNT_PATH.to_string()), + source: Some(docker_supervisor_volume_name(sandbox, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }]; + if runtime_initiated { + mounts.push(Mount { + target: Some(BOUNDARY_CONTROL_MOUNT_PATH.to_string()), + source: Some(docker_control_volume_name_by_id(&sandbox.id, config)), + typ: Some(MountTypeEnum::VOLUME), + read_only: Some(false), + volume_options: Some(MountVolumeOptions { + no_copy: Some(true), + ..Default::default() + }), + ..Default::default() + }); + } let created = docker .create_container( Some( @@ -5040,17 +5037,7 @@ async fn stage_docker_supervisor_bundle( )), host_config: Some(HostConfig { network_mode: Some("none".to_string()), - mounts: Some(vec![Mount { - target: Some(SUPERVISOR_STATE_MOUNT_PATH.to_string()), - source: Some(docker_supervisor_volume_name(sandbox, config)), - typ: Some(MountTypeEnum::VOLUME), - read_only: Some(false), - volume_options: Some(MountVolumeOptions { - no_copy: Some(true), - ..Default::default() - }), - ..Default::default() - }]), + mounts: Some(mounts), cap_drop: Some(vec!["ALL".to_string()]), security_opt: Some(vec!["no-new-privileges:true".to_string()]), ..Default::default() @@ -5068,14 +5055,50 @@ async fn stage_docker_supervisor_bundle( .path(SUPERVISOR_STATE_MOUNT_PATH) .copy_uidgid("true") .build(); - let result = docker - .upload_to_container( - &created.id, - Some(options), - bollard::body_full(Bytes::from(archive)), - ) - .await - .map_err(|error| Status::internal(format!("stage Docker supervisor bundle: {error}"))); + let result: Result<(), Status> = async { + docker + .upload_to_container( + &created.id, + Some(options), + bollard::body_full(Bytes::from(archive)), + ) + .await + .map_err(|error| { + Status::internal(format!("stage Docker supervisor bundle: {error}")) + })?; + if runtime_initiated { + // Initialize the socket volume using the supervisor's staging user. + // Docker preserves the existing volume root's ownership, so use + // a child directory created by the supervisor's staging user. + let mut archive = tar::Builder::new(Vec::new()); + append_docker_archive_directory( + &mut archive, + "supervisor", + 0o711, + SUPERVISOR_UID, + SUPERVISOR_GID, + )?; + let bytes = archive.into_inner().map_err(|error| { + Status::internal(format!("finish control socket archive: {error}")) + })?; + let options = UploadToContainerOptionsBuilder::default() + .path(BOUNDARY_CONTROL_MOUNT_PATH) + .copy_uidgid("true") + .build(); + docker + .upload_to_container( + &created.id, + Some(options), + bollard::body_full(Bytes::from(bytes)), + ) + .await + .map_err(|error| { + Status::internal(format!("stage Docker control socket volume: {error}")) + })?; + } + Ok(()) + } + .await; let cleanup = docker .remove_container( &created.id, From c4bccdaf0fa464997598b9b7404a9b56d4157b75 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Sun, 18 Oct 2026 13:09:58 -0700 Subject: [PATCH 9/9] refactor(sandbox): keep runtime transports behind the backend Signed-off-by: Drew Newberry --- .../src/isolation_registration.rs | 36 +------- crates/openshell-sandbox-backend/README.md | 4 + .../src/boundary_protocol.rs | 83 ++++++++++++++++++- .../src/delegated.rs | 8 +- 4 files changed, 91 insertions(+), 40 deletions(-) diff --git a/crates/openshell-core/src/isolation_registration.rs b/crates/openshell-core/src/isolation_registration.rs index 25b364e1af..1633dbb742 100644 --- a/crates/openshell-core/src/isolation_registration.rs +++ b/crates/openshell-core/src/isolation_registration.rs @@ -29,27 +29,12 @@ pub enum SandboxTransport { guest_cid: u32, port: u32, }, - /// The supervisor accepts a connection from its sandbox runtime. TLS and - /// gRPC roles stay unchanged. - RuntimeInitiatedTcp { - authority: String, - listen_address: SocketAddr, - }, - RuntimeInitiatedUnix { - socket_path: PathBuf, - }, - RuntimeInitiatedVsock { - guest_cid: u32, - port: u32, - }, } impl SandboxTransport { pub fn validate(&self) -> Result<(), &'static str> { let valid = match self { - Self::Unix { socket_path } | Self::RuntimeInitiatedUnix { socket_path } => { - socket_path.is_absolute() - } + Self::Unix { socket_path } => socket_path.is_absolute(), Self::Tcp { authority, addresses, @@ -60,11 +45,7 @@ impl SandboxTransport { .iter() .all(|address| address.port() != 0 && !address.ip().is_unspecified()) } - Self::RuntimeInitiatedTcp { - authority, - listen_address, - } => !authority.is_empty() && listen_address.port() != 0, - Self::Vsock { guest_cid, port } | Self::RuntimeInitiatedVsock { guest_cid, port } => { + Self::Vsock { guest_cid, port } => { *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX } }; @@ -101,14 +82,6 @@ pub fn validate_registrations(registrations: &[BackendRegistration]) -> Result<( let mut names = BTreeSet::new(); for registration in registrations { validate_backend_name(®istration.name)?; - if matches!( - registration.endpoint, - SandboxTransport::RuntimeInitiatedTcp { .. } - | SandboxTransport::RuntimeInitiatedUnix { .. } - | SandboxTransport::RuntimeInitiatedVsock { .. } - ) { - return Err("runtime-initiated control is not a registered backend endpoint"); - } if !names.insert(®istration.name) { return Err("duplicate isolation backend registration"); } @@ -128,11 +101,6 @@ impl TryFrom<&BackendRegistration> for crate::proto::compute::v1::IsolationBacke validate_backend_name(®istration.name)?; registration.endpoint.validate()?; let endpoint = match ®istration.endpoint { - SandboxTransport::RuntimeInitiatedTcp { .. } - | SandboxTransport::RuntimeInitiatedUnix { .. } - | SandboxTransport::RuntimeInitiatedVsock { .. } => { - return Err("runtime-initiated control is not a registered backend endpoint"); - } SandboxTransport::Unix { socket_path } => Endpoint::UnixSocketPath( socket_path .to_str() diff --git a/crates/openshell-sandbox-backend/README.md b/crates/openshell-sandbox-backend/README.md index 01f89b3b7d..a63d52ebde 100644 --- a/crates/openshell-sandbox-backend/README.md +++ b/crates/openshell-sandbox-backend/README.md @@ -102,6 +102,10 @@ retain generation, session, resource, identity, and provisioning evidence checks ## Boundary transport establishment +`SandboxTransport` belongs to this backend's runtime descriptor. Core retains +only operator-registered endpoints; `DelegatedLaunch` converts those endpoints +into the backend's transport without changing their serialized representation. + The supervisor chooses one `BoundaryTransport` per runtime descriptor and uses clones of it for policy discovery and attachment. It dials the descriptor's endpoint, uses a backend-supplied `BoundaryTransportConnector` for raw stream diff --git a/crates/openshell-sandbox-backend/src/boundary_protocol.rs b/crates/openshell-sandbox-backend/src/boundary_protocol.rs index d4e2cf0ac7..65527d5eb1 100644 --- a/crates/openshell-sandbox-backend/src/boundary_protocol.rs +++ b/crates/openshell-sandbox-backend/src/boundary_protocol.rs @@ -238,7 +238,88 @@ impl<'de> Deserialize<'de> for SupervisorInstanceId { #[error("supervisor instance ID must be a non-nil canonical lowercase UUID")] pub struct SupervisorInstanceIdError; -pub use openshell_core::isolation_registration::SandboxTransport; +/// Backend-owned transport for a driver-provisioned runtime. The serialized +/// variants preserve existing runtime descriptors independently of operator +/// endpoint registration in core. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "kebab-case", deny_unknown_fields)] +pub enum SandboxTransport { + Unix { + socket_path: PathBuf, + }, + Tcp { + /// Logical HTTP authority used for routing. + authority: String, + /// Concrete connection candidates pinned by the operator or driver. + addresses: Vec, + }, + Vsock { + guest_cid: u32, + port: u32, + }, + /// The supervisor accepts a connection from its sandbox runtime. TLS and + /// gRPC roles stay unchanged. + RuntimeInitiatedTcp { + authority: String, + listen_address: std::net::SocketAddr, + }, + RuntimeInitiatedUnix { + socket_path: PathBuf, + }, + RuntimeInitiatedVsock { + guest_cid: u32, + port: u32, + }, +} + +impl SandboxTransport { + pub fn validate(&self) -> Result<(), &'static str> { + let valid = match self { + Self::Unix { socket_path } | Self::RuntimeInitiatedUnix { socket_path } => { + socket_path.is_absolute() + } + Self::Tcp { + authority, + addresses, + } => { + !authority.is_empty() + && !addresses.is_empty() + && addresses + .iter() + .all(|address| address.port() != 0 && !address.ip().is_unspecified()) + } + Self::RuntimeInitiatedTcp { + authority, + listen_address, + } => !authority.is_empty() && listen_address.port() != 0, + Self::Vsock { guest_cid, port } | Self::RuntimeInitiatedVsock { guest_cid, port } => { + *guest_cid >= 3 && *guest_cid != u32::MAX && *port != 0 && *port != u32::MAX + } + }; + if valid { + Ok(()) + } else { + Err("invalid boundary control transport endpoint") + } + } +} + +impl From for SandboxTransport { + fn from(endpoint: openshell_core::isolation_registration::SandboxTransport) -> Self { + use openshell_core::isolation_registration::SandboxTransport as RegisteredEndpoint; + match endpoint { + RegisteredEndpoint::Unix { socket_path } => Self::Unix { socket_path }, + RegisteredEndpoint::Tcp { + authority, + addresses, + } => Self::Tcp { + authority, + addresses, + }, + RegisteredEndpoint::Vsock { guest_cid, port } => Self::Vsock { guest_cid, port }, + } + } +} /// Supervisor-side, generation-pinned TLS server authentication. #[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] diff --git a/crates/openshell-sandbox-backend/src/delegated.rs b/crates/openshell-sandbox-backend/src/delegated.rs index 349141e406..96e2047ecb 100644 --- a/crates/openshell-sandbox-backend/src/delegated.rs +++ b/crates/openshell-sandbox-backend/src/delegated.rs @@ -12,9 +12,7 @@ use openshell_isolation_interface::contract::{ }; use serde::{Deserialize, Serialize}; -use crate::boundary_protocol::{ - SandboxRuntimeDescriptor, SandboxTlsClientConfig, SandboxTransport, -}; +use crate::boundary_protocol::{SandboxRuntimeDescriptor, SandboxTlsClientConfig}; /// Delivered by the trusted driver separately from its opaque backend payload. /// Endpoint selection belongs to operator configuration, not launch data. @@ -37,7 +35,7 @@ impl DelegatedLaunch { /// Combine driver-owned identity and TLS with the operator-owned endpoint. pub fn runtime_descriptor( self, - transport: SandboxTransport, + transport: openshell_core::isolation_registration::SandboxTransport, ) -> Result { validate_backend_name(&self.backend_name)?; let descriptor = SandboxRuntimeDescriptor { @@ -45,7 +43,7 @@ impl DelegatedLaunch { generation: self.generation, session_id: self.session_id, workload_identity: self.workload_identity, - transport, + transport: transport.into(), tls: self.tls, host_gateway_ip: self.host_gateway_ip, resource_claims: self.resource_claims,