diff --git a/crates/openshell-driver-kubernetes/src/config.rs b/crates/openshell-driver-kubernetes/src/config.rs index 6bd7d93af3..acbd0d006f 100644 --- a/crates/openshell-driver-kubernetes/src/config.rs +++ b/crates/openshell-driver-kubernetes/src/config.rs @@ -263,6 +263,19 @@ pub struct KubernetesComputeConfig { /// Applied when a `CreateSandbox` request does not specify one. /// Empty string (default) = omit the field, using the cluster default. pub default_runtime_class_name: String, + /// Pod annotation keys that sandbox templates may set. Each entry is an + /// exact key (`example.com/team`) or a prefix wildcard (`example.com/*`). + /// Empty (default) rejects every caller-supplied pod annotation. + /// + /// Keys in `openshell.ai` and `agents.x-k8s.io` (and their subdomains) + /// are always rejected and cannot be allowlisted. `OpenShell` writes sandbox + /// identity under `openshell.ai`, and the Agent Sandbox controller owns + /// `agents.x-k8s.io` and strips those keys from pod templates. + pub allowed_pod_annotations: Vec, + /// Pod label keys that sandbox templates may set, using the same entry + /// format and reserved domains as `allowed_pod_annotations`. Empty + /// (default) rejects every caller-supplied pod label. + pub allowed_pod_labels: Vec, /// Lifetime (seconds) of the projected `ServiceAccount` token kubelet /// writes into each sandbox pod. Used only for the one-shot /// `IssueSandboxToken` bootstrap exchange — the gateway-minted JWT @@ -364,6 +377,8 @@ impl Default for KubernetesComputeConfig { workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(), workspace_storage_class: String::new(), default_runtime_class_name: String::new(), + allowed_pod_annotations: Vec::new(), + allowed_pod_labels: Vec::new(), sa_token_ttl_secs: 3600, provider_spiffe_workload_api_socket_path: String::new(), sandbox_uid: None, @@ -379,7 +394,29 @@ impl KubernetesComputeConfig { self.validate_provider_spiffe_workload_api_socket_path()?; self.validate_sandbox_identity_config()?; self.validate_proxy_uid()?; - self.validate_upstream_proxy_config() + self.validate_upstream_proxy_config()?; + self.validate_allowed_pod_annotations()?; + self.validate_allowed_pod_labels() + } + + pub fn validate_allowed_pod_annotations(&self) -> Result<(), String> { + validate_pod_metadata_allowlist("allowed_pod_annotations", &self.allowed_pod_annotations) + } + + pub fn validate_allowed_pod_labels(&self) -> Result<(), String> { + validate_pod_metadata_allowlist("allowed_pod_labels", &self.allowed_pod_labels) + } + + /// Whether a caller-supplied pod annotation key passes the operator allowlist. + #[must_use] + pub fn pod_annotation_allowed(&self, key: &str) -> bool { + pod_metadata_key_allowed(&self.allowed_pod_annotations, key) + } + + /// Whether a caller-supplied pod label key passes the operator allowlist. + #[must_use] + pub fn pod_label_allowed(&self, key: &str) -> bool { + pod_metadata_key_allowed(&self.allowed_pod_labels, key) } /// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS, @@ -788,6 +825,87 @@ fn is_dns1123_subdomain(value: &str) -> bool { }) } +/// Validate an `allowed_pod_annotations` or `allowed_pod_labels` list. Each +/// entry is an exact qualified key or a `/*` wildcard outside the +/// reserved domains. +fn validate_pod_metadata_allowlist(field: &str, entries: &[String]) -> Result<(), String> { + for entry in entries { + let key = entry.strip_suffix("/*").unwrap_or(entry); + let wildcard = key.len() != entry.len(); + let (prefix, name) = match key.split_once('/') { + Some((prefix, name)) if !wildcard => (Some(prefix), Some(name)), + Some(_) => { + return Err(format!( + "{field} entry {entry:?}: wildcard entries must be `/*`" + )); + } + None if wildcard => (Some(key), None), + None => (None, Some(key)), + }; + if let Some(prefix) = prefix + && !is_dns1123_subdomain(prefix) + { + return Err(format!( + "{field} entry {entry:?}: invalid key prefix {prefix:?}" + )); + } + if let Some(name) = name + && !is_qualified_name_segment(name) + { + return Err(format!( + "{field} entry {entry:?}: invalid key name {name:?}" + )); + } + if prefix.is_some_and(is_reserved_pod_metadata_domain) { + return Err(format!( + "{field} entry {entry:?}: domain is reserved for OpenShell and Agent Sandbox" + )); + } + } + Ok(()) +} + +fn pod_metadata_key_allowed(entries: &[String], key: &str) -> bool { + let prefix = key.split_once('/').map(|(prefix, _)| prefix); + if prefix.is_some_and(is_reserved_pod_metadata_domain) { + return false; + } + entries.iter().any(|entry| { + entry + .strip_suffix("/*") + .map_or_else(|| entry == key, |domain| prefix == Some(domain)) + }) +} + +/// Pod annotation and label domains owned by `OpenShell` (sandbox identity +/// such as `openshell.ai/sandbox-id` and boundary labels) and the Agent +/// Sandbox controller (which strips them from pod templates). Sandbox +/// templates can never set keys in these domains or their subdomains, +/// regardless of `allowed_pod_annotations` or `allowed_pod_labels`. +const RESERVED_POD_METADATA_DOMAINS: &[&str] = &["openshell.ai", "agents.x-k8s.io"]; + +fn is_reserved_pod_metadata_domain(domain: &str) -> bool { + RESERVED_POD_METADATA_DOMAINS.iter().any(|reserved| { + domain == *reserved + || domain + .strip_suffix(reserved) + .is_some_and(|rest| rest.ends_with('.')) + }) +} + +/// Kubernetes qualified-name segment: at most 63 alphanumeric, `-`, `_`, or +/// `.` characters, beginning and ending with an alphanumeric character. +fn is_qualified_name_segment(value: &str) -> bool { + let bytes = value.as_bytes(); + !bytes.is_empty() + && bytes.len() <= 63 + && bytes.first().is_some_and(u8::is_ascii_alphanumeric) + && bytes.last().is_some_and(u8::is_ascii_alphanumeric) + && bytes + .iter() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(b)) +} + fn validate_provider_spiffe_workload_api_socket_path_value( socket_path: &str, ) -> Result<(), String> { @@ -961,6 +1079,81 @@ mod tests { assert_eq!(cfg.default_runtime_class_name, "nvidia"); } + fn with_allowed_pod_annotations(entries: &[&str]) -> KubernetesComputeConfig { + KubernetesComputeConfig { + allowed_pod_annotations: entries.iter().map(ToString::to_string).collect(), + ..KubernetesComputeConfig::default() + } + } + + #[test] + fn pod_annotation_allowed_matches_exact_and_domain_entries() { + assert!(!KubernetesComputeConfig::default().pod_annotation_allowed("example.com/a")); + + let cfg = with_allowed_pod_annotations(&["example.com/*", "team.org/owner", "owner"]); + for (key, allowed) in [ + ("example.com/a", true), + ("sub.example.com/a", false), + ("team.org/owner", true), + ("team.org/other", false), + ("owner", true), + ] { + assert_eq!(cfg.pod_annotation_allowed(key), allowed, "{key}"); + } + + // Reserved domains stay blocked even if validation was bypassed. + let cfg = with_allowed_pod_annotations(&["openshell.ai/sandbox-id"]); + assert!(!cfg.pod_annotation_allowed("openshell.ai/sandbox-id")); + } + + #[test] + fn validate_allowed_pod_annotations_rejects_malformed_and_reserved_entries() { + with_allowed_pod_annotations(&["example.com/*", "team.org/owner", "owner"]) + .validate_allowed_pod_annotations() + .unwrap(); + for entry in [ + "*", + "", + "example.com/", + "Example.com/owner", + "example.com/a/*", + "example.com/*/owner", + "-bad", + "openshell.ai/*", + "agents.x-k8s.io/pod-name", + "extensions.agents.x-k8s.io/*", + ] { + assert!( + with_allowed_pod_annotations(&[entry]) + .validate_allowed_pod_annotations() + .is_err(), + "{entry:?}" + ); + } + } + + #[test] + fn pod_label_allowlist_is_separate_from_annotation_allowlist() { + let cfg = KubernetesComputeConfig { + allowed_pod_annotations: vec!["annotations.example.com/*".into()], + allowed_pod_labels: vec!["app".into(), "labels.example.com/*".into()], + ..KubernetesComputeConfig::default() + }; + assert!(cfg.pod_label_allowed("app")); + assert!(cfg.pod_label_allowed("labels.example.com/team")); + assert!(!cfg.pod_label_allowed("annotations.example.com/team")); + assert!(!cfg.pod_annotation_allowed("labels.example.com/team")); + assert!(!KubernetesComputeConfig::default().pod_label_allowed("app")); + + let cfg = KubernetesComputeConfig { + allowed_pod_labels: vec!["openshell.ai/managed-by".into()], + ..KubernetesComputeConfig::default() + }; + assert!(!cfg.pod_label_allowed("openshell.ai/managed-by")); + let err = cfg.validate_allowed_pod_labels().unwrap_err(); + assert!(err.starts_with("allowed_pod_labels entry"), "{err}"); + } + #[test] fn default_runtime_class_name_is_empty() { let cfg = KubernetesComputeConfig::default(); diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 9676a17be0..f6546e7b8f 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -775,6 +775,12 @@ impl KubernetesComputeDriver { config .validate_upstream_proxy_config() .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_annotations() + .map_err(KubernetesDriverError::Precondition)?; + config + .validate_allowed_pod_labels() + .map_err(KubernetesDriverError::Precondition)?; // Read the bundle once at startup so an unreadable path or a // certificate-free file fails the gateway here, rather than as a // repeated sandbox-create failure. The authoritative read still @@ -1428,6 +1434,17 @@ impl KubernetesComputeDriver { kubernetes_driver_config_for_spec(sandbox.spec.as_ref()) } + fn validate_pod_metadata_for_sandbox(&self, sandbox: &Sandbox) -> Result<(), String> { + let Some(template) = sandbox + .spec + .as_ref() + .and_then(|spec| spec.template.as_ref()) + else { + return Ok(()); + }; + validate_pod_metadata(template, &self.config) + } + fn agent_sandbox_api( client: Client, sandbox_api_version: &str, @@ -1620,6 +1637,8 @@ impl KubernetesComputeDriver { .map_err(tonic::Status::failed_precondition)?; let _ = Self::validate_driver_config_for_sandbox(sandbox) .map_err(tonic::Status::invalid_argument)?; + self.validate_pod_metadata_for_sandbox(sandbox) + .map_err(tonic::Status::invalid_argument)?; match self.config.workspace_mode { WorkspaceMode::Shared => { validate_kube_resource_name_length(&sandbox.workspace, &sandbox.name)?; @@ -1875,6 +1894,8 @@ impl KubernetesComputeDriver { sandbox, ) .map_err(admission_error)?; + self.validate_pod_metadata_for_sandbox(sandbox) + .map_err(KubernetesDriverError::InvalidArgument)?; let gpu_requirements = sandbox .spec .as_ref() @@ -6811,6 +6832,47 @@ fn platform_config_string(template: &SandboxTemplate, key: &str) -> Option Result<(), String> { + if let Some(serde_json::Value::Object(annotations)) = + platform_config_struct(template, "annotations") + { + reject_disallowed_pod_metadata_keys( + "annotations", + "allowed_pod_annotations", + annotations.keys().map(String::as_str), + |key| config.pod_annotation_allowed(key), + )?; + } + reject_disallowed_pod_metadata_keys( + "labels", + "allowed_pod_labels", + template.labels.keys().map(String::as_str), + |key| config.pod_label_allowed(key), + ) +} + +fn reject_disallowed_pod_metadata_keys<'a>( + kind: &str, + field: &str, + keys: impl Iterator, + allowed: impl Fn(&str) -> bool, +) -> Result<(), String> { + let mut rejected: Vec<&str> = keys.filter(|key| !allowed(key)).collect(); + if rejected.is_empty() { + return Ok(()); + } + rejected.sort_unstable(); + Err(format!( + "pod {kind} not permitted by the gateway's {field}: {}", + rejected.join(", ") + )) +} + /// Extract a nested Struct value from the template's `platform_config`, /// converting it to `serde_json::Value`. fn platform_config_struct(template: &SandboxTemplate, key: &str) -> Option { @@ -9658,6 +9720,80 @@ mod tests { assert!(err.message().contains("gpu count must be greater than 0")); } + fn template_with_pod_annotations(keys: &[&str]) -> SandboxTemplate { + let annotations: serde_json::Map<_, _> = keys + .iter() + .map(|key| ((*key).to_string(), "v".into())) + .collect(); + SandboxTemplate { + platform_config: Some(json_struct( + serde_json::json!({ "annotations": annotations }), + )), + ..SandboxTemplate::default() + } + } + + fn template_with_pod_labels(keys: &[&str]) -> SandboxTemplate { + SandboxTemplate { + labels: keys + .iter() + .map(|key| ((*key).to_string(), "v".to_string())) + .collect(), + ..SandboxTemplate::default() + } + } + + #[test] + fn validate_pod_metadata_reports_keys_outside_allowlists() { + let config = KubernetesComputeConfig { + allowed_pod_annotations: vec!["example.com/*".into()], + allowed_pod_labels: vec!["app".into()], + ..KubernetesComputeConfig::default() + }; + validate_pod_metadata(&SandboxTemplate::default(), &config).unwrap(); + validate_pod_metadata(&template_with_pod_annotations(&["example.com/a"]), &config).unwrap(); + validate_pod_metadata(&template_with_pod_labels(&["app"]), &config).unwrap(); + + let template = + template_with_pod_annotations(&["example.com/a", "sub.example.com/a", "other.net/a"]); + let err = validate_pod_metadata(&template, &config).unwrap_err(); + assert!( + err.ends_with("allowed_pod_annotations: other.net/a, sub.example.com/a"), + "{err}" + ); + + let template = template_with_pod_labels(&["app", "team", "openshell.ai/managed-by"]); + let err = validate_pod_metadata(&template, &config).unwrap_err(); + assert!( + err.ends_with("allowed_pod_labels: openshell.ai/managed-by, team"), + "{err}" + ); + } + + #[tokio::test] + async fn create_sandbox_rejects_disallowed_pod_metadata() { + let driver = KubernetesComputeDriver::new_for_test(KubernetesComputeConfig::default()); + for (template, key) in [ + ( + template_with_pod_annotations(&["example.com/owner"]), + "example.com/owner", + ), + (template_with_pod_labels(&["app"]), "app"), + ] { + let sandbox = Sandbox { + spec: Some(SandboxSpec { + template: Some(template), + ..Default::default() + }), + ..Default::default() + }; + assert!(matches!( + Box::pin(driver.create_sandbox_inner(&sandbox)).await, + Err(KubernetesDriverError::InvalidArgument(message)) if message.contains(key) + )); + } + } + #[test] fn kube_pulling_event_adds_image_progress_metadata() { let mut metadata = std::collections::HashMap::new(); diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index d70f706f5f..448147af6c 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -87,6 +87,16 @@ struct Args { )] sandbox_image_pull_secrets: Vec, + #[arg( + long, + env = "OPENSHELL_K8S_ALLOWED_POD_ANNOTATIONS", + value_delimiter = ',' + )] + allowed_pod_annotations: Vec, + + #[arg(long, env = "OPENSHELL_K8S_ALLOWED_POD_LABELS", value_delimiter = ',')] + allowed_pod_labels: Vec, + #[arg(long, env = "OPENSHELL_MANAGED_SSH_INGRESS_ENABLED")] managed_ssh_ingress_enabled: bool, @@ -288,6 +298,8 @@ async fn main() -> Result<()> { .unwrap_or_default(), default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") .unwrap_or_default(), + allowed_pod_annotations: args.allowed_pod_annotations, + allowed_pod_labels: args.allowed_pod_labels, sa_token_ttl_secs: args.sa_token_ttl_secs, provider_spiffe_workload_api_socket_path: args .provider_spiffe_workload_api_socket_path diff --git a/deploy/helm/openshell/tests/gateway_config_test.yaml b/deploy/helm/openshell/tests/gateway_config_test.yaml index df69d5b7aa..faf2b07a7e 100644 --- a/deploy/helm/openshell/tests/gateway_config_test.yaml +++ b/deploy/helm/openshell/tests/gateway_config_test.yaml @@ -725,6 +725,34 @@ tests: path: data["gateway.toml"] pattern: 'image_pull_secrets\s*=' + - it: renders allowed pod annotations and labels for the Kubernetes driver + template: templates/gateway-config.yaml + set: + gatewayConfig: + openshell.drivers.kubernetes: + allowed_pod_annotations: + - example.com/* + - team.example.org/owner + allowed_pod_labels: + - app + asserts: + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?allowed_pod_annotations\s*=\s*\[\s*"example\.com/\*",\s*"team\.example\.org/owner",?\s*\]' + - matchRegex: + path: data["gateway.toml"] + pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?allowed_pod_labels\s*=\s*\[\s*"app",?\s*\]' + + - it: omits allowed pod annotations and labels by default + template: templates/gateway-config.yaml + asserts: + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'allowed_pod_annotations\s*=' + - notMatchRegex: + path: data["gateway.toml"] + pattern: 'allowed_pod_labels\s*=' + - it: does not render local mTLS user auth for Kubernetes deployments template: templates/gateway-config.yaml asserts: diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index e14131264e..38f873413e 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -860,6 +860,18 @@ workspace_default_storage_size = "10Gi" # Kubernetes RuntimeClass applied to sandbox pods when the API request does # not specify one. Empty (default) = omit the field, using the cluster default. # default_runtime_class_name = "kata-containers" +# Pod annotation keys that sandbox templates may set. Each entry is an exact +# key or a `/*` wildcard. Empty (default) rejects every template +# annotation. Keys under openshell.ai (OpenShell sandbox identity) and +# agents.x-k8s.io (owned by the Agent Sandbox controller), including +# subdomains, are reserved: templates cannot set them and the allowlist +# cannot include them. +# allowed_pod_annotations = ["example.com/*", "team.example.org/owner"] +# Pod label keys that sandbox templates may set, in the same format and with +# the same reserved domains. Empty (default) rejects every template label. +# Labels select pods for NetworkPolicy rules and Services, so only allow keys +# that no other rule in the sandbox namespace selects on. +# allowed_pod_labels = ["app", "team.example.org/*"] # Kubelet clamps projected tokens below 600 seconds. The driver caps values at 86400. sa_token_ttl_secs = 3600 # Optional SPIFFE Workload API socket mounted into sandbox pods for dynamic diff --git a/docs/how-it-works/sandboxes/runtimes.mdx b/docs/how-it-works/sandboxes/runtimes.mdx index e8ef578363..cc76cf98d2 100644 --- a/docs/how-it-works/sandboxes/runtimes.mdx +++ b/docs/how-it-works/sandboxes/runtimes.mdx @@ -253,6 +253,8 @@ Only the OpenShell gateway and the Agent Sandbox controller should be able to ma | `supervisor_image` | `supervisor.image.*` | Override the supervisor image. | | `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Default workspace PVC size. | | `workspace_storage_class` | `server.workspaceStorageClass` | `StorageClass` for workspace PVCs. Set this if the cluster has no default `StorageClass`. | +| `allowed_pod_annotations` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_annotations` | Pod annotation keys that sandbox templates may set, as exact keys or `/*` wildcards. Empty (default) rejects all template annotations. Keys under `openshell.ai` and `agents.x-k8s.io` are reserved and can't be allowlisted. | +| `allowed_pod_labels` | `gatewayConfig["openshell.drivers.kubernetes"].allowed_pod_labels` | Pod label keys that sandbox templates may set, in the same format as `allowed_pod_annotations`. Empty (default) rejects all template labels. Labels select pods for NetworkPolicy rules and Services, so only allow keys that no other rule in the sandbox namespace selects on. | | `sandbox_uid` / `sandbox_gid` | `server.sandboxUid` / `server.sandboxGid` | Sandbox pod UID and GID. Takes priority over OpenShift SCC namespace annotations. `sandbox_gid` defaults to the UID. | | `https_proxy` | `upstreamProxy.url` | Corporate proxy for sandbox egress. | | `no_proxy` | `upstreamProxy.noProxy` | Destinations that bypass the corporate proxy. |