From df7f9a7b0e59bcf0ff388d406ace98ffe0b69414 Mon Sep 17 00:00:00 2001 From: Shiju Date: Mon, 12 Oct 2026 23:36:35 +0530 Subject: [PATCH 01/13] feat(providers): report bounded token-grant failures Signed-off-by: Shiju --- Cargo.lock | 1 + crates/openshell-cli/src/commands/provider.rs | 2 +- crates/openshell-cli/src/run.rs | 262 ++- crates/openshell-core/Cargo.toml | 1 + crates/openshell-core/build.rs | 2 + crates/openshell-core/src/endpoint_failure.rs | 389 ++++ crates/openshell-core/src/endpoint_status.rs | 531 ++++- crates/openshell-core/src/grpc_client.rs | 298 ++- crates/openshell-core/src/lib.rs | 2 + crates/openshell-core/src/oauth.rs | 945 ++++++-- .../openshell-core/src/token_grant_failure.rs | 601 +++++ crates/openshell-server/src/compute/mod.rs | 24 +- crates/openshell-server/src/grpc/policy.rs | 3 +- .../src/grpc/policy/endpoint_status.rs | 121 +- .../src/grpc/policy/endpoint_status_tests.rs | 584 ++++- crates/openshell-server/src/grpc/provider.rs | 1142 ++++++++-- crates/openshell-server/src/storage_proto.rs | 19 +- .../src/supervisor_session.rs | 39 +- .../openshell-supervisor-network/Cargo.toml | 2 +- .../src/l7/mod.rs | 38 +- .../src/l7/relay.rs | 190 +- .../l7/relay/token_grant_ownership_tests.rs | 200 +- .../src/l7/token_grant_injection.rs | 1041 +++++++-- .../token_grant_injection/multiple_grants.rs | 107 +- .../openshell-supervisor-network/src/opa.rs | 6 +- .../openshell-supervisor-network/src/proxy.rs | 218 +- .../src/proxy/tests/token_grants.rs | 4 + .../src/token_grant.rs | 504 +++-- .../src/endpoint_status.rs | 10 +- docs/how-it-works/providers/profiles.mdx | 49 + docs/how-it-works/sandboxes/overview.mdx | 33 +- docs/observability/logging.mdx | 4 +- e2e/rust/Cargo.lock | 1 + e2e/rust/Cargo.toml | 2 +- e2e/rust/src/harness/gateway.rs | 11 + e2e/rust/tests/provider_token_exchange.rs | 49 +- .../tests/provider_token_exchange/closure.rs | 805 +++++++ proto/openshell.proto | 69 + sdk/go/docs/src/api/sandboxes.md | 29 + sdk/go/openshell/v1/fake/fake_test.go | 69 + sdk/go/openshell/v1/fake/sandbox.go | 18 + .../v1/internal/converter/coverage_test.go | 26 + .../v1/internal/converter/sandbox.go | 34 + .../v1/internal/converter/sandbox_test.go | 142 ++ sdk/go/openshell/v1/refresh.go | 12 + sdk/go/openshell/v1/sandbox.go | 15 + sdk/go/openshell/v1/types/sandbox.go | 64 +- sdk/go/proto/openshellv1/openshell.pb.go | 1976 ++++++++++------- skills/debug-openshell-cluster/SKILL.md | 2 +- 49 files changed, 9053 insertions(+), 1643 deletions(-) create mode 100644 crates/openshell-core/src/endpoint_failure.rs create mode 100644 crates/openshell-core/src/token_grant_failure.rs create mode 100644 e2e/rust/tests/provider_token_exchange/closure.rs diff --git a/Cargo.lock b/Cargo.lock index 810bd6ed0f..861e22825b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4336,6 +4336,7 @@ dependencies = [ "tempfile", "thiserror 2.0.20", "tokio", + "tokio-rustls", "tokio-stream", "tonic", "tonic-prost", diff --git a/crates/openshell-cli/src/commands/provider.rs b/crates/openshell-cli/src/commands/provider.rs index 0098297b38..897f4268ad 100644 --- a/crates/openshell-cli/src/commands/provider.rs +++ b/crates/openshell-cli/src/commands/provider.rs @@ -2118,7 +2118,7 @@ fn refresh_status_row(status: &ProviderCredentialRefreshStatus) -> String { ) } -fn provider_refresh_recovery_action_name( +pub fn provider_refresh_recovery_action_name( action: ProviderCredentialRefreshRecoveryAction, ) -> &'static str { match action { diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 49f6f05d81..37e66d1725 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -1808,6 +1808,17 @@ where ); } + if let Some(observation) = sandbox + .status + .as_ref() + .and_then(|status| status.last_token_grant_failure.as_deref()) + { + println!(" {}", "Last token grant failure:".dimmed()); + for line in token_grant_failure_display_lines(observation) { + println!(" {line}"); + } + } + let policy_from_global = config.policy_source == PolicySource::Global as i32; println!( " {} {}", @@ -3037,10 +3048,22 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "path": endpoint.path, "last_result": endpoint_result_name(endpoint.last_result()), "last_reported_at": endpoint.last_reported_time.as_ref().map(ToString::to_string).unwrap_or_default(), + "last_failure": endpoint.last_failure.as_ref().map(endpoint_failure_to_json), }) }) .collect::>() }); + let last_token_grant_failure = sandbox + .status + .as_ref() + .and_then(|status| status.last_token_grant_failure.as_deref()) + .map(|observation| { + serde_json::json!({ + "destination_host": observation.destination_host, + "destination_port": observation.destination_port, + "failure": observation.failure.as_ref().map(endpoint_failure_to_json), + }) + }); let admission = sandbox .status .as_ref() @@ -3097,6 +3120,7 @@ fn sandbox_to_json(sandbox: &Sandbox) -> serde_json::Value { "exit_code": sandbox.status.as_ref().and_then(|status| status.exit_code), "conditions": conditions, "endpoint_statuses": endpoint_statuses, + "last_token_grant_failure": last_token_grant_failure, "configuration_admission": admission, "provisioning": provisioning, "created_from_workload_template": created_from_workload_template, @@ -3154,6 +3178,92 @@ fn endpoint_result_name(result: EndpointResult) -> &'static str { } } +fn endpoint_failure_domain_name(domain: i32) -> &'static str { + match openshell_core::proto::EndpointFailureDomain::try_from(domain) { + Ok(openshell_core::proto::EndpointFailureDomain::TokenGrant) => "token_grant", + Ok(openshell_core::proto::EndpointFailureDomain::Unspecified) | Err(_) => "unknown", + } +} + +fn endpoint_failure_recovery( + failure: &openshell_core::proto::EndpointFailure, +) -> openshell_core::proto::ProviderCredentialRefreshRecoveryAction { + openshell_core::proto::ProviderCredentialRefreshRecoveryAction::try_from( + failure.recovery_action, + ) + .unwrap_or(openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Unspecified) +} + +fn endpoint_failure_recovery_name( + action: openshell_core::proto::ProviderCredentialRefreshRecoveryAction, +) -> &'static str { + match action { + // Structured output names every value; "-" is a table placeholder. + openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Unspecified => { + "unspecified" + } + action => crate::commands::provider::provider_refresh_recovery_action_name(action), + } +} + +fn endpoint_failure_to_json(failure: &openshell_core::proto::EndpointFailure) -> serde_json::Value { + serde_json::json!({ + "domain": endpoint_failure_domain_name(failure.domain), + "code": failure.code, + "recovery_action": endpoint_failure_recovery_name(endpoint_failure_recovery(failure)), + "provider": failure.provider, + "credential_key": failure.credential_key, + "stage": failure.stage, + "upstream_http_status": failure.upstream_http_status, + "request_id": failure.request_id, + }) +} + +fn endpoint_failure_display_lines(failure: &openshell_core::proto::EndpointFailure) -> Vec { + use openshell_core::proto::ProviderCredentialRefreshRecoveryAction as Recovery; + let upstream = failure + .upstream_http_status + .map(|status| format!(", upstream HTTP {status}")) + .unwrap_or_default(); + let recovery = match endpoint_failure_recovery(failure) { + Recovery::Retry => "Retry later; the failure can be temporary.", + Recovery::Reauthorize => "Reauthorize the provider credential.", + Recovery::FixConfiguration => { + "An operator must fix the provider or identity configuration." + } + Recovery::Investigate => "An operator must investigate gateway and supervisor logs.", + Recovery::Unspecified => "No recovery action provided.", + }; + let mut lines = vec![format!( + "Cause: {} ({}, stage {}{upstream})", + failure.code, + endpoint_failure_domain_name(failure.domain).replace('_', " "), + failure.stage, + )]; + if !failure.provider.is_empty() { + lines.push(format!( + "Credential: {}/{}", + failure.provider, failure.credential_key + )); + } + lines.push(format!("Recovery: {recovery}")); + lines.push(format!("Request ID: {}", failure.request_id)); + lines +} + +fn token_grant_failure_display_lines( + observation: &openshell_core::proto::TokenGrantFailureObservation, +) -> Vec { + let mut lines = vec![format!( + "Destination: {} (port {})", + observation.destination_host, observation.destination_port + )]; + if let Some(failure) = observation.failure.as_ref() { + lines.extend(endpoint_failure_display_lines(failure)); + } + lines +} + fn endpoint_status_display_lines(endpoint: &EndpointStatus) -> Vec { let ports = endpoint .ports @@ -3179,14 +3289,18 @@ fn endpoint_status_display_lines(endpoint: &EndpointStatus) -> Vec { .last_reported_time .as_ref() .map_or_else(|| "no report yet".to_string(), ToString::to_string); - vec![ + let mut lines = vec![ format!( "{} (ports: {ports}; path: {})", endpoint.host, endpoint.path ), format!("Last result: {result}"), - format!("Reported at (gateway acceptance): {reported_at}"), - ] + ]; + if let Some(failure) = endpoint.last_failure.as_ref() { + lines.extend(endpoint_failure_display_lines(failure)); + } + lines.push(format!("Reported at (gateway acceptance): {reported_at}")); + lines } fn restart_policy_name(policy: i32) -> &'static str { @@ -8627,6 +8741,7 @@ mod tests { path: "/mcp".to_string(), last_result: EndpointResult::TransportFailed as i32, last_reported_time: Some("2026-09-05T10:01:00Z".parse().unwrap()), + last_failure: None, }], conditions: vec![SandboxCondition { r#type: "Ready".to_string(), @@ -8674,6 +8789,7 @@ mod tests { "path": "/mcp", "last_result": "TransportFailed", "last_reported_at": "2026-09-05T10:01:00Z", + "last_failure": null, }]) ); } @@ -8688,6 +8804,7 @@ mod tests { path: "/mcp".to_string(), last_result: EndpointResult::TransportFailed as i32, last_reported_time: Some("2026-09-05T11:01:00Z".parse().unwrap()), + last_failure: None, }; assert_eq!( @@ -8709,6 +8826,7 @@ mod tests { path: "/**".to_string(), last_result: EndpointResult::NoObservedExchange as i32, last_reported_time: None, + last_failure: None, }; assert_eq!( @@ -8736,11 +8854,149 @@ mod tests { "path": "/**", "last_result": "NoObservedExchange", "last_reported_at": "", + "last_failure": null, }]) ); assert_eq!(output["conditions"], serde_json::json!([])); } + #[test] + fn endpoint_status_explains_a_classified_credential_failure() { + let failure = openshell_core::proto::EndpointFailure { + domain: openshell_core::proto::EndpointFailureDomain::TokenGrant as i32, + code: "provider_authorization_required".to_string(), + recovery_action: + openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Reauthorize as i32, + provider: "github".to_string(), + credential_key: "access_token".to_string(), + stage: "gateway_rpc".to_string(), + upstream_http_status: Some(409), + request_id: "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908".to_string(), + }; + let endpoint = EndpointStatus { + endpoint_id: "endpoint:example".to_string(), + host: "tools.example.test".to_string(), + ports: vec![443], + path: "/mcp".to_string(), + last_result: EndpointResult::CredentialUnavailable as i32, + last_reported_time: Some("2026-09-05T11:01:00Z".parse().unwrap()), + last_failure: Some(failure), + }; + + assert_eq!( + super::endpoint_status_display_lines(&endpoint), + vec![ + "tools.example.test (ports: 443; path: /mcp)", + "Last result: Required OpenShell-managed credential was unavailable.", + "Cause: provider_authorization_required (token grant, stage gateway_rpc, upstream HTTP 409)", + "Credential: github/access_token", + "Recovery: Reauthorize the provider credential.", + "Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + "Reported at (gateway acceptance): 2026-09-05T11:01:00Z", + ] + ); + let sandbox = Sandbox { + status: Some(SandboxStatus { + endpoint_statuses: vec![endpoint], + ..Default::default() + }), + ..Default::default() + }; + let json = super::sandbox_to_json(&sandbox); + let yaml: serde_json::Value = + serde_yml::from_str(&serde_yml::to_string(&json).unwrap()).unwrap(); + for output in [json, yaml] { + assert_eq!( + output["endpoint_statuses"][0]["last_failure"], + serde_json::json!({ + "domain": "token_grant", + "code": "provider_authorization_required", + "recovery_action": "reauthorize", + "provider": "github", + "credential_key": "access_token", + "stage": "gateway_rpc", + "upstream_http_status": 409, + "request_id": "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + }) + ); + } + } + + #[test] + fn sandbox_reports_rest_token_grant_failure_without_mcp_inventory() { + let observation = openshell_core::proto::TokenGrantFailureObservation { + destination_host: "api.example.test".to_string(), + destination_port: 8443, + failure: Some(openshell_core::proto::EndpointFailure { + domain: openshell_core::proto::EndpointFailureDomain::TokenGrant.into(), + code: "provider_authorization_required".to_string(), + recovery_action: + openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Reauthorize + .into(), + provider: "github".to_string(), + credential_key: "access_token".to_string(), + stage: "token_endpoint".to_string(), + upstream_http_status: Some(409), + request_id: "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908".to_string(), + }), + }; + assert_eq!( + super::token_grant_failure_display_lines(&observation), + vec![ + "Destination: api.example.test (port 8443)", + "Cause: provider_authorization_required (token grant, stage token_endpoint, upstream HTTP 409)", + "Credential: github/access_token", + "Recovery: Reauthorize the provider credential.", + "Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + ] + ); + let sandbox = Sandbox { + status: Some(SandboxStatus { + phase: SandboxPhase::Ready.into(), + last_token_grant_failure: Some(Box::new(observation)), + ..Default::default() + }), + ..Default::default() + }; + let json = super::sandbox_to_json(&sandbox); + let yaml: serde_json::Value = + serde_yml::from_str(&serde_yml::to_string(&json).unwrap()).unwrap(); + for output in [json, yaml] { + assert_eq!(output["phase"], "Ready"); + assert_eq!(output["endpoint_statuses"], serde_json::json!([])); + assert_eq!( + output["last_token_grant_failure"], + serde_json::json!({ + "destination_host": "api.example.test", + "destination_port": 8443, + "failure": { + "domain": "token_grant", + "code": "provider_authorization_required", + "recovery_action": "reauthorize", + "provider": "github", + "credential_key": "access_token", + "stage": "token_endpoint", + "upstream_http_status": 409, + "request_id": "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + }, + }) + ); + } + assert!(super::sandbox_to_json(&Sandbox::default())["last_token_grant_failure"].is_null()); + } + + #[test] + fn endpoint_failure_json_names_an_unspecified_recovery() { + let failure = openshell_core::proto::EndpointFailure { + domain: 99, + recovery_action: 99, + ..Default::default() + }; + let json = super::endpoint_failure_to_json(&failure); + assert_eq!(json["domain"], "unknown"); + assert_eq!(json["recovery_action"], "unspecified"); + } + #[test] fn endpoint_status_http_response_does_not_claim_a_successful_tool_call() { let endpoint = EndpointStatus { diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index bcaf213f51..f0c1d85847 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -67,6 +67,7 @@ protoc-bin-vendored = { workspace = true } [dev-dependencies] tempfile = "3" rcgen = { workspace = true } +tokio-rustls = { workspace = true } [lints] workspace = true diff --git a/crates/openshell-core/build.rs b/crates/openshell-core/build.rs index d3b078941a..5ff822affb 100644 --- a/crates/openshell-core/build.rs +++ b/crates/openshell-core/build.rs @@ -43,6 +43,8 @@ fn main() -> Result<(), Box> { // the build process environment. let mut proto_config = tonic_prost_build::Config::new(); proto_config.protoc_executable(protoc_bin_vendored::protoc_bin_path()?); + // Keep optional grant evidence off sandbox lifecycle futures' stacks. + proto_config.boxed(".openshell.v1.SandboxStatus.last_token_grant_failure"); let proto_include = protoc_bin_vendored::include_path()?; let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?); diff --git a/crates/openshell-core/src/endpoint_failure.rs b/crates/openshell-core/src/endpoint_failure.rs new file mode 100644 index 0000000000..dba48aaa07 --- /dev/null +++ b/crates/openshell-core/src/endpoint_failure.rs @@ -0,0 +1,389 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Bounded causes for failed endpoint results. +//! +//! A cause holds a code and a stage from the closed sets of its domain, the +//! recovery action documented for the code, configured provider and +//! credential names, an optional HTTP status, and the supervisor's request ID. +//! The supervisor builds causes from typed failures and the gateway validates +//! reported causes against the same sets here, so neither side can accept a +//! code the other does not define. Each domain also names the endpoint results +//! it can explain. No cause carries remote text. +//! +//! A new producer adds a domain variant and one validation function in this +//! module that checks its result, code and stage sets, and code-to-recovery +//! mapping. Names, status, and the request ID are checked the same way for +//! every domain. + +use crate::proto::{self, ProviderCredentialRefreshRecoveryAction}; +use crate::token_grant_failure::{TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage}; + +/// Longest provider or credential name a reported cause may carry. +/// +/// A reported name is also limited to printable ASCII, so status renderers +/// cannot be given control, bidirectional, or format characters. +pub const MAX_ENDPOINT_FAILURE_NAME_BYTES: usize = 253; + +/// The bounded cause of one failed endpoint observation. +/// +/// This explains why an exchange failed; it does not establish that the remote +/// endpoint is unhealthy. A token grant can fail before the request reaches it. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EndpointFailure { + domain: proto::EndpointFailureDomain, + code: &'static str, + stage: &'static str, + recovery: ProviderCredentialRefreshRecoveryAction, + provider: String, + credential_key: String, + upstream_http_status: Option, + request_id: String, +} + +impl EndpointFailure { + /// Describe a refused token grant on the exchange named by `request_id`. + /// + /// `provider` and `credential_key` are empty when the grant failed before + /// a credential was selected. Names the gateway would reject are reported + /// as empty too: one unreportable name must not discard the sandbox's + /// whole status report. + #[must_use] + pub fn token_grant( + failure: TokenGrantFailure, + provider: &str, + credential_key: &str, + request_id: &str, + ) -> Self { + let (provider, credential_key) = if provider.is_empty() == credential_key.is_empty() + && valid_name(provider) + && valid_name(credential_key) + { + (provider, credential_key) + } else { + ("", "") + }; + Self { + domain: proto::EndpointFailureDomain::TokenGrant, + code: failure.code(), + stage: failure.stage().code(), + recovery: failure.recovery(), + provider: provider.to_string(), + credential_key: credential_key.to_string(), + upstream_http_status: failure.upstream_status(), + request_id: request_id.to_string(), + } + } + + /// Return the protobuf form reported to the gateway. + #[must_use] + pub fn to_proto(&self) -> proto::EndpointFailure { + proto::EndpointFailure { + domain: self.domain.into(), + code: self.code.to_string(), + recovery_action: self.recovery.into(), + provider: self.provider.clone(), + credential_key: self.credential_key.clone(), + stage: self.stage.to_string(), + upstream_http_status: self.upstream_http_status.map(u32::from), + request_id: self.request_id.clone(), + } + } +} + +/// Why a reported cause was rejected. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum EndpointFailureValidationError { + /// The domain is unspecified or unknown. + Domain, + /// The domain does not explain the endpoint result it accompanies. + Result, + /// The code is not in the domain's closed set. + Code, + /// The stage is not in the domain's closed set. + Stage, + /// The recovery action is not the one documented for the code. + Recovery, + /// A provider or credential name is malformed, or only one is present. + Name, + /// The HTTP status is outside 100 through 599. + UpstreamStatus, + /// The request ID is not a canonical lowercase UUID. + RequestId, +} + +impl EndpointFailureValidationError { + /// Return a fixed description that contains none of the rejected values. + #[must_use] + pub const fn message(self) -> &'static str { + match self { + Self::Domain => "endpoint failure domain is invalid", + Self::Result => "endpoint failure domain does not explain this endpoint result", + Self::Code => "endpoint failure code is not defined for its domain", + Self::Stage => "endpoint failure stage is not defined for its domain", + Self::Recovery => "endpoint failure recovery action does not match its code", + Self::Name => "endpoint failure provider and credential names are invalid", + Self::UpstreamStatus => "endpoint failure upstream HTTP status is invalid", + Self::RequestId => "endpoint failure request_id must be a canonical UUID", + } + } +} + +/// Check a reported cause, and the result it accompanies, against the closed +/// sets of its domain. +/// +/// The gateway separately checks that a named provider is attached to the +/// reporting sandbox. +/// +/// # Errors +/// +/// Returns the first rule the cause breaks. +pub fn validate_endpoint_failure( + failure: &proto::EndpointFailure, + result: proto::EndpointResult, +) -> Result<(), EndpointFailureValidationError> { + match proto::EndpointFailureDomain::try_from(failure.domain) { + Ok(proto::EndpointFailureDomain::TokenGrant) => validate_token_grant(failure, result)?, + Ok(proto::EndpointFailureDomain::Unspecified) | Err(_) => { + return Err(EndpointFailureValidationError::Domain); + } + } + let named = !failure.provider.is_empty(); + if named == failure.credential_key.is_empty() + || !valid_name(&failure.provider) + || !valid_name(&failure.credential_key) + { + return Err(EndpointFailureValidationError::Name); + } + if failure + .upstream_http_status + .is_some_and(|status| !(100..=599).contains(&status)) + { + return Err(EndpointFailureValidationError::UpstreamStatus); + } + if !is_canonical_uuid(&failure.request_id) { + return Err(EndpointFailureValidationError::RequestId); + } + Ok(()) +} + +/// Check the parts of a token-grant cause that its domain defines. +fn validate_token_grant( + failure: &proto::EndpointFailure, + result: proto::EndpointResult, +) -> Result<(), EndpointFailureValidationError> { + // A refused grant means the endpoint's credential was unavailable. + if result != proto::EndpointResult::CredentialUnavailable { + return Err(EndpointFailureValidationError::Result); + } + let kind = TokenGrantFailureKind::from_code(&failure.code) + .ok_or(EndpointFailureValidationError::Code)?; + TokenGrantStage::from_code(&failure.stage).ok_or(EndpointFailureValidationError::Stage)?; + if failure.recovery_action != i32::from(kind.recovery()) { + return Err(EndpointFailureValidationError::Recovery); + } + Ok(()) +} + +fn valid_name(name: &str) -> bool { + name.len() <= MAX_ENDPOINT_FAILURE_NAME_BYTES + && name.bytes().all(|byte| byte.is_ascii_graphic()) +} + +fn is_canonical_uuid(value: &str) -> bool { + uuid::Uuid::parse_str(value).is_ok_and(|parsed| parsed.hyphenated().to_string() == value) +} + +#[cfg(test)] +mod tests { + use super::*; + + const REQUEST_ID: &str = "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9"; + const CREDENTIAL_UNAVAILABLE: proto::EndpointResult = + proto::EndpointResult::CredentialUnavailable; + + fn reported() -> proto::EndpointFailure { + EndpointFailure::token_grant( + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ) + .with_upstream_status(409), + "github", + "access_token", + REQUEST_ID, + ) + .to_proto() + } + + #[test] + fn token_grant_failure_reports_its_code_stage_recovery_and_names() { + assert_eq!( + reported(), + proto::EndpointFailure { + domain: proto::EndpointFailureDomain::TokenGrant.into(), + code: "provider_authorization_required".into(), + recovery_action: ProviderCredentialRefreshRecoveryAction::Reauthorize.into(), + provider: "github".into(), + credential_key: "access_token".into(), + stage: "gateway_rpc".into(), + upstream_http_status: Some(409), + request_id: REQUEST_ID.into(), + } + ); + assert_eq!( + validate_endpoint_failure(&reported(), CREDENTIAL_UNAVAILABLE), + Ok(()) + ); + } + + #[test] + fn every_token_grant_classification_validates() { + for kind in [ + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantFailureKind::Configuration, + TokenGrantFailureKind::Timeout, + TokenGrantFailureKind::Unavailable, + TokenGrantFailureKind::Unclassified, + ] { + for stage in [ + TokenGrantStage::Configuration, + TokenGrantStage::WorkloadIdentity, + TokenGrantStage::GatewayRpc, + TokenGrantStage::IntermediateExchange, + TokenGrantStage::TokenEndpoint, + TokenGrantStage::Injection, + ] { + let failure = EndpointFailure::token_grant( + TokenGrantFailure::new(kind, stage), + "", + "", + REQUEST_ID, + ); + assert_eq!( + validate_endpoint_failure(&failure.to_proto(), CREDENTIAL_UNAVAILABLE), + Ok(()) + ); + } + } + } + + type Mutation = fn(&mut proto::EndpointFailure); + + #[test] + fn token_grant_causes_explain_only_credential_failures() { + for result in [ + proto::EndpointResult::Unspecified, + proto::EndpointResult::NoObservedExchange, + proto::EndpointResult::HttpResponseReceived, + proto::EndpointResult::PolicyDenied, + proto::EndpointResult::TlsFailed, + proto::EndpointResult::TransportFailed, + proto::EndpointResult::UpstreamRejected, + ] { + assert_eq!( + validate_endpoint_failure(&reported(), result), + Err(EndpointFailureValidationError::Result), + "{result:?}" + ); + } + assert_eq!( + validate_endpoint_failure(&reported(), CREDENTIAL_UNAVAILABLE), + Ok(()) + ); + } + + #[test] + fn constructed_causes_always_pass_gateway_validation() { + let oversized = "k".repeat(MAX_ENDPOINT_FAILURE_NAME_BYTES + 1); + for (provider, credential_key) in [ + ("github", oversized.as_str()), + ("git\u{202e}hub", "access_token"), + ("github", "access token"), + ("github", ""), + ("", "access_token"), + ] { + let failure = EndpointFailure::token_grant( + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Injection, + ), + provider, + credential_key, + REQUEST_ID, + ) + .to_proto(); + assert_eq!( + validate_endpoint_failure(&failure, CREDENTIAL_UNAVAILABLE), + Ok(()) + ); + assert!(failure.provider.is_empty() && failure.credential_key.is_empty()); + } + } + + #[test] + fn rejects_values_outside_the_closed_sets() { + let cases: [(Mutation, EndpointFailureValidationError); 15] = [ + (|f| f.domain = 0, EndpointFailureValidationError::Domain), + (|f| f.domain = 99, EndpointFailureValidationError::Domain), + ( + |f| f.code = "reauthorization_required".into(), + EndpointFailureValidationError::Code, + ), + ( + |f| f.code = "Provider_Authorization_Required".into(), + EndpointFailureValidationError::Code, + ), + ( + |f| f.stage = "token service said no".into(), + EndpointFailureValidationError::Stage, + ), + ( + |f| f.recovery_action = ProviderCredentialRefreshRecoveryAction::Retry.into(), + EndpointFailureValidationError::Recovery, + ), + (|f| f.provider.clear(), EndpointFailureValidationError::Name), + ( + |f| f.credential_key.clear(), + EndpointFailureValidationError::Name, + ), + ( + |f| f.provider = "git\nhub".into(), + EndpointFailureValidationError::Name, + ), + ( + |f| f.provider = "git\u{202e}hub".into(), + EndpointFailureValidationError::Name, + ), + ( + |f| f.credential_key = "k".repeat(MAX_ENDPOINT_FAILURE_NAME_BYTES + 1), + EndpointFailureValidationError::Name, + ), + ( + |f| f.upstream_http_status = Some(600), + EndpointFailureValidationError::UpstreamStatus, + ), + ( + |f| f.upstream_http_status = Some(99), + EndpointFailureValidationError::UpstreamStatus, + ), + ( + |f| f.request_id = REQUEST_ID.to_ascii_uppercase(), + EndpointFailureValidationError::RequestId, + ), + ( + |f| f.request_id.clear(), + EndpointFailureValidationError::RequestId, + ), + ]; + for (mutate, expected) in cases { + let mut failure = reported(); + mutate(&mut failure); + assert_eq!( + validate_endpoint_failure(&failure, CREDENTIAL_UNAVAILABLE), + Err(expected) + ); + } + } +} diff --git a/crates/openshell-core/src/endpoint_status.rs b/crates/openshell-core/src/endpoint_status.rs index 66bcf278c4..4cb604cf2b 100644 --- a/crates/openshell-core/src/endpoint_status.rs +++ b/crates/openshell-core/src/endpoint_status.rs @@ -8,7 +8,8 @@ //! //! The types in this module deliberately cannot carry request URLs, headers, //! bodies, tool arguments, credentials, or upstream error text. Network -//! enforcement reports only the configured endpoint identifier and a typed result. +//! enforcement reports configured endpoint results and the latest refused token +//! grant with a bounded host, port, and [`EndpointFailure`]. use std::collections::BTreeMap; use std::fmt::Write as _; @@ -17,6 +18,8 @@ use std::sync::Arc; use sha2::{Digest, Sha256}; use tokio::sync::{mpsc, watch}; +use crate::endpoint_failure::EndpointFailure; + /// Maximum number of pending endpoint observations between network enforcement /// and the sandbox orchestrator. /// @@ -85,6 +88,7 @@ pub fn initial_endpoint_status( path: path.to_string(), last_result: crate::proto::EndpointResult::NoObservedExchange.into(), last_reported_time: None, + last_failure: None, } } @@ -186,6 +190,24 @@ pub struct EndpointObservationContext { authority_generation: EndpointObservationGeneration, } +impl EndpointObservationContext { + /// Check that selected request state belongs to the captured configuration. + #[must_use] + pub fn matches_config(&self, policy_hash: &str, provider_env_revision: Option) -> bool { + self.inventory.config_version.policy_hash == policy_hash + && provider_env_revision.is_none_or(|revision| { + revision == self.inventory.config_version.provider_env_revision + }) + } + + /// Check a provider snapshot when forwarding has no L7 endpoint selection. + #[must_use] + pub fn matches_provider_revision(&self, revision: Option) -> bool { + revision + .is_none_or(|revision| revision == self.inventory.config_version.provider_env_revision) + } +} + /// Commands consumed in FIFO order by the sandbox endpoint-status tracker. #[derive(Debug)] pub enum EndpointStatusCommand { @@ -206,11 +228,24 @@ pub enum EndpointStatusCommand { endpoint_id: String, /// Typed, non-sensitive terminal result. result: EndpointResult, + /// Bounded cause of the result, when the enforcement point classified it. + failure: Option, /// Inventory installation captured when the request started. inventory_generation: EndpointObservationGeneration, /// Supervisor observation authority captured when the request started. authority_generation: EndpointObservationGeneration, }, + /// Report a refused grant, or a successful grant for a named credential. + ObserveTokenGrant { + /// Authority captured before request selection and asynchronous work. + context: EndpointObservationContext, + /// Canonical destination, without paths, query strings, or headers. + destination_host: String, + /// Destination port. + destination_port: u16, + /// Success identifies the credential; failure carries its bounded cause. + result: Result<(String, String), EndpointFailure>, + }, } /// Receiving half of the bounded tool server endpoint-status channel. @@ -232,6 +267,7 @@ impl EndpointStatusReceiver { inventory_generation: None, current: self.current.clone(), endpoints: BTreeMap::new(), + last_token_grant_failure: None, } } @@ -262,6 +298,29 @@ pub struct EndpointObservationSender { } impl EndpointObservationSender { + /// Report a grant outcome without delaying traffic. Success clears only a + /// failure for the same destination and credential (or an unselected grant). + /// Invalid destinations and saturated queues lose telemetry, never traffic. + #[must_use] + pub fn try_observe_token_grant( + &self, + context: EndpointObservationContext, + destination_host: &str, + destination_port: u16, + result: Result<(String, String), EndpointFailure>, + ) -> bool { + if !valid_token_grant_destination(destination_host, u32::from(destination_port)) { + return false; + } + self.commands + .try_send(EndpointStatusCommand::ObserveTokenGrant { + context, + destination_host: destination_host.to_ascii_lowercase(), + destination_port, + result, + }) + .is_ok() + } /// Capture the installed observation authority before selecting request state. /// /// Returns `None` until the first complete endpoint inventory is installed. @@ -375,6 +434,20 @@ impl EndpointObservationSender { &self, observation: EndpointObservationHandle, result: EndpointResult, + ) -> bool { + self.try_observe_with_failure(observation, result, None) + } + + /// Try to enqueue a terminal observation and its bounded cause. + /// + /// The cause replaces any earlier cause for the endpoint; `None` clears it. + /// Capacity and rejection behave as in [`Self::try_observe`]. + #[must_use] + pub fn try_observe_with_failure( + &self, + observation: EndpointObservationHandle, + result: EndpointResult, + failure: Option, ) -> bool { if result == EndpointResult::NoObservedExchange { return false; @@ -384,6 +457,7 @@ impl EndpointObservationSender { config_version: observation.config_version, endpoint_id: observation.endpoint_id, result, + failure, inventory_generation: observation.inventory_generation, authority_generation: observation.authority_generation, }) @@ -391,6 +465,25 @@ impl EndpointObservationSender { } } +/// Accept only a bounded DNS host or IP address, never a URL or arbitrary text. +#[must_use] +pub fn valid_token_grant_destination(host: &str, port: u32) -> bool { + if host.is_empty() || host.len() > 253 || !(1..=65535).contains(&port) { + return false; + } + host.parse::().is_ok() + || host + .strip_prefix('[') + .and_then(|host| host.strip_suffix(']')) + .is_some_and(|host| host.parse::().is_ok()) + || host + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'-')) + && host + .split('.') + .all(|label| !label.is_empty() && !label.starts_with('-') && !label.ends_with('-')) +} + /// Create the bounded channel used for tool server endpoint inventory and observations. #[must_use] pub fn endpoint_status_channel() -> (EndpointObservationSender, EndpointStatusReceiver) { @@ -418,6 +511,18 @@ pub struct EndpointObservation { pub endpoint_id: String, /// Latest result observed for the endpoint in the current configuration version. pub result: EndpointResult, + /// Bounded cause of `result`, when the enforcement point classified it. + pub failure: Option, +} + +/// The latest result for one endpoint and the cause reported with it. +/// +/// Both fields change together, so a later result can never inherit an +/// earlier result's cause. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +struct LatestObservation { + result: EndpointResult, + failure: Option, } /// Complete tool server endpoint status for one installed policy and provider revision. @@ -427,6 +532,8 @@ pub struct EndpointStatusSnapshot { pub config_version: EndpointConfigVersion, /// Complete, identifier-sorted set of configured endpoint statuses. pub endpoints: Vec, + /// Latest refused grant across all HTTP protocols, independent of MCP inventory. + pub last_token_grant_failure: Option, /// Endpoints that carried at least one new result in this report batch. /// /// The gateway uses this set to advance observation timestamps only for @@ -452,7 +559,8 @@ pub struct EndpointStatusTracker { config_version: Option, inventory_generation: Option, current: watch::Sender, - endpoints: BTreeMap, + endpoints: BTreeMap, + last_token_grant_failure: Option, } impl EndpointStatusTracker { @@ -480,6 +588,7 @@ impl EndpointStatusTracker { config_version, endpoint_id, result, + failure, inventory_generation, authority_generation, } => { @@ -503,9 +612,59 @@ impl EndpointStatusTracker { // Repeated results are material observations: the gateway // advances endpoint freshness even when the classification is // unchanged. - *endpoint = result; + *endpoint = LatestObservation { result, failure }; true } + EndpointStatusCommand::ObserveTokenGrant { + context, + destination_host, + destination_port, + result, + } => { + if self.config_version.as_ref() != Some(&context.inventory.config_version) + || !self + .inventory_generation + .as_ref() + .is_some_and(|generation| generation.matches(&context.inventory.generation)) + || !self + .current + .borrow() + .authority_generation + .matches(&context.authority_generation) + { + return false; + } + match result { + Err(failure) => { + self.last_token_grant_failure = + Some(crate::proto::TokenGrantFailureObservation { + destination_host, + destination_port: destination_port.into(), + failure: Some(failure.to_proto()), + }); + true + } + Ok((provider, credential_key)) => { + let matches = + self.last_token_grant_failure + .as_ref() + .is_some_and(|observation| { + observation.destination_host == destination_host + && observation.destination_port + == u32::from(destination_port) + && observation.failure.as_ref().is_some_and(|failure| { + failure.provider.is_empty() + || (failure.provider == provider + && failure.credential_key == credential_key) + }) + }); + if matches { + self.last_token_grant_failure = None; + } + matches + } + } + } } } @@ -518,12 +677,14 @@ impl EndpointStatusTracker { .iter() .map(|(endpoint_id, endpoint)| EndpointObservation { endpoint_id: endpoint_id.clone(), - result: *endpoint, + result: endpoint.result, + failure: endpoint.failure.clone(), }) .collect(); Some(EndpointStatusSnapshot { config_version, endpoints, + last_token_grant_failure: self.last_token_grant_failure.clone(), observed_endpoint_ids: Vec::new(), supervisor_session_id: String::new(), report_sequence: 0, @@ -546,8 +707,9 @@ impl EndpointStatusTracker { return false; }; for endpoint in self.endpoints.values_mut() { - *endpoint = EndpointResult::NoObservedExchange; + *endpoint = LatestObservation::default(); } + self.last_token_grant_failure = None; true } @@ -563,6 +725,9 @@ impl EndpointStatusTracker { let provider_changed = self.config_version.as_ref().is_some_and(|current| { current.provider_env_revision != config_version.provider_env_revision }); + if policy_changed || provider_changed { + self.last_token_grant_failure = None; + } let previous = std::mem::take(&mut self.endpoints); self.endpoints = inventory @@ -573,15 +738,15 @@ impl EndpointStatusTracker { // depend on provider-managed credentials. let preserve = !policy_changed && (!provider_changed || !endpoint.uses_provider_credentials); - let result = if preserve { + let latest = if preserve { previous .get(&endpoint.endpoint_id) - .copied() - .unwrap_or(EndpointResult::NoObservedExchange) + .cloned() + .unwrap_or_default() } else { - EndpointResult::NoObservedExchange + LatestObservation::default() }; - (endpoint.endpoint_id, result) + (endpoint.endpoint_id, latest) }) .collect(); self.config_version = Some(config_version); @@ -705,6 +870,16 @@ mod tests { config_version: EndpointConfigVersion, endpoint_id: &str, result: EndpointResult, + ) -> EndpointStatusCommand { + observe_with_failure(tracker, config_version, endpoint_id, result, None) + } + + fn observe_with_failure( + tracker: &EndpointStatusTracker, + config_version: EndpointConfigVersion, + endpoint_id: &str, + result: EndpointResult, + failure: Option, ) -> EndpointStatusCommand { EndpointStatusCommand::Observe { inventory_generation: tracker @@ -715,9 +890,83 @@ mod tests { config_version, endpoint_id: endpoint_id.to_string(), result, + failure, } } + #[test] + fn failure_travels_with_its_result_and_clears_with_it() { + use crate::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + }; + let failure = EndpointFailure::token_grant( + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ), + "github", + "access_token", + "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", + ); + let latest = |tracker: &EndpointStatusTracker| { + tracker + .snapshot() + .expect("inventory installed") + .endpoints + .into_iter() + .find(|endpoint| endpoint.endpoint_id == "endpoint:credentialed") + .expect("credentialed endpoint") + }; + let fail = |tracker: &EndpointStatusTracker, version| { + observe_with_failure( + tracker, + version, + "endpoint:credentialed", + EndpointResult::CredentialUnavailable, + Some(failure.clone()), + ) + }; + let mut tracker = endpoint_status_channel().1.tracker(); + tracker.apply(EndpointStatusCommand::Reset { + inventory_generation: EndpointObservationGeneration::new(), + config_version: config_version("policy-a", 1), + endpoints: inventory(), + }); + + assert!(tracker.apply(fail(&tracker, config_version("policy-a", 1)))); + assert_eq!(latest(&tracker).failure.as_ref(), Some(&failure)); + + // A later success replaces the result and its cause together. + tracker.apply(observe( + &tracker, + config_version("policy-a", 1), + "endpoint:credentialed", + EndpointResult::HttpResponseReceived, + )); + assert_eq!(latest(&tracker).failure, None); + + // A replacement supervisor session forgets the cause. + tracker.apply(fail(&tracker, config_version("policy-a", 1))); + assert!(tracker.clear_observations()); + assert_eq!(latest(&tracker).failure, None); + + // A provider revision, such as a reauthorization, clears credentialed causes. + tracker.apply(fail(&tracker, config_version("policy-a", 1))); + tracker.apply(EndpointStatusCommand::Reset { + inventory_generation: EndpointObservationGeneration::new(), + config_version: config_version("policy-a", 2), + endpoints: inventory(), + }); + assert_eq!( + latest(&tracker), + EndpointObservation { + endpoint_id: "endpoint:credentialed".to_string(), + result: EndpointResult::NoObservedExchange, + failure: None, + } + ); + } + #[test] fn policy_change_resets_every_endpoint() { let mut tracker = endpoint_status_channel().1.tracker(); @@ -793,10 +1042,12 @@ mod tests { EndpointObservation { endpoint_id: "endpoint:new".to_string(), result: EndpointResult::NoObservedExchange, + failure: None, }, EndpointObservation { endpoint_id: "endpoint:public".to_string(), result: EndpointResult::HttpResponseReceived, + failure: None, }, ] ); @@ -832,10 +1083,12 @@ mod tests { EndpointObservation { endpoint_id: "endpoint:credentialed".to_string(), result: EndpointResult::NoObservedExchange, + failure: None, }, EndpointObservation { endpoint_id: "endpoint:public".to_string(), result: EndpointResult::HttpResponseReceived, + failure: None, }, ] ); @@ -1181,4 +1434,262 @@ mod tests { }) if endpoint_id == "endpoint:public" )); } + + fn grant_refusal(provider: &str, credential: &str) -> EndpointFailure { + use crate::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + }; + EndpointFailure::token_grant( + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, + ), + provider, + credential, + "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", + ) + } + + #[tokio::test] + async fn grant_failure_without_mcp_inventory_clears_only_on_matching_success() { + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset(config_version("policy-a", 1), Vec::new()) + .await + .unwrap(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let context = sender.capture().unwrap(); + let failure = grant_refusal("github", "token"); + assert!(sender.try_observe_token_grant( + context.clone(), + "API.EXAMPLE.TEST", + 8443, + Err(failure.clone()), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let expected = crate::proto::TokenGrantFailureObservation { + destination_host: "api.example.test".into(), + destination_port: 8443, + failure: Some(failure.to_proto()), + }; + assert!(tracker.snapshot().unwrap().endpoints.is_empty()); + assert_eq!( + tracker.snapshot().unwrap().last_token_grant_failure, + Some(expected.clone()) + ); + for (host, port, provider, credential) in [ + ("other.example.test", 8443, "github", "token"), + ("api.example.test", 443, "github", "token"), + ("api.example.test", 8443, "other", "token"), + ("api.example.test", 8443, "github", "other"), + ] { + assert!(sender.try_observe_token_grant( + context.clone(), + host, + port, + Ok((provider.into(), credential.into())), + )); + assert!(!tracker.apply(receiver.recv().await.unwrap())); + assert_eq!( + tracker.snapshot().unwrap().last_token_grant_failure, + Some(expected.clone()) + ); + } + assert!(sender.try_observe_token_grant( + context.clone(), + "api.example.test", + 8443, + Ok(("github".into(), "token".into())), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + + assert!(sender.try_observe_token_grant( + context.clone(), + "api.example.test", + 8443, + Err(grant_refusal("", "")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!(sender.try_observe_token_grant( + context, + "api.example.test", + 8443, + Ok(("selected".into(), "token".into())), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + } + + #[tokio::test] + async fn grant_context_stays_retired_across_configuration_and_authority_cycles() { + for replacement in [config_version("policy-b", 1), config_version("policy-a", 2)] { + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset(config_version("policy-a", 1), Vec::new()) + .await + .unwrap(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let retired = sender.capture().unwrap(); + assert!(sender.try_observe_token_grant( + retired.clone(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + for version in [replacement, config_version("policy-a", 1)] { + sender.reset(version, Vec::new()).await.unwrap(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + } + assert!(sender.try_observe_token_grant( + retired, + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(!tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + let retired = sender.capture().unwrap(); + assert!(sender.try_observe_token_grant( + retired.clone(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!(tracker.clear_observations()); + assert!(sender.try_observe_token_grant( + retired, + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(!tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + assert!(sender.try_observe_token_grant( + sender.capture().unwrap(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_some() + ); + } + } + + #[tokio::test] + async fn invalid_destinations_and_full_grant_queue_drop_only_telemetry() { + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset(config_version("policy-a", 1), Vec::new()) + .await + .unwrap(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let context = sender.capture().unwrap(); + let oversized = "a".repeat(254); + for (host, port) in [ + ("", 443), + ("api.example.test", 0), + ("api\n.example.test", 443), + ("api example.test", 443), + ("https://api.example.test/path?token=secret", 443), + ("user@api.example.test", 443), + ("api\u{202e}.example.test", 443), + (oversized.as_str(), 443), + ] { + assert!(!sender.try_observe_token_grant( + context.clone(), + host, + port, + Err(grant_refusal("github", "token")), + )); + } + assert!(receiver.try_recv().is_err()); + assert!(!valid_token_grant_destination("api.example.test", 65536)); + let longest = [ + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(61), + ] + .join("."); + for (host, port) in [(longest.as_str(), u16::MAX), ("::1", 443)] { + assert!(sender.try_observe_token_grant( + context.clone(), + host, + port, + Err(grant_refusal("github", "token")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + } + for _ in 0..ENDPOINT_STATUS_CHANNEL_CAPACITY { + assert!(sender.try_observe_token_grant( + context.clone(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + } + assert!(!sender.try_observe_token_grant( + context.clone(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!(sender.try_observe_token_grant( + context.clone(), + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + drop(receiver); + assert!(!sender.try_observe_token_grant( + context, + "api.example.test", + 443, + Err(grant_refusal("github", "token")), + )); + } } diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index ccaa774f16..8b07457086 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -35,6 +35,7 @@ use crate::proto::{ }; use crate::sandbox_env; use crate::time::{duration_to_std, timestamp_to_millis}; +use crate::token_grant_failure::{TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage}; use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_extension_core::{BearerTokenSlot, ExtensionCredentialStore}; use tonic::Status; @@ -280,7 +281,7 @@ impl ClientSpanStatus { } } - fn finish(mut self, result: Result) -> Result { + fn finish(mut self, result: std::result::Result) -> std::result::Result { self.finished = result.is_ok(); result } @@ -1349,6 +1350,12 @@ mod provider_environment_tests { } } +/// Exchange a provider credential without retaining gateway diagnostics. +/// +/// Only a verified TLS connection can establish the provenance of a precise +/// grant classification. Plaintext responses remain safe unknown failures. +/// A `request_id` travels as `x-request-id` metadata, which the gateway keeps +/// on its request span and failure log line. #[tracing::instrument( name = "supervisor.gateway.exchange_provider_subject_token", skip_all, @@ -1360,56 +1367,274 @@ pub async fn exchange_provider_subject_token( provider: &str, credential_key: &str, supervisor_jwt_svid: &str, -) -> Result { + request_id: Option<&str>, +) -> std::result::Result { let status = ClientSpanStatus::current(); - debug!( - endpoint = %endpoint, - sandbox_id = %sandbox_id, - provider = %provider, - credential_key = %credential_key, - "Exchanging provider subject token through gateway" - ); + status.finish( + exchange_provider_subject_token_inner( + endpoint, + sandbox_id, + provider, + credential_key, + supervisor_jwt_svid, + request_id, + ) + .await, + ) +} - let mut client = connect(endpoint).await?; - let response = client - .exchange_provider_subject_token(ExchangeProviderSubjectTokenRequest { +async fn exchange_provider_subject_token_inner( + endpoint: &str, + sandbox_id: &str, + provider: &str, + credential_key: &str, + supervisor_jwt_svid: &str, + request_id: Option<&str>, +) -> std::result::Result { + let endpoint = token_grant_gateway_endpoint(endpoint)?; + if sandbox_id.is_empty() || provider.is_empty() || credential_key.is_empty() { + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::GatewayRpc, + )); + } + if supervisor_jwt_svid.is_empty() { + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantStage::WorkloadIdentity, + )); + } + // TLS setup errors can contain paths and transport errors can contain URLs. + // Discard those sources at the grant boundary; never diagnose from prose. + let mut client = tokio::time::timeout(Duration::from_secs(35), connect(endpoint.as_str())) + .await + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))? + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))?; + let request = provider_subject_token_exchange_request( + ExchangeProviderSubjectTokenRequest { sandbox_id: sandbox_id.to_string(), provider: provider.to_string(), credential_key: credential_key.to_string(), supervisor_jwt_svid: supervisor_jwt_svid.to_string(), - }) - .await - .map_err(provider_subject_token_exchange_status)?; - let inner = response.into_inner(); + }, + request_id, + ); + let response = tokio::time::timeout( + Duration::from_secs(35), + client.exchange_provider_subject_token(request), + ) + .await + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))? + .map_err(|status| provider_subject_token_exchange_status(&status, &endpoint))?; + provider_subject_token_exchange_result(response.into_inner()) +} + +/// Attach the exchange's request ID as `x-request-id` gRPC metadata. +/// +/// A value that is not valid metadata is omitted, never rewritten. +fn provider_subject_token_exchange_request( + body: ExchangeProviderSubjectTokenRequest, + request_id: Option<&str>, +) -> tonic::Request { + let mut request = tonic::Request::new(body); + if let Some(request_id) = request_id.and_then(|value| value.parse().ok()) { + request.metadata_mut().insert("x-request-id", request_id); + } + request +} + +fn gateway_grant_failure(kind: TokenGrantFailureKind) -> TokenGrantFailure { + TokenGrantFailure::new(kind, TokenGrantStage::GatewayRpc) +} + +/// Validate a grant gateway authority before acquiring a workload assertion. +/// +/// Rejects endpoint components that cannot be used by the gRPC transport and +/// might otherwise carry sensitive values into connection diagnostics. +pub fn validate_token_grant_gateway_endpoint( + endpoint: &str, +) -> std::result::Result<(), TokenGrantFailure> { + token_grant_gateway_endpoint(endpoint).map(|_| ()) +} + +fn token_grant_gateway_endpoint( + endpoint: &str, +) -> std::result::Result { + let endpoint = url::Url::parse(endpoint) + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Configuration))?; + // gRPC endpoints identify an authority, not an arbitrary token-service URL. + // Reject sensitive URL components before shared connection code can use it. + if !matches!(endpoint.scheme(), "http" | "https") + || endpoint.host_str().is_none() + || !endpoint.username().is_empty() + || endpoint.password().is_some() + || endpoint.query().is_some() + || endpoint.fragment().is_some() + || endpoint.path() != "/" + { + return Err(gateway_grant_failure(TokenGrantFailureKind::Configuration)); + } + Ok(endpoint) +} + +fn provider_subject_token_exchange_result( + inner: crate::proto::ExchangeProviderSubjectTokenResponse, +) -> std::result::Result { let expires_in = inner .expires_after .as_ref() .map(duration_to_std) .transpose() - .into_diagnostic()? + .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))? .map_or(0, |value| { i64::try_from(value.as_secs()).unwrap_or(i64::MAX) }); - status.finish(Ok(ProviderSubjectTokenExchangeResult { + Ok(ProviderSubjectTokenExchangeResult { access_token: inner.access_token, expires_in, token_type: inner.token_type, - })) + }) } -fn provider_subject_token_exchange_status(status: Status) -> miette::Report { - let message = status.message(); - if message.is_empty() { - miette::miette!( - "gateway ExchangeProviderSubjectToken failed with status {}", - status.code() - ) +fn provider_subject_token_exchange_status( + status: &Status, + endpoint: &url::Url, +) -> TokenGrantFailure { + // The shared connector verifies the gateway certificate for https + // endpoints; only that channel can vouch for a precise classification. + if endpoint.scheme() == "https" { + TokenGrantFailure::from_status(status) } else { - miette::miette!( - "gateway ExchangeProviderSubjectToken failed with status {}: {}", - status.code(), - message + // HTTP provides no authenticated server identity, even when the caller + // supplied a valid bearer. Structured details cannot confer that trust. + gateway_grant_failure(TokenGrantFailureKind::Unclassified) + } +} + +#[cfg(test)] +mod token_grant_exchange_tests { + use super::*; + + #[test] + fn authenticated_gateway_preserves_typed_failure_and_http_downgrades_it() { + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::IntermediateExchange, ) + .with_upstream_status(409); + let status = failure.to_status(); + let https = url::Url::parse("https://gateway.example:8080/").unwrap(); + let http = url::Url::parse("http://gateway.example:8080/").unwrap(); + assert_eq!( + provider_subject_token_exchange_status(&status, &https), + failure + ); + let downgraded = provider_subject_token_exchange_status(&status, &http); + assert_eq!(downgraded.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(downgraded.stage(), TokenGrantStage::GatewayRpc); + assert_eq!(downgraded.upstream_status(), None); + } + + #[test] + fn exchange_request_carries_only_a_valid_request_id() { + let body = || ExchangeProviderSubjectTokenRequest { + sandbox_id: "sandbox".into(), + provider: "provider".into(), + credential_key: "credential".into(), + supervisor_jwt_svid: "svid".into(), + }; + let request_id = "5c4b3a29-1807-4f6e-8d5c-4b3a29180716"; + let request = provider_subject_token_exchange_request(body(), Some(request_id)); + assert_eq!( + request + .metadata() + .get("x-request-id") + .and_then(|value| value.to_str().ok()), + Some(request_id) + ); + for absent in [None, Some("line\nbreak")] { + let request = provider_subject_token_exchange_request(body(), absent); + assert!(request.metadata().get("x-request-id").is_none()); + } + } + + #[test] + fn gateway_error_text_is_never_retained() { + use std::error::Error; + + let status = Status::failed_precondition( + "SYNTHETIC_GATEWAY_SECRET https://user:secret@token.example/path?token=secret", + ); + let https = url::Url::parse("https://gateway.example:8080/").unwrap(); + let failure = provider_subject_token_exchange_status(&status, &https); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + assert!(failure.source().is_none()); + } + + #[test] + fn grant_gateway_endpoint_rejects_sensitive_components() { + for endpoint in [ + "https://user:SECRET@localhost/", + "https://localhost/?token=SECRET", + "https://localhost/#SECRET", + "https://localhost/SECRET", + "file:///SECRET", + "not an endpoint SECRET", + ] { + let failure = token_grant_gateway_endpoint(endpoint).unwrap_err(); + assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + } + assert_eq!( + token_grant_gateway_endpoint("https://localhost:8443") + .unwrap() + .scheme(), + "https" + ); + assert_eq!( + token_grant_gateway_endpoint("http://localhost:8080") + .unwrap() + .scheme(), + "http" + ); + } + + #[test] + fn malformed_gateway_duration_discards_token_and_diagnostic() { + let failure = provider_subject_token_exchange_result( + crate::proto::ExchangeProviderSubjectTokenResponse { + access_token: "SYNTHETIC_ACCESS_SECRET".into(), + token_type: "Bearer".into(), + expires_after: Some(prost_types::Duration { + seconds: 1, + nanos: -1, + }), + }, + ) + .err() + .expect("invalid duration is rejected"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + } + + #[test] + fn successful_gateway_response_preserves_cache_expiration() { + let result = provider_subject_token_exchange_result( + crate::proto::ExchangeProviderSubjectTokenResponse { + access_token: "valid-token".into(), + token_type: "Bearer".into(), + expires_after: Some(prost_types::Duration { + seconds: 120, + nanos: 0, + }), + }, + ) + .expect("valid response"); + assert_eq!(result.access_token, "valid-token"); + assert_eq!(result.expires_in, 120); + assert_eq!(result.token_type, "Bearer"); } } @@ -1555,9 +1780,15 @@ pub struct ProviderEnvironmentResult { pub stable_placeholder_environment_keys: Vec, } +/// Intermediate provider credential returned to the trusted supervisor. +/// +/// Tokens deliberately have no Debug representation. pub struct ProviderSubjectTokenExchangeResult { + /// Bearer material for the final token exchange; never log this field. pub access_token: String, + /// Remaining lifetime in seconds, or zero when the gateway omitted it. pub expires_in: i64, + /// Token type declared by the service. pub token_type: String, } @@ -1793,6 +2024,10 @@ impl CachedOpenShellClient { EndpointResult::UpstreamRejected => ProtoEndpointResult::UpstreamRejected, } .into(), + failure: endpoint + .failure + .as_ref() + .map(crate::endpoint_failure::EndpointFailure::to_proto), }) .collect(); @@ -1806,6 +2041,9 @@ impl CachedOpenShellClient { observed_endpoint_ids: snapshot.observed_endpoint_ids.clone(), supervisor_session_id: snapshot.supervisor_session_id.clone(), report_sequence: snapshot.report_sequence, + token_grant_status: Some(crate::proto::TokenGrantStatus { + last_failure: snapshot.last_token_grant_failure.clone(), + }), }) .await .into_diagnostic()?; diff --git a/crates/openshell-core/src/lib.rs b/crates/openshell-core/src/lib.rs index d4bfb4579a..99730a26cc 100644 --- a/crates/openshell-core/src/lib.rs +++ b/crates/openshell-core/src/lib.rs @@ -20,6 +20,7 @@ pub mod dynamic_credential_key; pub mod dynamic_string_allowlist; #[cfg(unix)] pub mod e2fsprogs; +pub mod endpoint_failure; pub mod endpoint_path; pub mod endpoint_status; pub mod error; @@ -61,6 +62,7 @@ pub mod shell; pub mod spiffe; pub mod telemetry; pub mod time; +pub mod token_grant_failure; pub mod transport_errors; pub mod upload; diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index d3a9e34725..cc4046bb23 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -8,27 +8,84 @@ //! that assemble form parameters, POST to a validated `OAuth2` token endpoint, //! parse the JSON response, and validate the returned access token. +use std::fmt; use std::net::IpAddr; +use std::time::Duration; -use miette::{IntoDiagnostic, Result, WrapErr}; use serde::Deserialize; +use crate::token_grant_failure::{ + REQUEST_ID_HEADER, TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, +}; + +/// JWT bearer client-assertion type used when the caller supplies no override. pub const DEFAULT_CLIENT_ASSERTION_TYPE: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"; +/// RFC 8693 access-token type used for unspecified exchange token types. pub const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_token"; const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; +const ERROR_CONTRACT: &str = "openshell.token-grant.v1"; +const MAX_TOKEN_RESPONSE_BYTES: usize = 64 * 1024; + +/// HTTP client that authenticates HTTPS endpoints and never redirects assertions. +/// +/// The private client fixes certificate/hostname verification, proxy policy and +/// redirects so callers cannot label an unverified response as authenticated. +pub struct TokenGrantClient { + client: reqwest::Client, +} -const MAX_OAUTH_ERROR_FIELD_LEN: usize = 256; +impl TokenGrantClient { + /// Build a verified client with a bounded request and connection lifetime. + /// + /// Construction errors contain no TLS configuration or dependency source. + pub fn new() -> Result { + Self::builder() + .build() + .map(|client| Self { client }) + .map_err(|_| { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ) + }) + } + + fn builder() -> reqwest::ClientBuilder { + // A process may already have installed this provider. Installation is + // idempotent for this caller and never relaxes peer authentication. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + reqwest::Client::builder() + .timeout(Duration::from_secs(30)) + .connect_timeout(Duration::from_secs(30)) + .no_proxy() + .redirect(reqwest::redirect::Policy::none()) + } +} /// `OAuth2` token response. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct OAuthTokenResponse { + /// Validated token68 credential; never include this field in diagnostics. pub access_token: String, + /// Lifetime in seconds, or zero when the endpoint omits the expiration. pub expires_in: i64, + /// Token type returned by the endpoint, or empty when omitted. pub token_type: String, } -#[derive(Debug, Deserialize)] +impl fmt::Debug for OAuthTokenResponse { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("OAuthTokenResponse") + .field("access_token", &"[REDACTED]") + .field("expires_in", &self.expires_in) + .field("token_type", &self.token_type) + .finish() + } +} + +#[derive(Deserialize)] struct RawTokenResponse { access_token: String, #[serde(default)] @@ -37,20 +94,24 @@ struct RawTokenResponse { token_type: String, } -#[derive(Debug, Deserialize)] +#[derive(Deserialize)] struct OAuthErrorResponse { error: Option, - error_description: Option, + openshell_error_contract: Option, } async fn post_oauth_token_request( - client: &reqwest::Client, + client: &TokenGrantClient, token_endpoint: &str, mut form_params: Vec<(&str, &str)>, audience: &str, scopes: &[String], -) -> Result { + request_id: Option<&str>, +) -> Result { let token_endpoint_url = parse_token_endpoint_url(token_endpoint)?; + // The owned client verifies certificates and cannot follow a redirect to + // another service. Plain HTTP is never evidence of service identity. + let authenticated = token_endpoint_url.scheme() == "https"; let audience_param; if !audience.is_empty() { @@ -64,29 +125,30 @@ async fn post_oauth_token_request( form_params.push(("scope", &scope_param)); } - let response = client - .post(token_endpoint_url) - .form(&form_params) + let mut request = client.client.post(token_endpoint_url).form(&form_params); + // The correlation ID is minted by the supervisor, never read from the + // workload. A value that is not a valid header is omitted, not rewritten. + if let Some(request_id) = + request_id.and_then(|value| reqwest::header::HeaderValue::from_str(value).ok()) + { + request = request.header(REQUEST_ID_HEADER, request_id); + } + let response = request .send() .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to POST to token endpoint {token_endpoint}"))?; - - if !response.status().is_success() { - let status = response.status(); - let body = response - .text() - .await - .unwrap_or_else(|_| "".to_string()); - return Err(miette::miette!("{}", failure_message(status, &body))); + .map_err(|error| transport_failure(&error))?; + let status = response.status(); + let body = read_bounded_body(response).await?; + if !status.is_success() { + return Err(classify_http_failure(status, &body, authenticated)); } - let raw = response - .json::() - .await - .into_diagnostic() - .wrap_err("failed to parse token response as JSON")?; - validate_access_token(&raw.access_token)?; + // Raw parse failures can quote response bytes. Retain only a closed failure + // value and never attach serde or HTTP errors as a diagnostic source. + let raw = serde_json::from_slice::(&body) + .map_err(|_| endpoint_failure(TokenGrantFailureKind::Unclassified, status))?; + validate_access_token(&raw.access_token) + .map_err(|error| error.with_upstream_status(status.as_u16()))?; Ok(OAuthTokenResponse { access_token: raw.access_token, expires_in: raw.expires_in, @@ -96,21 +158,27 @@ async fn post_oauth_token_request( /// Client credentials grant form fields. pub struct TokenGrantParams<'a> { + /// Secret JWT client assertion sent only to the configured endpoint. pub client_assertion: &'a str, + /// Assertion format; an empty value selects the JWT bearer default. pub client_assertion_type: &'a str, + /// Optional resource audience carried in the grant form. pub audience: &'a str, + /// Optional scopes carried in the grant form. pub scopes: &'a [String], } /// POST a client credentials grant request to an `OAuth2` token endpoint. /// /// Applies the default for `client_assertion_type` and conditionally includes -/// `audience` and `scope`. +/// `audience` and `scope`. A `request_id` is sent as the +/// `x-openshell-request-id` header so the token service can log the exchange. pub async fn post_oauth_token_grant( - client: &reqwest::Client, + client: &TokenGrantClient, token_endpoint: &str, params: &TokenGrantParams<'_>, -) -> Result { + request_id: Option<&str>, +) -> Result { let client_assertion_type = effective_client_assertion_type(params.client_assertion_type); let form_params = vec![ ("grant_type", "client_credentials"), @@ -123,18 +191,26 @@ pub async fn post_oauth_token_grant( form_params, params.audience, params.scopes, + request_id, ) .await } /// RFC 8693 token-exchange form fields. pub struct TokenExchangeParams<'a> { + /// Secret JWT client assertion sent only to the configured endpoint. pub client_assertion: &'a str, + /// Assertion format; an empty value selects the JWT bearer default. pub client_assertion_type: &'a str, + /// Secret subject credential to exchange for a resource token. pub subject_token: &'a str, + /// Subject format; an empty value selects the access-token default. pub subject_token_type: &'a str, + /// Optional resource audience carried in the exchange form. pub audience: &'a str, + /// Optional scopes carried in the exchange form. pub scopes: &'a [String], + /// Requested result format; an empty value selects the access-token default. pub requested_token_type: &'a str, } @@ -142,12 +218,14 @@ pub struct TokenExchangeParams<'a> { /// /// Assembles the standard token-exchange form fields, applies defaults for /// `client_assertion_type`, `subject_token_type`, and `requested_token_type`, -/// and conditionally includes `audience` and `scope`. +/// and conditionally includes `audience` and `scope`. A `request_id` is sent +/// as the `x-openshell-request-id` header. pub async fn post_oauth_token_exchange( - client: &reqwest::Client, + client: &TokenGrantClient, token_endpoint: &str, params: &TokenExchangeParams<'_>, -) -> Result { + request_id: Option<&str>, +) -> Result { let client_assertion_type = effective_client_assertion_type(params.client_assertion_type); let subject_token_type = effective_token_type(params.subject_token_type); let requested_token_type = effective_token_type(params.requested_token_type); @@ -165,10 +243,12 @@ pub async fn post_oauth_token_exchange( form_params, params.audience, params.scopes, + request_id, ) .await } +/// Select the JWT bearer assertion type when no nonblank override is supplied. pub fn effective_client_assertion_type(client_assertion_type: &str) -> &str { if client_assertion_type.trim().is_empty() { DEFAULT_CLIENT_ASSERTION_TYPE @@ -177,6 +257,7 @@ pub fn effective_client_assertion_type(client_assertion_type: &str) -> &str { } } +/// Select the RFC 8693 access-token type when no nonblank override is supplied. pub fn effective_token_type(token_type: &str) -> &str { if token_type.trim().is_empty() { ACCESS_TOKEN_TYPE @@ -185,16 +266,24 @@ pub fn effective_token_type(token_type: &str) -> &str { } } -fn parse_token_endpoint_url(token_endpoint: &str) -> Result { - let url = reqwest::Url::parse(token_endpoint) - .into_diagnostic() - .wrap_err("token_endpoint must be an absolute URL")?; +fn parse_token_endpoint_url(token_endpoint: &str) -> Result { + let invalid = TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ); + let url = reqwest::Url::parse(token_endpoint).map_err(|_| invalid)?; if token_endpoint_transport_allowed(&url) { return Ok(url); } - Err(miette::miette!( - "token_endpoint must use https, except http for loopback or in-cluster service hosts" - )) + Err(invalid) +} + +/// Validate the configured endpoint before acquiring any workload credentials. +/// +/// HTTPS is accepted; plain HTTP is limited to loopback and Kubernetes service +/// hosts and never authenticates a precise token-service error classification. +pub fn validate_token_endpoint(token_endpoint: &str) -> Result<(), TokenGrantFailure> { + parse_token_endpoint_url(token_endpoint).map(|_| ()) } fn token_endpoint_transport_allowed(url: &reqwest::Url) -> bool { @@ -230,10 +319,12 @@ fn is_kubernetes_service_host(host: &str) -> bool { (is_service_name || is_cluster_local_service) && labels.iter().all(|label| !label.is_empty()) } -pub fn validate_access_token(token: &str) -> Result<()> { +/// Reject credentials that cannot safely occupy a token68 HTTP header value. +pub fn validate_access_token(token: &str) -> Result<(), TokenGrantFailure> { if token.is_empty() || !is_token68(token) { - return Err(miette::miette!( - "token grant returned a malformed access token" + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::Unclassified, + TokenGrantStage::TokenEndpoint, )); } Ok(()) @@ -259,66 +350,158 @@ fn is_token68_value_byte(byte: u8) -> bool { byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~' | b'+' | b'/') } -fn failure_message(status: reqwest::StatusCode, body: &str) -> String { - let Ok(error_response) = serde_json::from_str::(body) else { - return format!("token grant failed with status {status}"); +fn endpoint_failure(kind: TokenGrantFailureKind, status: reqwest::StatusCode) -> TokenGrantFailure { + TokenGrantFailure::new(kind, TokenGrantStage::TokenEndpoint) + .with_upstream_status(status.as_u16()) +} + +fn transport_failure(error: &reqwest::Error) -> TokenGrantFailure { + let kind = if error.is_timeout() { + TokenGrantFailureKind::Timeout + } else if error.is_connect() { + TokenGrantFailureKind::Unavailable + } else { + TokenGrantFailureKind::Unclassified }; - let error = error_response - .error - .as_deref() - .map(sanitize_oauth_error_field) - .filter(|value| !value.is_empty()); - let description = error_response - .error_description - .as_deref() - .map(sanitize_oauth_error_field) - .filter(|value| !value.is_empty()); - match (error, description) { - (Some(error), Some(description)) => { - format!( - "token grant failed with status {status}: error={error}; error_description={description}" - ) - } - (Some(error), None) => { - format!("token grant failed with status {status}: error={error}") - } - (None, Some(description)) => { - format!("token grant failed with status {status}: error_description={description}") + TokenGrantFailure::new(kind, TokenGrantStage::TokenEndpoint) +} + +async fn read_bounded_body(mut response: reqwest::Response) -> Result, TokenGrantFailure> { + let status = response.status(); + let oversized = endpoint_failure(TokenGrantFailureKind::Unclassified, status); + if response + .content_length() + .is_some_and(|length| length > MAX_TOKEN_RESPONSE_BYTES as u64) + { + return Err(oversized); + } + let mut body = Vec::with_capacity(MAX_TOKEN_RESPONSE_BYTES); + while let Some(chunk) = response + .chunk() + .await + .map_err(|error| transport_failure(&error).with_upstream_status(status.as_u16()))? + { + // Content-Length may be absent or inaccurate. Check before allocation + // so a streamed response cannot grow our retained buffer past the cap. + if chunk.len() > MAX_TOKEN_RESPONSE_BYTES.saturating_sub(body.len()) { + return Err(oversized); } - (None, None) => format!("token grant failed with status {status}"), + body.extend_from_slice(&chunk); } + Ok(body) } -fn sanitize_oauth_error_field(value: &str) -> String { - value - .chars() - .map(|ch| if ch.is_control() { ' ' } else { ch }) - .take(MAX_OAUTH_ERROR_FIELD_LEN) - .collect::() - .trim() - .to_string() +fn classify_http_failure( + status: reqwest::StatusCode, + body: &[u8], + authenticated: bool, +) -> TokenGrantFailure { + let default = if status == reqwest::StatusCode::TOO_MANY_REQUESTS || status.is_server_error() { + TokenGrantFailureKind::Unavailable + } else { + TokenGrantFailureKind::Unclassified + }; + let Ok(response) = serde_json::from_slice::(body) else { + return endpoint_failure(default, status); + }; + if response.openshell_error_contract.as_deref() != Some(ERROR_CONTRACT) { + return endpoint_failure(default, status); + } + let (kind, expected_status) = match response.error.as_deref() { + Some("provider_authorization_required") => ( + TokenGrantFailureKind::ProviderAuthorizationRequired, + status == reqwest::StatusCode::CONFLICT, + ), + Some("workload_identity_rejected") => ( + TokenGrantFailureKind::WorkloadIdentityRejected, + matches!(status.as_u16(), 401 | 403), + ), + Some("configuration_error") => ( + TokenGrantFailureKind::Configuration, + status == reqwest::StatusCode::BAD_REQUEST, + ), + _ => return endpoint_failure(default, status), + }; + // An adopted extension is meaningful only from an authenticated service and + // with its defined status. Contradictions cannot claim availability either. + if !authenticated || !expected_status { + return endpoint_failure(TokenGrantFailureKind::Unclassified, status); + } + endpoint_failure(kind, status) } #[cfg(test)] mod tests { use super::*; use std::collections::HashMap; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use std::error::Error; + use std::sync::Arc; + use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpListener; #[derive(Debug)] struct CapturedRequest { form: HashMap, + request_id: Option, } - fn test_client() -> reqwest::Client { - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("test client") + fn test_client() -> TokenGrantClient { + TokenGrantClient { + client: TokenGrantClient::builder() + .timeout(Duration::from_secs(5)) + .build() + .expect("test client"), + } + } + + async fn capture_request(stream: &mut S) -> CapturedRequest { + let mut buf = Vec::new(); + let mut chunk = [0u8; 512]; + let mut expected_len = None; + loop { + let n = stream.read(&mut chunk).await.expect("read"); + assert!(n > 0); + buf.extend_from_slice(&chunk[..n]); + if expected_len.is_none() + && let Some(header_end) = header_end(&buf) + { + let headers = String::from_utf8_lossy(&buf[..header_end]); + let cl = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + expected_len = Some(header_end + cl); + } + if expected_len.is_some_and(|len| buf.len() >= len) { + break; + } + } + let header_end = header_end(&buf).expect("request headers"); + let headers = String::from_utf8_lossy(&buf[..header_end]); + let request_id = headers.lines().find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case(REQUEST_ID_HEADER) + .then(|| value.trim().to_string()) + }); + let form_body = String::from_utf8_lossy(&buf[header_end..]); + CapturedRequest { + form: parse_form_body(&form_body), + request_id, + } + } + + async fn send_response(stream: &mut S, status: &str, body: &str) { + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ); + stream.write_all(response.as_bytes()).await.expect("write"); + stream.shutdown().await.expect("shutdown response"); } async fn token_endpoint_once( @@ -333,45 +516,72 @@ mod tests { let body = body.to_string(); let handle = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.expect("accept"); - let mut buf = Vec::new(); - let mut chunk = [0u8; 512]; - let mut expected_len = None; - loop { - let n = stream.read(&mut chunk).await.expect("read"); - assert!(n > 0); - buf.extend_from_slice(&chunk[..n]); - if expected_len.is_none() - && let Some(header_end) = header_end(&buf) - { - let headers = String::from_utf8_lossy(&buf[..header_end]); - let cl = headers - .lines() - .find_map(|line| { - let (name, value) = line.split_once(':')?; - name.eq_ignore_ascii_case("content-length") - .then(|| value.trim().parse::().ok()) - .flatten() - }) - .unwrap_or(0); - expected_len = Some(header_end + cl); - } - if expected_len.is_some_and(|len| buf.len() >= len) { - break; - } - } - let header_end = header_end(&buf).unwrap(); - let form_body = String::from_utf8_lossy(&buf[header_end..]).to_string(); - let form = parse_form_body(&form_body); - let response = format!( - "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", - body.len(), - ); - stream.write_all(response.as_bytes()).await.expect("write"); - CapturedRequest { form } + let request = capture_request(&mut stream).await; + send_response(&mut stream, &status, &body).await; + request }); (format!("http://{addr}/token"), handle) } + async fn tls_token_endpoint_once( + status: &str, + body: &str, + ) -> ( + String, + reqwest::Certificate, + tokio::task::JoinHandle>, + ) { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).expect("CA params"); + ca_params.is_ca = rcgen::IsCa::Ca(rcgen::BasicConstraints::Unconstrained); + let ca_key = rcgen::KeyPair::generate().expect("CA key"); + let ca = ca_params.self_signed(&ca_key).expect("CA certificate"); + let key = rcgen::KeyPair::generate().expect("server key"); + let certificate = rcgen::CertificateParams::new(vec!["localhost".to_string()]) + .expect("server params") + .signed_by(&key, &ca, &ca_key) + .expect("server certificate"); + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert( + vec![certificate.der().clone()], + rustls::pki_types::PrivateKeyDer::try_from(key.serialize_der()) + .expect("server key"), + ) + .expect("server config"); + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = TcpListener::bind("127.0.0.1:0").await.expect("TLS bind"); + let port = listener.local_addr().expect("TLS addr").port(); + let status = status.to_string(); + let body = body.to_string(); + let task = tokio::spawn(async move { + let (socket, _) = listener.accept().await.expect("TLS accept"); + // A rejected trust root/hostname must fail during the handshake, + // before the server can receive the secret-bearing grant request. + let Ok(mut stream) = acceptor.accept(socket).await else { + return None; + }; + let request = capture_request(&mut stream).await; + send_response(&mut stream, &status, &body).await; + Some(request) + }); + ( + format!("https://localhost:{port}/token"), + reqwest::Certificate::from_der(ca.der()).expect("trusted CA"), + task, + ) + } + + fn trusted_test_client(root: reqwest::Certificate) -> TokenGrantClient { + TokenGrantClient { + client: TokenGrantClient::builder() + .timeout(Duration::from_secs(5)) + .add_root_certificate(root) + .build() + .expect("trusted test client"), + } + } + async fn token_endpoint_redirect_once(location: &str) -> (String, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") .await @@ -444,6 +654,310 @@ mod tests { } } + fn exchange_params() -> TokenExchangeParams<'static> { + TokenExchangeParams { + client_assertion: "ASSERTION_SECRET_CANARY", + client_assertion_type: "", + subject_token: "SUBJECT_SECRET_CANARY", + subject_token_type: "", + audience: "", + scopes: &[], + requested_token_type: "", + } + } + + fn assert_generic_failure(failure: TokenGrantFailure) { + assert_eq!(failure.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + assert!(failure.source().is_none()); + let report = miette::Report::new(failure); + let rendered = format!("{report:?}"); + for forbidden in [ + "CANARY", + "provider_authorization_required", + "workload_identity_rejected", + ] { + assert!( + !rendered.contains(forbidden), + "diagnostics must stay generic" + ); + } + } + + #[tokio::test] + async fn verified_https_signal_is_preserved_for_both_grants() { + let cases = [ + ( + "409 Conflict", + "provider_authorization_required", + TokenGrantFailureKind::ProviderAuthorizationRequired, + ), + ( + "401 Unauthorized", + "workload_identity_rejected", + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + "403 Forbidden", + "workload_identity_rejected", + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + "400 Bad Request", + "configuration_error", + TokenGrantFailureKind::Configuration, + ), + ]; + for (status, code, expected) in cases { + for exchange in [false, true] { + let body = format!( + r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# + ); + let (endpoint, root, request) = tls_token_endpoint_once(status, &body).await; + let client = trusted_test_client(root); + let failure = if exchange { + post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None).await + } else { + post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None).await + } + .expect_err("adopted failure"); + let request = request + .await + .expect("TLS fixture") + .expect("authenticated request"); + assert_eq!(request.form.contains_key("subject_token"), exchange); + assert_eq!(failure.kind(), expected); + assert_eq!(failure.stage(), TokenGrantStage::TokenEndpoint); + assert_generic_failure(failure); + } + } + } + + #[tokio::test] + async fn plain_http_cannot_assert_adopted_failure_for_either_grant() { + for (status, code) in [ + ("409 Conflict", "provider_authorization_required"), + ("401 Unauthorized", "workload_identity_rejected"), + ("400 Bad Request", "configuration_error"), + ] { + for exchange in [false, true] { + let body = format!( + r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# + ); + let (endpoint, request) = token_endpoint_once(status, &body).await; + let client = test_client(); + let failure = if exchange { + post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None).await + } else { + post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None).await + } + .expect_err("unauthenticated failure"); + request.await.expect("HTTP fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_generic_failure(failure); + } + } + } + + #[tokio::test] + async fn tls_requires_a_trusted_certificate_and_matching_hostname() { + for trust_certificate in [false, true] { + let body = r#"{"error":"provider_authorization_required","openshell_error_contract":"openshell.token-grant.v1"}"#; + let (endpoint, root, request) = tls_token_endpoint_once("409 Conflict", body).await; + let (client, endpoint) = if trust_certificate { + ( + trusted_test_client(root), + endpoint.replace("localhost", "127.0.0.1"), + ) + } else { + (test_client(), endpoint) + }; + let failure = post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None) + .await + .expect_err("TLS must authenticate the configured server"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unavailable); + assert_eq!(failure.upstream_status(), None); + assert!(request.await.expect("TLS fixture").is_none()); + assert_generic_failure(failure); + } + } + + #[tokio::test] + async fn secret_canaries_in_errors_and_endpoint_components_never_survive() { + for body in [ + r#"{"error":"ERROR_SECRET_CANARY","error_description":"DESCRIPTION_SECRET_CANARY"}"#, + r#"{"error":"invalid_client","error_description":"ASSERTION_SECRET_CANARY TOKEN_SECRET_CANARY"}"#, + "RAW_BODY_SECRET_CANARY", + ] { + let (endpoint, request) = token_endpoint_once("401 Unauthorized", body).await; + let endpoint = endpoint.replace( + "http://", + "http://USER_SECRET_CANARY:PASSWORD_SECRET_CANARY@", + ) + "?credential=QUERY_SECRET_CANARY#FRAGMENT_SECRET_CANARY"; + let failure = + post_oauth_token_exchange(&test_client(), &endpoint, &exchange_params(), None) + .await + .expect_err("safe endpoint failure"); + request.await.expect("HTTP fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_generic_failure(failure); + } + for endpoint in [ + "not a URL ENDPOINT_SECRET_CANARY", + "ftp://USER_SECRET_CANARY:PASSWORD_SECRET_CANARY@localhost/token?QUERY_SECRET_CANARY", + ] { + let failure = + post_oauth_token_grant(&test_client(), endpoint, &empty_grant_params(), None) + .await + .expect_err("invalid endpoint"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); + assert_generic_failure(failure); + } + } + + #[tokio::test] + async fn malformed_success_fields_are_generic_and_never_return_a_token() { + for body in [ + "{}", + r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":null}"#, + r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","expires_in":"DURATION_SECRET_CANARY"}"#, + r#"{"access_token":"TOKEN_SECRET_CANARY","access_token":"SECOND_SECRET_CANARY","token_type":"Bearer"}"#, + r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","token_type":"Bearer"}"#, + r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","expires_in":1,"expires_in":1}"#, + r#"{"access_token":"","token_type":"Bearer"}"#, + ] { + let (endpoint, request) = token_endpoint_once("200 OK", body).await; + let failure = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect_err("malformed success"); + request.await.expect("HTTP fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.upstream_status(), Some(200)); + assert_generic_failure(failure); + } + } + + #[test] + fn successful_token_response_debug_redacts_the_access_token() { + let response = OAuthTokenResponse { + access_token: "TOKEN_SECRET_CANARY".to_string(), + token_type: "Bearer".to_string(), + expires_in: 60, + }; + assert!(!format!("{response:?}").contains("TOKEN_SECRET_CANARY")); + } + + #[tokio::test] + async fn request_and_body_timeouts_drop_sources_and_keep_observed_status() { + for after_headers in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let (release, wait) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + capture_request(&mut stream).await; + if after_headers { + stream + .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\n\r\n") + .await + .expect("response headers"); + } + let _ = wait.await; + }); + let client = TokenGrantClient { + client: TokenGrantClient::builder() + .timeout(Duration::from_millis(100)) + .build() + .expect("timeout client"), + }; + let endpoint = format!("http://{address}/token?QUERY_SECRET_CANARY"); + let failure = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) + .await + .expect_err("bounded timeout"); + release.send(()).expect("release fixture"); + server.await.expect("timeout fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Timeout); + assert_eq!(failure.upstream_status(), after_headers.then_some(200)); + assert_generic_failure(failure); + } + } + + async fn raw_token_endpoint_once(response: Vec) -> (String, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let address = listener.local_addr().expect("addr"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + capture_request(&mut stream).await; + // Bound rejection can close the reader before an oversized fixture + // finishes writing. That is the behavior this fixture exercises. + let _ = stream.write_all(&response).await; + }); + (format!("http://{address}/token"), server) + } + + #[tokio::test] + async fn response_body_bound_applies_to_success_errors_and_chunked_bodies() { + let body = "X".repeat(MAX_TOKEN_RESPONSE_BYTES + 1); + for status in ["200 OK", "409 Conflict", "503 Service Unavailable"] { + for chunked in [false, true] { + let response = if chunked { + format!( + "HTTP/1.1 {status}\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n{:x}\r\n{body}\r\n0\r\n\r\n", + body.len() + ) + } else { + // The oversized declared length must suffice for rejection, + // without waiting for a body the server does not provide. + format!( + "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ) + }; + let (endpoint, server) = raw_token_endpoint_once(response.into_bytes()).await; + let failure = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect_err("bounded response"); + server.await.expect("bound fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert!(failure.upstream_status().is_some()); + assert_generic_failure(failure); + } + } + } + + #[tokio::test] + async fn exact_body_bound_preserves_success_and_default_expiration() { + let mut body = r#"{"access_token":"access-token","token_type":"Bearer"}"#.to_string(); + body.extend(std::iter::repeat_n( + ' ', + MAX_TOKEN_RESPONSE_BYTES - body.len(), + )); + let (endpoint, server) = token_endpoint_once("200 OK", &body).await; + let response = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect("exact bound is accepted"); + server.await.expect("bound fixture"); + assert_eq!(response.access_token, "access-token"); + assert_eq!(response.expires_in, 0); + } + + #[tokio::test] + async fn body_read_errors_discard_partial_secret_content() { + let response = b"HTTP/1.1 200 OK\r\nContent-Length: 1000\r\nConnection: close\r\n\r\nPARTIAL_BODY_SECRET_CANARY".to_vec(); + let (endpoint, server) = raw_token_endpoint_once(response).await; + let failure = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect_err("truncated body"); + server.await.expect("read fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.upstream_status(), Some(200)); + assert_generic_failure(failure); + } + #[tokio::test] async fn posts_form_params_and_parses_success_response() { let (endpoint, request) = token_endpoint_once( @@ -453,7 +967,7 @@ mod tests { .await; let client = test_client(); - let response = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let response = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect("should succeed"); let request = request.await.expect("endpoint task"); @@ -480,15 +994,14 @@ mod tests { .await; let client = test_client(); - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("malformed token should fail"); let _ = request.await; - assert_eq!( - err.to_string(), - "token grant returned a malformed access token" - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(200)); } #[tokio::test] @@ -496,16 +1009,17 @@ mod tests { let (endpoint, handle) = token_endpoint_redirect_once("http://127.0.0.1:1/stolen").await; let client = test_client(); - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("redirect should fail"); let _ = handle.await; - assert_eq!(err.to_string(), "token grant failed with status 302 Found"); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.upstream_status(), Some(302)); } #[tokio::test] - async fn reports_sanitized_oauth_error() { + async fn discards_oauth_error_descriptions_and_unrecognized_codes() { let (endpoint, request) = token_endpoint_once( "401 Unauthorized", r#"{"error":"invalid_client","error_description":"bad assertion"}"#, @@ -513,15 +1027,14 @@ mod tests { .await; let client = test_client(); - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("should fail on OAuth error"); let _ = request.await; - assert_eq!( - err.to_string(), - "token grant failed with status 401 Unauthorized: error=invalid_client; error_description=bad assertion" - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(401)); } #[tokio::test] @@ -533,33 +1046,50 @@ mod tests { .await; let client = test_client(); - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("should fail on server error"); let _ = request.await; let message = err.to_string(); - assert_eq!( - message, - "token grant failed with status 500 Internal Server Error" - ); + assert_eq!(message, "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unavailable); + assert_eq!(err.upstream_status(), Some(500)); assert!(!message.contains("stack trace")); } + #[tokio::test] + async fn success_accepts_any_token_type_and_lifetime() { + // The supervisor and gateway place the token by the profile's auth + // style, so token_type and a nonpositive lifetime never block a grant. + for body in [ + r#"{"access_token":"access-123"}"#, + r#"{"access_token":"access-123","token_type":"N_A"}"#, + r#"{"access_token":"access-123","token_type":"Bearer","expires_in":-1}"#, + ] { + let (endpoint, server) = token_endpoint_once("200 OK", body).await; + let response = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect(body); + server.await.expect("token endpoint task"); + assert_eq!(response.access_token, "access-123"); + } + } + #[tokio::test] async fn reports_malformed_success_json() { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; let client = test_client(); - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params()) + let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("should fail on malformed JSON"); let _ = request.await; - assert!( - err.to_string() - .contains("failed to parse token response as JSON") - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(200)); } #[tokio::test] @@ -584,6 +1114,7 @@ mod tests { scopes: &scopes, requested_token_type: "urn:ietf:params:oauth:token-type:id_token", }, + Some("0b6e9c3a-5d4f-4b1e-9a2c-7f8e1d2c3b4a"), ) .await .expect("token exchange should succeed"); @@ -591,6 +1122,10 @@ mod tests { assert_eq!(response.access_token, "exchanged-token"); assert_eq!(response.expires_in, 60); + assert_eq!( + request.request_id.as_deref(), + Some("0b6e9c3a-5d4f-4b1e-9a2c-7f8e1d2c3b4a") + ); assert_eq!( request.form.get("grant_type").map(String::as_str), Some(TOKEN_EXCHANGE_GRANT_TYPE) @@ -646,6 +1181,7 @@ mod tests { scopes: &[], requested_token_type: "", }, + None, ) .await .expect("token exchange should succeed"); @@ -668,6 +1204,7 @@ mod tests { ); assert!(!request.form.contains_key("audience")); assert!(!request.form.contains_key("scope")); + assert_eq!(request.request_id, None); } #[test] @@ -722,46 +1259,98 @@ mod tests { "==", ] { let err = validate_access_token(token).expect_err("should be rejected"); - assert_eq!( - err.to_string(), - "token grant returned a malformed access token" - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); } } #[test] - fn failure_message_reports_oauth_error_fields() { - let message = failure_message( - reqwest::StatusCode::UNAUTHORIZED, - r#"{"error":"invalid_client","error_description":"Invalid client credentials"}"#, - ); - assert_eq!( - message, - "token grant failed with status 401 Unauthorized: error=invalid_client; error_description=Invalid client credentials" - ); - } - - #[test] - fn failure_message_omits_unstructured_response_body() { - let message = failure_message( - reqwest::StatusCode::INTERNAL_SERVER_ERROR, - "internal error containing implementation details", - ); - assert_eq!( - message, - "token grant failed with status 500 Internal Server Error" - ); + fn response_matrix_requires_consistent_authenticated_extension() { + let cases = [ + ( + "provider_authorization_required", + 409, + TokenGrantFailureKind::ProviderAuthorizationRequired, + ), + ( + "workload_identity_rejected", + 401, + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + "workload_identity_rejected", + 403, + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + "configuration_error", + 400, + TokenGrantFailureKind::Configuration, + ), + ]; + for (code, valid_status, expected) in cases { + let body = + format!(r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}"}}"#); + for status in [300, 400, 401, 403, 409, 429, 500, 503, 599] { + for authenticated in [false, true] { + let failure = classify_http_failure( + reqwest::StatusCode::from_u16(status).expect("status"), + body.as_bytes(), + authenticated, + ); + // Workload rejection permits both authentication denial codes. + let status_matches = status == valid_status + || (code == "workload_identity_rejected" && matches!(status, 401 | 403)); + assert_eq!( + failure.kind(), + if authenticated && status_matches { + expected + } else { + TokenGrantFailureKind::Unclassified + }, + "code={code}, status={status}, authenticated={authenticated}" + ); + assert_eq!(failure.upstream_status(), Some(status)); + assert_eq!(failure.stage(), TokenGrantStage::TokenEndpoint); + } + } + } } #[test] - fn failure_message_sanitizes_oauth_error_fields() { - let long_description = "a".repeat(MAX_OAUTH_ERROR_FIELD_LEN + 20); - let body = - format!(r#"{{"error":"invalid_client\n","error_description":"{long_description}"}}"#); - let message = failure_message(reqwest::StatusCode::UNAUTHORIZED, &body); - assert!(!message.contains('\n')); - assert!(message.contains("error=invalid_client")); - assert!(message.contains(&"a".repeat(MAX_OAUTH_ERROR_FIELD_LEN))); - assert!(!message.contains(&"a".repeat(MAX_OAUTH_ERROR_FIELD_LEN + 1))); + fn malformed_ambiguous_and_unknown_signals_preserve_only_availability() { + let bodies = [ + "not JSON", + "{}", + r#"{"error":null}"#, + r#"{"error":42}"#, + r#"{"error":"invalid_client"}"#, + r#"{"error":"invalid_grant","error_description":"provider_authorization_required"}"#, + r#"{"error":"provider_authorization_required"}"#, + r#"{"error":"provider_authorization_required","openshell_error_contract":"unknown"}"#, + r#"{"error":"unknown","openshell_error_contract":"openshell.token-grant.v1"}"#, + r#"{"error":"provider_authorization_required","error":"provider_authorization_required","openshell_error_contract":"openshell.token-grant.v1"}"#, + r#"{"error":"provider_authorization_required","openshell_error_contract":"openshell.token-grant.v1","openshell_error_contract":"openshell.token-grant.v1"}"#, + r#"{"error":"provider_authorization_required","openshell_error_contract":true}"#, + r#"{"error":"provider_authorization_required","openshell_error_contract":"openshell.token-grant.v1"} trailing"#, + ]; + for body in bodies { + for status in [302, 400, 401, 403, 409, 422, 429, 500, 503, 599] { + for authenticated in [false, true] { + let failure = classify_http_failure( + reqwest::StatusCode::from_u16(status).expect("status"), + body.as_bytes(), + authenticated, + ); + let expected = if status == 429 || (500..600).contains(&status) { + TokenGrantFailureKind::Unavailable + } else { + TokenGrantFailureKind::Unclassified + }; + assert_eq!(failure.kind(), expected, "body={body}, status={status}"); + assert_eq!(failure.upstream_status(), Some(status)); + } + } + } } } diff --git a/crates/openshell-core/src/token_grant_failure.rs b/crates/openshell-core/src/token_grant_failure.rs new file mode 100644 index 0000000000..5038218bb7 --- /dev/null +++ b/crates/openshell-core/src/token_grant_failure.rs @@ -0,0 +1,601 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Credential-free failures shared by dynamic token grants and their RPC boundary. +//! +//! Printable errors contain only a fixed message. Trusted in-process consumers +//! can inspect allowlisted classifications without retaining upstream diagnostics. + +use std::collections::HashMap; +use std::fmt; + +use prost::Message; +use tonic::{Code, Status}; + +use crate::proto::ProviderCredentialRefreshRecoveryAction; +use crate::rpc_error::ERROR_DOMAIN; + +/// Header that carries the supervisor-minted request ID of an HTTP exchange. +/// +/// The supervisor sends it to token services and returns it on a refused +/// request, so every party can name the same exchange. The value is an opaque +/// random ID and grants nothing. +pub const REQUEST_ID_HEADER: &str = "x-openshell-request-id"; + +const FAILURE_MESSAGE: &str = "dynamic token grant failed"; +const FAILURE_REASON: &str = "TOKEN_GRANT_FAILURE"; +const ERROR_INFO_TYPE_URL: &str = "type.googleapis.com/google.rpc.ErrorInfo"; +const MAX_DETAILS_BYTES: usize = 1024; + +/// An allowlisted cause of a dynamic token-grant failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TokenGrantFailureKind { + /// The authenticated token service requires provider authorization. + ProviderAuthorizationRequired, + /// An authenticated component rejected the workload identity. + WorkloadIdentityRejected, + /// Local configuration or an authenticated configuration contract is invalid. + Configuration, + /// A grant operation exceeded its time limit. + Timeout, + /// A required service or connection is unavailable. + Unavailable, + /// Available evidence does not establish a more specific cause. + Unclassified, +} + +impl TokenGrantFailureKind { + const fn code(self) -> &'static str { + match self { + Self::ProviderAuthorizationRequired => "provider_authorization_required", + Self::WorkloadIdentityRejected => "workload_identity_rejected", + Self::Configuration => "configuration_error", + Self::Timeout => "timeout", + Self::Unavailable => "unavailable", + Self::Unclassified => "unclassified", + } + } + + pub(crate) fn from_code(code: &str) -> Option { + match code { + "provider_authorization_required" => Some(Self::ProviderAuthorizationRequired), + "workload_identity_rejected" => Some(Self::WorkloadIdentityRejected), + "configuration_error" => Some(Self::Configuration), + "timeout" => Some(Self::Timeout), + "unavailable" => Some(Self::Unavailable), + "unclassified" => Some(Self::Unclassified), + _ => None, + } + } + + pub(crate) const fn recovery(self) -> ProviderCredentialRefreshRecoveryAction { + match self { + Self::ProviderAuthorizationRequired => { + ProviderCredentialRefreshRecoveryAction::Reauthorize + } + Self::WorkloadIdentityRejected | Self::Configuration => { + ProviderCredentialRefreshRecoveryAction::FixConfiguration + } + Self::Timeout | Self::Unavailable => ProviderCredentialRefreshRecoveryAction::Retry, + Self::Unclassified => ProviderCredentialRefreshRecoveryAction::Investigate, + } + } + + const fn status_code(self) -> Code { + match self { + Self::ProviderAuthorizationRequired | Self::Configuration => Code::FailedPrecondition, + Self::WorkloadIdentityRejected => Code::PermissionDenied, + Self::Timeout => Code::DeadlineExceeded, + Self::Unavailable => Code::Unavailable, + Self::Unclassified => Code::Unknown, + } + } +} + +/// The trusted local operation that observed a token-grant failure. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum TokenGrantStage { + /// Validating the grant configuration. + Configuration, + /// Acquiring or validating workload identity. + WorkloadIdentity, + /// Calling the authenticated gateway RPC. + GatewayRpc, + /// Exchanging the workload identity for an intermediate token. + IntermediateExchange, + /// Calling the final token endpoint. + TokenEndpoint, + /// Preparing or injecting the token into the protected request. + Injection, +} + +impl TokenGrantStage { + /// Return the stable stage code for trusted local consumers. + pub const fn code(self) -> &'static str { + match self { + Self::Configuration => "configuration", + Self::WorkloadIdentity => "workload_identity", + Self::GatewayRpc => "gateway_rpc", + Self::IntermediateExchange => "intermediate_exchange", + Self::TokenEndpoint => "token_endpoint", + Self::Injection => "injection", + } + } + + pub(crate) fn from_code(code: &str) -> Option { + match code { + "configuration" => Some(Self::Configuration), + "workload_identity" => Some(Self::WorkloadIdentity), + "gateway_rpc" => Some(Self::GatewayRpc), + "intermediate_exchange" => Some(Self::IntermediateExchange), + "token_endpoint" => Some(Self::TokenEndpoint), + "injection" => Some(Self::Injection), + _ => None, + } + } +} + +/// A token-grant failure that cannot retain credentials or arbitrary diagnostics. +/// +/// Both printable representations are generic, and the error has no source. +/// Structured getters are intended for trusted in-process consumers, not output +/// to a workload or local diagnostic stream. +#[derive(Clone, Copy, Eq, PartialEq)] +pub struct TokenGrantFailure { + kind: TokenGrantFailureKind, + stage: TokenGrantStage, + upstream_status: Option, +} + +impl TokenGrantFailure { + /// Construct a failure from a trusted local classification and operation. + pub const fn new(kind: TokenGrantFailureKind, stage: TokenGrantStage) -> Self { + Self { + kind, + stage, + upstream_status: None, + } + } + + /// Attach an observed HTTP status, discarding values outside 100 through 599. + #[must_use] + pub fn with_upstream_status(mut self, status: u16) -> Self { + self.upstream_status = (100..=599).contains(&status).then_some(status); + self + } + + /// Identify the operation that observed the failure without changing its cause. + #[must_use] + pub const fn at_stage(mut self, stage: TokenGrantStage) -> Self { + self.stage = stage; + self + } + + /// Return the allowlisted failure classification for trusted local consumers. + pub const fn kind(&self) -> TokenGrantFailureKind { + self.kind + } + + /// Return the operation that observed this failure. + pub const fn stage(&self) -> TokenGrantStage { + self.stage + } + + /// Return an observed valid HTTP status without retaining its response body. + pub const fn upstream_status(&self) -> Option { + self.upstream_status + } + + /// Return the stable allowlisted failure code for trusted local consumers. + pub const fn code(&self) -> &'static str { + self.kind.code() + } + + /// Return who can resolve this failure, in the provider refresh vocabulary. + pub const fn recovery(&self) -> ProviderCredentialRefreshRecoveryAction { + self.kind.recovery() + } + + /// Encode this failure with its default gRPC status code and generic message. + pub fn to_status(self) -> Status { + self.to_status_with_code(self.kind.status_code()) + } + + /// Encode this failure while preserving a gateway guard's gRPC status code. + /// + /// A success code cannot represent a failure and is replaced with `Unknown`. + pub fn to_status_with_code(self, code: Code) -> Status { + let code = if code == Code::Ok { + Code::Unknown + } else { + code + }; + let mut metadata = HashMap::from([ + ("version".to_owned(), "1".to_owned()), + ("code".to_owned(), self.code().to_owned()), + ("stage".to_owned(), self.stage.code().to_owned()), + ]); + if let Some(status) = self.upstream_status { + metadata.insert("upstream_http_status".to_owned(), status.to_string()); + } + let info = tonic_types::pb::ErrorInfo { + reason: FAILURE_REASON.to_owned(), + domain: ERROR_DOMAIN.to_owned(), + metadata, + }; + let envelope = tonic_types::pb::Status { + code: code as i32, + message: FAILURE_MESSAGE.to_owned(), + details: vec![prost_types::Any { + type_url: ERROR_INFO_TYPE_URL.to_owned(), + value: info.encode_to_vec(), + }], + }; + Status::with_details(code, FAILURE_MESSAGE, envelope.encode_to_vec().into()) + } + + /// Decode strictly bounded internal details, discarding all other diagnostics. + /// + /// Malformed messages or unknown and inconsistent classifications become an + /// unclassified gateway RPC failure. This validates the failure contract, + /// not peer identity: callers must authenticate the gateway before trusting + /// a precise classification received over the network. + pub fn from_status(status: &Status) -> Self { + Self::decode_status(status).unwrap_or_else(|| { + Self::new( + TokenGrantFailureKind::Unclassified, + TokenGrantStage::GatewayRpc, + ) + }) + } + + fn decode_status(status: &Status) -> Option { + if status.details().len() > MAX_DETAILS_BYTES + || status.message() != FAILURE_MESSAGE + || status.code() == Code::Ok + { + return None; + } + let envelope = tonic_types::pb::Status::decode(status.details()).ok()?; + if envelope.code != status.code() as i32 || envelope.message != FAILURE_MESSAGE { + return None; + } + let [detail] = envelope.details.as_slice() else { + return None; + }; + if detail.type_url != ERROR_INFO_TYPE_URL { + return None; + } + let info = tonic_types::pb::ErrorInfo::decode(detail.value.as_slice()).ok()?; + if info.reason != FAILURE_REASON + || info.domain != ERROR_DOMAIN + || info.metadata.get("version")?.as_str() != "1" + || info.metadata.keys().any(|key| { + !matches!( + key.as_str(), + "version" | "code" | "stage" | "upstream_http_status" + ) + }) + { + return None; + } + let upstream_status = match info.metadata.get("upstream_http_status") { + Some(value) => { + if value.len() != 3 || !value.bytes().all(|byte| byte.is_ascii_digit()) { + return None; + } + let status = value.parse::().ok()?; + if !(100..=599).contains(&status) { + return None; + } + Some(status) + } + None => None, + }; + Some(Self { + kind: TokenGrantFailureKind::from_code(info.metadata.get("code")?)?, + stage: TokenGrantStage::from_code(info.metadata.get("stage")?)?, + upstream_status, + }) + } +} + +impl fmt::Display for TokenGrantFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(FAILURE_MESSAGE) + } +} + +impl fmt::Debug for TokenGrantFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(FAILURE_MESSAGE) + } +} + +impl std::error::Error for TokenGrantFailure {} + +impl miette::Diagnostic for TokenGrantFailure {} + +#[cfg(test)] +mod tests { + use super::*; + + const KINDS: [TokenGrantFailureKind; 6] = [ + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantFailureKind::Configuration, + TokenGrantFailureKind::Timeout, + TokenGrantFailureKind::Unavailable, + TokenGrantFailureKind::Unclassified, + ]; + const STAGES: [TokenGrantStage; 6] = [ + TokenGrantStage::Configuration, + TokenGrantStage::WorkloadIdentity, + TokenGrantStage::GatewayRpc, + TokenGrantStage::IntermediateExchange, + TokenGrantStage::TokenEndpoint, + TokenGrantStage::Injection, + ]; + + #[test] + fn every_kind_has_a_documented_code_and_recovery_action() { + use ProviderCredentialRefreshRecoveryAction as Recovery; + let table = KINDS.map(|kind| { + let failure = TokenGrantFailure::new(kind, TokenGrantStage::TokenEndpoint); + (failure.code(), failure.recovery()) + }); + assert_eq!( + table, + [ + ("provider_authorization_required", Recovery::Reauthorize), + ("workload_identity_rejected", Recovery::FixConfiguration), + ("configuration_error", Recovery::FixConfiguration), + ("timeout", Recovery::Retry), + ("unavailable", Recovery::Retry), + ("unclassified", Recovery::Investigate), + ] + ); + } + + fn sample() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::IntermediateExchange, + ) + .with_upstream_status(409) + } + + fn envelope(status: &Status) -> tonic_types::pb::Status { + tonic_types::pb::Status::decode(status.details()).unwrap() + } + + fn replace_envelope(original: &Status, value: &tonic_types::pb::Status) -> Status { + Status::with_details( + original.code(), + original.message(), + value.encode_to_vec().into(), + ) + } + + fn mutate_info(mutate: impl FnOnce(&mut tonic_types::pb::ErrorInfo)) -> Status { + let status = sample().to_status(); + let mut value = envelope(&status); + let mut info = + tonic_types::pb::ErrorInfo::decode(value.details[0].value.as_slice()).unwrap(); + mutate(&mut info); + value.details[0].value = info.encode_to_vec(); + replace_envelope(&status, &value) + } + + fn assert_rejected(status: &Status) { + let failure = TokenGrantFailure::from_status(status); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); + assert_eq!(failure.upstream_status(), None); + assert_eq!(failure.to_string(), FAILURE_MESSAGE); + assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); + assert!(std::error::Error::source(&failure).is_none()); + } + + #[test] + fn protobuf_normalization_still_validates_the_classification() { + let original = sample().to_status(); + let mut value = envelope(&original); + // Generated decoders ignore unknown wire fields. Only the validated + // classification survives; peer diagnostics never enter the failure. + value.details[0].value.extend([32, 1]); + assert_eq!( + TokenGrantFailure::from_status(&replace_envelope(&original, &value)), + sample() + ); + + // Protobuf map entries use the last value. Check that a replacement + // still passes the closed code set instead of bypassing validation. + for (code, expected) in [ + ("timeout", Some(TokenGrantFailureKind::Timeout)), + ("SYNTHETIC_SECRET", None), + ] { + let replacement = tonic_types::pb::ErrorInfo { + metadata: HashMap::from([("code".to_owned(), code.to_owned())]), + ..Default::default() + }; + let mut value = envelope(&original); + value.details[0].value.extend(replacement.encode_to_vec()); + let status = replace_envelope(&original, &value); + if let Some(kind) = expected { + let failure = TokenGrantFailure::from_status(&status); + assert_eq!( + failure, + TokenGrantFailure::new(kind, sample().stage()).with_upstream_status(409) + ); + assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); + } else { + assert_rejected(&status); + } + } + } + + #[test] + fn all_classifications_and_stages_round_trip_without_printable_detail() { + for kind in KINDS { + for stage in STAGES { + for upstream in [None, Some(100), Some(409), Some(599)] { + let mut failure = TokenGrantFailure::new(kind, stage); + if let Some(status) = upstream { + failure = failure.with_upstream_status(status); + } + let status = failure.to_status(); + assert!(status.details().len() <= MAX_DETAILS_BYTES); + assert_eq!(status.message(), FAILURE_MESSAGE); + assert_eq!(TokenGrantFailure::from_status(&status), failure); + assert_eq!(failure.to_string(), FAILURE_MESSAGE); + assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); + assert_eq!(format!("{failure:#?}"), FAILURE_MESSAGE); + assert!(std::error::Error::source(&failure).is_none()); + assert_eq!(miette::Report::new(failure).to_string(), FAILURE_MESSAGE); + } + } + } + } + + #[test] + fn status_mapping_preserves_guard_codes_and_never_encodes_success() { + let expected = [ + Code::FailedPrecondition, + Code::PermissionDenied, + Code::FailedPrecondition, + Code::DeadlineExceeded, + Code::Unavailable, + Code::Unknown, + ]; + for (kind, code) in KINDS.into_iter().zip(expected) { + let failure = TokenGrantFailure::new(kind, TokenGrantStage::GatewayRpc); + assert_eq!(failure.to_status().code(), code); + for override_code in [ + Code::PermissionDenied, + Code::Unauthenticated, + Code::NotFound, + ] { + let status = failure.to_status_with_code(override_code); + assert_eq!(status.code(), override_code); + assert_eq!(TokenGrantFailure::from_status(&status), failure); + } + } + assert_eq!(sample().to_status_with_code(Code::Ok).code(), Code::Unknown); + } + + #[test] + fn status_bounds_and_stage_changes_preserve_only_allowlisted_values() { + for status in [0, 99, 600, u16::MAX] { + assert_eq!( + sample().with_upstream_status(status).upstream_status(), + None + ); + } + let failure = sample().at_stage(TokenGrantStage::Injection); + assert_eq!(failure.stage(), TokenGrantStage::Injection); + assert_eq!(failure.kind(), sample().kind()); + assert_eq!(failure.upstream_status(), Some(409)); + } + + #[test] + fn rejects_unknown_missing_or_noncanonical_metadata() { + for (key, value) in [ + ("version", "2"), + ("version", "01"), + ("code", "SYNTHETIC_SECRET"), + ("stage", "SYNTHETIC_SECRET"), + ("unknown", "SYNTHETIC_SECRET"), + ("upstream_http_status", "099"), + ("upstream_http_status", "600"), + ("upstream_http_status", "0409"), + ("upstream_http_status", "+409"), + ("upstream_http_status", "409 "), + ("upstream_http_status", "4e2"), + ("upstream_http_status", ""), + ] { + assert_rejected(&mutate_info(|info| { + info.metadata.insert(key.into(), value.into()); + })); + } + for key in ["version", "code", "stage"] { + assert_rejected(&mutate_info(|info| { + info.metadata.remove(key); + })); + } + assert_rejected(&mutate_info(|info| info.reason = "SYNTHETIC_SECRET".into())); + assert_rejected(&mutate_info(|info| info.domain = "SYNTHETIC_SECRET".into())); + assert_rejected(&mutate_info(|info| info.reason.clear())); + assert_rejected(&mutate_info(|info| info.domain.clear())); + } + + #[test] + fn rejects_inconsistent_envelopes_and_arbitrary_status_diagnostics() { + let original = sample().to_status(); + for code in [Code::Ok, Code::Internal] { + assert_rejected(&Status::with_details( + code, + FAILURE_MESSAGE, + original.details().to_vec().into(), + )); + } + assert_rejected(&Status::with_details( + original.code(), + "SYNTHETIC_SECRET", + original.details().to_vec().into(), + )); + assert_rejected(&Status::unknown("SYNTHETIC_SECRET")); + for code in [-1, 0, 17, i32::MAX] { + let mut value = envelope(&original); + value.code = code; + assert_rejected(&replace_envelope(&original, &value)); + } + let mut value = envelope(&original); + value.message = "SYNTHETIC_SECRET".into(); + assert_rejected(&replace_envelope(&original, &value)); + for type_url in [ + "google.rpc.ErrorInfo", + "https://untrusted.invalid/google.rpc.ErrorInfo", + "type.googleapis.com/google.rpc.RetryInfo", + "", + ] { + let mut value = envelope(&original); + value.details[0].type_url = type_url.into(); + assert_rejected(&replace_envelope(&original, &value)); + } + let mut value = envelope(&original); + value.details.clear(); + assert_rejected(&replace_envelope(&original, &value)); + let mut value = envelope(&original); + value.details.push(value.details[0].clone()); + assert_rejected(&replace_envelope(&original, &value)); + } + + #[test] + fn rejects_truncation_oversize_invalid_utf8_and_malformed_messages() { + let original = sample().to_status(); + for length in 0..original.details().len() { + assert_rejected(&Status::with_details( + original.code(), + FAILURE_MESSAGE, + original.details()[..length].to_vec().into(), + )); + } + for details in [ + vec![0; MAX_DETAILS_BYTES + 1], + vec![8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2], + vec![18, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1], + vec![0], + vec![11], // Unterminated protobuf group. + ] { + assert_rejected(&Status::with_details( + original.code(), + FAILURE_MESSAGE, + details.into(), + )); + } + let mut value = envelope(&original); + value.details[0].value = vec![10, 1, 0xff]; + assert_rejected(&replace_envelope(&original, &value)); + } +} diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 9cacfc41ef..92cc060901 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -6494,6 +6494,7 @@ fn public_status_from_driver( restart_count: 0, next_restart_time: None, main_process_started_time: None, + last_token_grant_failure: None, } } @@ -6628,6 +6629,9 @@ fn apply_driver_snapshot( } if let (Some(status), Some(current_status)) = (status.as_mut(), sandbox.status.as_ref()) { + status + .last_token_grant_failure + .clone_from(¤t_status.last_token_grant_failure); status .main_process_instance_id .clone_from(¤t_status.main_process_instance_id); @@ -10739,17 +10743,34 @@ mod tests { #[test] fn driver_snapshot_preserves_endpoint_failure_and_ready_phase() { let mut sandbox = sandbox_record("sandbox-id", "sandbox-name", SandboxPhase::Ready); + let failure = openshell_core::endpoint_failure::EndpointFailure::token_grant( + openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::Timeout, + openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + ), + "github", + "api_token", + "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", + ) + .to_proto(); let endpoint = openshell_core::proto::EndpointStatus { endpoint_id: "endpoint:v1:gateway".to_string(), host: "api.example.com".to_string(), ports: vec![443], path: "/mcp".to_string(), - last_result: openshell_core::proto::EndpointResult::TransportFailed as i32, + last_result: openshell_core::proto::EndpointResult::CredentialUnavailable as i32, last_reported_time: Some("2026-09-05T01:01:00.000Z".parse().unwrap()), + last_failure: Some(failure.clone()), }; + let grant_failure = Box::new(openshell_core::proto::TokenGrantFailureObservation { + destination_host: "api.example.com".to_string(), + destination_port: 443, + failure: Some(failure), + }); sandbox.status = Some(SandboxStatus { phase: SandboxPhase::Ready as i32, endpoint_statuses: vec![endpoint.clone()], + last_token_grant_failure: Some(grant_failure.clone()), ..Default::default() }); let incoming = ready_driver_sandbox("sandbox-id", "sandbox-name"); @@ -10759,6 +10780,7 @@ mod tests { assert_eq!(sandbox.phase(), SandboxPhase::Ready as i32); let status = sandbox.status.expect("driver status applied"); assert_eq!(status.endpoint_statuses, vec![endpoint]); + assert_eq!(status.last_token_grant_failure, Some(grant_failure)); assert!( status .conditions diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 97318f00b2..0ebdb8c51d 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -4949,10 +4949,9 @@ pub(super) async fn handle_report_policy_status( |sandbox| { sandbox.set_current_policy_version(version_to_set); if let Some(endpoints) = &endpoint_reset { - endpoint_status::reconcile_endpoint_statuses( + endpoint_status::reset_endpoint_statuses( sandbox, endpoints, - &HashSet::new(), &prost_types::Timestamp::default(), ); } diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status.rs b/crates/openshell-server/src/grpc/policy/endpoint_status.rs index 462099198b..211457f727 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status.rs @@ -16,11 +16,13 @@ use crate::provider_profile_sources::EffectiveProviderProfileCatalog; use crate::supervisor_owner::{OWNER_TTL, SupervisorOwnerIndex}; use crate::supervisor_session::EndpointReportCursor; use openshell_core::GetResourceVersion; -use openshell_core::endpoint_status::initial_endpoint_status; +use openshell_core::endpoint_failure::validate_endpoint_failure; +use openshell_core::endpoint_status::{initial_endpoint_status, valid_token_grant_destination}; use openshell_core::mcp::is_mcp_protocol; use openshell_core::proto::{ EndpointResult, EndpointStatus, PolicySource, ReportEndpointStatusRequest, ReportEndpointStatusResponse, Sandbox, SandboxPolicy as ProtoSandboxPolicy, SandboxStatus, + TokenGrantStatus, }; use prost::Message; use sha2::{Digest, Sha256}; @@ -40,8 +42,8 @@ struct ExpectedEndpoint { // Credential attribution is derived from the effective policy so a // sandbox cannot claim credential failure for an unauthenticated endpoint. provider_credentialed: bool, - // The public status comes only from the validated policy endpoint, - // never from observed request data or supervisor-supplied text. + // The public address comes only from the validated policy endpoint, never + // from observed request data. Only a validated failure cause is reported. status: EndpointStatus, } @@ -54,7 +56,9 @@ struct EndpointContext { /// Accept the last observed result for each configured tool server endpoint. /// /// Addresses and identifiers come from the authoritative policy. The supervisor -/// supplies only typed results, so request data and upstream text cannot enter status. +/// supplies typed results and, for credential failures, a cause whose codes come +/// from closed sets and whose names are printable ASCII, so request data and +/// upstream text cannot enter status. pub(in crate::grpc) async fn handle_report_endpoint_status( state: &Arc, request: Request, @@ -140,6 +144,10 @@ async fn handle_report_endpoint_status_inner( let current_cursor = state .supervisor_sessions .endpoint_report_cursor(&req.sandbox_id, &req.supervisor_session_id); + let configuration_changed = current_cursor.as_ref().is_some_and(|cursor| { + cursor.policy_hash != req.policy_hash + || cursor.provider_env_revision != req.provider_env_revision + }); let report_digest: [u8; 32] = Sha256::digest(req.encode_to_vec()).into(); if let Some(cursor) = current_cursor.as_ref() { if req.report_sequence < cursor.report_sequence { @@ -166,10 +174,17 @@ async fn handle_report_endpoint_status_inner( // newer complete snapshot retires every lower sequence, including an RPC // cancelled after dispatch or a report from an earlier equal-valued epoch. + let attached_providers = sandbox + .spec + .as_ref() + .map(|spec| spec.providers.as_slice()) + .unwrap_or_default(); + validate_token_grant_status(req.token_grant_status.as_ref(), attached_providers)?; let (reports, observed_endpoint_ids) = validate_endpoint_snapshot( &req.observations, &req.observed_endpoint_ids, &context.endpoints, + attached_providers, )?; validate_endpoint_observation_markers(&sandbox, &reports, &observed_endpoint_ids)?; let now = openshell_core::time::timestamp_from_system_time(std::time::SystemTime::now()) @@ -179,6 +194,13 @@ async fn handle_report_endpoint_status_inner( .store .update_message_cas::(&req.sandbox_id, expected_resource_version, |sandbox| { reconcile_endpoint_statuses(sandbox, &reports, &observed_endpoint_ids, &now); + let status = sandbox.status.as_mut().expect("status reconciled"); + if let Some(grant_status) = &req.token_grant_status { + status.last_token_grant_failure = grant_status.last_failure.clone().map(Box::new); + } else if configuration_changed { + // A legacy report cannot retain evidence from another configuration. + status.last_token_grant_failure = None; + } }) .await .map_err(|error| { @@ -387,7 +409,7 @@ pub async fn reset_endpoint_status_for_supervisor_session( let updated = state .store .update_message_cas::(sandbox_id, expected_resource_version, |sandbox| { - reconcile_endpoint_statuses(sandbox, &reports, &HashSet::new(), &now); + reset_endpoint_statuses(sandbox, &reports, &now); }) .await .map_err(|error| { @@ -436,7 +458,7 @@ pub async fn reset_endpoint_status_after_supervisor_disconnect( let updated = state .store .update_message_cas::(sandbox_id, expected_resource_version, |sandbox| { - reconcile_endpoint_statuses(sandbox, &reports, &HashSet::new(), &now); + reset_endpoint_statuses(sandbox, &reports, &now); }) .await .map_err(|error| { @@ -507,10 +529,9 @@ pub async fn invalidate_endpoint_status_on_startup(state: &Arc) -> } for sandbox in &sandboxes { - let has_endpoint_status = sandbox - .status - .as_ref() - .is_some_and(|status| !status.endpoint_statuses.is_empty()); + let has_endpoint_status = sandbox.status.as_ref().is_some_and(|status| { + !status.endpoint_statuses.is_empty() || status.last_token_grant_failure.is_some() + }); if !has_endpoint_status { continue; } @@ -565,9 +586,11 @@ fn invalidate_endpoint_status_without_session(sandbox: &mut Sandbox) { }; // Losing observation authority clears evidence, while policy-derived identity // remains available so callers can still identify the configured endpoint. + status.last_token_grant_failure = None; for endpoint in &mut status.endpoint_statuses { endpoint.last_result = EndpointResult::NoObservedExchange as i32; endpoint.last_reported_time = None; + endpoint.last_failure = None; } } @@ -597,6 +620,7 @@ fn validate_endpoint_snapshot( reported: &[openshell_core::proto::EndpointObservation], reported_observed_endpoint_ids: &[String], expected: &BTreeMap, + attached_providers: &[String], ) -> Result<(BTreeMap, HashSet), Status> { let mut validated = BTreeMap::new(); for endpoint in reported { @@ -622,11 +646,15 @@ fn validate_endpoint_snapshot( "credential-unavailable requires a credentialed tool server endpoint", )); } + if let Some(failure) = endpoint.failure.as_ref() { + validate_reported_failure(result, failure, attached_providers)?; + } if validated .insert( endpoint.endpoint_id.clone(), EndpointStatus { last_result: result as i32, + last_failure: endpoint.failure.clone(), ..expected_endpoint.status.clone() }, ) @@ -666,6 +694,62 @@ fn validate_endpoint_snapshot( Ok((validated, observed_endpoint_ids)) } +/// Accept a cause only for a result its domain explains, from the closed sets, +/// and naming nothing outside the sandbox's own providers. +/// +/// The credential name is checked for shape only. It is a name the sandbox's +/// supervisor received with its provider environment, never a credential value. +fn validate_reported_failure( + result: EndpointResult, + failure: &openshell_core::proto::EndpointFailure, + attached_providers: &[String], +) -> Result<(), Status> { + validate_endpoint_failure(failure, result).map_err(|invalid| { + openshell_core::rpc_error::invalid_argument("observations.failure", invalid.message()) + })?; + if !failure.provider.is_empty() && !attached_providers.contains(&failure.provider) { + return Err(openshell_core::rpc_error::failed_precondition( + "ENDPOINT_FAILURE_PROVIDER_NOT_ATTACHED", + "an endpoint failure names a provider that is not attached to the sandbox", + )); + } + Ok(()) +} + +fn validate_token_grant_status( + status: Option<&TokenGrantStatus>, + attached_providers: &[String], +) -> Result<(), Status> { + let Some(observation) = status.and_then(|status| status.last_failure.as_ref()) else { + return Ok(()); + }; + if !valid_token_grant_destination(&observation.destination_host, observation.destination_port) + || observation.destination_host != observation.destination_host.to_ascii_lowercase() + { + return Err(openshell_core::rpc_error::invalid_argument( + "token_grant_status.last_failure", + "token grant destination is invalid", + )); + } + let failure = observation.failure.as_ref().ok_or_else(|| { + openshell_core::rpc_error::invalid_argument( + "token_grant_status.last_failure.failure", + "token grant failure cause is required", + ) + })?; + if failure.domain != i32::from(openshell_core::proto::EndpointFailureDomain::TokenGrant) { + return Err(openshell_core::rpc_error::invalid_argument( + "token_grant_status.last_failure.failure.domain", + "sandbox token grant status requires a token grant failure", + )); + } + validate_reported_failure( + EndpointResult::CredentialUnavailable, + failure, + attached_providers, + ) +} + fn validate_endpoint_observation_markers( sandbox: &Sandbox, reports: &BTreeMap, @@ -688,7 +772,8 @@ fn validate_endpoint_observation_markers( continue; } // A retained result may reuse its previous report time. New evidence, - // including evidence accepted after a reset, must carry a marker. + // including evidence accepted after a reset, must carry a marker. A + // cause can change without a marker; it then keeps the report time. if previous.get(endpoint_id.as_str()) != Some(&report.last_result) && !observed_endpoint_ids.contains(endpoint_id) { @@ -761,6 +846,20 @@ pub(super) async fn endpoint_status_reset_for_loaded_policy( Ok((!same_configuration).then(|| unknown_endpoint_reports(&context.endpoints))) } +/// Clear all supervised network evidence without changing lifecycle state. +pub(super) fn reset_endpoint_statuses( + sandbox: &mut Sandbox, + reports: &BTreeMap, + now: &prost_types::Timestamp, +) { + reconcile_endpoint_statuses(sandbox, reports, &HashSet::new(), now); + sandbox + .status + .as_mut() + .expect("status reconciled") + .last_token_grant_failure = None; +} + /// Replace the endpoint inventory and evidence without changing lifecycle state. pub(super) fn reconcile_endpoint_statuses( sandbox: &mut Sandbox, diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs index 4c951863a0..c034f04b78 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use super::super::tests::{mcp_policy_with_versions, test_sandbox, with_sandbox}; +use super::super::tests::{mcp_policy_with_versions, test_provider, test_sandbox, with_sandbox}; use super::super::{handle_get_sandbox_config, handle_report_policy_status, handle_update_config}; use super::*; use crate::grpc::test_support::{authed_request, test_server_state}; @@ -9,7 +9,7 @@ use openshell_core::endpoint_status::endpoint_id; use openshell_core::proto::{ EndpointObservation, GetSandboxConfigRequest, GetSandboxRequest, NetworkEndpoint, NetworkPolicyRule, PolicyStatus, ReportPolicyStatusRequest, SandboxCondition, SandboxPhase, - UpdateConfigRequest, + TokenGrantFailureObservation, UpdateConfigRequest, }; use tonic::Code; @@ -25,6 +25,7 @@ fn test_initial_endpoint_status(endpoint_id: &str, host: &str, path: &str) -> En path: path.to_string(), last_result: EndpointResult::NoObservedExchange as i32, last_reported_time: None, + last_failure: None, } } @@ -159,10 +160,12 @@ async fn sandbox_with_accepted_endpoint_result( observations: vec![EndpointObservation { endpoint_id: endpoint_id.clone(), result: EndpointResult::HttpResponseReceived as i32, + failure: None, }], observed_endpoint_ids: vec![endpoint_id], supervisor_session_id: "session-a".to_string(), report_sequence: 1, + token_grant_status: None, }; handle_report_endpoint_status( &state, @@ -540,6 +543,7 @@ async fn report_endpoint_status_rejects_stale_configuration_epoch() { let observation = EndpointObservation { endpoint_id: endpoint_id(endpoint), result: EndpointResult::NoObservedExchange as i32, + failure: None, }; let (session_tx, _session_rx) = tokio::sync::mpsc::channel(1); let (shutdown_tx, _shutdown_rx) = tokio::sync::oneshot::channel(); @@ -564,6 +568,7 @@ async fn report_endpoint_status_rejects_stale_configuration_epoch() { observed_endpoint_ids: Vec::new(), supervisor_session_id: "session-current".to_string(), report_sequence: 1, + token_grant_status: None, }, ReportEndpointStatusRequest { sandbox_id: sandbox_id.to_string(), @@ -573,6 +578,7 @@ async fn report_endpoint_status_rejects_stale_configuration_epoch() { observed_endpoint_ids: Vec::new(), supervisor_session_id: "session-current".to_string(), report_sequence: 1, + token_grant_status: None, }, ] { let error = @@ -612,8 +618,9 @@ fn endpoint_snapshot_rejects_credential_failure_for_uncredentialed_path() { let reported = vec![EndpointObservation { endpoint_id: "endpoint:v1:public".to_string(), result: EndpointResult::CredentialUnavailable as i32, + failure: None, }]; - let error = validate_endpoint_snapshot(&reported, &[], &expected) + let error = validate_endpoint_snapshot(&reported, &[], &expected, &[]) .expect_err("uncredentialed endpoint cannot report credential failure"); assert_eq!(error.code(), Code::InvalidArgument); assert_eq!( @@ -729,6 +736,572 @@ fn endpoint_reconciliation_advances_only_observed_endpoint_timestamp() { ); } +const FAILURE_REQUEST_ID: &str = "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908"; + +fn credentialed_expected() -> BTreeMap { + BTreeMap::from([( + "endpoint:v1:tools".to_string(), + ExpectedEndpoint { + provider_credentialed: true, + status: test_initial_endpoint_status( + "endpoint:v1:tools", + "tools.example.com", + "/mcp/**", + ), + }, + )]) +} + +fn reauthorize_failure() -> openshell_core::proto::EndpointFailure { + openshell_core::endpoint_failure::EndpointFailure::token_grant( + openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::ProviderAuthorizationRequired, + openshell_core::token_grant_failure::TokenGrantStage::GatewayRpc, + ), + "github", + "access_token", + FAILURE_REQUEST_ID, + ) + .to_proto() +} + +fn sandbox_grant_failure() -> TokenGrantFailureObservation { + TokenGrantFailureObservation { + destination_host: "api.example.com".to_string(), + destination_port: 443, + failure: Some(reauthorize_failure()), + } +} + +async fn rest_grant_status_fixture( + sandbox_id: &str, +) -> (Arc, ReportEndpointStatusRequest) { + let state = test_server_state().await; + state + .store + .put_message(&test_provider("github", "github")) + .await + .expect("store attached provider"); + let mut policy = openshell_policy::restrictive_default_policy(); + policy.network_policies.insert( + "rest".to_string(), + NetworkPolicyRule { + name: "rest".to_string(), + endpoints: vec![NetworkEndpoint { + host: "api.example.com".to_string(), + port: 443, + protocol: "rest".to_string(), + access: openshell_core::proto::NetworkAccessPreset::Full as i32, + ..Default::default() + }], + ..Default::default() + }, + ); + state + .store + .put_message(&test_sandbox( + sandbox_id, + sandbox_id, + policy.clone(), + vec!["github".to_string()], + )) + .await + .expect("store REST-only sandbox"); + state + .store + .put_policy_revision( + &format!("{sandbox_id}-v1"), + sandbox_id, + "default", + 1, + &policy.encode_to_vec(), + &deterministic_policy_hash(&policy), + ) + .await + .expect("store REST policy"); + acknowledge_loaded_policy(&state, sandbox_id, 1).await; + register_session(&state, sandbox_id, "session-a"); + reset_endpoint_status_for_supervisor_session(&state, sandbox_id, "session-a") + .await + .expect("initialize REST observation session"); + assert!( + state + .supervisor_sessions + .initialize_endpoint_status_authority(sandbox_id, "session-a") + ); + let sandbox = stored_sandbox(&state, sandbox_id).await; + let context = active_endpoint_context(&state, &sandbox).await.unwrap(); + assert!(context.endpoints.is_empty(), "fixture has no MCP endpoints"); + ( + state, + ReportEndpointStatusRequest { + sandbox_id: sandbox_id.to_string(), + policy_hash: context.policy_hash, + provider_env_revision: context.provider_env_revision, + supervisor_session_id: "session-a".to_string(), + report_sequence: 1, + token_grant_status: Some(TokenGrantStatus { + last_failure: Some(sandbox_grant_failure()), + }), + ..Default::default() + }, + ) +} + +#[test] +fn sandbox_grant_status_validates_destination_cause_and_provider() { + let attached = ["github".to_string()]; + let validate = |failure| { + validate_token_grant_status( + Some(&TokenGrantStatus { + last_failure: Some(failure), + }), + &attached, + ) + }; + for host in ["api.example.com", "127.0.0.1", "::1", "[2001:db8::1]"] { + let mut observation = sandbox_grant_failure(); + observation.destination_host = host.to_string(); + validate(observation).expect("bounded DNS and IP destinations are accepted"); + } + for host in [ + "", + "API.EXAMPLE.COM", + "https://SECRET.example/token", + ":", + "bad\nSECRET", + ] { + let mut observation = sandbox_grant_failure(); + observation.destination_host = host.to_string(); + let error = validate(observation).unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(!error.message().contains("SECRET")); + } + for port in [0, 65536] { + let mut observation = sandbox_grant_failure(); + observation.destination_port = port; + assert_eq!( + validate(observation).unwrap_err().code(), + Code::InvalidArgument + ); + } + let mut missing_cause = sandbox_grant_failure(); + missing_cause.failure = None; + assert_eq!( + validate(missing_cause).unwrap_err().code(), + Code::InvalidArgument + ); + let mut unknown_domain = sandbox_grant_failure(); + unknown_domain.failure.as_mut().unwrap().domain = 99; + assert_eq!( + validate(unknown_domain).unwrap_err().code(), + Code::InvalidArgument + ); + let mut foreign_provider = sandbox_grant_failure(); + foreign_provider.failure.as_mut().unwrap().provider = "other-team".to_string(); + assert_eq!( + validate(foreign_provider).unwrap_err().code(), + Code::FailedPrecondition + ); + let mut unnamed = sandbox_grant_failure(); + let cause = unnamed.failure.as_mut().unwrap(); + cause.provider.clear(); + cause.credential_key.clear(); + validate_token_grant_status( + Some(&TokenGrantStatus { + last_failure: Some(unnamed), + }), + &[], + ) + .expect("selection failures may have no credential name"); +} + +#[tokio::test] +async fn rest_grant_failure_is_retrievable_retained_and_explicitly_cleared() { + let sandbox_id = "rest-grant-status"; + let (state, report) = rest_grant_status_fixture(sandbox_id).await; + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .expect("REST failure is reported without MCP observations"); + let first = stored_sandbox(&state, sandbox_id).await; + assert_eq!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure, + Some(Box::new(sandbox_grant_failure())) + ); + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .expect("identical retry"); + assert_eq!(stored_sandbox(&state, sandbox_id).await, first); + let mut changed_retry = report.clone(); + changed_retry + .token_grant_status + .as_mut() + .unwrap() + .last_failure + .as_mut() + .unwrap() + .destination_port = 8443; + assert_eq!( + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(changed_retry), sandbox_id) + ) + .await + .unwrap_err() + .code(), + Code::InvalidArgument + ); + let mut legacy = report.clone(); + legacy.report_sequence = 2; + legacy.token_grant_status = None; + handle_report_endpoint_status(&state, with_sandbox(Request::new(legacy), sandbox_id)) + .await + .expect("missing extension retains current evidence"); + assert_eq!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure, + Some(Box::new(sandbox_grant_failure())) + ); + let mut cleared = report.clone(); + cleared.report_sequence = 3; + cleared.token_grant_status = Some(TokenGrantStatus::default()); + handle_report_endpoint_status(&state, with_sandbox(Request::new(cleared), sandbox_id)) + .await + .expect("success explicitly clears the previous failure"); + assert!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure + .is_none() + ); + assert_eq!( + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id) + ) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + assert_eq!( + handle_report_endpoint_status(&state, with_sandbox(Request::new(report), "other-sandbox")) + .await + .unwrap_err() + .code(), + Code::PermissionDenied + ); +} + +#[tokio::test] +async fn rest_grant_failure_resets_on_provider_and_policy_changes() { + let sandbox_id = "rest-grant-revisions"; + let (state, mut report) = rest_grant_status_fixture(sandbox_id).await; + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .unwrap(); + state + .store + .update_message_cas::( + "provider-github", + 0, + |provider| { + provider.credentials.insert( + "GITHUB_TOKEN".to_string(), + "changed-inert-token".to_string(), + ); + }, + ) + .await + .unwrap(); + let sandbox = stored_sandbox(&state, sandbox_id).await; + let context = active_endpoint_context(&state, &sandbox).await.unwrap(); + assert_ne!(context.provider_env_revision, report.provider_env_revision); + let old_report = report.clone(); + report.report_sequence = 2; + report.policy_hash = context.policy_hash; + report.provider_env_revision = context.provider_env_revision; + report.token_grant_status = None; + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .expect("a new provider revision clears even a legacy report's cause"); + assert!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure + .is_none() + ); + assert_eq!( + handle_report_endpoint_status(&state, with_sandbox(Request::new(old_report), sandbox_id)) + .await + .unwrap_err() + .code(), + Code::FailedPrecondition + ); + report.report_sequence = 3; + report.token_grant_status = Some(TokenGrantStatus { + last_failure: Some(sandbox_grant_failure()), + }); + handle_report_endpoint_status(&state, with_sandbox(Request::new(report), sandbox_id)) + .await + .unwrap(); + let mut policy = sandbox.spec.unwrap().policy.unwrap(); + policy.network_policies.get_mut("rest").unwrap().endpoints[0].host = + "changed.example.com".to_string(); + state + .store + .put_policy_revision( + "rest-grant-revisions-v2", + sandbox_id, + "default", + 2, + &policy.encode_to_vec(), + &deterministic_policy_hash(&policy), + ) + .await + .unwrap(); + acknowledge_loaded_policy(&state, sandbox_id, 2).await; + let status = public_status(&state, sandbox_id).await; + assert!(status.endpoint_statuses.is_empty()); + assert!(status.last_token_grant_failure.is_none()); +} + +#[tokio::test] +async fn rest_grant_failure_resets_on_session_loss_replacement_and_startup() { + let sandbox_id = "rest-grant-sessions"; + let (state, mut report) = rest_grant_status_fixture(sandbox_id).await; + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .unwrap(); + state + .supervisor_sessions + .remove_if_current(sandbox_id, "session-a") + .unwrap(); + reset_endpoint_status_after_supervisor_disconnect(&state, sandbox_id) + .await + .unwrap(); + assert!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure + .is_none() + ); + for session in ["session-b", "session-c"] { + register_session(&state, sandbox_id, session); + reset_endpoint_status_for_supervisor_session(&state, sandbox_id, session) + .await + .unwrap(); + assert!( + public_status(&state, sandbox_id) + .await + .last_token_grant_failure + .is_none() + ); + assert!( + state + .supervisor_sessions + .initialize_endpoint_status_authority(sandbox_id, session) + ); + let predecessor = report.clone(); + report.supervisor_session_id = session.to_string(); + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(report.clone()), sandbox_id), + ) + .await + .unwrap(); + assert_eq!( + handle_report_endpoint_status( + &state, + with_sandbox(Request::new(predecessor), sandbox_id) + ) + .await + .unwrap_err() + .code(), + Code::PermissionDenied + ); + } + state + .supervisor_sessions + .remove_if_current(sandbox_id, "session-c") + .unwrap(); + invalidate_endpoint_status_on_startup(&state).await.unwrap(); + let status = public_status(&state, sandbox_id).await; + assert!(status.endpoint_statuses.is_empty()); + assert!(status.last_token_grant_failure.is_none()); +} + +fn failed_observation( + result: EndpointResult, + failure: openshell_core::proto::EndpointFailure, +) -> Vec { + vec![EndpointObservation { + endpoint_id: "endpoint:v1:tools".to_string(), + result: result as i32, + failure: Some(failure), + }] +} + +#[test] +fn endpoint_snapshot_accepts_bounded_failure_for_credential_failure() { + let attached = ["github".to_string()]; + let (reports, _) = validate_endpoint_snapshot( + &failed_observation(EndpointResult::CredentialUnavailable, reauthorize_failure()), + &["endpoint:v1:tools".to_string()], + &credentialed_expected(), + &attached, + ) + .expect("a classified credential failure is accepted"); + let status = &reports["endpoint:v1:tools"]; + assert_eq!( + status.last_result, + EndpointResult::CredentialUnavailable as i32 + ); + assert_eq!(status.last_failure, Some(reauthorize_failure())); + assert_eq!(status.host, "tools.example.com"); + + // A failure that has not selected a credential names no provider. + let mut unnamed = reauthorize_failure(); + unnamed.provider.clear(); + unnamed.credential_key.clear(); + validate_endpoint_snapshot( + &failed_observation(EndpointResult::CredentialUnavailable, unnamed), + &["endpoint:v1:tools".to_string()], + &credentialed_expected(), + &[], + ) + .expect("an unnamed failure is accepted"); +} + +#[test] +fn endpoint_snapshot_rejects_failures_outside_the_contract() { + let attached = ["github".to_string()]; + let invalid = |result, failure| { + validate_endpoint_snapshot( + &failed_observation(result, failure), + &["endpoint:v1:tools".to_string()], + &credentialed_expected(), + &attached, + ) + .expect_err("invalid failure is rejected") + }; + + let error = invalid(EndpointResult::TransportFailed, reauthorize_failure()); + assert_eq!(error.code(), Code::InvalidArgument); + assert_eq!( + error.message(), + "endpoint failure domain does not explain this endpoint result" + ); + + let mut unknown_code = reauthorize_failure(); + unknown_code.code = "issuer said: token SECRET expired".to_string(); + let error = invalid(EndpointResult::CredentialUnavailable, unknown_code); + assert_eq!(error.code(), Code::InvalidArgument); + assert!(!error.message().contains("SECRET")); + + let mut mismatched = reauthorize_failure(); + mismatched.recovery_action = + openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Retry as i32; + let error = invalid(EndpointResult::CredentialUnavailable, mismatched); + assert_eq!( + error.message(), + "endpoint failure recovery action does not match its code" + ); + + let mut foreign = reauthorize_failure(); + foreign.provider = "other-team".to_string(); + let error = invalid(EndpointResult::CredentialUnavailable, foreign); + assert_eq!(error.code(), Code::FailedPrecondition); + let info = openshell_core::rpc_error::decode_details(&error) + .and_then(|details| details.error_info().cloned()) + .expect("stable error reason"); + assert_eq!(info.reason, "ENDPOINT_FAILURE_PROVIDER_NOT_ATTACHED"); +} + +#[test] +fn endpoint_failure_change_keeps_report_time_and_session_loss_clears_it() { + let mut sandbox = test_sandbox( + "sandbox-id", + "sandbox-name", + ProtoSandboxPolicy::default(), + Vec::new(), + ); + let endpoint = EndpointStatus { + last_result: EndpointResult::CredentialUnavailable as i32, + last_failure: Some(reauthorize_failure()), + ..test_initial_endpoint_status("endpoint", "api.example.com", "/**") + }; + // A gateway replica that predates last_failure can rewrite the record + // without it: the stored result remains and its cause is gone. + let rewritten = BTreeMap::from([( + "endpoint".to_string(), + EndpointStatus { + last_failure: None, + ..endpoint.clone() + }, + )]); + let observed = HashSet::from(["endpoint".to_string()]); + reconcile_endpoint_statuses( + &mut sandbox, + &rewritten, + &observed, + ×tamp("2026-09-05T01:00:00.000Z"), + ); + + // The supervisor's next report carries a cause for the same result without + // marking the endpoint observed. Only a new result needs a marker, so the + // cause is stored and keeps the report time. + let mut retry = reauthorize_failure(); + retry.request_id = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d".to_string(); + let changed = BTreeMap::from([( + "endpoint".to_string(), + EndpointStatus { + last_failure: Some(retry.clone()), + ..endpoint + }, + )]); + validate_endpoint_observation_markers(&sandbox, &changed, &HashSet::new()) + .expect("a changed cause for the same result needs no marker"); + reconcile_endpoint_statuses( + &mut sandbox, + &changed, + &HashSet::new(), + ×tamp("2026-09-05T02:00:00.000Z"), + ); + let status = &sandbox.status.as_ref().unwrap().endpoint_statuses[0]; + assert_eq!(status.last_failure, Some(retry)); + assert_eq!( + status.last_reported_time, + Some(timestamp("2026-09-05T01:00:00.000Z")) + ); + + invalidate_endpoint_status_without_session(&mut sandbox); + let status = &sandbox.status.as_ref().unwrap().endpoint_statuses[0]; + assert_eq!( + status.last_result, + EndpointResult::NoObservedExchange as i32 + ); + assert_eq!(status.last_failure, None); +} + #[test] fn endpoint_snapshot_requires_marker_for_new_observed_result() { let mut sandbox = test_sandbox( @@ -905,15 +1478,18 @@ async fn report_endpoint_status_is_session_bound_and_retry_idempotent() { EndpointObservation { endpoint_id: observed_id.clone(), result: EndpointResult::TransportFailed as i32, + failure: None, }, EndpointObservation { endpoint_id: unobserved_id, result: EndpointResult::NoObservedExchange as i32, + failure: None, }, ], observed_endpoint_ids: vec![observed_id.clone()], supervisor_session_id: "session-a".to_string(), report_sequence: 1, + token_grant_status: None, }; handle_report_endpoint_status( &state, @@ -1145,10 +1721,12 @@ async fn loaded_policy_and_unknown_endpoint_inventory_commit_atomically() { observations: vec![EndpointObservation { endpoint_id: initial_active.endpoint_id.clone(), result: EndpointResult::HttpResponseReceived as i32, + failure: None, }], observed_endpoint_ids: vec![initial_active.endpoint_id.clone()], supervisor_session_id: "active-session".to_string(), report_sequence: 1, + token_grant_status: None, }), sandbox_id, ), diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index f77f6a20b7..e3aeda2edf 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -2501,33 +2501,26 @@ use crate::auth::workspace_authz::{ authorize_workspace, require_platform_admin, }; use openshell_core::oauth::{ - self, TokenExchangeParams, effective_client_assertion_type, effective_token_type, + self, TokenExchangeParams, TokenGrantClient, effective_client_assertion_type, + effective_token_type, +}; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; const DEFAULT_INTERMEDIATE_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const MAX_INTERMEDIATE_TOKEN_CACHE_TTL_SECONDS: i64 = 3600; const INTERMEDIATE_TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; const MAX_INTERMEDIATE_TOKEN_CACHE_ENTRIES: usize = 1024; -static TOKEN_EXCHANGE_HTTP_CLIENT: LazyLock> = - LazyLock::new(|| { - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .connect_timeout(std::time::Duration::from_secs(30)) - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(|err| { - format!("provider token exchange HTTP client configuration failed: {err}") - }) - }); +static TOKEN_EXCHANGE_HTTP_CLIENT: LazyLock> = + LazyLock::new(TokenGrantClient::new); static INTERMEDIATE_TOKEN_CACHE: LazyLock = LazyLock::new(IntermediateTokenCache::new); -fn token_exchange_http_client() -> Result<&'static reqwest::Client, Status> { +fn token_exchange_http_client() -> Result<&'static TokenGrantClient, Status> { TOKEN_EXCHANGE_HTTP_CLIENT .as_ref() - .map_err(|err| Status::internal(err.clone())) + .map_err(|failure| failure.to_status_with_code(tonic::Code::Internal)) } #[derive(Clone)] @@ -4055,43 +4048,181 @@ pub(super) async fn handle_update_provider( } } +/// Exchange a supervisor's attached provider credential with safe failure details. pub(super) async fn handle_exchange_provider_subject_token( state: &Arc, request: Request, ) -> Result, Status> { - let req = request.get_ref().clone(); - let principal = crate::auth::guard::enforce_sandbox_scope(&request, &req.sandbox_id)?; - crate::auth::guard::ensure_sandbox_principal_scope(&principal, &req.sandbox_id)?; - drop(request); + let request_id = canonical_request_id(request.metadata()); + // A supervisor's failures go to its own authenticated sandbox's log stream, + // never to the sandbox named in the request. Other callers are logged on + // the gateway only. + let log_sandbox_id = match request.extensions().get::() { + Some(Principal::Sandbox(principal)) => Some(principal.sandbox_id.clone()), + _ => None, + }; + exchange_provider_subject_token_inner(state, request, request_id.as_deref()) + .await + .map_err(|status| { + let status = sanitize_token_grant_status(status); + log_token_grant_failure( + log_sandbox_id.as_deref(), + request_id.as_deref(), + TokenGrantFailure::from_status(&status), + ); + status + }) +} - if req.provider.trim().is_empty() { - return Err(Status::invalid_argument("provider is required")); +fn sanitize_token_grant_status(status: Status) -> Status { + // Rebuild even valid details so foreign metadata, sources and diagnostics + // cannot escape an early return from a shared dependency. + TokenGrantFailure::from_status(&status).to_status_with_code(status.code()) +} + +/// Return the request's `x-request-id` only when it is a canonical UUID. +/// +/// The supervisor sends the ID of the HTTP exchange that needs the grant. Any +/// other value is not echoed into logs or a token-service request. +fn canonical_request_id(metadata: &tonic::metadata::MetadataMap) -> Option { + let value = metadata.get("x-request-id")?.to_str().ok()?; + let parsed = uuid::Uuid::parse_str(value).ok()?; + (parsed.hyphenated().to_string() == value).then(|| value.to_string()) +} + +/// Log a grant failure for operators with its bounded classification. +/// +/// The gateway log stream keeps only an event's message and `sandbox_id`, so +/// the classification goes into the message. It never contains remote text. +fn log_token_grant_failure( + sandbox_id: Option<&str>, + request_id: Option<&str>, + failure: TokenGrantFailure, +) { + let upstream_http_status = failure + .upstream_status() + .map_or_else(|| "none".to_string(), |status| status.to_string()); + let message = format!( + "dynamic token grant failed request_id={} code={} stage={} upstream_http_status={upstream_http_status}", + request_id.unwrap_or("none"), + failure.code(), + failure.stage().code(), + ); + if let Some(sandbox_id) = sandbox_id { + warn!(sandbox_id = %sandbox_id, "{message}"); + } else { + warn!("{message}"); } - if req.credential_key.trim().is_empty() { - return Err(Status::invalid_argument("credential_key is required")); +} + +fn token_grant_configuration_status(code: tonic::Code) -> Status { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ) + .to_status_with_code(code) +} + +fn token_grant_identity_status(kind: TokenGrantFailureKind, code: tonic::Code) -> Status { + TokenGrantFailure::new(kind, TokenGrantStage::WorkloadIdentity).to_status_with_code(code) +} + +fn token_grant_workload_api_status(error: spiffe::WorkloadApiError, code: tonic::Code) -> Status { + use spiffe::WorkloadApiError; + let kind = match error { + WorkloadApiError::MissingEndpointSocket + | WorkloadApiError::NotUnicodeEndpointSocket(_) + | WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, + WorkloadApiError::NoIdentityIssued | WorkloadApiError::PermissionDenied(_) => { + TokenGrantFailureKind::WorkloadIdentityRejected + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Status(status)) => { + match status.code() { + tonic::Code::DeadlineExceeded => TokenGrantFailureKind::Timeout, + tonic::Code::Unavailable => TokenGrantFailureKind::Unavailable, + tonic::Code::InvalidArgument => TokenGrantFailureKind::Configuration, + _ => TokenGrantFailureKind::Unclassified, + } + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Tonic(_)) => { + TokenGrantFailureKind::Unavailable + } + _ => TokenGrantFailureKind::Unclassified, + }; + token_grant_identity_status(kind, code) +} + +fn token_grant_dependency_status(status: Status) -> Status { + // Dependency status text is untrusted; only transport codes establish + // availability. No provider authorization meaning is inferred from prose. + let kind = match status.code() { + tonic::Code::DeadlineExceeded => TokenGrantFailureKind::Timeout, + tonic::Code::Unavailable => TokenGrantFailureKind::Unavailable, + _ => TokenGrantFailureKind::Unclassified, + }; + TokenGrantFailure::new(kind, TokenGrantStage::GatewayRpc).to_status_with_code(status.code()) +} + +async fn exchange_provider_subject_token_inner( + state: &Arc, + request: Request, + request_id: Option<&str>, +) -> Result, Status> { + let req = request.get_ref().clone(); + // This RPC requires the exact sandbox supervisor. Check without emitting + // caller-controlled identity fields through the general scope guard's log. + let auth_code = match request.extensions().get::() { + Some(Principal::Sandbox(principal)) if principal.sandbox_id == req.sandbox_id => None, + Some( + Principal::Sandbox(_) + | Principal::User(_) + | Principal::Operator(_) + | Principal::Peer(_), + ) => Some(tonic::Code::PermissionDenied), + Some(Principal::Anonymous) | None => Some(tonic::Code::Unauthenticated), + }; + if let Some(code) = auth_code { + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(code)); } - if req.supervisor_jwt_svid.trim().is_empty() { - return Err(Status::invalid_argument("supervisor_jwt_svid is required")); + drop(request); + + if req.provider.trim().is_empty() + || req.credential_key.trim().is_empty() + || req.supervisor_jwt_svid.trim().is_empty() + { + return Err(token_grant_configuration_status( + tonic::Code::InvalidArgument, + )); } let sandbox = state .store .get_message::(&req.sandbox_id) .await - .map_err(|e| Status::internal(format!("fetch sandbox failed: {e}")))? - .ok_or_else(|| Status::not_found("sandbox not found"))?; + .map_err(|_| { + TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(tonic::Code::Internal) + })? + .ok_or_else(|| token_grant_configuration_status(tonic::Code::NotFound))?; let workspace = sandbox.object_workspace().to_string(); let spec = sandbox .spec .as_ref() - .ok_or_else(|| Status::internal("sandbox has no spec"))?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::Internal))?; if !spec .providers .iter() .any(|provider| provider == &req.provider) { - return Err(Status::permission_denied( - "provider is not attached to this sandbox", + return Err(token_grant_configuration_status( + tonic::Code::PermissionDenied, )); } @@ -4099,44 +4230,49 @@ pub(super) async fn handle_exchange_provider_subject_token( .store .get_message_by_name::(&workspace, &req.provider) .await - .map_err(|e| Status::internal(format!("fetch provider failed: {e}")))? - .ok_or_else(|| Status::not_found("provider not found"))?; + .map_err(|_| { + TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(tonic::Code::Internal) + })? + .ok_or_else(|| token_grant_configuration_status(tonic::Code::NotFound))?; let catalog = state .provider_profile_sources .snapshot_catalog(state.store.as_ref(), &workspace) - .await?; + .await + .map_err(token_grant_dependency_status)?; let profile = get_provider_type_profile_for_scope( &catalog, &provider.r#type, &provider.profile_workspace, ) - .ok_or_else(|| Status::failed_precondition("provider profile not found"))?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let profile_proto = profile.to_proto(); let credential = profile_proto .credentials .iter() .find(|credential| credential.name == req.credential_key) - .ok_or_else(|| { - Status::failed_precondition("credential not declared by provider profile") - })?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let token_grant = credential .token_grant .as_ref() - .ok_or_else(|| Status::failed_precondition("credential does not declare token_grant"))?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let grant_type = ProviderCredentialTokenGrantType::try_from(token_grant.grant_type) .unwrap_or(ProviderCredentialTokenGrantType::ClientCredentials); if grant_type != ProviderCredentialTokenGrantType::TokenExchange { - return Err(Status::failed_precondition( - "credential token_grant is not token_exchange", + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, )); } let subject_token = token_grant .subject_token .as_ref() - .ok_or_else(|| Status::failed_precondition("token_exchange subject_token is missing"))?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; if subject_token.source != "provider_credential" { - return Err(Status::failed_precondition( - "unsupported subject_token source", + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, )); } if !profile_proto @@ -4144,10 +4280,17 @@ pub(super) async fn handle_exchange_provider_subject_token( .iter() .any(|credential| credential.name == subject_token.credential) { - return Err(Status::failed_precondition( - "subject token credential not declared by provider profile", + return Err(token_grant_configuration_status( + tonic::Code::FailedPrecondition, )); } + // Validate configuration before identity acquisition, cache lookup or any + // token exchange; invalid configuration must never trigger an outbound grant. + let (cache_ttl, subject_token_expires_at_ms) = validate_intermediate_grant_configuration( + token_grant, + &provider, + &subject_token.credential, + )?; let stored_subject_token = resolve_subject_token_credential(&state.credentials, &provider, &subject_token.credential) .await?; @@ -4189,42 +4332,22 @@ pub(super) async fn handle_exchange_provider_subject_token( let token_response = perform_intermediate_token_exchange( &token_grant.token_endpoint, - &gateway_jwt_svid, - &token_grant.client_assertion_type, - &stored_subject_token, - &subject_token.subject_token_type, - &supervisor_claims.sub, - &token_grant.requested_token_type, + &TokenExchangeParams { + client_assertion: &gateway_jwt_svid, + client_assertion_type: &token_grant.client_assertion_type, + subject_token: &stored_subject_token, + subject_token_type: &subject_token.subject_token_type, + audience: &supervisor_claims.sub, + scopes: &[], + requested_token_type: &token_grant.requested_token_type, + }, + request_id, ) - .await - .inspect_err(|status| { - warn!( - sandbox_id = %req.sandbox_id, - provider = %req.provider, - credential_key = %req.credential_key, - subject_credential = %subject_token.credential, - client_assertion_type = %effective_client_assertion_type(&token_grant.client_assertion_type), - gateway_svid_issuer = %gateway_claims.iss, - gateway_svid_subject = %gateway_claims.sub, - gateway_svid_audience = ?gateway_claims.aud, - supervisor_svid_issuer = %supervisor_claims.iss, - supervisor_svid_subject = %supervisor_claims.sub, - supervisor_svid_audience = ?supervisor_claims.aud, - status = ?status.code(), - error = %status.message(), - "intermediate provider token exchange failed" - ); - })?; + .await?; let cache_expires_at_ms = intermediate_token_cache_expires_at_ms( &token_response, - token_grant - .cache_ttl - .as_ref() - .map(openshell_core::time::duration_to_std) - .transpose() - .map_err(|error| Status::invalid_argument(error.to_string()))?, - provider_credential_expiration_ms(&provider, &subject_token.credential)? - .unwrap_or_default(), + cache_ttl, + subject_token_expires_at_ms.unwrap_or_default(), supervisor_claims.exp, ); if cache_expires_at_ms > crate::persistence::current_time_ms() { @@ -4241,24 +4364,43 @@ pub(super) async fn handle_exchange_provider_subject_token( })) } +fn validate_intermediate_grant_configuration( + grant: &openshell_core::proto::ProviderCredentialTokenGrant, + provider: &Provider, + subject_credential: &str, +) -> Result<(Option, Option), Status> { + oauth::validate_token_endpoint(&grant.token_endpoint).map_err(TokenGrantFailure::to_status)?; + let cache_ttl = grant + .cache_ttl + .as_ref() + .map(openshell_core::time::duration_to_std) + .transpose() + .map_err(|_| token_grant_configuration_status(tonic::Code::InvalidArgument))?; + let subject_expires_at_ms = provider_credential_expiration_ms(provider, subject_credential)?; + Ok((cache_ttl, subject_expires_at_ms)) +} + async fn resolve_subject_token_credential( credentials: &crate::credentials::CredentialRuntime, provider: &Provider, credential_key: &str, ) -> Result { + // Check the provider's expiry before asking a credential driver to resolve + // a handle; stale stored material cannot authorize another grant attempt. + ensure_subject_token_credential_not_expired(provider, credential_key)?; if let Some(value) = provider .credentials .get(credential_key) .filter(|value| !value.is_empty()) { - ensure_subject_token_credential_not_expired(provider, credential_key)?; return Ok(value.clone()); } if provider.credential_handles.contains_key(credential_key) { let resolved = credentials .resolve_provider_handles(provider, crate::persistence::current_time_ms()) - .await?; + .await + .map_err(token_grant_dependency_status)?; if let Some(value) = resolved .values .get(credential_key) @@ -4268,9 +4410,7 @@ async fn resolve_subject_token_credential( } } - Err(Status::failed_precondition( - "subject token credential is not configured", - )) + Err(token_grant_authorization_required_status()) } fn ensure_subject_token_credential_not_expired( @@ -4279,13 +4419,21 @@ fn ensure_subject_token_credential_not_expired( ) -> Result<(), Status> { let expires_at_ms = provider_credential_expiration_ms(provider, credential_key)?; if expires_at_ms.is_some_and(|value| value <= crate::persistence::current_time_ms()) { - return Err(Status::failed_precondition( - "subject token credential has expired", - )); + return Err(token_grant_authorization_required_status()); } Ok(()) } +/// The user's stored subject credential is absent or expired. Only the user +/// can restore it, by authorizing the provider again. +fn token_grant_authorization_required_status() -> Status { + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(tonic::Code::FailedPrecondition) +} + fn provider_credential_expiration_ms( provider: &Provider, credential_key: &str, @@ -4295,7 +4443,7 @@ fn provider_credential_expiration_ms( .get(credential_key) .map(openshell_core::time::timestamp_to_millis) .transpose() - .map_err(|error| Status::failed_precondition(error.to_string())) + .map_err(|_| token_grant_configuration_status(tonic::Code::FailedPrecondition)) } struct IntermediateTokenCacheKeyInput<'a> { @@ -4403,21 +4551,15 @@ async fn fetch_gateway_jwt_svid(audience: &str) -> Result { let socket_path = std::env::var(GATEWAY_SPIFFE_WORKLOAD_API_SOCKET) .ok() .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| { - Status::failed_precondition(format!( - "{GATEWAY_SPIFFE_WORKLOAD_API_SOCKET} is required for provider token exchange" - )) - })?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let endpoint = workload_api_endpoint(std::path::Path::new(&socket_path)); let client = spiffe::WorkloadApiClient::connect_to(&endpoint) .await - .map_err(|e| { - Status::failed_precondition(format!("SPIFFE Workload API unavailable: {e}")) - })?; + .map_err(|error| token_grant_workload_api_status(error, tonic::Code::FailedPrecondition))?; client .fetch_jwt_token([audience], None) .await - .map_err(|e| Status::failed_precondition(format!("failed to fetch gateway JWT-SVID: {e}"))) + .map_err(|error| token_grant_workload_api_status(error, tonic::Code::FailedPrecondition)) } fn validate_gateway_jwt_svid_claims( @@ -4425,17 +4567,22 @@ fn validate_gateway_jwt_svid_claims( expected_audience: &str, ) -> Result<(), Status> { if !claims.aud.contains(expected_audience) { - return Err(Status::failed_precondition( - "gateway SVID audience does not match token grant audience", + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::FailedPrecondition, )); } if spiffe_trust_domain(&claims.sub).is_none() { - return Err(Status::failed_precondition( - "gateway SVID subject is not a SPIFFE ID", + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::FailedPrecondition, )); } if claims.exp > 0 && claims.exp.saturating_mul(1000) <= crate::persistence::current_time_ms() { - return Err(Status::failed_precondition("gateway SVID has expired")); + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::FailedPrecondition, + )); } Ok(()) } @@ -4447,45 +4594,54 @@ async fn validate_supervisor_jwt_svid( ) -> Result { let unverified = parse_unverified_spiffe_claims(token)?; if unverified.iss != gateway_claims.iss { - return Err(Status::permission_denied( - "supervisor SVID issuer does not match gateway SVID issuer", + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, )); } if !unverified.aud.contains(expected_audience) { - return Err(Status::permission_denied( - "supervisor SVID audience does not match token grant audience", + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, )); } - let supervisor_trust_domain = spiffe_trust_domain(&unverified.sub) - .ok_or_else(|| Status::permission_denied("supervisor SVID subject is not a SPIFFE ID"))?; - let gateway_trust_domain = spiffe_trust_domain(&gateway_claims.sub) - .ok_or_else(|| Status::failed_precondition("gateway SVID subject is not a SPIFFE ID"))?; + let supervisor_trust_domain = spiffe_trust_domain(&unverified.sub).ok_or_else(|| { + token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, + ) + })?; + let gateway_trust_domain = spiffe_trust_domain(&gateway_claims.sub).ok_or_else(|| { + token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::FailedPrecondition, + ) + })?; if supervisor_trust_domain != gateway_trust_domain { - return Err(Status::permission_denied( - "supervisor SVID trust domain does not match gateway SVID trust domain", + return Err(token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, )); } let socket_path = std::env::var(GATEWAY_SPIFFE_WORKLOAD_API_SOCKET) .ok() .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| { - Status::failed_precondition(format!( - "{GATEWAY_SPIFFE_WORKLOAD_API_SOCKET} is required for supervisor JWT-SVID validation" - )) - })?; + .ok_or_else(|| token_grant_configuration_status(tonic::Code::FailedPrecondition))?; let endpoint = workload_api_endpoint(std::path::Path::new(&socket_path)); let client = spiffe::WorkloadApiClient::connect_to(&endpoint) .await - .map_err(|e| { - Status::failed_precondition(format!("SPIFFE Workload API unavailable: {e}")) - })?; + .map_err(|error| token_grant_workload_api_status(error, tonic::Code::FailedPrecondition))?; let bundles = client .fetch_jwt_bundles() .await - .map_err(|e| Status::internal(format!("SPIFFE JWT bundle fetch failed: {e}")))?; - spiffe::JwtSvid::parse_and_validate(token, &bundles, &[expected_audience]) - .map_err(|e| Status::permission_denied(format!("invalid supervisor JWT-SVID: {e}")))?; + .map_err(|error| token_grant_workload_api_status(error, tonic::Code::Internal))?; + spiffe::JwtSvid::parse_and_validate(token, &bundles, &[expected_audience]).map_err(|_| { + token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, + ) + })?; Ok(unverified) } @@ -4493,35 +4649,26 @@ fn parse_unverified_spiffe_claims(token: &str) -> Result Status { - Status::permission_denied(error.to_string()) +fn jwt_svid_parse_error_status(_error: JwtSvidParseError) -> Status { + token_grant_identity_status( + TokenGrantFailureKind::WorkloadIdentityRejected, + tonic::Code::PermissionDenied, + ) } async fn perform_intermediate_token_exchange( token_endpoint: &str, - gateway_jwt_svid: &str, - client_assertion_type: &str, - subject_token: &str, - subject_token_type: &str, - audience: &str, - requested_token_type: &str, + params: &TokenExchangeParams<'_>, + request_id: Option<&str>, ) -> Result { let client = token_exchange_http_client()?; - oauth::post_oauth_token_exchange( - client, - token_endpoint, - &TokenExchangeParams { - client_assertion: gateway_jwt_svid, - client_assertion_type, - subject_token, - subject_token_type, - audience, - scopes: &[], - requested_token_type, - }, - ) - .await - .map_err(|e| Status::failed_precondition(e.to_string())) + oauth::post_oauth_token_exchange(client, token_endpoint, params, request_id) + .await + .map_err(|failure| { + failure + .at_stage(TokenGrantStage::IntermediateExchange) + .to_status() + }) } fn effective_jwt_svid_audience(token_endpoint: &str, jwt_svid_audience: &str) -> String { @@ -5342,6 +5489,709 @@ mod tests { .expect("an absent expiration remains non-expiring"); } + mod token_grant_failures { + use super::*; + use crate::auth::principal::{SandboxIdentitySource, SandboxPrincipal}; + use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; + + const SECRET: &str = "GRANT_SECRET_CANARY_3319"; + + #[derive(Clone, Default)] + struct GrantTraceBuffer(Arc>>); + + impl std::io::Write for GrantTraceBuffer { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + fn capture_grant_warnings() -> ( + GrantTraceBuffer, + crate::otel_tracing::test_exporter::ScopedTracingTestGuard, + ) { + let output = GrantTraceBuffer::default(); + let writer = output.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .without_time() + .with_writer(move || writer.clone()) + .finish(); + let guard = crate::otel_tracing::test_exporter::install_scoped(subscriber); + // A captured sentinel proves the subscriber was active for WARN + // events, so an empty capture cannot hide a broken test fixture. + tracing::warn!("grant capture active"); + (output, guard) + } + + /// Credential values and handles never reach logs. Provider and + /// credential names are not secrets and can appear in diagnostics. + fn assert_no_credential_warnings(output: &GrantTraceBuffer) { + let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + assert!(logs.contains("grant capture active")); + assert!(!logs.contains(SECRET), "{logs}"); + } + + #[cfg(unix)] + mod expired_driver { + use super::*; + use openshell_core::proto::credentials::v1::{ + DeleteCredentialRequest, DeleteCredentialResponse, + GetCredentialDriverCapabilitiesRequest, GetCredentialDriverCapabilitiesResponse, + ListCredentialsRequest, ListCredentialsResponse, ResolveCredentialsRequest, + ResolveCredentialsResponse, ResolvedCredential, StoreCredentialRequest, + StoreCredentialResponse, + credential_driver_server::{CredentialDriver, CredentialDriverServer}, + }; + + #[derive(Debug)] + struct ExpiredDriver; + + #[tonic::async_trait] + impl CredentialDriver for ExpiredDriver { + async fn get_capabilities( + &self, + _request: Request, + ) -> Result, Status> + { + Ok(Response::new(GetCredentialDriverCapabilitiesResponse { + driver_name: "expired-fixture".to_string(), + supports_expires_at: true, + extension: Some(openshell_core::extension_protocol::extension_metadata( + openshell_core::extension_protocol::ExtensionFamily::Credentials, + "expired-fixture", + "0.0.0", + [], + )), + ..Default::default() + })) + } + + async fn store_credential( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("fixture only resolves credentials")) + } + + async fn delete_credential( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("fixture only resolves credentials")) + } + + async fn list_credentials( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented("fixture only resolves credentials")) + } + + async fn resolve_credentials( + &self, + request: Request, + ) -> Result, Status> { + Ok(Response::new(ResolveCredentialsResponse { + credentials: request + .into_inner() + .credentials + .into_iter() + .map(|request| ResolvedCredential { + request_id: request.request_id, + value: SECRET.to_string(), + expiration_time: Some(ts(0)), + }) + .collect(), + })) + } + } + + #[tokio::test] + async fn driver_returned_subject_expiry_is_generic_and_not_logged() { + let directory = tempfile::tempdir().unwrap(); + let socket = directory.path().join("driver.sock"); + let listener = tokio::net::UnixListener::bind(&socket).unwrap(); + let incoming = futures::stream::unfold(listener, |listener| async move { + Some((listener.accept().await.map(|(stream, _)| stream), listener)) + }); + let (shutdown, stop) = tokio::sync::oneshot::channel::<()>(); + let server = tokio::spawn(async move { + tonic::transport::Server::builder() + .add_service(CredentialDriverServer::new(ExpiredDriver)) + .serve_with_incoming_shutdown(incoming, async { + let _ = stop.await; + }) + .await + }); + let config = + openshell_core::Config::new(None).with_credential_drivers(["expired-fixture"]); + let socket_path = toml::Value::String(socket.to_string_lossy().into_owned()); + let file: crate::config_file::ConfigFile = toml::from_str(&format!( + "[openshell.credential_drivers.expired-fixture]\ntransport = \"uds\"\nsocket_path = {socket_path}\n" + )).unwrap(); + let credentials = + crate::credentials::CredentialRuntime::from_config_file(&config, Some(&file)) + .await + .unwrap(); + let mut provider = provider_with_values("grant-provider", "custom"); + provider.credentials.clear(); + provider.credential_handles.insert( + "subject".to_string(), + CredentialHandle { + driver: "expired-fixture".to_string(), + handle: SECRET.to_string(), + metadata: HashMap::new(), + }, + ); + assert!(provider.credential_expiration_times.is_empty()); + let (output, capture) = capture_grant_warnings(); + let status = resolve_subject_token_credential(&credentials, &provider, "subject") + .await + .unwrap_err(); + drop(capture); + let failure = assert_generic(&status, Code::FailedPrecondition); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::ProviderAuthorizationRequired + ); + assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); + assert_no_credential_warnings(&output); + drop(credentials); + shutdown.send(()).unwrap(); + server.await.unwrap().unwrap(); + } + } + + fn sandbox_request( + body: ExchangeProviderSubjectTokenRequest, + principal_id: &str, + ) -> Request { + let mut request = Request::new(body); + request + .extensions_mut() + .insert(Principal::Sandbox(SandboxPrincipal { + sandbox_id: principal_id.to_string(), + source: SandboxIdentitySource::BootstrapJwt { + issuer: SECRET.to_string(), + }, + trust_domain: None, + })); + request + } + + fn request_body() -> ExchangeProviderSubjectTokenRequest { + ExchangeProviderSubjectTokenRequest { + sandbox_id: "grant-sandbox".to_string(), + provider: SECRET.to_string(), + credential_key: SECRET.to_string(), + supervisor_jwt_svid: SECRET.to_string(), + } + } + + fn assert_generic(status: &Status, code: Code) -> TokenGrantFailure { + assert_eq!(status.code(), code); + assert_eq!(status.message(), "dynamic token grant failed"); + assert!(!format!("{status:?} {status}").contains(SECRET)); + assert!(!String::from_utf8_lossy(status.details()).contains(SECRET)); + assert!(status.metadata().is_empty()); + TokenGrantFailure::from_status(status) + } + + #[test] + fn residual_status_drops_message_details_metadata_and_source() { + let mut raw = + Status::with_details(Code::Internal, SECRET, SECRET.as_bytes().to_vec().into()); + raw.metadata_mut() + .insert("x-upstream-secret", SECRET.parse().unwrap()); + raw.set_source(Arc::new(std::io::Error::other(SECRET))); + assert!(std::error::Error::source(&raw).is_some()); + let status = sanitize_token_grant_status(raw); + let failure = assert_generic(&status, Code::Internal); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); + assert!(std::error::Error::source(&status).is_none()); + } + + #[test] + fn strict_typed_failure_survives_final_boundary() { + for kind in [ + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantFailureKind::Configuration, + TokenGrantFailureKind::Timeout, + TokenGrantFailureKind::Unavailable, + TokenGrantFailureKind::Unclassified, + ] { + let failure = TokenGrantFailure::new(kind, TokenGrantStage::IntermediateExchange) + .with_upstream_status(409); + let raw = failure.to_status(); + let code = raw.code(); + let status = sanitize_token_grant_status(raw); + assert_eq!(assert_generic(&status, code), failure); + } + } + + #[test] + fn only_canonical_request_ids_are_echoed() { + const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; + let mut metadata = tonic::metadata::MetadataMap::new(); + assert_eq!(canonical_request_id(&metadata), None); + metadata.insert("x-request-id", REQUEST_ID.parse().unwrap()); + assert_eq!(canonical_request_id(&metadata).as_deref(), Some(REQUEST_ID)); + for rejected in [ + "2F3E4D5C-6B7A-4891-A0B1-C2D3E4F5A6B7", + "2f3e4d5c6b7a4891a0b1c2d3e4f5a6b7", + "request code=provider_authorization_required", + ] { + metadata.insert("x-request-id", rejected.parse().unwrap()); + assert_eq!(canonical_request_id(&metadata), None, "{rejected}"); + } + } + + #[tokio::test] + async fn handler_failure_log_names_request_and_classification_without_secrets() { + const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; + let state = test_server_state_without_provider_profiles().await; + let mut request = sandbox_request(request_body(), "grant-sandbox"); + request + .metadata_mut() + .insert("x-request-id", REQUEST_ID.parse().unwrap()); + let (output, capture) = capture_grant_warnings(); + let error = handle_exchange_provider_subject_token(&state, request) + .await + .unwrap_err(); + drop(capture); + let failure = assert_generic(&error, Code::NotFound); + let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + assert!( + logs.contains(&format!( + "dynamic token grant failed request_id={REQUEST_ID} code={} stage={} upstream_http_status=none", + failure.code(), + failure.stage().code(), + )), + "{logs}" + ); + // The authenticated supervisor's own sandbox receives the line. + assert!(logs.contains("sandbox_id=grant-sandbox"), "{logs}"); + assert!(!logs.contains(SECRET), "{logs}"); + + // Another sandbox's supervisor cannot add a line to this sandbox's + // stream; its attempt is attributed to its own sandbox. + let (output, capture) = capture_grant_warnings(); + handle_exchange_provider_subject_token( + &state, + sandbox_request(request_body(), "other"), + ) + .await + .unwrap_err(); + drop(capture); + let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); + assert!( + logs.contains("dynamic token grant failed request_id=none"), + "{logs}" + ); + assert!(logs.contains("sandbox_id=other"), "{logs}"); + assert!(!logs.contains("sandbox_id=grant-sandbox"), "{logs}"); + } + + #[tokio::test] + async fn handler_auth_and_required_fields_are_generic() { + let state = test_server_state_without_provider_profiles().await; + let missing = + handle_exchange_provider_subject_token(&state, Request::new(request_body())) + .await + .unwrap_err(); + let failure = assert_generic(&missing, Code::Unauthenticated); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::WorkloadIdentityRejected + ); + let user = + handle_exchange_provider_subject_token(&state, authed_request(request_body())) + .await + .unwrap_err(); + assert_generic(&user, Code::PermissionDenied); + let wrong_sandbox = handle_exchange_provider_subject_token( + &state, + sandbox_request(request_body(), SECRET), + ) + .await + .unwrap_err(); + assert_generic(&wrong_sandbox, Code::PermissionDenied); + + for missing_field in 0..3 { + let mut body = request_body(); + match missing_field { + 0 => body.provider.clear(), + 1 => body.credential_key.clear(), + _ => body.supervisor_jwt_svid.clear(), + } + let error = handle_exchange_provider_subject_token( + &state, + sandbox_request(body, "grant-sandbox"), + ) + .await + .unwrap_err(); + let failure = assert_generic(&error, Code::InvalidArgument); + assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); + } + } + + #[tokio::test] + async fn handler_missing_sandbox_and_unattached_provider_are_generic() { + let state = test_server_state_without_provider_profiles().await; + let missing = handle_exchange_provider_subject_token( + &state, + sandbox_request(request_body(), "grant-sandbox"), + ) + .await + .unwrap_err(); + assert_generic(&missing, Code::NotFound); + state + .store + .put_message(&Sandbox { + metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { + id: "grant-sandbox".to_string(), + name: "grant-sandbox".to_string(), + workspace: "default".to_string(), + ..Default::default() + }), + spec: Some(SandboxSpec::default()), + ..Default::default() + }) + .await + .unwrap(); + let unattached = handle_exchange_provider_subject_token( + &state, + sandbox_request(request_body(), "grant-sandbox"), + ) + .await + .unwrap_err(); + assert_generic(&unattached, Code::PermissionDenied); + } + + #[tokio::test] + async fn credential_driver_diagnostics_and_expiration_are_generic() { + let config = openshell_core::Config::new(None).with_credential_drivers(["test-static"]); + let credentials = crate::credentials::CredentialRuntime::from_config(&config).unwrap(); + let mut provider = provider_with_values("grant-provider", "custom"); + provider.credentials.clear(); + provider.credential_handles.insert( + "subject".to_string(), + CredentialHandle { + driver: SECRET.to_string(), + handle: SECRET.to_string(), + metadata: HashMap::new(), + }, + ); + let raw = credentials + .resolve_provider_handles(&provider, 0) + .await + .unwrap_err(); + assert!(raw.message().contains(SECRET)); + let failure = resolve_subject_token_credential(&credentials, &provider, "subject") + .await + .unwrap_err(); + assert_generic(&failure, Code::InvalidArgument); + + provider + .credential_expiration_times + .insert("subject".to_string(), ts(0)); + let expired = resolve_subject_token_credential(&credentials, &provider, "subject") + .await + .unwrap_err(); + let failure = assert_generic(&expired, Code::FailedPrecondition); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::ProviderAuthorizationRequired + ); + assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); + } + + #[tokio::test] + async fn expired_sibling_handle_is_skipped_without_logging_subject() { + let config = openshell_core::Config::new(None).with_credential_drivers(["test-static"]); + let credentials = crate::credentials::CredentialRuntime::from_config(&config).unwrap(); + let mut provider = provider_with_values("grant-provider", "custom"); + provider.credentials.clear(); + provider.credential_handles = credentials + .store_provider_credentials( + provider.object_name(), + provider.object_workspace(), + "grant-fixture-provider", + &HashMap::from([ + ("subject".to_string(), SECRET.to_string()), + ("expired-sibling".to_string(), SECRET.to_string()), + ]), + &HashMap::new(), + ) + .await + .unwrap(); + provider + .credential_expiration_times + .insert("expired-sibling".to_string(), ts(0)); + let (output, capture) = capture_grant_warnings(); + let resolved = credentials + .resolve_provider_handles(&provider, crate::persistence::current_time_ms()) + .await + .unwrap(); + assert_eq!( + resolved.values.get("subject").map(String::as_str), + Some(SECRET) + ); + assert!(!resolved.values.contains_key("expired-sibling")); + let subject = resolve_subject_token_credential(&credentials, &provider, "subject") + .await + .unwrap(); + drop(capture); + assert_eq!(subject, SECRET); + assert_no_credential_warnings(&output); + } + + #[test] + fn invalid_grant_configuration_fails_before_acquisition() { + let mut grant = token_grant_credential("access_token").token_grant.unwrap(); + let mut provider = provider_with_values("grant-provider", "custom"); + grant.cache_ttl = Some(prost_types::Duration { + seconds: -1, + nanos: 0, + }); + let invalid_ttl = + validate_intermediate_grant_configuration(&grant, &provider, "subject") + .unwrap_err(); + assert_eq!( + assert_generic(&invalid_ttl, Code::InvalidArgument).kind(), + TokenGrantFailureKind::Configuration, + ); + grant.cache_ttl = None; + provider.credential_expiration_times.insert( + "subject".to_string(), + prost_types::Timestamp { + seconds: 0, + nanos: -1, + }, + ); + let invalid_expiration = + validate_intermediate_grant_configuration(&grant, &provider, "subject") + .unwrap_err(); + assert_generic(&invalid_expiration, Code::FailedPrecondition); + provider.credential_expiration_times.clear(); + grant.token_endpoint = format!("ftp://example.com/{SECRET}"); + let invalid_endpoint = + validate_intermediate_grant_configuration(&grant, &provider, "subject") + .unwrap_err(); + assert_eq!( + assert_generic(&invalid_endpoint, Code::FailedPrecondition).kind(), + TokenGrantFailureKind::Configuration, + ); + } + + #[test] + fn workload_api_status_preserves_recovery_and_discards_diagnostics() { + use openshell_core::proto::ProviderCredentialRefreshRecoveryAction as Recovery; + use spiffe::WorkloadApiError; + for code in [Code::FailedPrecondition, Code::Internal] { + for (error, kind, recovery) in [ + ( + WorkloadApiError::from(Status::permission_denied(SECRET)), + TokenGrantFailureKind::WorkloadIdentityRejected, + Recovery::FixConfiguration, + ), + ( + WorkloadApiError::NoIdentityIssued, + TokenGrantFailureKind::WorkloadIdentityRejected, + Recovery::FixConfiguration, + ), + ( + WorkloadApiError::from(Status::deadline_exceeded(SECRET)), + TokenGrantFailureKind::Timeout, + Recovery::Retry, + ), + ( + WorkloadApiError::from(Status::unavailable(SECRET)), + TokenGrantFailureKind::Unavailable, + Recovery::Retry, + ), + ( + WorkloadApiError::from(Status::invalid_argument(SECRET)), + TokenGrantFailureKind::Configuration, + Recovery::FixConfiguration, + ), + ( + WorkloadApiError::MissingEndpointSocket, + TokenGrantFailureKind::Configuration, + Recovery::FixConfiguration, + ), + ( + WorkloadApiError::HintNotFound(SECRET.to_string()), + TokenGrantFailureKind::Unclassified, + Recovery::Investigate, + ), + ] { + let status = token_grant_workload_api_status(error, code); + let failure = assert_generic(&status, code); + assert_eq!(failure.kind(), kind); + assert_eq!(failure.recovery(), recovery); + assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); + } + } + } + + #[tokio::test] + async fn malformed_svid_and_claim_denials_are_generic() { + let malformed = parse_unverified_spiffe_claims(SECRET).unwrap_err(); + let failure = assert_generic(&malformed, Code::PermissionDenied); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::WorkloadIdentityRejected + ); + assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); + let claims = SpiffeJwtClaims { + iss: SECRET.to_string(), + sub: SECRET.to_string(), + aud: openshell_core::spiffe::AudienceClaim::One(SECRET.to_string()), + exp: 0, + }; + let gateway = + validate_gateway_jwt_svid_claims(&claims, "expected-audience").unwrap_err(); + assert_generic(&gateway, Code::FailedPrecondition); + let supervisor = validate_supervisor_jwt_svid(SECRET, &claims, "expected-audience") + .await + .unwrap_err(); + assert_generic(&supervisor, Code::PermissionDenied); + } + + fn secret_exchange_params() -> TokenExchangeParams<'static> { + TokenExchangeParams { + client_assertion: SECRET, + client_assertion_type: "", + subject_token: SECRET, + subject_token_type: "", + audience: SECRET, + scopes: &[], + requested_token_type: "", + } + } + + /// Serve one token response and return the request headers it received. + async fn token_endpoint_capturing_headers( + body: String, + ) -> (String, tokio::task::JoinHandle) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/token", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let mut request = Vec::new(); + let mut chunk = [0_u8; 1024]; + while !request.windows(4).any(|window| window == b"\r\n\r\n") { + let count = stream.read(&mut chunk).await.unwrap(); + assert_ne!(count, 0, "request headers ended early"); + request.extend_from_slice(&chunk[..count]); + } + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + String::from_utf8_lossy(&request).into_owned() + }); + (endpoint, task) + } + + #[tokio::test] + async fn intermediate_exchange_forwards_the_supervisor_request_id() { + const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; + let body = format!("{{\"access_token\":\"{SECRET}\",\"token_type\":\"Bearer\"}}"); + let (endpoint, task) = token_endpoint_capturing_headers(body).await; + perform_intermediate_token_exchange( + &endpoint, + &secret_exchange_params(), + Some(REQUEST_ID), + ) + .await + .unwrap(); + let headers = task.await.unwrap().to_ascii_lowercase(); + assert!( + headers.contains(&format!("x-openshell-request-id: {REQUEST_ID}")), + "{headers}" + ); + } + + async fn token_endpoint_response( + status: u16, + body: String, + ) -> (String, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}/token", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + // This fixture returns an error as soon as the request starts. + let mut request = [0_u8; 1]; + stream.read_exact(&mut request).await.unwrap(); + let response = format!( + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + }); + (endpoint, task) + } + + #[tokio::test] + async fn intermediate_http_failure_keeps_stage_and_status_without_body() { + for (http_status, expected_kind, adopted_extension) in [ + (409, TokenGrantFailureKind::Unclassified, true), + (503, TokenGrantFailureKind::Unavailable, false), + (503, TokenGrantFailureKind::Unclassified, true), + ] { + let body = if adopted_extension { + format!( + "{{\"error\":\"provider_authorization_required\",\"openshell_error_contract\":\"openshell.token-grant.v1\",\"error_description\":\"{SECRET}\"}}" + ) + } else { + format!("{{\"error\":\"{SECRET}\",\"error_description\":\"{SECRET}\"}}") + }; + let (endpoint, task) = token_endpoint_response(http_status, body).await; + let result = + perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) + .await; + task.await.unwrap(); + let status = result.unwrap_err(); + let failure = assert_generic(&status, status.code()); + assert_eq!(failure.kind(), expected_kind); + assert_eq!(failure.stage(), TokenGrantStage::IntermediateExchange); + assert_eq!(failure.upstream_status(), Some(http_status)); + } + } + + #[tokio::test] + async fn intermediate_success_remains_cacheable() { + let body = format!( + "{{\"access_token\":\"{SECRET}\",\"token_type\":\"Bearer\",\"expires_in\":300}}" + ); + let (endpoint, task) = token_endpoint_response(200, body).await; + let token = + perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) + .await + .unwrap(); + task.await.unwrap(); + assert_eq!(token.access_token, SECRET); + let cache = IntermediateTokenCache::new(); + let expires_at_ms = intermediate_token_cache_expires_at_ms(&token, None, 0, 0); + cache.set("cache-key".to_string(), &token, expires_at_ms); + let cached = cache.get("cache-key").unwrap(); + assert_eq!(cached.access_token, SECRET); + assert_eq!(cached.token_type, "Bearer"); + assert!(cached.expires_in > 0 && cached.expires_in <= 270); + } + } + #[test] fn create_validation_accepts_broker_only_credential_by_logical_name() { let profile = ProviderTypeProfile::from_proto(&ProviderProfile { diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index 3e0bad0afe..fdfd57fe2a 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -148,12 +148,18 @@ mod tests { // Operator credential export adds four public messages and no durable types. // ListProvidersRequest field 4 adds a public-only label selector. Older // requests decode an empty selector; durable schemas remain unchanged. + // Endpoint failure causes add EndpointFailure and EndpointFailureDomain to + // both closures through EndpointStatus.last_failure. Stored statuses decode + // without a cause, and the next accepted report or reset replaces them. + // Sandbox-wide grant reporting adds TokenGrantFailureObservation to both + // closures and TokenGrantStatus to public reports. Older rows have no grant + // record; absent report extensions retain compatibility without a rewrite. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "b6734ba353b4ffe81240309f7971ef48f6473e14ce2196412beabe1984b0548d"; + "10fce7e572f1513551d6b7abec7dcc8fc2cb40c92fb394f8321232f5b9921fd4"; const DURABLE_SCHEMA_SHA256: &str = - "5281143e2825539af6148785c91f9df67f654eb74d215f57940af9314867a55f"; + "d4dc7abdb91b919db1f7be0d4652e30cbfda86f3b2ed9b59d5de1f76a2bf691d"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = - "761dea31a521b0650840fe2a823ad6e36a265ed323ba4506889781d630df0ee3"; + "ac1ffc4b5b81467de27270a4e33f6db1e31eeb2c52f79750dbaaf19515eadbb2"; // Encoded with the schema that omitted stable_placeholder. Keep these // bytes fixed so the current encoder cannot hide a compatibility change. const LEGACY_STATIC_PROFILE: &str = "0a240a116c65676163792d70726f66696c652d6964120d6c65676163792d737461746963280712510a0d6c65676163792d73746174696312184c6567616379207374617469632063726564656e7469616c2a260a076170695f6b65791a1153594e5448455449435f4150495f4b455920012a06626561726572"; @@ -621,9 +627,9 @@ mod tests { overlap_hash.as_str(), ), ( - (311, 27), - (93, 21), - (81, 21), + (314, 28), + (95, 22), + (83, 22), PUBLIC_RPC_SCHEMA_SHA256, DURABLE_SCHEMA_SHA256, PUBLIC_DURABLE_OVERLAP_SHA256 @@ -724,6 +730,7 @@ mod tests { assert_eq!(status.main_process_instance_id, "supervisor-id"); assert_eq!(status.exit_code, None); assert_eq!(status.restart_count, 0); + assert!(status.last_token_grant_failure.is_none()); assert!(status.next_restart_time.is_none()); assert!(status.main_process_started_time.is_none()); assert_eq!(status.conditions.len(), 1); diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index a1a844824e..77bcc89aba 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -684,9 +684,11 @@ impl SupervisorSessionRegistry { }; // Status without its live observation authority is unknown. Keep the // configured address so a caller can still identify each endpoint. + status.last_token_grant_failure = None; for endpoint in &mut status.endpoint_statuses { endpoint.last_result = openshell_core::proto::EndpointResult::NoObservedExchange as i32; endpoint.last_reported_time = None; + endpoint.last_failure = None; } } @@ -3263,22 +3265,42 @@ mod tests { host: "api.example.com".to_string(), ports: vec![443], path: "/mcp".to_string(), - last_result: EndpointResult::HttpResponseReceived as i32, + last_result: EndpointResult::CredentialUnavailable as i32, last_reported_time: Some("2026-09-05T01:01:00.000Z".parse().unwrap()), + // Evidence without live authority must drop its cause with the result. + last_failure: Some( + openshell_core::endpoint_failure::EndpointFailure::token_grant( + openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::Timeout, + openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + ), + "github", + "access_token", + "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", + ) + .to_proto(), + ), }; let ready = SandboxCondition { r#type: "Ready".to_string(), status: "True".to_string(), ..Default::default() }; + let grant_failure = Box::new(openshell_core::proto::TokenGrantFailureObservation { + destination_host: "api.example.com".to_string(), + destination_port: 443, + failure: endpoint.last_failure.clone(), + }); sandbox.status = Some(SandboxStatus { endpoint_statuses: vec![endpoint.clone()], + last_token_grant_failure: Some(grant_failure.clone()), conditions: vec![ready.clone()], ..Default::default() }); let unknown = EndpointStatus { last_result: EndpointResult::NoObservedExchange as i32, last_reported_time: None, + last_failure: None, ..endpoint.clone() }; @@ -3287,12 +3309,22 @@ mod tests { let projected_status = without_session.status.expect("status"); assert_eq!(projected_status.endpoint_statuses, vec![unknown.clone()]); assert_eq!(projected_status.conditions, vec![ready.clone()]); + assert!(projected_status.last_token_grant_failure.is_none()); + + let mut rest_only = sandbox.clone(); + rest_only.status.as_mut().unwrap().endpoint_statuses.clear(); + registry.project_endpoint_status(&mut rest_only, false); + assert!(rest_only.status.unwrap().last_token_grant_failure.is_none()); let mut remotely_owned = sandbox.clone(); registry.project_endpoint_status(&mut remotely_owned, true); let remote_status = remotely_owned.status.expect("status"); assert_eq!(remote_status.endpoint_statuses, vec![endpoint.clone()]); assert_eq!(remote_status.conditions, vec![ready.clone()]); + assert_eq!( + remote_status.last_token_grant_failure, + Some(grant_failure.clone()) + ); let (session_tx, _session_rx) = mpsc::channel(1); registry.register( @@ -3306,12 +3338,17 @@ mod tests { let uninitialized_status = before_initialization.status.expect("status"); assert_eq!(uninitialized_status.endpoint_statuses, vec![unknown]); assert_eq!(uninitialized_status.conditions, vec![ready.clone()]); + assert!(uninitialized_status.last_token_grant_failure.is_none()); assert!(registry.initialize_endpoint_status_authority("sandbox-1", "session-1")); registry.project_endpoint_status(&mut sandbox, false); let initialized_status = sandbox.status.expect("status"); assert_eq!(initialized_status.endpoint_statuses, vec![endpoint]); assert_eq!(initialized_status.conditions, vec![ready]); + assert_eq!( + initialized_status.last_token_grant_failure, + Some(grant_failure) + ); } fn sandbox_principal(sandbox_id: &str) -> Principal { diff --git a/crates/openshell-supervisor-network/Cargo.toml b/crates/openshell-supervisor-network/Cargo.toml index 239466358b..5d5a73502e 100644 --- a/crates/openshell-supervisor-network/Cargo.toml +++ b/crates/openshell-supervisor-network/Cargo.toml @@ -52,6 +52,7 @@ tempfile = "3" thiserror = { workspace = true } tokio = { workspace = true } tokio-rustls = { workspace = true } +tonic = { workspace = true } tower-mcp-types = { workspace = true } tracing = { workspace = true } uuid = { workspace = true } @@ -67,7 +68,6 @@ openshell-ocsf = { path = "../openshell-ocsf", features = ["test-support"] } openshell-otel = { path = "../openshell-otel" } openshell-supervisor-middleware-builtins = { path = "../openshell-supervisor-middleware-builtins" } opentelemetry_sdk = { workspace = true, features = ["testing"] } -tonic = { workspace = true } temp-env = "0.3" tokio = { workspace = true, features = ["test-util"] } tokio-tungstenite = { workspace = true } diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index 89c580ed8a..21abccd511 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -235,6 +235,40 @@ impl EndpointObserver { /// Report the first terminal exchange result, dropping later attempts. pub(crate) fn observe(&self, result: EndpointResult) { + self.observe_with_failure(result, None); + } + + /// Report a refused token grant as an unavailable credential with its cause. + /// + /// As with [`Self::observe_credential_failure`], an endpoint that does not + /// use provider credentials records nothing. + pub(crate) fn observe_token_grant_refusal( + &self, + refusal: &token_grant_injection::TokenGrantInjectionError, + request_id: &str, + ) { + if !self.uses_provider_credentials { + return; + } + let (provider, credential_key) = refusal.credential().unwrap_or_default(); + self.observe_with_failure( + EndpointResult::CredentialUnavailable, + Some( + openshell_core::endpoint_failure::EndpointFailure::token_grant( + refusal.failure(), + provider, + credential_key, + request_id, + ), + ), + ); + } + + fn observe_with_failure( + &self, + result: EndpointResult, + failure: Option, + ) { let Ok(mut pending) = self.observation.lock() else { // Status reporting cannot interrupt traffic if another observer panicked. return; @@ -253,7 +287,9 @@ impl EndpointObserver { // that activation gap, including a return to equal policy values. return; } - let _accepted = self.sender.try_observe(observation, result); + let _accepted = self + .sender + .try_observe_with_failure(observation, result, failure); } /// Report an endpoint mismatch or an unavailable provider credential. diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 31ac623352..5bbaf8e50b 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -145,12 +145,17 @@ fn credential_generation_guard( /// Resolve a grant from the provider installation and policy owners admitted /// for this request. Provider refresh is independent of tunnel lifetime: the /// connection's original dynamic map cannot authorize a later request. +/// +/// A refused grant returns a [`TokenGrantInjectionError`](crate::l7::token_grant_injection::TokenGrantInjectionError) +/// inside the report, so callers can tell it apart from other failures. pub(crate) async fn prepare_inspected_request( req: crate::l7::provider::L7Request, ctx: &L7EvalContext, engine: &TunnelPolicyEngine, config: &L7EndpointConfig, request_info: &L7RequestInfo, + request_id: &str, + observation_context: Option<&openshell_core::endpoint_status::EndpointObservationContext>, ) -> Result<(crate::l7::provider::L7Request, L7EvalContext)> { let mut scoped = ctx.clone(); let snapshot = ctx @@ -201,7 +206,14 @@ pub(crate) async fn prepare_inspected_request( } let owners = admitted_token_grant_owners(engine, &scoped, ¤t_info)?; crate::l7::token_grant_injection::inject_for_admitted_owners( - req, &scoped, snapshot, &owners, + req, + &scoped, + snapshot, + &owners, + request_id, + observation_context.filter(|context| { + context.matches_config(&config.policy_hash, Some(snapshot.revision)) + }), ) .await? } else { @@ -659,15 +671,42 @@ where websocket_middleware, observation_context, } = forwarding; - let (request, grant_ctx) = - match prepare_inspected_request(request, ctx, engine, config, request_info).await { - Ok(prepared) => prepared, - Err(error) => { + let (request, grant_ctx) = match prepare_inspected_request( + request, + ctx, + engine, + config, + request_info, + request_id, + observation_context, + ) + .await + { + Ok(prepared) => prepared, + Err(error) => { + if let Some(refusal) = + error.downcast_ref::() + { + // The refusal event already names the credential and request + // ID. The provider revision is not pinned here: a revision + // change resets the inventory, which drops this observation. + if let Some(observer) = EndpointObserver::begin_captured( + ctx.endpoint_observation_tx.as_ref(), + config, + observation_context, + None, + Some(engine.generation_guard()), + ) { + observer.observe_token_grant_refusal(refusal, request_id); + } + write_token_grant_failure_response(client, request_id).await?; + } else { warn!(error = %error, "Token grant failed before forwarding"); write_bad_gateway_response(client).await?; - return Ok(None); } - }; + return Ok(None); + } + }; *ctx = grant_ctx; let observer = EndpointObserver::begin_captured( ctx.endpoint_observation_tx.as_ref(), @@ -1998,6 +2037,10 @@ where return Ok(()); } + let observation_context = ctx + .endpoint_observation_tx + .as_ref() + .and_then(EndpointObservationSender::capture); // Parse one HTTP request from client let req = match provider.parse_request(client).await { Ok(Some(req)) => req, @@ -2228,7 +2271,7 @@ where request_id: &request_id, response_chain: &response_chain, websocket_middleware: middleware_session.is_some(), - observation_context: None, + observation_context: observation_context.as_ref(), }, ) .await; @@ -2688,6 +2731,10 @@ where return Ok(()); } + let observation_context = ctx + .endpoint_observation_tx + .as_ref() + .and_then(EndpointObservationSender::capture); // Validate the head, including body framing, before deciding whether // this endpoint can inspect the requested protocol. Upgrade refusal // must not wait for a body or depend on its inspection size limit. @@ -2900,7 +2947,7 @@ where request_id: &request_id, response_chain: &response_chain, websocket_middleware: false, - observation_context: None, + observation_context: observation_context.as_ref(), }, ) .await? @@ -3472,6 +3519,10 @@ where return Ok(()); } + let observation_context = ctx + .endpoint_observation_tx + .as_ref() + .and_then(EndpointObservationSender::capture); // Read next request from client. let req = match provider.parse_request(client).await { Ok(Some(req)) => req, @@ -3597,20 +3648,21 @@ where req }; - let req_with_auth = match crate::l7::token_grant_injection::inject_if_needed(req, ctx).await - { - Ok(req) => req, - Err(e) => { - warn!( - host = %ctx.host, - port = ctx.port, - error = %e, - "Token grant failed in passthrough relay" - ); - write_bad_gateway_response(client).await?; - return Ok(()); - } + let mut grant_ctx = ctx.clone(); + let Ok(req_with_auth) = crate::l7::token_grant_injection::inject_if_needed_captured( + req, + &mut grant_ctx, + &request_id, + observation_context.as_ref(), + ) + .await + else { + // The refusal event names the credential and request ID. + // The sandbox-level grant record does not require an MCP endpoint identity. + write_token_grant_failure_response(client, &request_id).await?; + return Ok(()); }; + let ctx = &grant_ctx; let scoped_ctx = scoped_context_for_request(ctx, &req_with_auth); let ctx = scoped_ctx.as_ref().unwrap_or(ctx); let resolver = ctx.secret_resolver.as_deref(); @@ -3676,17 +3728,72 @@ where Ok(()) } +/// Refuse a request whose dynamic token grant failed. +/// +/// The response is the generic empty `502` plus the exchange's request ID, so +/// the caller can quote it to an operator. It never carries the cause. +async fn write_token_grant_failure_response(client: &mut W, request_id: &str) -> Result<()> +where + W: AsyncWrite + Unpin, +{ + let response = format!( + "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n{}: {request_id}\r\nConnection: close\r\n\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ); + client + .write_all(response.as_bytes()) + .await + .into_diagnostic()?; + client.flush().await.into_diagnostic()?; + Ok(()) +} + #[cfg(test)] mod tests { use super::*; use crate::opa::{NetworkInput, OpaEngine}; use openshell_core::proto::{StaticCredentialBinding, StaticCredentialEndpointBinding}; use openshell_core::provider_credentials::ProviderCredentialState; + use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, + }; use std::collections::HashMap as TestHashMap; use std::fmt::Write as _; use std::path::PathBuf; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWriteExt}; + fn relay_grant_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ) + .with_upstream_status(409) + } + + /// A refused grant returns the generic empty 502 plus a request ID, and no + /// part of the response reveals the classification. + fn assert_grant_refusal_response(response: &str) { + assert!( + response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), + "{response}" + ); + assert!(response.contains("Content-Length: 0\r\n"), "{response}"); + let request_id = response + .lines() + .find_map(|line| { + line.strip_prefix(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + .and_then(|value| value.strip_prefix(": ")) + }) + .expect("refusal carries the request ID"); + assert!(uuid::Uuid::parse_str(request_id).is_ok(), "{request_id}"); + // The random ID can contain any digits; check everything else. + let without_id = response.replace(request_id, ""); + let failure = relay_grant_failure(); + for classification in [failure.code(), failure.stage().code(), "409"] { + assert!(!without_id.contains(classification), "{response}"); + } + } + #[tokio::test] async fn body_denial_returns_actionable_local_json() { let req = crate::l7::provider::L7Request { @@ -4311,7 +4418,7 @@ mod tests { } fn rest_token_grant_relay_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result<&str, TokenGrantFailure>, ) -> ( L7EndpointConfig, TunnelPolicyEngine, @@ -4383,7 +4490,7 @@ network_policies: } fn rest_token_exchange_relay_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result<&str, TokenGrantFailure>, ) -> ( L7EndpointConfig, TunnelPolicyEngine, @@ -4481,7 +4588,7 @@ network_policies: } fn passthrough_token_grant_relay_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result<&str, TokenGrantFailure>, ) -> ( PolicyGenerationGuard, L7EvalContext, @@ -4837,7 +4944,7 @@ network_policies: } fn passthrough_token_exchange_relay_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result<&str, TokenGrantFailure>, ) -> ( PolicyGenerationGuard, L7EvalContext, @@ -5170,7 +5277,7 @@ network_policies: #[tokio::test] async fn l7_rest_tls_relay_second_grant_failure_forwards_nothing() { - assert_multiple_grants_tls_relay(Err("issuer echoed identity-secret")).await; + assert_multiple_grants_tls_relay(Err(relay_grant_failure())).await; } async fn token_grant_tls_pair() -> ( @@ -5205,7 +5312,9 @@ network_policies: (client.unwrap(), server.unwrap()) } - async fn assert_multiple_grants_tls_relay(identity_result: std::result::Result<&str, &str>) { + async fn assert_multiple_grants_tls_relay( + identity_result: std::result::Result<&str, TokenGrantFailure>, + ) { let (config, tunnel_engine, mut ctx, fixture) = rest_token_grant_relay_context(Ok("service-token")); let service_key = "api.example.test\t8080\t/v1/**\tprovider:access_token"; @@ -5260,13 +5369,12 @@ network_policies: .unwrap() .unwrap(); let response = String::from_utf8_lossy(&response[..n]); - assert!(response.contains(if identity_result.is_ok() { - "204 No Content" + if identity_result.is_ok() { + assert!(response.contains("204 No Content")); } else { - "502 Bad Gateway" - })); + assert_grant_refusal_response(&response); + } assert!(!response.contains("service-token")); - assert!(!response.contains("identity-secret")); drop(app); tokio::time::timeout(std::time::Duration::from_secs(2), relay) .await @@ -5358,7 +5466,7 @@ network_policies: #[tokio::test] async fn l7_rest_relay_token_grant_failure_does_not_forward_request() { let (config, tunnel_engine, ctx, fixture) = - rest_token_grant_relay_context(Err("oauth unavailable")); + rest_token_grant_relay_context(Err(relay_grant_failure())); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -5392,7 +5500,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); + assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -5478,7 +5586,7 @@ network_policies: #[tokio::test] async fn l7_rest_relay_token_exchange_failure_does_not_forward_request() { let (config, tunnel_engine, ctx, fixture) = - rest_token_exchange_relay_context(Err("oauth unavailable")); + rest_token_exchange_relay_context(Err(relay_grant_failure())); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -5512,7 +5620,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); + assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -7900,7 +8008,7 @@ network_policies: #[tokio::test] async fn passthrough_relay_token_grant_failure_returns_bad_gateway_without_forwarding() { let (generation_guard, ctx, fixture) = - passthrough_token_grant_relay_context(Err("oauth unavailable")); + passthrough_token_grant_relay_context(Err(relay_grant_failure())); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -7934,7 +8042,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); + assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -8017,7 +8125,7 @@ network_policies: #[tokio::test] async fn passthrough_relay_token_exchange_failure_returns_bad_gateway_without_forwarding() { let (generation_guard, ctx, fixture) = - passthrough_token_exchange_relay_context(Err("oauth unavailable")); + passthrough_token_exchange_relay_context(Err(relay_grant_failure())); let (mut app, mut relay_client) = tokio::io::duplex(8192); let (mut relay_upstream, mut upstream) = tokio::io::duplex(8192); let relay = tokio::spawn(async move { @@ -8051,7 +8159,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); + assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 477d46ff6e..5fd18eb00d 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -53,7 +53,17 @@ impl TokenGrantResolver for ScriptedResolver { fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>> { + ) -> Pin< + Box< + dyn Future< + Output = std::result::Result< + String, + openshell_core::token_grant_failure::TokenGrantFailure, + >, + > + Send + + 'a, + >, + > { Box::pin(async move { self.calls .lock() @@ -70,7 +80,10 @@ impl TokenGrantResolver for ScriptedResolver { .expect("test releases the held grant"); } if self.fail_once.swap(false, Ordering::SeqCst) { - return Err(miette!("synthetic grant service unavailable")); + return Err(openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, + openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + )); } Ok(format!("inert-{}", request.audience)) }) @@ -154,6 +167,7 @@ struct Fixture { configs: Vec, state: ProviderCredentialState, resolver: Arc, + observation: Option, } impl Fixture { @@ -224,6 +238,7 @@ impl Fixture { credentials(grants), ), resolver: Arc::new(ScriptedResolver::default()), + observation: None, } } @@ -303,6 +318,7 @@ impl Fixture { // state. Refresh tests require requests to use the live ownership. dynamic_credentials: Some(Arc::new(RwLock::new(dynamic_credentials))), token_grant_resolver: Some(resolver), + endpoint_observation_tx: self.observation.clone(), ..Default::default() }; let (app, client) = tokio::io::duplex(16 * 1024); @@ -682,6 +698,176 @@ async fn native_protocol_grant_failures_close_before_upstream_bytes() { } } +/// Refuse one request through the real inspected relay with a failing +/// grant, and return the workload response with the status commands it caused. +async fn refused_grant( + protocol: &str, + provider_credentialed: bool, +) -> ( + String, + Vec, +) { + use openshell_core::endpoint_status::{ + EndpointConfigVersion, EndpointInventoryEntry, EndpointStatusCommand, + endpoint_status_channel, + }; + + let mut fixture = if protocol == "rest" { + let mut endpoint = route("native", "/native"); + endpoint.method = "POST"; + Fixture::new(&[endpoint], &[("/native", "native", "aud-native")], 1) + } else { + native_protocol_fixture(protocol, false) + }; + for config in &mut fixture.configs { + // The gateway derives these from policy; OPA fixtures carry none. + config.endpoint_id = "endpoint:v1:native".into(); + config.policy_hash = "policy".into(); + config.provider_credentialed = provider_credentialed; + } + let (sender, mut receiver) = endpoint_status_channel(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: 1, + }, + if protocol == "mcp" { + vec![EndpointInventoryEntry { + endpoint_id: "endpoint:v1:native".into(), + uses_provider_credentials: provider_credentialed, + }] + } else { + Vec::new() + }, + ) + .await + .expect("install endpoint inventory"); + assert!(matches!( + receiver.recv().await, + Some(EndpointStatusCommand::Reset { .. }) + )); + fixture.observation = Some(sender); + fixture.resolver.fail_once.store(true, Ordering::SeqCst); + + let mut connection = fixture.connect(); + let body = if protocol == "rest" { + Vec::new() + } else { + native_protocol_body(protocol, "echo") + }; + let response = connection.exchange_body("/native", &body).await; + let capture = connection.finish().await; + assert_eq!(capture.bytes_seen, 0, "failed grant forwards nothing"); + let mut commands = Vec::new(); + while let Ok(command) = receiver.try_recv() { + commands.push(command); + } + (response, commands) +} + +fn refusal_request_id(response: &str) -> String { + assert!(response.starts_with("HTTP/1.1 502"), "{response}"); + let request_id = response + .lines() + .find_map(|line| line.strip_prefix("x-openshell-request-id: ")) + .expect("refusal carries the request ID") + .to_string(); + let without_id = response.replace(&request_id, ""); + for classification in ["unavailable", "token_endpoint", "RETRY"] { + assert!(!without_id.contains(classification), "{response}"); + } + request_id +} + +#[tokio::test] +async fn mcp_grant_failure_reports_its_cause_to_endpoint_status_only() { + use openshell_core::endpoint_status::EndpointStatusCommand; + use openshell_core::proto::ProviderCredentialRefreshRecoveryAction; + + let (response, commands) = refused_grant("mcp", true).await; + // The workload sees the generic refusal and an opaque request ID. + let request_id = refusal_request_id(&response); + + // The trusted status path receives the classified cause for that request. + let [ + EndpointStatusCommand::ObserveTokenGrant { + result: Err(grant_failure), + .. + }, + EndpointStatusCommand::Observe { + endpoint_id, + result, + failure: Some(failure), + .. + }, + ] = commands.as_slice() + else { + panic!("one grant failure observation with a cause: {commands:?}"); + }; + assert_eq!(endpoint_id, "endpoint:v1:native"); + assert_eq!(*result, EndpointResult::CredentialUnavailable); + let failure = failure.to_proto(); + assert_eq!(failure.code, "unavailable"); + assert_eq!(failure.stage, "token_endpoint"); + assert_eq!( + failure.recovery_action, + ProviderCredentialRefreshRecoveryAction::Retry as i32 + ); + assert_eq!(failure.provider, "native"); + assert_eq!(failure.credential_key, "access_token"); + assert_eq!(failure.request_id, request_id); + assert_eq!(grant_failure.to_proto(), failure); + assert_eq!( + openshell_core::endpoint_failure::validate_endpoint_failure( + &failure, + openshell_core::proto::EndpointResult::CredentialUnavailable + ), + Ok(()) + ); +} + +#[tokio::test] +async fn grant_failure_on_uncredentialed_endpoint_reports_only_sandbox_cause() { + // The gateway rejects credential failures for endpoints that do not use + // provider credentials, so the supervisor must not report one. + let (response, commands) = refused_grant("mcp", false).await; + refusal_request_id(&response); + assert!( + matches!( + commands.as_slice(), + [openshell_core::endpoint_status::EndpointStatusCommand::ObserveTokenGrant { .. }] + ), + "{commands:?}" + ); +} + +#[tokio::test] +async fn non_mcp_grant_failures_report_without_an_endpoint_inventory() { + for protocol in ["rest", "graphql", "json-rpc"] { + let (response, commands) = refused_grant(protocol, true).await; + let request_id = refusal_request_id(&response); + let [ + openshell_core::endpoint_status::EndpointStatusCommand::ObserveTokenGrant { + destination_host, + destination_port, + result: Err(failure), + .. + }, + ] = commands.as_slice() + else { + panic!("{protocol}: bounded grant cause without MCP inventory: {commands:?}"); + }; + assert_eq!(destination_host, HOST); + assert_eq!(*destination_port, PORT); + let failure = failure.to_proto(); + assert_eq!(failure.request_id, request_id); + assert_eq!(failure.provider, "native"); + assert_eq!(failure.credential_key, "access_token"); + assert_eq!(failure.code, "unavailable"); + } +} + #[tokio::test] async fn grant_selection_uses_canonical_path_and_ignores_query_selector_text() { let fixture = Fixture::new(&[route("a", "/a/**")], &[("/a/**", "a", "aud-a")], 1); @@ -1317,7 +1503,15 @@ async fn transformed_jsonrpc_body_selects_final_operation_owner() { let transformed = body_request("/api/rpc", final_body); let (injected, _) = timeout( WAIT, - prepare_inspected_request(transformed, &ctx, &engine, &config, &info), + prepare_inspected_request( + transformed, + &ctx, + &engine, + &config, + &info, + "request-final-body", + None, + ), ) .await .expect("final-body grant finishes") diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 0afa9e4d88..85bfaf4638 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -7,14 +7,21 @@ mod multiple_grants; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::fmt; use std::future::Future; use std::pin::Pin; use std::sync::Arc; -use miette::{Result, miette}; use openshell_core::dynamic_credential_key::{credential_identity, endpoint_selector}; +use openshell_core::endpoint_failure::EndpointFailure; +use openshell_core::endpoint_status::EndpointObservationContext; +#[cfg(test)] +use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::proto::{ProviderCredentialTokenGrant, ProviderProfileCredential}; -use openshell_core::provider_credentials::ProviderCredentialSnapshot; +use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, +}; use openshell_ocsf::{ ActionId, ActivityId, DispositionId, Endpoint, HttpActivityBuilder, HttpRequest, SeverityId, StatusId, Url as OcsfUrl, ctx::ctx as ocsf_ctx, ocsf_emit, @@ -24,6 +31,7 @@ use tracing::warn; use crate::l7::provider::L7Request; use crate::l7::relay::L7EvalContext; +/// Borrowed grant configuration for one endpoint-bound provider credential. pub struct TokenGrantRequest<'a> { pub provider_key: &'a str, pub token_endpoint: &'a str, @@ -34,15 +42,20 @@ pub struct TokenGrantRequest<'a> { pub cache_ttl: Option, pub grant_type: i32, pub requested_token_type: &'a str, + /// Supervisor-minted ID of the HTTP exchange that needs the token. + pub request_id: &'a str, } +/// Resolves credentials without retaining arbitrary upstream diagnostics in failures. pub trait TokenGrantResolver: Send + Sync { + /// Obtain a token or a bounded failure whose classification survives injection. fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>>; + ) -> Pin> + Send + 'a>>; } +/// Resolves provider tokens using the supervisor's authenticated grant flow. #[derive(Default)] pub struct SpiffeTokenGrantResolver; @@ -50,7 +63,7 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>> { + ) -> Pin> + Send + 'a>> { Box::pin(async move { crate::token_grant::obtain_provider_token( crate::token_grant::ObtainProviderTokenRequest { @@ -63,6 +76,7 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { cache_ttl_override: request.cache_ttl, grant_type: request.grant_type, requested_token_type: request.requested_token_type, + request_id: request.request_id, }, ) .await @@ -70,36 +84,141 @@ impl TokenGrantResolver for SpiffeTokenGrantResolver { } } +/// Construct the production SPIFFE-backed resolver. pub fn default_resolver() -> Arc { Arc::new(SpiffeTokenGrantResolver) } +/// A refused request: the grant failure and the credential whose grant failed. +/// +/// The credential is the `provider:credential` name pair of the selected +/// binding, absent when the failure happened before a binding was selected. +/// Display and Debug print only the fixed grant failure message, so callers +/// can log this error without classifying it. +#[derive(Clone, PartialEq, Eq)] +pub struct TokenGrantInjectionError { + failure: TokenGrantFailure, + credential: Option, +} + +impl TokenGrantInjectionError { + fn new(failure: TokenGrantFailure) -> Self { + Self { + failure, + credential: None, + } + } + + fn for_binding(failure: TokenGrantFailure, binding_key: &str) -> Self { + Self { + failure, + credential: credential_identity(binding_key).map(str::to_string), + } + } + + /// Return the bounded classification for trusted status reporting. + pub fn failure(&self) -> TokenGrantFailure { + self.failure + } + + /// Return the provider and credential names of the failed grant, if known. + pub fn credential(&self) -> Option<(&str, &str)> { + self.credential.as_deref()?.split_once(':') + } +} + +impl fmt::Display for TokenGrantInjectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.failure, formatter) + } +} + +impl fmt::Debug for TokenGrantInjectionError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.failure, formatter) + } +} + +impl std::error::Error for TokenGrantInjectionError {} + +impl miette::Diagnostic for TokenGrantInjectionError {} + /// Resolves one endpoint-bound grant per protected header before rewriting a request. /// /// Each header independently uses its most-specific matching binding. Every selected /// grant must succeed; callers must not forward the request when this returns an error. -pub async fn inject_if_needed(req: L7Request, ctx: &L7EvalContext) -> Result { - let request_path = req.target.split('?').next().unwrap_or(req.target.as_str()); - let credentials = match ctx.dynamic_credentials.as_ref() { - Some(dynamic_credentials) => { - let credentials = dynamic_credentials - .read() - .map_err(|_| miette!("dynamic credential snapshot unavailable"))?; - select_token_grants(token_grant_candidates( - &credentials, - ctx, - request_path, - None, - )?)? - } - None => Vec::new(), +/// `request_id` is the supervisor-minted ID of this exchange; it is sent to the +/// token service and gateway and recorded on the refusal event. +#[cfg(test)] +async fn inject_if_needed( + req: L7Request, + ctx: &L7EvalContext, + request_id: &str, +) -> Result { + let observation_context = ctx + .endpoint_observation_tx + .as_ref() + .and_then(EndpointObservationSender::capture); + inject_if_needed_captured( + req, + &mut ctx.clone(), + request_id, + observation_context.as_ref(), + ) + .await +} + +/// Use authority captured before the proxy selected policy or awaited input. +pub async fn inject_if_needed_captured( + req: L7Request, + ctx: &mut L7EvalContext, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, +) -> Result { + let request_path = request_path(&req); + // A connection can sit idle across a refresh before its first request. + // Select and pin the live installation, not its accept-time credential map. + let snapshot = ctx + .provider_credentials + .as_ref() + .map(ProviderCredentialState::snapshot); + if let Some(snapshot) = &snapshot { + ctx.provider_credential_revision = Some(snapshot.revision); + ctx.provider_credential_installation_id = Some(snapshot.installation_id.clone()); + } + let selected = if let Some(snapshot) = snapshot { + token_grant_candidates(&snapshot.dynamic_credentials, ctx, request_path, None) + .and_then(select_token_grants) + .map(|selected| { + selected + .into_iter() + .map(|(key, credential)| (snapshot.scoped_key(&key), credential)) + .collect() + }) + } else { + ctx.dynamic_credentials.as_ref().map_or_else( + || Ok(Vec::new()), + |dynamic_credentials| { + dynamic_credentials + .read() + .map_err(|_| TokenGrantInjectionError::new(credential_lock_failure())) + .and_then(|credentials| { + select_token_grants(token_grant_candidates( + &credentials, + ctx, + request_path, + None, + )?) + }) + }, + ) }; - inject_selected(req, ctx, credentials).await + inject_or_report(req, ctx, selected, request_id, observation_context).await } /// Inject only grants owned by endpoints that admitted the inspected request. /// An empty owner set permits no acquisition, including audit-only forwarding. -/// L4-only forwarding continues to use `inject_if_needed` with its existing +/// L4-only forwarding uses `inject_if_needed_captured` with its existing /// endpoint-selector contract; it has no per-request L7 admission decision. /// /// Credentials come from the pinned `snapshot`, and each selected key is scoped @@ -110,101 +229,207 @@ pub(super) async fn inject_for_admitted_owners( ctx: &L7EvalContext, snapshot: &ProviderCredentialSnapshot, admitted_owners: &HashSet, -) -> Result { - let request_path = req.target.split('?').next().unwrap_or(req.target.as_str()); - if snapshot + request_id: &str, + observation_context: Option<&EndpointObservationContext>, +) -> Result { + let request_path = request_path(&req); + let unowned = snapshot .dynamic_credentials .iter() - .any(|(key, credential)| { + .find(|(key, credential)| { credential.token_grant.is_some() && credential.token_grant_owners.is_empty() && dynamic_credential_key_match_score(key, &ctx.host, ctx.port, request_path) .is_some() + }); + let selected = if let Some((key, _)) = unowned { + // A gateway that predates endpoint ownership metadata must be upgraded + // before the supervisor; without owners no grant is admitted. + Err(TokenGrantInjectionError::for_binding( + injection_configuration_failure(), + key, + )) + } else { + token_grant_candidates( + &snapshot.dynamic_credentials, + ctx, + request_path, + Some(admitted_owners), + ) + .and_then(select_token_grants) + .map(|selected| { + selected + .into_iter() + .map(|(key, cred)| (snapshot.scoped_key(&key), cred)) + .collect() }) + }; + inject_or_report(req, ctx, selected, request_id, observation_context).await +} + +/// Inject the selected grants, or record why the request was refused. +/// +/// Every refusal emits one generic event that carries the request ID. The +/// classification stays in the returned error for trusted status reporting. +async fn inject_or_report( + req: L7Request, + ctx: &L7EvalContext, + selected: Result, TokenGrantInjectionError>, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, +) -> Result { + if selected.as_ref().is_ok_and(Vec::is_empty) { + // Most requests need no grant; keep that path free of allocation. + return Ok(req); + } + let method = req.action.clone(); + let request_path = request_path(&req).to_string(); + let observation_context = observation_context + .filter(|context| context.matches_provider_revision(ctx.provider_credential_revision)); + let successful_credentials: Vec<_> = if observation_context.is_some() { + selected + .as_ref() + .ok() + .into_iter() + .flatten() + .filter_map(|(key, _)| { + credential_identity(key)? + .split_once(':') + .map(|(provider, credential)| (provider.to_string(), credential.to_string())) + }) + .collect() + } else { + Vec::new() + }; + let result = match selected { + Ok(credentials) => inject_selected(req, ctx, credentials, request_id).await, + Err(error) => Err(error), + }; + if let Err(error) = &result { + report_refused_grant(ctx, &method, &request_path, request_id, error); + } + if let Some((sender, context)) = ctx + .endpoint_observation_tx + .as_ref() + .zip(observation_context) { - return Err(miette!( - "dynamic token grant is missing endpoint ownership metadata; upgrade the gateway before the supervisor" - )); + match &result { + Err(error) => { + let (provider, credential) = error.credential().unwrap_or(("", "")); + let failure = + EndpointFailure::token_grant(error.failure(), provider, credential, request_id); + let _ = sender.try_observe_token_grant( + context.clone(), + &ctx.host, + ctx.port, + Err(failure), + ); + } + Ok(_) => { + for credential in successful_credentials { + let _ = sender.try_observe_token_grant( + context.clone(), + &ctx.host, + ctx.port, + Ok(credential), + ); + } + } + } } - let candidates = token_grant_candidates( - &snapshot.dynamic_credentials, - ctx, - request_path, - Some(admitted_owners), - )?; - let credentials = select_token_grants(candidates)? - .into_iter() - .map(|(key, cred)| (snapshot.scoped_key(&key), cred)) - .collect(); - inject_selected(req, ctx, credentials).await + result +} + +/// Record a refused grant. +/// +/// The OCSF event's shorthand form can reach the sandboxed agent through +/// `policy.local` denials, so it names only the credential, the destination, +/// and the opaque request ID. The classification goes only into structured +/// fields of the operator log line, which the shorthand log never renders. +fn report_refused_grant( + ctx: &L7EvalContext, + method: &str, + request_path: &str, + request_id: &str, + error: &TokenGrantInjectionError, +) { + let credential = error + .credential + .as_deref() + .map_or_else(|| "a dynamic credential".to_string(), ocsf_message_field); + let failure = error.failure; + warn!( + host = %ctx.host, + port = ctx.port, + provider = %credential, + request_id = %request_id, + code = failure.code(), + stage = failure.stage().code(), + upstream_http_status = %failure + .upstream_status() + .map_or_else(|| "none".to_string(), |status| status.to_string()), + "Token grant failed" + ); + ocsf_emit!( + HttpActivityBuilder::new(ocsf_ctx()) + .activity(ActivityId::Fail) + .action(ActionId::Denied) + .disposition(DispositionId::Blocked) + .severity(SeverityId::Medium) + .status(StatusId::Failure) + .http_request(HttpRequest::new( + method, + OcsfUrl::new("http", &ctx.host, request_path, ctx.port), + )) + .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) + .message(format!( + "Token grant failed for {credential} to {}:{}", + ctx.host, ctx.port + )) + .unmapped("request_id", request_id) + .build() + ); } /// Acquires every selected grant and rewrites the request headers. /// -/// An empty selection forwards the request unchanged. Any failed acquisition -/// returns an error before any header is written. +/// Any failed acquisition returns an error before any header is written. async fn inject_selected( req: L7Request, ctx: &L7EvalContext, credentials: Vec<(String, ProviderProfileCredential)>, -) -> Result { - let request_path = req.target.split('?').next().unwrap_or(req.target.as_str()); - if credentials.is_empty() { - return Ok(req); - } + request_id: &str, +) -> Result { + let request_path = request_path(&req).to_string(); let resolver = ctx .token_grant_resolver .as_ref() - .ok_or_else(|| miette!("token grant resolver unavailable"))?; + .ok_or_else(|| TokenGrantInjectionError::new(injection_configuration_failure()))?; + // A request that cannot carry a header never acquires a credential. + parse_header_block(&req.raw_header).map_err(TokenGrantInjectionError::new)?; // Release the credential snapshot lock before acquisition. Keep all token values // local until every selected grant and header has been validated, so a later // failure cannot expose a partially authenticated request to a relay caller. let mut headers = Vec::with_capacity(credentials.len()); for (provider_key, cred) in &credentials { + let refused = |failure| TokenGrantInjectionError::for_binding(failure, provider_key); let token_grant = cred .token_grant .as_ref() - .ok_or_else(|| miette!("selected credential has no token grant"))?; - let request = token_grant_request(provider_key, token_grant)?; - if let Ok(access_token) = resolver.obtain(request).await { - crate::token_grant::validate_access_token(&access_token)?; - headers.push(token_grant_header(cred, &access_token)?); - } else { - // An issuer may echo credentials in its error description. Only the - // binding identity is safe to include in diagnostics or relay errors. - let provider_key = ocsf_message_field(provider_key); - warn!( - host = %ctx.host, - port = ctx.port, - provider = %provider_key, - "Token grant failed" - ); - ocsf_emit!( - HttpActivityBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .http_request(HttpRequest::new( - &req.action, - OcsfUrl::new("http", &ctx.host, request_path, ctx.port), - )) - .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) - .message(format!( - "Token grant failed for {} to {}:{}", - provider_key, ctx.host, ctx.port - )) - .build() - ); - return Err(miette!("Token grant failed")); - } + .ok_or_else(|| refused(injection_configuration_failure()))?; + let request = + token_grant_request(provider_key, token_grant, request_id).map_err(refused)?; + let access_token = resolver.obtain(request).await.map_err(refused)?; + crate::token_grant::validate_access_token(&access_token) + .map_err(|failure| refused(failure.at_stage(TokenGrantStage::Injection)))?; + headers.push(token_grant_header(cred, &access_token).map_err(refused)?); } let mut raw_header = req.raw_header; for (name, value) in headers { - raw_header = inject_header(&raw_header, &name, &value)?; + raw_header = + inject_header(&raw_header, &name, &value).map_err(TokenGrantInjectionError::new)?; } for (provider_key, _) in credentials { ocsf_emit!( @@ -215,12 +440,14 @@ async fn inject_selected( .severity(SeverityId::Informational) .http_request(HttpRequest::new( &req.action, - OcsfUrl::new("http", &ctx.host, request_path, ctx.port), + OcsfUrl::new("http", &ctx.host, &request_path, ctx.port), )) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .message(format!( "Token grant successful for {} to {}:{}", - ocsf_message_field(&provider_key), + ocsf_message_field( + credential_identity(&provider_key).unwrap_or("a dynamic credential") + ), ctx.host, ctx.port )) @@ -230,23 +457,44 @@ async fn inject_selected( Ok(L7Request { raw_header, ..req }) } +fn request_path(req: &L7Request) -> &str { + req.target.split('?').next().unwrap_or(req.target.as_str()) +} + +fn credential_lock_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::Injection, + ) +} + +fn injection_configuration_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Injection, + ) +} + fn select_token_grants( mut candidates: Vec<(u32, String, ProviderProfileCredential)>, -) -> Result> { +) -> Result, TokenGrantInjectionError> { // Examine the strongest bindings first. The complete binding key stays with // its grant because the resolver uses it to separate cache entries and revisions. candidates.sort_unstable_by(|a, b| b.0.cmp(&a.0).then_with(|| b.1.cmp(&a.1))); let mut selected = BTreeMap::::new(); for (score, key, credential) in candidates { - let identity = - credential_identity(&key).ok_or_else(|| miette!("invalid dynamic credential key"))?; - let header = token_grant_header_name(&credential)?.to_ascii_lowercase(); + let refused = + || TokenGrantInjectionError::for_binding(injection_configuration_failure(), &key); + let identity = credential_identity(&key).ok_or_else(refused)?; + let header = token_grant_header_name(&credential) + .map_err(|_| refused())? + .to_ascii_lowercase(); if let Some((selected_score, selected_key, _)) = selected.get(&header) { // Equal-specificity selectors of the same credential can overlap. A // different credential cannot win a tie for the same protected header. let selected_identity = credential_identity(selected_key); if score == *selected_score && Some(identity) != selected_identity { - return Err(miette!("ambiguous dynamic token grants for one header")); + return Err(refused()); } continue; } @@ -268,7 +516,7 @@ fn token_grant_candidates( ctx: &L7EvalContext, request_path: &str, admitted_owners: Option<&HashSet>, -) -> Result> { +) -> Result, TokenGrantInjectionError> { credentials .iter() .filter_map(|(key, cred)| { @@ -283,7 +531,10 @@ fn token_grant_candidates( return None; } if endpoint_selector(key).is_none() { - return Some(Err(miette!("invalid dynamic credential endpoint key"))); + return Some(Err(TokenGrantInjectionError::for_binding( + injection_configuration_failure(), + key, + ))); } let score = dynamic_credential_key_match_score(key, &ctx.host, ctx.port, request_path)?; Some(Ok((score, key.clone(), cred.clone()))) @@ -301,13 +552,19 @@ fn ocsf_message_field(value: &str) -> String { fn token_grant_request<'a>( provider_key: &'a str, token_grant: &'a ProviderCredentialTokenGrant, -) -> Result> { + request_id: &'a str, +) -> Result, TokenGrantFailure> { let cache_ttl = token_grant .cache_ttl .as_ref() .map(openshell_core::time::duration_to_std) .transpose() - .map_err(|error| miette!("invalid token grant cache_ttl: {error}"))?; + .map_err(|_| { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ) + })?; Ok(TokenGrantRequest { provider_key, token_endpoint: &token_grant.token_endpoint, @@ -318,6 +575,7 @@ fn token_grant_request<'a>( cache_ttl, grant_type: token_grant.grant_type, requested_token_type: &token_grant.requested_token_type, + request_id, }) } @@ -376,8 +634,9 @@ fn inject_token_grant_header( raw_header: &[u8], credential: &ProviderProfileCredential, access_token: &str, -) -> Result> { - crate::token_grant::validate_access_token(access_token)?; +) -> Result, TokenGrantFailure> { + crate::token_grant::validate_access_token(access_token) + .map_err(|failure| failure.at_stage(TokenGrantStage::Injection))?; let (header_name, header_value) = token_grant_header(credential, access_token)?; inject_header(raw_header, &header_name, &header_value) } @@ -385,7 +644,7 @@ fn inject_token_grant_header( fn token_grant_header( credential: &ProviderProfileCredential, access_token: &str, -) -> Result<(String, String)> { +) -> Result<(String, String), TokenGrantFailure> { let header_name = token_grant_header_name(credential)?; let header_value = if credential.auth_style.trim().eq_ignore_ascii_case("header") { access_token.to_string() @@ -395,34 +654,26 @@ fn token_grant_header( Ok((header_name.to_string(), header_value)) } -fn token_grant_header_name(credential: &ProviderProfileCredential) -> Result<&str> { - match credential.auth_style.trim().to_ascii_lowercase().as_str() { +fn token_grant_header_name( + credential: &ProviderProfileCredential, +) -> Result<&str, TokenGrantFailure> { + let header_name = match credential.auth_style.trim().to_ascii_lowercase().as_str() { "" | "bearer" => { - let header_name = if credential.header_name.trim().is_empty() { + if credential.header_name.trim().is_empty() { "Authorization" } else { credential.header_name.trim() - }; - validate_header_name(header_name)?; - Ok(header_name) - } - "header" => { - let header_name = credential.header_name.trim(); - if header_name.is_empty() { - return Err(miette!( - "token grant auth_style header requires header_name" - )); } - validate_header_name(header_name)?; - Ok(header_name) } - other => Err(miette!( - "token grant auth_style '{other}' is not supported; use bearer or header" - )), - } + // Header auth names its own header; there is no default to fall back to. + "header" if !credential.header_name.trim().is_empty() => credential.header_name.trim(), + _ => return Err(injection_configuration_failure()), + }; + validate_header_name(header_name)?; + Ok(header_name) } -fn validate_header_name(header_name: &str) -> Result<()> { +fn validate_header_name(header_name: &str) -> Result<(), TokenGrantFailure> { let valid = !header_name.is_empty() && header_name.bytes().all(|byte| { byte.is_ascii_alphanumeric() @@ -444,31 +695,39 @@ fn validate_header_name(header_name: &str) -> Result<()> { | b'~' ) }); - if !valid { - return Err(miette!( - "token grant header_name is not a valid HTTP header name" - )); - } - match header_name.to_ascii_lowercase().as_str() { - "host" | "content-length" | "transfer-encoding" | "connection" => Err(miette!( - "token grant header_name may not override HTTP framing or connection headers" - )), - _ => Ok(()), + // A grant may not override HTTP framing or connection headers. + if !valid + || matches!( + header_name.to_ascii_lowercase().as_str(), + "host" | "content-length" | "transfer-encoding" | "connection" + ) + { + return Err(injection_configuration_failure()); } + Ok(()) } -fn inject_header(raw_header: &[u8], header_name: &str, header_value: &str) -> Result> { +fn parse_header_block(raw_header: &[u8]) -> Result<(&str, usize), TokenGrantFailure> { let header_end = raw_header .windows(4) .position(|w| w == b"\r\n\r\n") - .ok_or_else(|| miette!("HTTP headers missing final CRLF CRLF"))?; - + .ok_or_else(injection_configuration_failure)?; let header_block = std::str::from_utf8(&raw_header[..header_end]) - .map_err(|_| miette!("HTTP headers contain invalid UTF-8"))?; + .map_err(|_| injection_configuration_failure())?; + if header_block.split("\r\n").next().is_none_or(str::is_empty) { + return Err(injection_configuration_failure()); + } + Ok((header_block, header_end)) +} + +fn inject_header( + raw_header: &[u8], + header_name: &str, + header_value: &str, +) -> Result, TokenGrantFailure> { + let (header_block, header_end) = parse_header_block(raw_header)?; let mut lines = header_block.split("\r\n"); - let request_line = lines - .next() - .ok_or_else(|| miette!("HTTP headers missing request line"))?; + let request_line = lines.next().ok_or_else(injection_configuration_failure)?; let inserted_header = format!("{header_name}: {header_value}"); let mut new_raw_header = Vec::with_capacity(raw_header.len() + inserted_header.len() + 2); @@ -505,9 +764,11 @@ pub mod test_support { use std::collections::HashMap; use std::sync::{Arc, Mutex}; + type ScriptedResponse = Result; + struct FakeTokenGrantResolver { requests: Arc>>, - responses: Arc>>>, + responses: Arc>>, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -521,13 +782,14 @@ pub mod test_support { cache_ttl: Option, grant_type: i32, requested_token_type: String, + request_id: String, } pub struct TokenGrantTestFixture { dynamic_credentials: Arc>>, resolver: Arc, requests: Arc>>, - responses: Arc>>>, + responses: Arc>>, /// Revision-scoped key -> installed key, so assertions can name the key a /// test installed even when injection requests its scoped form. scoped_keys: Mutex>, @@ -550,27 +812,27 @@ pub mod test_support { Self::new_with_grant(key, Ok(token), token_exchange_grant()) } - pub fn failure(key: &str, error: &str) -> Self { - Self::new(key, Err(error)) + pub fn failure(key: &str, failure: TokenGrantFailure) -> Self { + Self::new(key, Err(failure)) } - pub fn failure_token_exchange(key: &str, error: &str) -> Self { - Self::new_with_grant(key, Err(error), token_exchange_grant()) + pub fn failure_token_exchange(key: &str, failure: TokenGrantFailure) -> Self { + Self::new_with_grant(key, Err(failure), token_exchange_grant()) } - fn new(key: &str, response: std::result::Result<&str, &str>) -> Self { + fn new(key: &str, response: Result<&str, TokenGrantFailure>) -> Self { Self::new_with_grant(key, response, token_grant()) } fn new_with_grant( key: &str, - response: std::result::Result<&str, &str>, + response: Result<&str, TokenGrantFailure>, token_grant: ProviderCredentialTokenGrant, ) -> Self { let requests = Arc::new(Mutex::new(Vec::new())); let responses = Arc::new(Mutex::new(HashMap::from([( key.to_string(), - response.map(str::to_string).map_err(str::to_string), + response.map(str::to_string), )]))); let resolver = Arc::new(FakeTokenGrantResolver { requests: requests.clone(), @@ -603,16 +865,26 @@ pub mod test_support { &self, key: &str, credential: ProviderProfileCredential, - response: std::result::Result<&str, &str>, + response: Result<&str, TokenGrantFailure>, ) { self.dynamic_credentials .write() .unwrap() .insert(key.to_string(), credential); - self.responses.lock().unwrap().insert( - key.to_string(), - response.map(str::to_string).map_err(str::to_string), - ); + self.responses + .lock() + .unwrap() + .insert(key.to_string(), response.map(str::to_string)); + } + + /// Return the request IDs the resolver received, in acquisition order. + pub fn requested_ids(&self) -> Vec { + self.requests + .lock() + .unwrap() + .iter() + .map(|request| request.request_id.clone()) + .collect() } pub fn assert_requested_keys(&self, expected: &[&str]) { @@ -667,17 +939,14 @@ pub mod test_support { /// Installs this fixture's credentials as a pinned provider-credential /// snapshot. Admission-gated injection requests grants by revision-scoped /// key, so each scripted response is also registered under that key. - pub fn provider_credentials( - &self, - ) -> openshell_core::provider_credentials::ProviderCredentialState { + pub fn provider_credentials(&self) -> ProviderCredentialState { let credentials = self.dynamic_credentials.read().unwrap().clone(); - let state = - openshell_core::provider_credentials::ProviderCredentialState::from_environment( - 1, - HashMap::new(), - HashMap::new(), - credentials.clone(), - ); + let state = ProviderCredentialState::from_environment( + 1, + HashMap::new(), + HashMap::new(), + credentials.clone(), + ); let snapshot = state.snapshot(); let mut responses = self.responses.lock().unwrap(); for key in credentials.keys() { @@ -796,7 +1065,7 @@ pub mod test_support { fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>> { + ) -> Pin> + Send + 'a>> { let owned = OwnedTokenGrantRequest { provider_key: request.provider_key.to_string(), token_endpoint: request.token_endpoint.to_string(), @@ -807,6 +1076,7 @@ pub mod test_support { cache_ttl: request.cache_ttl, grant_type: request.grant_type, requested_token_type: request.requested_token_type.to_string(), + request_id: request.request_id.to_string(), }; Box::pin(async move { self.requests @@ -822,7 +1092,6 @@ pub mod test_support { .get(request.provider_key) .expect("fake grant response configured") .clone() - .map_err(|err| miette!("{err}")) }) } } @@ -834,6 +1103,8 @@ mod tests { use crate::l7::provider::{BodyLength, L7Request}; use crate::l7::token_grant_injection::test_support::TokenGrantTestFixture; + const REQUEST_ID: &str = "6a1f0e2d-3c4b-4a59-8e7f-0123456789ab"; + fn credential(auth_style: &str, header_name: &str) -> ProviderProfileCredential { ProviderProfileCredential { auth_style: auth_style.to_string(), @@ -1011,10 +1282,7 @@ mod tests { for header_name in ["Host", "Content-Length", "Transfer-Encoding", "Connection"] { let err = token_grant_header(&credential("header", header_name), "grant-token") .expect_err("framing header override should be rejected"); - assert_eq!( - err.to_string(), - "token grant header_name may not override HTTP framing or connection headers" - ); + assert_eq!(err, injection_configuration_failure()); } } @@ -1056,10 +1324,9 @@ mod tests { ) .expect_err("malformed token must not be injected"); - assert_eq!( - err.to_string(), - "token grant returned a malformed access token" - ); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.stage(), TokenGrantStage::Injection); + assert_eq!(err.to_string(), "dynamic token grant failed"); } #[tokio::test] @@ -1089,7 +1356,7 @@ mod tests { body_length: BodyLength::None, }; - let rewritten = inject_if_needed(req, &ctx) + let rewritten = inject_if_needed(req, &ctx, REQUEST_ID) .await .expect("fake token grant should inject"); let rewritten = @@ -1097,6 +1364,7 @@ mod tests { assert!(rewritten.contains("Authorization: Bearer grant-token\r\n")); fixture.assert_one_request("api.example.com\t443\t/v1/**\tprovider:access_token"); + assert_eq!(fixture.requested_ids(), [REQUEST_ID]); } #[tokio::test] @@ -1127,7 +1395,7 @@ mod tests { body_length: BodyLength::None, }; - let rewritten = inject_if_needed(req, &ctx) + let rewritten = inject_if_needed(req, &ctx, REQUEST_ID) .await .expect("fake token exchange grant should inject"); let rewritten = @@ -1166,14 +1434,439 @@ mod tests { body_length: BodyLength::None, }; - let err = inject_if_needed(req, &ctx) + let err = inject_if_needed(req, &ctx, REQUEST_ID) .await .expect_err("malformed resolver token should fail closed"); + assert_eq!(err.failure().stage(), TokenGrantStage::Injection); + assert_eq!(err.credential(), Some(("provider", "access_token"))); + assert_eq!(err.to_string(), "dynamic token grant failed"); + fixture.assert_one_request("api.example.com\t443\t/v1/**\tprovider:access_token"); + } + + #[tokio::test] + async fn refused_grant_keeps_resolver_classification_and_failed_credential() { + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ); + let fixture = TokenGrantTestFixture::failure_token_exchange( + "api.example.com\t443\t/v1/**\tprovider:access_token", + failure, + ); + let ctx = L7EvalContext { + host: "api.example.com".into(), + port: 443, + dynamic_credentials: Some(fixture.dynamic_credentials()), + token_grant_resolver: Some(fixture.resolver()), + ..Default::default() + }; + let req = L7Request { + action: "GET".to_string(), + target: "/v1/projects".to_string(), + query_params: HashMap::new(), + raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + + let err = inject_if_needed(req, &ctx, REQUEST_ID) + .await + .expect_err("failed grant refuses the request"); + + assert_eq!(err.failure(), failure); + assert_eq!(err.credential(), Some(("provider", "access_token"))); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{err:?}"), "dynamic token grant failed"); + assert_eq!(fixture.requested_ids(), [REQUEST_ID]); + } + + fn grant_context(fixture: &TokenGrantTestFixture) -> L7EvalContext { + L7EvalContext { + host: "api.example.com".into(), + port: 443, + dynamic_credentials: Some(fixture.dynamic_credentials()), + token_grant_resolver: Some(fixture.resolver()), + ..Default::default() + } + } + + fn grant_request(raw_header: &[u8]) -> L7Request { + L7Request { + action: "GET".to_string(), + target: "/v1/projects".to_string(), + query_params: HashMap::new(), + raw_header: raw_header.to_vec(), + body_length: BodyLength::None, + } + } + + #[tokio::test] + async fn malformed_request_framing_fails_before_acquisition() { + for raw_header in [ + b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com".as_slice(), + b"GET /v1/projects HTTP/1.1\r\nX-Secret: \xff\r\n\r\n".as_slice(), + b"\r\n\r\n".as_slice(), + ] { + let fixture = TokenGrantTestFixture::success( + "api.example.com\t443\t/v1/**\tprovider:access_token", + "grant-token", + ); + let err = inject_if_needed( + grant_request(raw_header), + &grant_context(&fixture), + REQUEST_ID, + ) + .await + .expect_err("invalid request framing refuses the request"); + assert_eq!(err.failure(), injection_configuration_failure()); + fixture.assert_no_requests(); + } + } + + #[tokio::test] + async fn poisoned_credential_snapshot_fails_closed() { + use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + let fixture = TokenGrantTestFixture::success( + "api.example.com\t443\t/v1/**\tprovider:access_token", + "grant-token", + ); + let credentials = fixture.dynamic_credentials(); + assert!( + std::thread::spawn(move || { + let _guard = credentials.write().expect("lock fixture credentials"); + panic!("poison fixture credentials"); + }) + .join() + .is_err() + ); + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: 1, + }, + Vec::new(), + ) + .await + .unwrap(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let mut ctx = grant_context(&fixture); + ctx.endpoint_observation_tx = Some(sender); + let err = inject_if_needed( + grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), + &ctx, + REQUEST_ID, + ) + .await + .expect_err("a poisoned snapshot cannot prove no grant applies"); + assert_eq!(err.failure(), credential_lock_failure()); + assert_eq!(err.credential(), None); + fixture.assert_no_requests(); + assert!(tracker.apply(receiver.recv().await.unwrap())); + let observation = tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .unwrap(); + let failure = observation.failure.unwrap(); + assert_eq!(failure.code, "unavailable"); + assert_eq!(failure.request_id, REQUEST_ID); + assert!(failure.provider.is_empty()); + assert!(failure.credential_key.is_empty()); + } + + #[tokio::test] + async fn missing_resolver_refuses_without_naming_a_credential() { + let fixture = TokenGrantTestFixture::success( + "api.example.com\t443\t/v1/**\tprovider:access_token", + "grant-token", + ); + let ctx = L7EvalContext { + host: "api.example.com".into(), + port: 443, + dynamic_credentials: Some(fixture.dynamic_credentials()), + token_grant_resolver: None, + ..Default::default() + }; + let req = L7Request { + action: "GET".to_string(), + target: "/v1/projects".to_string(), + query_params: HashMap::new(), + raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), + body_length: BodyLength::None, + }; + + let err = inject_if_needed(req, &ctx, REQUEST_ID) + .await + .expect_err("a grant without a resolver refuses the request"); + + assert_eq!(err.failure(), injection_configuration_failure()); + assert_eq!(err.credential(), None); + fixture.assert_no_requests(); + } + + fn refreshed_grant_context( + accepted_key: &str, + current: &TokenGrantTestFixture, + ) -> L7EvalContext { + let accepted = TokenGrantTestFixture::success(accepted_key, "stale-token"); + let accepted_snapshot = ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + accepted.dynamic_credentials().read().unwrap().clone(), + ) + .snapshot(); + let live_state = current.provider_credentials(); + // A socket retains scoped credentials from accept time. Register that + // old key too, so selecting it succeeds instead of failing in the fake. + current.add_credential( + &accepted_snapshot.scoped_key(accepted_key), + accepted_snapshot.dynamic_credentials[accepted_key].clone(), + Ok("stale-token"), + ); + let mut ctx = grant_context(current); + ctx.dynamic_credentials = Some(Arc::new(std::sync::RwLock::new( + accepted_snapshot + .dynamic_credentials + .iter() + .map(|(key, credential)| (accepted_snapshot.scoped_key(key), credential.clone())) + .collect(), + ))); + ctx.provider_credentials = Some(live_state); + ctx + } + + #[tokio::test] + async fn fallback_reports_live_credential_after_accept_time_refresh() { + use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, + ); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let mut ctx = refreshed_grant_context( + "api.example.com\t443\t/v1/**\told-provider:access_token", + &fixture, + ); + let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: snapshot.revision, + }, + Vec::new(), + ) + .await + .unwrap(); + assert!(tracker.apply(receiver.try_recv().unwrap())); + let authority = sender.capture().unwrap(); + ctx.endpoint_observation_tx = Some(sender); + + let error = inject_if_needed_captured( + grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), + &mut ctx, + REQUEST_ID, + Some(&authority), + ) + .await + .expect_err("the live credential refuses, even though the accepted one succeeds"); + assert_eq!(error.failure(), failure); assert_eq!( - err.to_string(), - "token grant returned a malformed access token" + error.credential(), + Some(("current-provider", "access_token")) + ); + assert_eq!(ctx.provider_credential_revision, Some(snapshot.revision)); + assert_eq!( + ctx.provider_credential_installation_id.as_deref(), + Some(snapshot.installation_id.as_str()) + ); + fixture.assert_requested_keys(&[CURRENT_KEY]); + assert!(tracker.apply(receiver.try_recv().expect("live refusal is reported"))); + let observation = tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .unwrap(); + assert_eq!(observation.destination_host, "api.example.com"); + assert_eq!(observation.destination_port, 443); + let cause = observation.failure.unwrap(); + assert_eq!(cause.provider, "current-provider"); + assert_eq!(cause.credential_key, "access_token"); + assert_eq!(cause.request_id, REQUEST_ID); + } + + #[tokio::test] + async fn fallback_cannot_clear_live_failure_with_stale_credential_success() { + use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + const CURRENT_KEY: &str = "api.example.com\t443\t/v1/projects\tprovider:access_token"; + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, + ); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let mut ctx = refreshed_grant_context( + "api.example.com\t443\t/v1/**\tprovider:access_token", + &fixture, + ); + let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: snapshot.revision, + }, + Vec::new(), + ) + .await + .unwrap(); + assert!(tracker.apply(receiver.try_recv().unwrap())); + let authority = sender.capture().unwrap(); + ctx.endpoint_observation_tx = Some(sender); + + // Establish B's failure through the production entry point with B's map. + let mut fresh = ctx.clone(); + fresh.dynamic_credentials = Some(Arc::new(std::sync::RwLock::new( + snapshot.dynamic_credentials.clone(), + ))); + let raw_header = b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"; + inject_if_needed_captured( + grant_request(raw_header), + &mut fresh, + REQUEST_ID, + Some(&authority), + ) + .await + .expect_err("current grant first records its failure"); + assert!(tracker.apply(receiver.try_recv().unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_some() + ); + + // A has the same credential identity, so its old success would clear B + // if fallback injection used the socket's copied map after the refresh. + let error = inject_if_needed_captured( + grant_request(raw_header), + &mut ctx, + REQUEST_ID, + Some(&authority), + ) + .await + .expect_err("the stale successful credential cannot replace the live refusal"); + assert_eq!(error.failure(), failure); + assert!(tracker.apply(receiver.try_recv().unwrap())); + let observation = tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .unwrap(); + assert_eq!(observation.failure.unwrap().provider, "provider"); + + fixture.add_credential( + &snapshot.scoped_key(CURRENT_KEY), + snapshot.dynamic_credentials[CURRENT_KEY].clone(), + Ok("current-token"), + ); + let injected = inject_if_needed_captured( + grant_request(raw_header), + &mut ctx, + REQUEST_ID, + Some(&authority), + ) + .await + .expect("the live credential now succeeds"); + assert!( + String::from_utf8(injected.raw_header) + .unwrap() + .contains("Authorization: Bearer current-token\r\n") + ); + fixture.assert_requested_keys(&[CURRENT_KEY, CURRENT_KEY, CURRENT_KEY]); + assert!(tracker.apply(receiver.try_recv().unwrap())); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() + ); + } + + #[tokio::test] + async fn fallback_rejects_telemetry_authority_captured_before_provider_refresh() { + use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, + ); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let mut ctx = refreshed_grant_context( + "api.example.com\t443\t/v1/**\told-provider:access_token", + &fixture, + ); + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: 0, + }, + Vec::new(), + ) + .await + .unwrap(); + assert!(tracker.apply(receiver.try_recv().unwrap())); + let accepted_authority = sender.capture().unwrap(); + sender + .reset( + EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: 1, + }, + Vec::new(), + ) + .await + .unwrap(); + assert!(tracker.apply(receiver.try_recv().unwrap())); + ctx.endpoint_observation_tx = Some(sender); + + let error = inject_if_needed_captured( + grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), + &mut ctx, + REQUEST_ID, + Some(&accepted_authority), + ) + .await + .expect_err("the live grant still fails closed under old reporting authority"); + assert_eq!(error.failure(), failure); + assert_eq!( + error.credential(), + Some(("current-provider", "access_token")) + ); + fixture.assert_requested_keys(&[CURRENT_KEY]); + assert!(matches!( + receiver.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Empty) + )); + assert!( + tracker + .snapshot() + .unwrap() + .last_token_grant_failure + .is_none() ); - fixture.assert_one_request("api.example.com\t443\t/v1/**\tprovider:access_token"); } } diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs index 77b44abaaa..4dc0346f47 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs @@ -11,6 +11,7 @@ use tracing_subscriber::layer::SubscriberExt; const SERVICE: &str = "api.example.com\t443\t/v1/**\trev:7\tprovider:service"; const IDENTITY: &str = "api.example.com\t443\t/v1/**\trev:7\tprovider:identity"; +const REQUEST_ID: &str = "0d9c8b7a-6f5e-4d3c-8b2a-1f0e9d8c7b6a"; fn request() -> L7Request { L7Request { @@ -23,7 +24,7 @@ fn request() -> L7Request { } fn fixture( - identity_result: std::result::Result<&str, &str>, + identity_result: Result<&str, TokenGrantFailure>, ) -> (TokenGrantTestFixture, L7EvalContext) { let fixture = TokenGrantTestFixture::success(SERVICE, "service-token"); let mut identity = fixture.dynamic_credentials().read().unwrap()[SERVICE].clone(); @@ -62,14 +63,14 @@ async fn malformed_endpoint_key_fails_before_acquisition() { credential, ); } - assert!(inject_if_needed(request(), &ctx).await.is_err()); + assert!(inject_if_needed(request(), &ctx, REQUEST_ID).await.is_err()); fixture.assert_no_requests(); } #[tokio::test] async fn injects_independent_grants_and_replaces_all_protected_headers() { let (fixture, ctx) = fixture(Ok("identity-token")); - let rewritten = inject_if_needed(request(), &ctx).await.unwrap(); + let rewritten = inject_if_needed(request(), &ctx, REQUEST_ID).await.unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); assert_eq!( bytes @@ -101,7 +102,7 @@ async fn chooses_specific_binding_independently_for_each_header() { let mut specific = fixture.dynamic_credentials().read().unwrap()[SERVICE].clone(); specific.token_grant.as_mut().unwrap().audience = "projects-only".into(); fixture.add_credential(specific_key, specific, Ok("specific-service")); - let rewritten = inject_if_needed(request(), &ctx).await.unwrap(); + let rewritten = inject_if_needed(request(), &ctx, REQUEST_ID).await.unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); assert!(bytes.contains("Authorization: Bearer specific-service\r\n")); assert!(bytes.contains("X-Workload-Jwt: identity-token\r\n")); @@ -115,7 +116,7 @@ async fn audience_override_changes_only_its_own_credential() { let mut identity = fixture.dynamic_credentials().read().unwrap()[IDENTITY].clone(); identity.token_grant.as_mut().unwrap().audience = "project-identity".into(); fixture.add_credential(override_key, identity.clone(), Ok("project-identity-token")); - let rewritten = inject_if_needed(request(), &ctx).await.unwrap(); + let rewritten = inject_if_needed(request(), &ctx, REQUEST_ID).await.unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); assert!(bytes.contains("Authorization: Bearer service-token\r\n")); assert!(bytes.contains("X-Workload-Jwt: project-identity-token\r\n")); @@ -135,11 +136,12 @@ async fn rejects_tied_credentials_for_same_header_before_acquisition() { collision, Ok("collision-token"), ); - let error = inject_if_needed(request(), &ctx).await.unwrap_err(); - assert_eq!( - error.to_string(), - "ambiguous dynamic token grants for one header" - ); + let error = inject_if_needed(request(), &ctx, REQUEST_ID) + .await + .unwrap_err(); + assert_eq!(error.failure().kind(), TokenGrantFailureKind::Configuration); + assert_eq!(error.failure().stage(), TokenGrantStage::Injection); + assert!(error.credential().is_some()); fixture.assert_no_requests(); } } @@ -150,18 +152,18 @@ async fn rejects_invalid_second_header_before_acquisition() { let mut identity = fixture.dynamic_credentials().read().unwrap()[IDENTITY].clone(); identity.header_name = "Content-Length".into(); fixture.add_credential(IDENTITY, identity, Ok("identity-token")); - assert!(inject_if_needed(request(), &ctx).await.is_err()); + assert!(inject_if_needed(request(), &ctx, REQUEST_ID).await.is_err()); fixture.assert_no_requests(); } #[tokio::test] async fn rejects_malformed_second_token_after_first_grant_succeeds() { let (fixture, ctx) = fixture(Ok("identity-token\r\nInjected: yes")); - let error = inject_if_needed(request(), &ctx).await.unwrap_err(); - assert_eq!( - error.to_string(), - "token grant returned a malformed access token" - ); + let error = inject_if_needed(request(), &ctx, REQUEST_ID) + .await + .unwrap_err(); + assert_eq!(error.failure().stage(), TokenGrantStage::Injection); + assert_eq!(error.credential(), Some(("provider", "identity"))); fixture.assert_requested_keys(&[SERVICE, IDENTITY]); } @@ -202,9 +204,15 @@ async fn second_grant_failure_redacts_errors_and_emits_no_success() { ); return; } - let (fixture, ctx) = fixture(Err("issuer echoed service-token and identity-secret")); + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::GatewayRpc, + ) + .with_upstream_status(409); + let (fixture, ctx) = fixture(Err(failure)); let logs = Arc::new(Mutex::new(Vec::new())); let events = Arc::new(Mutex::new(Vec::new())); + let shorthand = Arc::new(Mutex::new(Vec::new())); let log_writer = Capture(logs.clone()); let subscriber = tracing_subscriber::registry() .with( @@ -213,21 +221,50 @@ async fn second_grant_failure_redacts_errors_and_emits_no_success() { .without_time() .with_writer(move || log_writer.clone()), ) - .with(openshell_ocsf::OcsfJsonlLayer::new(Capture(events.clone()))); - let error = inject_if_needed(request(), &ctx) + .with(openshell_ocsf::OcsfJsonlLayer::new(Capture(events.clone()))) + .with(openshell_ocsf::OcsfShorthandLayer::new(Capture( + shorthand.clone(), + ))); + let error = inject_if_needed(request(), &ctx, REQUEST_ID) .with_subscriber(subscriber) .await .unwrap_err(); - assert_eq!(error.to_string(), "Token grant failed"); + assert_eq!(error.failure(), failure); + assert_eq!(error.to_string(), "dynamic token grant failed"); let logs = String::from_utf8(logs.lock().unwrap().clone()).unwrap(); let events = String::from_utf8(events.lock().unwrap().clone()).unwrap(); - assert!(logs.contains("Token grant failed")); - assert!(events.contains("Token grant failed")); - for output in [&logs, &events] { + let shorthand = String::from_utf8(shorthand.lock().unwrap().clone()).unwrap(); + // The policy advisor serves the agent exactly these shorthand lines. + let denials = shorthand + .lines() + .filter(|line| line.contains(" OCSF ") && line.contains(" DENIED ")) + .collect::>() + .join("\n"); + assert!( + denials.contains("Token grant failed for provider:identity"), + "{shorthand}" + ); + // Every sink names the exchange by request ID. Only the operator log line + // carries the classification; agent-readable sinks never do. + for output in [&logs, &events, &shorthand] { + assert!(output.contains(REQUEST_ID), "request ID missing: {output}"); assert!(!output.contains("service-token")); - assert!(!output.contains("identity-secret")); assert!(!output.contains("Token grant successful")); } + assert!(logs.contains(failure.code()), "{logs}"); + for output in [&events, &shorthand, &denials] { + for classification in [ + failure.code(), + failure.stage().code(), + "upstream_http_status", + "REAUTHORIZE", + ] { + assert!( + !output.contains(classification), + "classification {classification} leaked: {output}" + ); + } + } fixture.assert_requested_keys(&[SERVICE, IDENTITY]); } @@ -245,7 +282,10 @@ async fn does_not_acquire_grants_outside_endpoint_bindings() { req.target = path.into(); let original = req.raw_header.clone(); assert_eq!( - inject_if_needed(req, &ctx).await.unwrap().raw_header, + inject_if_needed(req, &ctx, REQUEST_ID) + .await + .unwrap() + .raw_header, original ); fixture.assert_no_requests(); @@ -257,8 +297,8 @@ async fn concurrent_requests_keep_credential_snapshots_separate() { let (first, first_ctx) = fixture(Ok("first-identity")); let (second, second_ctx) = fixture(Ok("second-identity")); let (first_result, second_result) = tokio::join!( - inject_if_needed(request(), &first_ctx), - inject_if_needed(request(), &second_ctx), + inject_if_needed(request(), &first_ctx, REQUEST_ID), + inject_if_needed(request(), &second_ctx, REQUEST_ID), ); for (result, expected, absent) in [ (first_result, "first-identity", "second-identity"), @@ -299,9 +339,16 @@ async fn admitted_owners_select_grants_independently_per_header() { .map(|owner| (*owner).to_string()) .collect::>(); - let rewritten = inject_for_admitted_owners(request(), &ctx, &state.snapshot(), &owners) - .await - .unwrap(); + let rewritten = inject_for_admitted_owners( + request(), + &ctx, + &state.snapshot(), + &owners, + REQUEST_ID, + None, + ) + .await + .unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); // A grant whose owner did not admit the request is never acquired, and its diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index 62cf0582e5..f3d12d8569 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -2452,6 +2452,9 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St // from overlapping host and path selectors. ep["token_grant_owner"] = e.token_grant_owner.clone().into(); } + // Every inspected protocol must bind grant reporting to the + // policy selected before credential acquisition. + ep["policy_hash"] = policy_hash.clone().into(); if is_mcp_protocol(&e.protocol) { // Derive endpoint identity from the policy endpoint while // it is still available. Request handling carries this @@ -2459,9 +2462,6 @@ fn proto_to_opa_data_json(proto: &ProtoSandboxPolicy, entrypoint_pid: u32) -> St // recomputes identity from a concrete request host. ep["endpoint_id"] = openshell_core::endpoint_status::endpoint_id(e).into(); - // The selected endpoint must retain its policy identity - // so it cannot bind to a replacement observation inventory. - ep["policy_hash"] = policy_hash.clone().into(); } if !e.credential_signing.is_empty() { ep["credential_signing"] = e.credential_signing.clone().into(); diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 7cf630e420..4fd73af354 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5222,6 +5222,8 @@ async fn inject_token_grant_for_forward_request( forward_request_bytes: Vec, l7_ctx: &mut crate::l7::relay::L7EvalContext, inspection: Option>, + request_id: &str, + observation_context: Option<&EndpointObservationContext>, ) -> Result> { let (path, _) = crate::l7::rest::parse_target_query(upstream_target)?; let request = crate::l7::rest::request_from_buffered_http( @@ -5237,6 +5239,8 @@ async fn inject_token_grant_for_forward_request( inspection.engine, inspection.config, inspection.request_info, + request_id, + observation_context, ) .await?; *l7_ctx = prepared_ctx; @@ -5245,9 +5249,14 @@ async fn inject_token_grant_for_forward_request( // L4-only forwarding keeps selector-based credentials; it has no inspected // endpoint owner decision. Inspected requests always use preparation above. - crate::l7::token_grant_injection::inject_if_needed(request, l7_ctx) - .await - .map(|req| req.raw_header) + Ok(crate::l7::token_grant_injection::inject_if_needed_captured( + request, + l7_ctx, + request_id, + observation_context, + ) + .await? + .raw_header) } /// Handle a plain HTTP forward proxy request (non-CONNECT). @@ -6334,32 +6343,46 @@ async fn handle_forward_proxy( engine, request_info, }), + &request_id, + endpoint_observation_context.as_ref(), ) .await { Ok(bytes) => bytes, Err(e) => { - warn!( - dst_host = %host_lc, - dst_port = port, - error = %e, - "token grant failed in forward proxy" - ); + let refusal = + e.downcast_ref::(); + if refusal.is_none() { + warn!( + dst_host = %host_lc, + dst_port = port, + error = %e, + "token grant failed in forward proxy" + ); + } if let Some(session) = middleware_session.take() { session .end(openshell_core::proto::MiddlewareSessionEndReason::Cancellation) .await; } - respond( - client, - &build_json_error_response( - 502, - "Bad Gateway", - "token_grant_failed", - "dynamic token grant failed", - ), - ) - .await?; + let response = refusal.map_or_else( + || { + build_json_error_response( + 502, + "Bad Gateway", + "token_grant_failed", + "dynamic token grant failed", + ) + }, + |refusal| { + // The refusal event already names the credential and request ID. + if let Some(observer) = endpoint_observer.as_ref() { + observer.observe_token_grant_refusal(refusal, &request_id); + } + build_token_grant_failure_response(&request_id) + }, + ); + respond(client, &response).await?; return Ok(()); } }; @@ -6829,6 +6852,30 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail .into_bytes() } +/// Build the forward proxy's refusal for a failed dynamic token grant. +/// +/// The JSON body matches every other grant refusal. The request ID header lets +/// the caller quote the exchange to an operator without learning its cause. +fn build_token_grant_failure_response(request_id: &str) -> Vec { + let body = serde_json::json!({ + "error": "token_grant_failed", + "detail": "dynamic token grant failed", + }) + .to_string(); + format!( + "HTTP/1.1 502 Bad Gateway\r\n\ + Content-Type: application/json\r\n\ + Content-Length: {}\r\n\ + {}: {request_id}\r\n\ + Connection: close\r\n\ + \r\n\ + {body}", + body.len(), + openshell_core::token_grant_failure::REQUEST_ID_HEADER, + ) + .into_bytes() +} + fn build_json_error_response_with_reason( status: u16, status_text: &str, @@ -10970,8 +11017,47 @@ network_policies: .map_err(|e| miette::miette!("upstream task failed: {e}")) } + const FORWARD_REQUEST_ID: &str = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; + + #[test] + fn forward_grant_refusal_keeps_its_body_and_adds_only_the_request_id() { + let response = String::from_utf8(build_token_grant_failure_response(FORWARD_REQUEST_ID)) + .expect("UTF-8 response"); + let (head, body) = response.split_once("\r\n\r\n").expect("header terminator"); + let generic = String::from_utf8(build_json_error_response( + 502, + "Bad Gateway", + "token_grant_failed", + "dynamic token grant failed", + )) + .expect("UTF-8 response"); + let (generic_head, generic_body) = + generic.split_once("\r\n\r\n").expect("header terminator"); + assert_eq!(body, generic_body, "the refusal body is unchanged"); + let mut added = head + .lines() + .filter(|line| !generic_head.lines().any(|generic| generic == *line)) + .collect::>(); + added.sort_unstable(); + assert_eq!( + added, + [format!("x-openshell-request-id: {FORWARD_REQUEST_ID}").as_str()] + ); + } + + fn forward_grant_failure() -> openshell_core::token_grant_failure::TokenGrantFailure { + openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, + openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + ) + .with_upstream_status(503) + } + fn forward_token_grant_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result< + &str, + openshell_core::token_grant_failure::TokenGrantFailure, + >, ) -> ( crate::l7::relay::L7EvalContext, crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, @@ -10996,7 +11082,10 @@ network_policies: } fn forward_token_exchange_context( - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result< + &str, + openshell_core::token_grant_failure::TokenGrantFailure, + >, ) -> ( crate::l7::relay::L7EvalContext, crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, @@ -11012,7 +11101,10 @@ network_policies: fn forward_token_grant_fixture( provider_key: &str, - resolver_response: std::result::Result<&str, &str>, + resolver_response: std::result::Result< + &str, + openshell_core::token_grant_failure::TokenGrantFailure, + >, token_exchange: bool, ) -> crate::l7::token_grant_injection::test_support::TokenGrantTestFixture { match (resolver_response, token_exchange) { @@ -13198,10 +13290,17 @@ network_policies: let (mut ctx, fixture) = forward_token_grant_context(Ok("grant-token")); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer stale-token\r\nConnection: close\r\n\r\n".to_vec(); - let with_token = - inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &mut ctx, None) - .await - .expect("forward token grant should inject"); + let with_token = inject_token_grant_for_forward_request( + "GET", + "/v1/projects", + raw, + &mut ctx, + None, + FORWARD_REQUEST_ID, + None, + ) + .await + .expect("forward token grant should inject"); let rewritten = rewrite_forward_request( &with_token, with_token.len(), @@ -13224,10 +13323,17 @@ network_policies: let (mut ctx, fixture) = forward_token_exchange_context(Ok("grant-token")); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer stale-token\r\nConnection: close\r\n\r\n".to_vec(); - let with_token = - inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &mut ctx, None) - .await - .expect("forward token exchange should inject"); + let with_token = inject_token_grant_for_forward_request( + "GET", + "/v1/projects", + raw, + &mut ctx, + None, + FORWARD_REQUEST_ID, + None, + ) + .await + .expect("forward token exchange should inject"); let rewritten = rewrite_forward_request( &with_token, with_token.len(), @@ -13249,29 +13355,53 @@ network_policies: #[tokio::test] async fn forward_proxy_token_grant_failure_returns_error_before_rewrite() { - let (mut ctx, fixture) = forward_token_grant_context(Err("oauth unavailable")); + let (mut ctx, fixture) = forward_token_grant_context(Err(forward_grant_failure())); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - let err = - inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &mut ctx, None) - .await - .expect_err("forward token grant failure should stop request rewriting"); - - assert_eq!(err.to_string(), "Token grant failed"); + let err = inject_token_grant_for_forward_request( + "GET", + "/v1/projects", + raw, + &mut ctx, + None, + FORWARD_REQUEST_ID, + None, + ) + .await + .expect_err("forward token grant failure should stop request rewriting"); + + let refusal = err + .downcast_ref::() + .expect("typed grant refusal"); + assert_eq!(refusal.failure(), forward_grant_failure()); + assert_eq!(refusal.credential(), Some(("provider", "access_token"))); + assert_eq!(err.to_string(), "dynamic token grant failed"); fixture.assert_one_request("api.example.test\t8080\t/v1/**\tprovider:access_token"); } #[tokio::test] async fn forward_proxy_token_exchange_failure_returns_error_before_rewrite() { - let (mut ctx, fixture) = forward_token_exchange_context(Err("oauth unavailable")); + let (mut ctx, fixture) = forward_token_exchange_context(Err(forward_grant_failure())); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - let err = - inject_token_grant_for_forward_request("GET", "/v1/projects", raw, &mut ctx, None) - .await - .expect_err("forward token exchange failure should stop request rewriting"); - - assert_eq!(err.to_string(), "Token grant failed"); + let err = inject_token_grant_for_forward_request( + "GET", + "/v1/projects", + raw, + &mut ctx, + None, + FORWARD_REQUEST_ID, + None, + ) + .await + .expect_err("forward token exchange failure should stop request rewriting"); + + let refusal = err + .downcast_ref::() + .expect("typed grant refusal"); + assert_eq!(refusal.failure(), forward_grant_failure()); + assert_eq!(refusal.credential(), Some(("provider", "access_token"))); + assert_eq!(err.to_string(), "dynamic token grant failed"); fixture.assert_one_token_exchange_request( "api.example.test\t8080\t/v1/**\tprovider:access_token", ); diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index 6ba4270973..13e3938fb6 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -79,6 +79,8 @@ async fn forward_inspection_selects_only_the_owner_admitting_the_request() { engine: &engine, request_info: &info, }), + "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", + None, ) .await .expect("prepare admitted forward request"); @@ -127,6 +129,8 @@ async fn forward_inspection_retains_installation_through_guarded_write() { engine: &engine, request_info: &info, }), + "5c4b3a29-1807-4f6e-8d5c-4b3a29180716", + None, ) .await .expect("prepare authenticated request"); diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index f0f92632f9..c0ce14798f 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -37,33 +37,28 @@ use std::future::Future; use std::sync::{Arc, LazyLock, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use miette::{IntoDiagnostic, Result, WrapErr}; use openshell_core::oauth::{ - self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantParams, - effective_client_assertion_type, effective_token_type, + self, ACCESS_TOKEN_TYPE, OAuthTokenResponse, TokenExchangeParams, TokenGrantClient, + TokenGrantParams, effective_client_assertion_type, effective_token_type, }; use openshell_core::proto::ProviderCredentialTokenGrantType; use openshell_core::sandbox_env; -use spiffe::WorkloadApiClient; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, +}; +use spiffe::{WorkloadApiClient, WorkloadApiError}; /// Token cache shared across all provider token grants. static TOKEN_CACHE: LazyLock = LazyLock::new(TokenCache::new); -static TOKEN_GRANT_HTTP_CLIENT: LazyLock = LazyLock::new(|| { - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - reqwest::Client::builder() - .timeout(Duration::from_secs(30)) - .connect_timeout(Duration::from_secs(30)) - .no_proxy() - .redirect(reqwest::redirect::Policy::none()) - .build() - .expect("token grant HTTP client configuration should be valid") -}); +// Initialization failures stay bounded and typed; credential acquisition must not panic. +static TOKEN_GRANT_HTTP_CLIENT: LazyLock> = + LazyLock::new(TokenGrantClient::new); const DEFAULT_TOKEN_CACHE_TTL_SECONDS: i64 = 300; const TOKEN_CACHE_EXPIRY_SKEW_SECONDS: i64 = 30; const MAX_TOKEN_EXPIRES_IN_SECONDS: i64 = 3600; /// Cached access token with expiration metadata. -#[derive(Debug, Clone)] +#[derive(Clone)] struct CachedToken { access_token: String, expires_at_ms: i64, @@ -82,28 +77,32 @@ impl TokenCache { } /// Get a cached token if it exists and is not expired. - fn get(&self, provider_name: &str) -> Option { + fn get(&self, provider_name: &str) -> Result, TokenGrantFailure> { let now_ms = current_time_ms(); - let tokens = self.tokens.read().ok()?; - let cached = tokens.get(provider_name)?; - if cached.expires_at_ms > now_ms { - Some(cached.access_token.clone()) - } else { - None - } + let tokens = self.tokens.read().map_err(|_| cache_failure())?; + Ok(tokens + .get(provider_name) + .filter(|cached| cached.expires_at_ms > now_ms) + .map(|cached| cached.access_token.clone())) } /// Store a token with expiration time. - fn set(&self, provider_name: String, access_token: String, expires_at_ms: i64) { - if let Ok(mut tokens) = self.tokens.write() { - tokens.insert( - provider_name, - CachedToken { - access_token, - expires_at_ms, - }, - ); - } + fn set( + &self, + provider_name: String, + access_token: String, + expires_at_ms: i64, + ) -> Result<(), TokenGrantFailure> { + // Poisoning invalidates cache integrity; never silently bypass it. + let mut tokens = self.tokens.write().map_err(|_| cache_failure())?; + tokens.insert( + provider_name, + CachedToken { + access_token, + expires_at_ms, + }, + ); + Ok(()) } } @@ -144,21 +143,31 @@ pub struct ObtainProviderTokenRequest<'a> { pub cache_ttl_override: Option, pub grant_type: i32, pub requested_token_type: &'a str, + /// Supervisor-minted ID of the HTTP exchange that needs the token. It is + /// sent to the gateway and token service; a cache hit sends nothing. + pub request_id: &'a str, } -pub async fn obtain_provider_token(request: ObtainProviderTokenRequest<'_>) -> Result { - let grant_type = - ProviderCredentialTokenGrantType::try_from(request.grant_type).map_err(|_| { - tracing::error!( - grant_type = request.grant_type, - provider = request.provider_name, - "unrecognised token grant_type" - ); - miette::miette!( - "unrecognised token grant_type {} for provider credential", - request.grant_type - ) - })?; +/// Obtain and cache a provider token, retaining only bounded grant failures. +pub async fn obtain_provider_token( + request: ObtainProviderTokenRequest<'_>, +) -> Result { + let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) + .map_err(|_| configuration_failure())?; + // Resolve local exchange prerequisites before fetching any workload credential. + let exchange_context = if grant_type == ProviderCredentialTokenGrantType::TokenExchange { + let (provider, credential_key) = parse_provider_credential_key(request.provider_name)?; + let endpoint = supervisor_gateway_endpoint_from_env()?; + openshell_core::grpc_client::validate_token_grant_gateway_endpoint(&endpoint)?; + Some(( + provider, + credential_key, + endpoint, + supervisor_sandbox_id_from_env()?, + )) + } else { + None + }; obtain_provider_token_with_grant( ObtainProviderTokenInput { cache: &TOKEN_CACHE, @@ -188,14 +197,13 @@ pub async fn obtain_provider_token(request: ObtainProviderTokenRequest<'_>) -> R request.client_assertion_type, request.audience, request.scopes, + request.request_id, ) .await } ProviderCredentialTokenGrantType::TokenExchange => { - let (provider, credential_key) = - parse_provider_credential_key(request.provider_name)?; - let endpoint = supervisor_gateway_endpoint_from_env()?; - let sandbox_id = supervisor_sandbox_id_from_env()?; + let (provider, credential_key, endpoint, sandbox_id) = + exchange_context.ok_or_else(configuration_failure)?; let intermediate = openshell_core::grpc_client::exchange_provider_subject_token( &endpoint, @@ -203,14 +211,12 @@ pub async fn obtain_provider_token(request: ObtainProviderTokenRequest<'_>) -> R provider, credential_key, &jwt_svid, + Some(request.request_id), ) - .await - .map_err(|err| { - miette::miette!( - "gateway intermediate provider token exchange failed: {err}" - ) - })?; - validate_access_token(&intermediate.access_token)?; + .await?; + validate_access_token(&intermediate.access_token).map_err(|failure| { + failure.at_stage(TokenGrantStage::IntermediateExchange) + })?; perform_token_exchange( request.token_endpoint, &TokenExchangeParams { @@ -222,6 +228,7 @@ pub async fn obtain_provider_token(request: ObtainProviderTokenRequest<'_>) -> R scopes: request.scopes, requested_token_type: request.requested_token_type, }, + request.request_id, ) .await } @@ -247,11 +254,19 @@ struct ObtainProviderTokenInput<'a> { async fn obtain_provider_token_with_grant( input: ObtainProviderTokenInput<'_>, grant: F, -) -> Result +) -> Result where F: FnOnce(String) -> Fut, - Fut: Future>, + Fut: Future>, { + // Even a cache hit cannot authorize an invalid endpoint configuration. + oauth::validate_token_endpoint(input.token_endpoint)?; + if input + .cache_ttl_override + .is_some_and(|ttl| i64::try_from(ttl.as_millis()).is_err()) + { + return Err(configuration_failure()); + } let jwt_audience = effective_jwt_svid_audience(input.token_endpoint, input.jwt_svid_audience); let cache_key = token_cache_key(TokenCacheKeyInput { provider_name: input.provider_name, @@ -265,12 +280,13 @@ where }); if input.cache_ttl_override != Some(Duration::ZERO) - && let Some(cached) = input.cache.get(&cache_key) + && let Some(cached) = input.cache.get(&cache_key)? { return Ok(cached); } let token_response = grant(jwt_audience).await?; + validate_access_token(&token_response.access_token)?; let cache_ttl = token_cache_ttl(input.cache_ttl_override, token_response.expires_in); if !cache_ttl.is_zero() { @@ -280,7 +296,7 @@ where cache_key, token_response.access_token.clone(), expires_at_ms, - ); + )?; } Ok(token_response.access_token) @@ -291,39 +307,75 @@ where /// This function connects to the local SPIRE agent via the Workload API and /// requests a JWT-SVID with the specified audience. The JWT-SVID is used as /// the client assertion in the `OAuth2` grant request. -async fn fetch_jwt_svid_for_token_grant(audience: &str) -> Result { +async fn fetch_jwt_svid_for_token_grant(audience: &str) -> Result { let socket_path = provider_spiffe_workload_api_socket_from_env()?; + fetch_jwt_svid_from_socket(&socket_path, audience).await +} - let endpoint = - openshell_core::spiffe::workload_api_endpoint(std::path::Path::new(&socket_path)); +async fn fetch_jwt_svid_from_socket( + socket_path: &str, + audience: &str, +) -> Result { + let endpoint = openshell_core::spiffe::workload_api_endpoint(std::path::Path::new(socket_path)); // Connect to SPIRE agent let client = WorkloadApiClient::connect_to(&endpoint) .await - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to connect to SPIFFE Workload API endpoint {endpoint}") - })?; + .map_err(workload_api_failure)?; // Fetch JWT-SVID with token service audience // None = use the sandbox's default SPIFFE ID client .fetch_jwt_token([audience], None) .await - .into_diagnostic() - .wrap_err("failed to fetch JWT-SVID for token grant") + .map_err(workload_api_failure) +} + +fn workload_api_failure(error: WorkloadApiError) -> TokenGrantFailure { + // Classify the local Workload API's typed rejection without retaining its + // endpoint, response text, JWT parsing details, or transport source. + let kind = match error { + WorkloadApiError::MissingEndpointSocket + | WorkloadApiError::NotUnicodeEndpointSocket(_) + | WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, + WorkloadApiError::NoIdentityIssued | WorkloadApiError::PermissionDenied(_) => { + TokenGrantFailureKind::WorkloadIdentityRejected + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Status(status)) => { + match status.code() { + tonic::Code::DeadlineExceeded => TokenGrantFailureKind::Timeout, + tonic::Code::Unavailable => TokenGrantFailureKind::Unavailable, + tonic::Code::InvalidArgument => TokenGrantFailureKind::Configuration, + _ => TokenGrantFailureKind::Unclassified, + } + } + WorkloadApiError::Transport(spiffe::transport::TransportError::Tonic(_)) => { + TokenGrantFailureKind::Unavailable + } + _ => TokenGrantFailureKind::Unclassified, + }; + TokenGrantFailure::new(kind, TokenGrantStage::WorkloadIdentity) +} + +fn configuration_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::Configuration, + TokenGrantStage::Configuration, + ) +} + +fn cache_failure() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::Configuration, + ) } -fn provider_spiffe_workload_api_socket_from_env() -> Result { +fn provider_spiffe_workload_api_socket_from_env() -> Result { std::env::var(sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) .ok() .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| { - miette::miette!( - "{} not set — SPIFFE authentication unavailable for token grant", - sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET - ) - }) + .ok_or_else(configuration_failure) } async fn perform_token_grant( @@ -332,9 +384,12 @@ async fn perform_token_grant( client_assertion_type: &str, audience: &str, scopes: &[String], -) -> Result { + request_id: &str, +) -> Result { oauth::post_oauth_token_grant( - &TOKEN_GRANT_HTTP_CLIENT, + TOKEN_GRANT_HTTP_CLIENT + .as_ref() + .map_err(|failure| *failure)?, token_endpoint, &TokenGrantParams { client_assertion: jwt_svid, @@ -342,6 +397,7 @@ async fn perform_token_grant( audience, scopes, }, + Some(request_id), ) .await } @@ -349,8 +405,17 @@ async fn perform_token_grant( async fn perform_token_exchange( token_endpoint: &str, params: &TokenExchangeParams<'_>, -) -> Result { - oauth::post_oauth_token_exchange(&TOKEN_GRANT_HTTP_CLIENT, token_endpoint, params).await + request_id: &str, +) -> Result { + oauth::post_oauth_token_exchange( + TOKEN_GRANT_HTTP_CLIENT + .as_ref() + .map_err(|failure| *failure)?, + token_endpoint, + params, + Some(request_id), + ) + .await } pub use oauth::validate_access_token; @@ -406,26 +471,24 @@ fn effective_jwt_svid_audience(token_endpoint: &str, jwt_svid_audience: &str) -> } } -fn supervisor_gateway_endpoint_from_env() -> Result { +fn supervisor_gateway_endpoint_from_env() -> Result { std::env::var(sandbox_env::ENDPOINT) .ok() .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| miette::miette!("{} not set", sandbox_env::ENDPOINT)) + .ok_or_else(configuration_failure) } -fn supervisor_sandbox_id_from_env() -> Result { +fn supervisor_sandbox_id_from_env() -> Result { std::env::var(sandbox_env::SANDBOX_ID) .ok() .filter(|value| !value.trim().is_empty()) - .ok_or_else(|| miette::miette!("{} not set", sandbox_env::SANDBOX_ID)) + .ok_or_else(configuration_failure) } -fn parse_provider_credential_key(key: &str) -> Result<(&str, &str)> { +fn parse_provider_credential_key(key: &str) -> Result<(&str, &str), TokenGrantFailure> { openshell_core::dynamic_credential_key::credential_identity(key) .and_then(|identity| identity.split_once(':')) - .ok_or_else(|| { - miette::miette!("dynamic token grant key is missing provider credential identity") - }) + .ok_or_else(configuration_failure) } struct TokenCacheKeyInput<'a> { @@ -550,10 +613,10 @@ pub mod test_support { fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin> + Send + 'a>> { + ) -> Pin> + Send + 'a>> { Box::pin(async move { let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) - .map_err(|_| miette::miette!("unknown test token grant type"))?; + .map_err(|_| configuration_failure())?; obtain_provider_token_with_grant( ObtainProviderTokenInput { cache: &self.cache, @@ -619,6 +682,8 @@ mod tests { use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; + const TEST_REQUEST_ID: &str = "4f1c2b3a-9d8e-4c7b-a6f5-e4d3c2b1a098"; + #[derive(Debug)] struct CapturedTokenRequest { request_line: String, @@ -814,7 +879,9 @@ mod tests { grant_calls: Arc, } - async fn obtain_counted_test_token(input: CountedTokenGrantInput<'_>) -> Result { + async fn obtain_counted_test_token( + input: CountedTokenGrantInput<'_>, + ) -> Result { obtain_provider_token_with_grant( ObtainProviderTokenInput { cache: input.cache, @@ -851,7 +918,7 @@ mod tests { audience: &str, scopes: &[String], cache_ttl_override: Option, - ) -> Result { + ) -> Result { obtain_provider_token_with_grant( ObtainProviderTokenInput { cache, @@ -865,11 +932,156 @@ mod tests { grant_type: ProviderCredentialTokenGrantType::ClientCredentials, requested_token_type: "", }, - |_| async { Err(miette::miette!("grant should not be called on cache hit")) }, + |_| async { panic!("grant should not be called on cache hit") }, ) .await } + #[tokio::test] + async fn invalid_endpoint_and_cache_ttl_fail_before_credential_acquisition() { + let cache = TokenCache::new(); + let grant_calls = Arc::new(AtomicUsize::new(0)); + for (endpoint, ttl) in [ + ("https://secret-endpoint-canary@", None), + ("https://auth.example.com/token", Some(Duration::MAX)), + ] { + let failure = obtain_counted_test_token(CountedTokenGrantInput { + cache: &cache, + provider_name: "provider:access_token", + token_endpoint: endpoint, + jwt_svid_audience: "https://auth.example.com", + audience: "api://resource", + scopes: &[], + cache_ttl_override: ttl, + expires_in: 60, + grant_calls: grant_calls.clone(), + }) + .await + .expect_err("invalid local configuration must fail closed"); + assert_eq!(failure, configuration_failure()); + assert_eq!(failure.to_string(), "dynamic token grant failed"); + assert!(std::error::Error::source(&failure).is_none()); + } + assert_eq!(grant_calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn workload_api_failures_discard_arbitrary_diagnostics() { + for (raw, kind) in [ + ( + WorkloadApiError::PermissionDenied("secret-spiffe-canary".to_string()), + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + WorkloadApiError::NoIdentityIssued, + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + WorkloadApiError::MissingEndpointSocket, + TokenGrantFailureKind::Configuration, + ), + ( + WorkloadApiError::HintNotFound("secret-jwt-canary".to_string()), + TokenGrantFailureKind::Unclassified, + ), + ( + WorkloadApiError::from(tonic::Status::deadline_exceeded("secret-timeout-canary")), + TokenGrantFailureKind::Timeout, + ), + ( + WorkloadApiError::from(tonic::Status::unavailable("secret-socket-canary")), + TokenGrantFailureKind::Unavailable, + ), + ( + WorkloadApiError::from(tonic::Status::unknown("secret-rpc-canary")), + TokenGrantFailureKind::Unclassified, + ), + ] { + let failure = workload_api_failure(raw); + assert_eq!(failure.kind(), kind); + assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); + assert_eq!(failure.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + assert!(std::error::Error::source(&failure).is_none()); + } + } + + #[tokio::test] + async fn spiffe_connection_failure_does_not_expose_socket_or_audience() { + let failure = fetch_jwt_svid_from_socket( + "/private/tmp/openshell-nonexistent-secret-socket-canary/agent.sock", + "secret-audience-canary", + ) + .await + .expect_err("missing socket must fail closed"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unavailable); + assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); + assert_eq!(failure.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + assert!(std::error::Error::source(&failure).is_none()); + } + + #[tokio::test] + async fn grant_failure_preserves_classification_stage_and_status() { + let cache = TokenCache::new(); + for stage in [ + TokenGrantStage::IntermediateExchange, + TokenGrantStage::TokenEndpoint, + ] { + let expected = + TokenGrantFailure::new(TokenGrantFailureKind::ProviderAuthorizationRequired, stage) + .with_upstream_status(409); + let failure = obtain_provider_token_with_grant( + ObtainProviderTokenInput { + cache: &cache, + provider_name: "provider:access_token", + token_endpoint: "https://auth.example.com/token", + jwt_svid_audience: "", + client_assertion_type: "", + audience: "", + scopes: &[], + cache_ttl_override: None, + grant_type: ProviderCredentialTokenGrantType::ClientCredentials, + requested_token_type: "", + }, + |_| async { Err(expected) }, + ) + .await + .expect_err("grant must fail"); + assert_eq!(failure, expected); + } + } + + #[tokio::test] + async fn poisoned_token_cache_fails_before_credential_acquisition() { + let cache = TokenCache::new(); + let tokens = cache.tokens.clone(); + assert!( + std::thread::spawn(move || { + let _guard = tokens.write().expect("lock fixture cache"); + panic!("poison fixture cache"); + }) + .join() + .is_err() + ); + let grant_calls = Arc::new(AtomicUsize::new(0)); + let failure = obtain_counted_test_token(CountedTokenGrantInput { + cache: &cache, + provider_name: "provider:access_token", + token_endpoint: "https://auth.example.com/token", + jwt_svid_audience: "", + audience: "", + scopes: &[], + cache_ttl_override: None, + expires_in: 60, + grant_calls: grant_calls.clone(), + }) + .await + .expect_err("poisoned cache must fail closed"); + assert_eq!(failure, cache_failure()); + assert_eq!(grant_calls.load(Ordering::SeqCst), 0); + } + #[test] fn test_derive_issuer_from_keycloak_token_endpoint() { let token_endpoint = @@ -1083,7 +1295,9 @@ mod tests { grant_type: ProviderCredentialTokenGrantType::ClientCredentials, requested_token_type: ACCESS_TOKEN_TYPE, }); - cache.set(identity_cache_key, "expired".into(), current_time_ms() - 1); + cache + .set(identity_cache_key, "expired".into(), current_time_ms() - 1) + .expect("test cache accepts an expired entry"); let service_token = obtain_token_without_grant_call( &cache, service, @@ -1222,11 +1436,13 @@ mod tests { grant_type: ProviderCredentialTokenGrantType::ClientCredentials, requested_token_type: ACCESS_TOKEN_TYPE, }); - cache.set( - cache_key, - "expired-token".to_string(), - current_time_ms() - 1, - ); + cache + .set( + cache_key, + "expired-token".to_string(), + current_time_ms() - 1, + ) + .expect("cache should accept expired fixture"); let token = obtain_counted_test_token(CountedTokenGrantInput { cache: &cache, @@ -1313,10 +1529,16 @@ mod tests { .await; let scopes = vec!["read".to_string(), "write".to_string()]; - let response = - perform_token_grant(&endpoint, "jwt-svid-token", "", "api://resource", &scopes) - .await - .expect("token grant should succeed"); + let response = perform_token_grant( + &endpoint, + "jwt-svid-token", + "", + "api://resource", + &scopes, + TEST_REQUEST_ID, + ) + .await + .expect("token grant should succeed"); let request = request.await.expect("token endpoint task should finish"); assert_eq!(response.access_token, "access-123"); @@ -1326,6 +1548,13 @@ mod tests { request.headers.get("content-type").map(String::as_str), Some("application/x-www-form-urlencoded") ); + assert_eq!( + request + .headers + .get(openshell_core::token_grant_failure::REQUEST_ID_HEADER) + .map(String::as_str), + Some(TEST_REQUEST_ID) + ); assert_eq!( request.form.get("grant_type").map(String::as_str), Some("client_credentials") @@ -1366,6 +1595,7 @@ mod tests { "urn:ietf:params:oauth:client-assertion-type:jwt-spiffe", "", &[], + TEST_REQUEST_ID, ) .await .expect("token grant should succeed"); @@ -1389,27 +1619,28 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[]) + let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) .await .expect_err("malformed access token should fail closed"); let _request = request.await.expect("token endpoint task should finish"); - assert_eq!( - err.to_string(), - "token grant returned a malformed access token" - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(200)); } #[tokio::test] async fn perform_token_grant_does_not_follow_redirects() { let (endpoint, request) = token_endpoint_redirect_once("http://127.0.0.1:1/stolen").await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[]) + let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) .await .expect_err("redirect response should fail closed"); let _request = request.await.expect("token endpoint task should finish"); - assert_eq!(err.to_string(), "token grant failed with status 302 Found"); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(302)); } #[tokio::test] @@ -1417,9 +1648,10 @@ mod tests { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":"access-123"}"#).await; - let response = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[]) - .await - .expect("token grant should succeed without audience or scopes"); + let response = + perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) + .await + .expect("token grant should succeed without audience or scopes"); let request = request.await.expect("token endpoint task should finish"); assert_eq!(response.access_token, "access-123"); @@ -1432,26 +1664,34 @@ mod tests { } #[tokio::test] - async fn perform_token_grant_reports_sanitized_oauth_error_response() { + async fn perform_token_grant_discards_oauth_error_description() { let (endpoint, request) = token_endpoint_once( "401 Unauthorized", - r#"{"error":"invalid_client","error_description":"bad jwt assertion"}"#, + r#"{"error":"invalid_client","error_description":"secret-jwt-assertion-canary"}"#, ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "api://resource", &[]) - .await - .expect_err("token grant should fail on OAuth error"); + let err = perform_token_grant( + &endpoint, + "jwt-svid-token", + "", + "api://resource", + &[], + TEST_REQUEST_ID, + ) + .await + .expect_err("token grant should fail on OAuth error"); let request = request.await.expect("token endpoint task should finish"); assert_eq!( request.form.get("audience").map(String::as_str), Some("api://resource") ); - assert_eq!( - err.to_string(), - "token grant failed with status 401 Unauthorized: error=invalid_client; error_description=bad jwt assertion" - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(format!("{err:?}"), "dynamic token grant failed"); + assert!(std::error::Error::source(&err).is_none()); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(401)); } #[tokio::test] @@ -1462,16 +1702,15 @@ mod tests { ) .await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[]) + let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) .await .expect_err("token grant should fail on server error"); let _request = request.await.expect("token endpoint task should finish"); let message = err.to_string(); - assert_eq!( - message, - "token grant failed with status 500 Internal Server Error" - ); + assert_eq!(message, "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unavailable); + assert_eq!(err.upstream_status(), Some(500)); assert!(!message.contains("stack trace")); assert!(!message.contains("implementation details")); } @@ -1480,14 +1719,13 @@ mod tests { async fn perform_token_grant_reports_malformed_success_json() { let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; - let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[]) + let err = perform_token_grant(&endpoint, "jwt-svid-token", "", "", &[], TEST_REQUEST_ID) .await .expect_err("malformed token response should fail"); let _request = request.await.expect("token endpoint task should finish"); - assert!( - err.to_string() - .contains("failed to parse token response as JSON") - ); + assert_eq!(err.to_string(), "dynamic token grant failed"); + assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(err.upstream_status(), Some(200)); } } diff --git a/crates/openshell-supervisor/src/endpoint_status.rs b/crates/openshell-supervisor/src/endpoint_status.rs index c598c94e83..d95b9b8a72 100644 --- a/crates/openshell-supervisor/src/endpoint_status.rs +++ b/crates/openshell-supervisor/src/endpoint_status.rs @@ -3,7 +3,7 @@ //! Report the last observed network result for configured external tool endpoints. //! -//! Only MCP-over-HTTP traffic currently supplies observations. Reporting is +//! MCP outcomes and token grant attempts supply observations. Reporting is //! passive, bounded, and independent of policy enforcement and sandbox readiness. use std::collections::{BTreeMap, BTreeSet}; @@ -329,11 +329,15 @@ fn apply_command( .collect::>(), ) } - EndpointStatusCommand::Observe { .. } => None, + EndpointStatusCommand::Observe { .. } | EndpointStatusCommand::ObserveTokenGrant { .. } => { + None + } }; let observed_endpoint_id = match &command { EndpointStatusCommand::Observe { endpoint_id, .. } => Some(endpoint_id.clone()), - EndpointStatusCommand::Reset { .. } => None, + EndpointStatusCommand::Reset { .. } | EndpointStatusCommand::ObserveTokenGrant { .. } => { + None + } }; if !tracker.apply(command) { return false; diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 7ff7d7c8e6..43ebbcd3f7 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -658,6 +658,55 @@ Token grants require the sandbox supervisor to have access to a SPIFFE Workload The gateway only brokers an intermediate token for a sandbox principal, and only when the requested provider is attached to that sandbox. It verifies the supervisor JWT-SVID issuer, audience, signature, and SPIFFE trust domain against the gateway's own JWT-SVID, then uses the verified supervisor SVID subject as the intermediate-token audience. +### Token Grant Failures + +When a token grant fails, OpenShell refuses the protected request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header. L7 relays return an empty body, and the forward proxy returns the JSON body `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`. The response, the supervisor's OCSF events, and `policy.local` denials never say why the grant failed, because the sandboxed agent can read them. Users authorized to read the sandbox can retrieve the cause from `Sandbox.status.last_token_grant_failure` and `openshell logs`. + +The request ID is a random UUID that the supervisor mints for each HTTP exchange. It names the same exchange everywhere: + +- The supervisor sends it to the token service in the `x-openshell-request-id` request header. +- For `token_exchange` grants, the supervisor sends it to the gateway as gRPC `x-request-id` metadata, and the gateway forwards it to the intermediate token endpoint. +- The supervisor's `Token grant failed` OCSF event carries it as `request_id`. In `openshell logs`, the supervisor's log line with the same message adds the failure code, stage, and upstream HTTP status. +- When the gateway hop fails, the gateway's `dynamic token grant failed` log line carries it with the failure code, stage, and upstream HTTP status. + +OpenShell classifies each failure into one of these codes. The recovery action uses the same values as provider refresh status: + +| Code | Meaning | Recovery action | +|---|---|---| +| `provider_authorization_required` | The user must authorize the provider again. | `reauthorize` | +| `workload_identity_rejected` | The SPIFFE Workload API, the gateway, or the token service rejected the sandbox's workload identity. | `fix_configuration` | +| `configuration_error` | The grant, provider, or token-service configuration is invalid. | `fix_configuration` | +| `timeout` | A grant step exceeded its time limit. | `retry` | +| `unavailable` | A required service or connection was unavailable. | `retry` | +| `unclassified` | The available evidence does not establish a more specific cause. | `investigate` | + +The stage names the step that observed the failure: `configuration`, `workload_identity`, `gateway_rpc`, `intermediate_exchange`, `token_endpoint`, or `injection`. OpenShell maps token-service responses as follows: + +- HTTP 429 and 5xx, and connection failures, are `unavailable`. A timeout, including a slow response body, is `timeout`. +- A bare HTTP 401, 403, or 409 is `unclassified`, with its status recorded. OpenShell never infers `provider_authorization_required` from a 401. +- For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these causes only over its verified TLS connection to the gateway. + +OpenShell keeps only the code, stage, and HTTP status. It discards response descriptions, headers, endpoint diagnostics, and error chains. Success and failure response bodies are limited to 64 KiB. A successful response must contain a nonempty header-compatible `access_token`. + +A token service can also state a cause by adopting the `openshell.token-grant.v1` error extension. For example, a service that requires the user to authorize a provider returns HTTP 409 with: + +```json +{ + "error": "provider_authorization_required", + "openshell_error_contract": "openshell.token-grant.v1" +} +``` + +OpenShell accepts this cause only from the configured endpoint over verified HTTPS. It verifies the server certificate and hostname and does not follow redirects. Loopback and Kubernetes-service HTTP endpoints cannot authenticate these signals. The extension recognizes these status and code pairs: + +- HTTP 409 with `provider_authorization_required`. +- HTTP 401 or 403 with `workload_identity_rejected`. +- HTTP 400 with `configuration_error`. + +Each response also requires the exact `openshell_error_contract` marker. Unknown versions, inconsistent status and code pairs, malformed responses, and signals received over plain HTTP are `unclassified`, except that HTTP 429 and 5xx stay `unavailable`. OpenShell does not infer identity rejection from generic OAuth errors such as `invalid_client` or `invalid_grant`, which can have more than one cause. + +The latest classified grant refusal is available as `last_token_grant_failure` for every destination, including REST requests and failures before an HTTP path is known. Its `destination_host` and `destination_port` identify the attempted call, and `failure` contains the bounded cause. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) for retrieval and clearing rules. Find correlated events in `openshell logs`: the supervisor's `Token grant failed` line or, when the gateway hop failed, the gateway's `dynamic token grant failed` line. The token service receives the ID with every request. The no-bytes guarantee excludes connection setup: an inspected CONNECT path can already have completed an upstream TLS handshake before a grant fails. + ## Provider Instances A provider instance stores concrete credentials and config for a profile type. `--type` accepts the ID of any imported profile, matched exactly. Creating a provider for a profile the gateway does not serve fails and names the import command. diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 2094a7280e..cddd4a3ebf 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -724,12 +724,31 @@ Read `last_result` to choose the next check: - `NoObservedExchange`: no result has been reported for the current configuration and supervisor session. Try the operation and inspect its logs if no result appears. - `HttpResponseReceived`: the server returned a final HTTP status below 400, including a protocol upgrade. Informational responses alone do not establish success. Check the client's response for protocol or tool errors; an HTTP 200 response can still contain an error. - `PolicyDenied`: OpenShell denied the request, including MCP protocol-version, request-header, request-body, and HTTP-method checks. Check the sandbox policy and denial logs. -- `CredentialUnavailable`: required credentials were unavailable. Check the endpoint's attached provider. +- `CredentialUnavailable`: required credentials were unavailable. Check the endpoint's attached provider and `last_failure`. - `TlsFailed`: TLS setup or the handshake failed. Check certificates and TLS configuration. - `TransportFailed`: the network exchange failed. Check name resolution, connectivity, and the server process. - `UpstreamRejected`: the server returned an HTTP status of 400 or higher. Check the server's response and logs. - `Unspecified`: the result is unrecognized or absent. Do not infer success from it. +For a provider-credentialed MCP-over-HTTP endpoint, a failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) also carries `last_failure`, which explains the `CredentialUnavailable` result. It describes why OpenShell could not complete the exchange; the tool server can still be healthy. `openshell sandbox get` prints it under the result: + +```text +Last result: Required OpenShell-managed credential was unavailable. +Cause: provider_authorization_required (token grant, stage gateway_rpc) +Credential: github/access_token +Recovery: Reauthorize the provider credential. +Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908 +``` + +In JSON and YAML output, `last_failure` has `domain`, `code`, `recovery_action`, `provider`, `credential_key`, `stage`, `upstream_http_status`, and `request_id`; it is `null` when no cause was reported. `recovery_action` says who acts next: + +- `reauthorize`: the user authorizes the provider again. +- `fix_configuration`: an operator repairs the provider, grant, or workload identity configuration. +- `retry`: the failure can be temporary; retry later. +- `investigate`: an operator inspects the supervisor and gateway logs for the request ID. + +`request_id` matches the `x-openshell-request-id` header of the refused response. `provider` and `credential_key` are names, never credential values; they are empty when the grant failed before a credential was selected. Like `last_result`, `last_failure` describes only the last observation: a later exchange replaces it, and a change to the attached provider, such as storing a new credential, clears it. + For example, a stopped server can produce `TransportFailed` while the sandbox stays `Ready`. After the server recovers, an HTTP response below 400 updates the result to `HttpResponseReceived`. Results come from actual traffic and do not expire. A result can remain unchanged after the server stops until another observed exchange or configuration/session reset. Reports combine recent observations and can drop them when full, so this list is not a request history. The address remains available when a result resets to `NoObservedExchange`. To verify current tool availability, run the actual operation. @@ -738,6 +757,18 @@ Results come from actual traffic and do not expire. A result can remain unchange Addresses use lowercase hosts, canonical paths, and sorted distinct effective ports. Equivalent addresses share one record that combines observations from all callers and ports; it does not establish that every caller or port works. A failure before the HTTP path is known, such as a TLS failure during CONNECT, updates status only when the host and port identify one distinct endpoint. If several configured paths share that host and port, inspect the logs for the failure. +### Read Token Grant Failures + +`openshell sandbox get` shows `last_token_grant_failure` for the latest classified grant refusal, including REST requests and failures before an HTTP path is known. The protobuf field is `Sandbox.status.last_token_grant_failure`. It carries `destination_host`, `destination_port`, and a `failure` with the bounded fields described above. It describes the attempted call; the remote server can still be healthy. + +Read it in JSON or YAML, or select it directly: + +```shell +openshell sandbox get my-sandbox --output json | jq '.last_token_grant_failure' +``` + +Match `failure.request_id` with the `x-openshell-request-id` from the refused response before starting recovery. A newer grant failure can replace this record. A successful grant for the same destination, provider, and credential clears it; when a failure happened before credential selection, the next successful grant to the same destination clears it. Policy, provider, and supervisor-session resets also clear it. JSON/YAML show `null` when no cause is retained. This is one latest observation, not a request history. + ### Inspect Logs and Activity Stream sandbox logs to monitor agent activity and diagnose policy decisions: diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index cddc35fa3d..755c0d14a2 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -228,10 +228,12 @@ An upstream that the proxy cannot reach returns `502 Bad Gateway`: } ``` -The `error` field is a short machine-readable code (`policy_denied`, `middleware_denied`, `middleware_failed`, `ssrf_denied`, `upstream_unreachable`, `unsupported_l7_protocol`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. The optional `reason` field, when present, provides the specific denial cause from the policy engine (for example, which binary was not allowed or which rule was missing). +The `error` field is a short machine-readable code (`policy_denied`, `middleware_denied`, `middleware_failed`, `ssrf_denied`, `upstream_unreachable`, `unsupported_l7_protocol`, `token_grant_failed`). The `detail` field is a human-readable explanation suitable for display in an agent transcript. The optional `reason` field, when present, provides the specific denial cause from the policy engine (for example, which binary was not allowed or which rule was missing). For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When the policy advisor is enabled, the body also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware` and can include a validated `reason_code`. A fail-closed runtime failure uses `middleware_failed` with platform-owned text. Both middleware responses omit `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. +A failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) returns `502 Bad Gateway` with an `x-openshell-request-id` header. The forward proxy returns `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`; L7 relays return an empty body. The response does not say why the grant failed. Give the request ID to someone who can read the sandbox. They can find the cause in `last_token_grant_failure` from `openshell sandbox get --output json` or in `openshell logs`, on the supervisor's `Token grant failed` line or, when the gateway hop failed, the gateway's `dynamic token grant failed` line. + ## Filesystem Sandbox Logs Landlock filesystem restrictions emit `CONFIG:` events at startup and whenever the sandbox has to skip a requested path. diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index b5403e2ac3..f68c7dade5 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -1590,6 +1590,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 01315baef5..47c7387518 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -256,7 +256,7 @@ rustls-pemfile = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } -tonic = { version = "0.14", features = ["transport"] } +tonic = { version = "0.14", features = ["transport", "tls-aws-lc"] } tonic-prost = "0.14" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "aws_lc_rs"] } tower = "0.5" diff --git a/e2e/rust/src/harness/gateway.rs b/e2e/rust/src/harness/gateway.rs index edb6f4570a..849807e8b0 100644 --- a/e2e/rust/src/harness/gateway.rs +++ b/e2e/rust/src/harness/gateway.rs @@ -22,6 +22,7 @@ pub struct ManagedGateway { log: PathBuf, pid_file: PathBuf, supervisor_image: Option, + native_ca_bundle: Option, } impl ManagedGateway { @@ -38,6 +39,7 @@ impl ManagedGateway { log: log.into(), pid_file: pid_file.into(), supervisor_image: None, + native_ca_bundle: None, } } @@ -56,6 +58,7 @@ impl ManagedGateway { log: env_path("OPENSHELL_E2E_GATEWAY_LOG")?, pid_file: env_path("OPENSHELL_E2E_GATEWAY_PID_FILE")?, supervisor_image: None, + native_ca_bundle: None, })) } @@ -64,6 +67,11 @@ impl ManagedGateway { self.supervisor_image = Some(image.to_owned()); } + /// Set the native TLS roots for this fixture's gateway child process. + pub fn set_native_ca_bundle(&mut self, path: impl Into) { + self.native_ca_bundle = Some(path.into()); + } + /// Start the gateway if it is not already running. pub fn start(&self) -> Result<(), String> { if let Some(pid) = self.current_pid()? { @@ -96,6 +104,9 @@ impl ManagedGateway { if let Some(image) = &self.supervisor_image { command.env("OPENSHELL_SUPERVISOR_IMAGE", image); } + if let Some(bundle) = &self.native_ca_bundle { + command.env("SSL_CERT_FILE", bundle); + } let child = command .spawn() .map_err(|err| format!("start openshell-gateway '{}': {err}", self.bin.display()))?; diff --git a/e2e/rust/tests/provider_token_exchange.rs b/e2e/rust/tests/provider_token_exchange.rs index 5c84423ae8..017ba10e50 100644 --- a/e2e/rust/tests/provider_token_exchange.rs +++ b/e2e/rust/tests/provider_token_exchange.rs @@ -3,6 +3,9 @@ #![cfg(feature = "e2e-podman")] +#[path = "provider_token_exchange/closure.rs"] +mod closure; + use std::collections::HashMap; use std::convert::Infallible; use std::fs; @@ -143,10 +146,14 @@ struct SpiffeWorkloadApi { subject: Arc, jwks: Arc>, encoding_key: Arc, + deny_identity: Arc, } impl SpiffeWorkloadApi { fn jwt_svid(&self, audience: &[String]) -> Result { + if self.deny_identity.load(Ordering::SeqCst) { + return Err(Status::permission_denied("identity-secret-canary")); + } let now = unix_timestamp(); let mut header = Header::new(Algorithm::RS256); header.kid = Some(KEY_ID.to_string()); @@ -314,6 +321,14 @@ fn jwks() -> Vec { } async fn start_spiffe_workload_api(path: &Path, subject: &str) -> FixtureHandle { + start_controlled_spiffe_workload_api(path, subject, Arc::default()).await +} + +async fn start_controlled_spiffe_workload_api( + path: &Path, + subject: &str, + deny_identity: Arc, +) -> FixtureHandle { let api = SpiffeWorkloadApi { subject: Arc::::from(subject), jwks: Arc::new(jwks()), @@ -321,6 +336,7 @@ async fn start_spiffe_workload_api(path: &Path, subject: &str) -> FixtureHandle EncodingKey::from_rsa_pem(TEST_RSA_PRIVATE_KEY.as_bytes()) .expect("test RSA key should parse"), ), + deny_identity, }; let endpoint = path.to_string_lossy(); if endpoint.starts_with("tcp:") { @@ -773,7 +789,7 @@ async fn sandbox_exec_http( let url = format!("http://host.openshell.internal:{target_port}{path}"); let expect_denied_python = if expect_denied { "True" } else { "False" }; let script = format!( - r#"import base64, json, os, re, urllib.error, urllib.request + r#"import base64, json, os, re, urllib.error, urllib.request, uuid def contains_credential(value): if any(marker in value for marker in ( @@ -810,6 +826,10 @@ except urllib.error.HTTPError as error: raise RuntimeError("provider credential material reached the workload error") if not {expect_denied_python} or error.code != 502: raise RuntimeError("unexpected HTTP response") from None + # The refusal names the exchange with an opaque ID and nothing else. + request_id = error.headers.get("x-openshell-request-id", "") + if str(uuid.UUID(request_id)) != request_id: + raise RuntimeError("refusal did not carry a canonical request ID") print("grant-denied") "# ); @@ -1001,8 +1021,8 @@ async fn podman_provider_token_exchange_injects_independent_grants_across_sandbo // Read one whole fixture request without consuming the following keepalive // request. OAuth forms can arrive separately from their headers. -async fn read_route_fixture_request( - stream: &mut TcpStream, +async fn read_route_fixture_request( + stream: &mut S, observed_bytes: Option<&AtomicUsize>, ) -> Result, String> { tokio::time::timeout(Duration::from_secs(30), async { @@ -1062,6 +1082,7 @@ async fn read_route_fixture_request( struct RouteGrantState { fail_next_b: AtomicBool, supervisor_requests: Mutex>, + failed_request_id: Mutex>, } async fn start_route_token_endpoint(port: u16, state: Arc) -> FixtureHandle { @@ -1097,6 +1118,14 @@ async fn start_route_token_endpoint(port: u16, state: Arc) -> F route == "b" && state.fail_next_b.swap(false, Ordering::SeqCst) } else { false }; let (status, body) = if fail { + let request_id = request.split_once("\r\n\r\n") + .map(|(headers, _)| headers) + .unwrap_or_default() + .lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("x-openshell-request-id")) + .map(|(_, value)| value.trim().to_owned()); + *state.failed_request_id.lock().expect("failed grant request ID lock") = request_id; ("HTTP/1.1 503 Service Unavailable", json!({"error": "temporary_failure"}).to_string()) } else if valid { let stage = if supervisor { "final" } else { "intermediate" }; @@ -1299,7 +1328,7 @@ async fn podman_provider_routes_recover_and_select_a_b_a_on_one_connection() { // The supervisor mediates direct workload connections as CONNECT. // HTTPConnection keeps one client socket for the successful A/B/A run. let script = format!( - r#"import http.client + r#"import http.client, uuid host = 'host.openshell.internal' port = {target_port} first = http.client.HTTPConnection(host, port, timeout=10) @@ -1307,6 +1336,9 @@ first.request('GET', '/b/failure', headers={{'Authorization': 'Bearer stale-valu failed = first.getresponse() failed.read() assert failed.status == 502, 'the controlled grant failure must return 502' +request_id = failed.getheader('x-openshell-request-id', '') +assert str(uuid.UUID(request_id)) == request_id, 'the refusal must carry a canonical request ID' +print('failed-request-id=' + request_id) assert failed.will_close, 'the failed tunnel must close' first.close() recovered = http.client.HTTPConnection(host, port, timeout=10) @@ -1345,6 +1377,15 @@ print('route-recovery-a-b-a-ok') let output = result.expect("gateway-configured route workload succeeds"); target_result.expect("route target observes EOF on every accepted connection"); assert!(output.contains("route-recovery-a-b-a-ok")); + let response_request_id = output + .lines() + .find_map(|line| line.strip_prefix("failed-request-id=")) + .expect("workload prints the refused response's request ID"); + assert_eq!( + grants.failed_request_id.lock().unwrap().as_deref(), + Some(response_request_id), + "the token service and refused response must identify the same exchange" + ); let requests = observed.lock().expect("route target observations lock"); assert_eq!(requests.len(), 3, "failed grant must not reach the target"); assert_eq!( diff --git a/e2e/rust/tests/provider_token_exchange/closure.rs b/e2e/rust/tests/provider_token_exchange/closure.rs new file mode 100644 index 0000000000..4709a03767 --- /dev/null +++ b/e2e/rust/tests/provider_token_exchange/closure.rs @@ -0,0 +1,805 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Exercise trusted grant failure retrieval through a real REST workload. + +use super::*; +use openshell_e2e::harness::container::ContainerEngine; +use openshell_e2e::harness::gateway::ManagedGateway; +use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; +use serde_json::Value; +use tempfile::TempDir; +use tokio::process::Command; +use tonic::transport::{Certificate, ClientTlsConfig, Endpoint as GrpcEndpoint}; + +const ERROR_CANARY: &str = "grant-response-secret-canary"; +const INTERMEDIATE: &str = "intermediate-closure"; + +async fn checked(command: &mut Command, label: &str) -> Result { + command.kill_on_drop(true); + let output = tokio::time::timeout(Duration::from_secs(300), command.output()) + .await + .map_err(|_| format!("{label} timed out"))? + .map_err(|_| format!("{label} could not start"))?; + if !output.status.success() { + return Err(format!("{label} failed")); + } + Ok(String::from_utf8_lossy(&output.stdout).into_owned()) +} + +async fn cli(args: &[&str]) -> Result { + checked(openshell_cmd().args(args), "closure CLI operation").await +} + +fn no_secrets(output: &str) -> Result<(), String> { + if [ + ERROR_CANARY, + "identity-secret-canary", + "stored-closure", + INTERMEDIATE, + ] + .iter() + .any(|secret| output.contains(secret)) + { + return Err("grant diagnostics exposed fixture credential material".into()); + } + Ok(()) +} + +// Both OAuth clients use native roots. Install the public fixture CA in the +// supervisor image and in the gateway child's own SSL_CERT_FILE. Never change +// verification flags or the test runner's process-wide environment. +struct GatewayTrust { + directory: TempDir, + engine: ContainerEngine, + image: String, + health_port: u16, + changed: bool, +} + +impl GatewayTrust { + async fn new() -> Result { + if std::env::var_os("OPENSHELL_GATEWAY_ENDPOINT").is_some() + || std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("podman") + || std::env::var("OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER") + .is_ok_and(|value| value != "0") + { + return Err("grant closure requires a wrapper-owned bundled Podman gateway".into()); + } + let args_path = std::env::var_os("OPENSHELL_E2E_GATEWAY_ARGS_FILE") + .ok_or("managed gateway arguments are missing")?; + let bytes = fs::read(args_path).map_err(|_| "could not read gateway arguments")?; + let args = bytes + .split(|byte| *byte == 0) + .filter(|arg| !arg.is_empty()) + .map(std::str::from_utf8) + .collect::, _>>() + .map_err(|_| "gateway arguments are not UTF-8")?; + let argument = |name| { + args.windows(2) + .find(|pair| pair[0] == name) + .map(|pair| pair[1]) + .ok_or("gateway argument is missing") + }; + let health_port = argument("--health-port")? + .parse() + .map_err(|_| "gateway health port is invalid")?; + let config = fs::read_to_string(argument("--config")?) + .map_err(|_| "could not read fixture gateway configuration")?; + let base_image = std::env::var("OPENSHELL_SUPERVISOR_IMAGE") + .ok() + .or_else(|| { + config.lines().find_map(|line| { + let value = line.trim().strip_prefix("supervisor_image = ")?; + serde_json::from_str::(value).ok() + }) + }) + .ok_or("fixture supervisor image is missing")?; + let directory = TempDir::new().map_err(|_| "could not create private TLS fixture")?; + generate_tls(directory.path()).await?; + let roots = std::env::var_os("SSL_CERT_FILE").map_or_else( + || PathBuf::from("/etc/ssl/certs/ca-certificates.crt"), + PathBuf::from, + ); + let mut bundle = fs::read(roots).map_err(|_| "native host CA bundle is required")?; + bundle.push(b'\n'); + bundle.extend( + fs::read(directory.path().join("ca.crt")).map_err(|_| "fixture CA is missing")?, + ); + fs::write(directory.path().join("host-roots.crt"), bundle) + .map_err(|_| "could not stage gateway roots")?; + let context = directory.path().join("image"); + fs::create_dir(&context).map_err(|_| "could not create public image context")?; + fs::copy( + directory.path().join("ca.crt"), + context.join("fixture-ca.crt"), + ) + .map_err(|_| "could not stage public fixture CA")?; + fs::write(context.join("Dockerfile"), format!( + "FROM {base_image} AS original\nFROM {E2E_WORKLOAD_IMAGE} AS bundle\nUSER 0\nCOPY --from=original /etc/ssl/certs/ca-certificates.crt /tmp/roots.crt\nCOPY fixture-ca.crt /tmp/fixture-ca.crt\nRUN [\"/usr/bin/python3\", \"-c\", \"from pathlib import Path; p=Path('/tmp/roots.crt'); p.write_bytes(p.read_bytes()+b'\\n'+Path('/tmp/fixture-ca.crt').read_bytes())\"]\nFROM {base_image}\nCOPY --from=bundle /tmp/roots.crt /etc/ssl/certs/ca-certificates.crt\n" + )).map_err(|_| "could not write supervisor fixture Dockerfile")?; + let engine = ContainerEngine::from_env()?; + let image = format!( + "localhost/openshell-grant-closure-{:016x}:fixture", + rand::random::() + ); + let mut build = Command::from(engine.command()); + build.args(["build", "--tag", &image]).arg(&context); + checked(&mut build, "build trusted supervisor fixture").await?; + Ok(Self { + directory, + engine, + image, + health_port, + changed: false, + }) + } + + async fn install(&mut self) -> Result<(), String> { + self.changed = true; + let mut gateway = ManagedGateway::from_env()?.ok_or("managed gateway is missing")?; + gateway.set_supervisor_image(&self.image); + gateway.set_native_ca_bundle(self.directory.path().join("host-roots.crt")); + gateway.stop()?; + gateway.start()?; + self.wait_ready().await + } + + async fn wait_ready(&self) -> Result<(), String> { + let url = format!("http://127.0.0.1:{}/healthz", self.health_port); + let deadline = tokio::time::Instant::now() + Duration::from_secs(120); + loop { + if checked( + Command::new("curl").args([ + "--noproxy", + "*", + "--silent", + "--fail", + "--max-time", + "2", + &url, + ]), + "gateway health probe", + ) + .await + .is_ok() + { + return Ok(()); + } + if tokio::time::Instant::now() >= deadline { + return Err("trusted fixture gateway did not become healthy".into()); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + } + + async fn restore(&mut self) -> Result<(), String> { + if self.changed { + let gateway = ManagedGateway::from_env()?.ok_or("managed gateway is missing")?; + gateway.stop()?; + gateway.start()?; + self.wait_ready().await?; + self.changed = false; + } + let mut remove = Command::from(self.engine.command()); + checked( + remove.args(["image", "rm", "--force", &self.image]), + "remove supervisor fixture image", + ) + .await?; + Ok(()) + } +} + +async fn generate_tls(directory: &Path) -> Result<(), String> { + checked( + Command::new("openssl") + .args([ + "req", + "-x509", + "-newkey", + "rsa:2048", + "-nodes", + "-days", + "1", + "-subj", + "/CN=OpenShell grant fixture CA", + "-addext", + "basicConstraints=critical,CA:TRUE", + "-keyout", + ]) + .arg(directory.join("ca.key")) + .arg("-out") + .arg(directory.join("ca.crt")), + "generate fixture CA", + ) + .await?; + checked( + Command::new("openssl") + .args([ + "req", + "-newkey", + "rsa:2048", + "-nodes", + "-subj", + "/CN=localhost", + "-keyout", + ]) + .arg(directory.join("server.key")) + .arg("-out") + .arg(directory.join("server.csr")), + "generate token service CSR", + ) + .await?; + fs::write(directory.join("extensions"), "subjectAltName=IP:127.0.0.1,DNS:localhost\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n") + .map_err(|_| "could not write token service certificate extensions")?; + checked( + Command::new("openssl") + .args(["x509", "-req", "-days", "1", "-in"]) + .arg(directory.join("server.csr")) + .arg("-CA") + .arg(directory.join("ca.crt")) + .arg("-CAkey") + .arg(directory.join("ca.key")) + .arg("-CAcreateserial") + .arg("-extfile") + .arg(directory.join("extensions")) + .arg("-out") + .arg(directory.join("server.crt")), + "sign token service certificate", + ) + .await?; + Ok(()) +} + +#[derive(Clone, Copy)] +enum Mode { + IntermediateAuthorization, + FinalAuthorization, + Bare401, + Success, +} + +struct TokenService { + mode: Mutex, + requests: Mutex>, +} + +async fn token_server( + directory: &Path, + port: u16, + state: Arc, +) -> Result { + let certificate = + fs::read(directory.join("server.crt")).map_err(|_| "token certificate is missing")?; + let key = fs::read(directory.join("server.key")).map_err(|_| "token key is missing")?; + let certificates = rustls_pemfile::certs(&mut certificate.as_slice()) + .collect::, _>>() + .map_err(|_| "invalid token certificate")?; + let private_key = rustls_pemfile::private_key(&mut key.as_slice()) + .map_err(|_| "invalid token key")? + .ok_or("empty token key")?; + let config = rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(certificates, private_key) + .map_err(|_| "invalid token TLS configuration")?; + let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); + let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) + .await + .map_err(|_| "could not bind HTTPS token fixture")?; + let endpoint = format!("https://127.0.0.1:{port}/token"); + let task = tokio::spawn(async move { + let mut requests = tokio::task::JoinSet::new(); + loop { + tokio::select! { + accepted = listener.accept() => { + let Ok((stream, _)) = accepted else { break; }; + let (acceptor, state, endpoint) = (acceptor.clone(), state.clone(), endpoint.clone()); + requests.spawn(async move { + let Ok(mut stream) = acceptor.accept(stream).await else { return; }; + let Ok(Some(request)) = read_route_fixture_request(&mut stream, None).await else { return; }; + let fields = token_form(&request); + let assertion = fields.get("client_assertion").map_or("", String::as_str); + let mut validation = Validation::new(Algorithm::RS256); + validation.set_audience(&[&endpoint]); + validation.set_issuer(&[ISSUER]); + let jwks: Value = serde_json::from_slice(&jwks()).unwrap(); + let key = DecodingKey::from_rsa_components(jwks["keys"][0]["n"].as_str().unwrap(), jwks["keys"][0]["e"].as_str().unwrap()).unwrap(); + if jsonwebtoken::decode::(assertion, &key, &validation).is_err() { return; } + let subject = fields.get("subject_token").map_or("", String::as_str); + let final_hop = subject == INTERMEDIATE; + if !final_hop && !subject.starts_with("stored-closure") { return; } + let request_id = request.split_once("\r\n\r\n").unwrap().0.lines() + .filter_map(|line| line.split_once(':')) + .find(|(name, _)| name.eq_ignore_ascii_case("x-openshell-request-id")) + .map_or("", |(_, value)| value.trim()).to_owned(); + state.requests.lock().unwrap().push((final_hop, request_id)); + let mode = *state.mode.lock().unwrap(); + let authorization = matches!((mode, final_hop), (Mode::IntermediateAuthorization, false) | (Mode::FinalAuthorization, true)); + let (status, body) = if authorization { + ("409 Conflict", json!({"error":"provider_authorization_required", "openshell_error_contract":"openshell.token-grant.v1", "error_description":ERROR_CANARY, "authorization_url":format!("https://{ERROR_CANARY}.invalid/")})) + } else if matches!((mode, final_hop), (Mode::Bare401, true)) { + ("401 Unauthorized", json!({"error":"invalid_client", "error_description":ERROR_CANARY})) + } else { + ("200 OK", json!({"access_token":if final_hop {"final-b"} else {INTERMEDIATE}, "token_type":"Bearer", "expires_in":0})) + }; + let body = body.to_string(); + let response = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); + let _ = stream.write_all(response.as_bytes()).await; + }); + } + _ = requests.join_next(), if !requests.is_empty() => {} + } + } + }); + Ok(FixtureHandle { task }) +} + +async fn probe(sandbox: &SandboxGuard, target_port: u16, denied: bool) -> Result { + let script = format!( + r"import urllib.request, urllib.error, uuid +try: + response = urllib.request.urlopen('http://host.openshell.internal:{target_port}/closure/probe', timeout=10) + response.read() + assert not {denied}, 'grant unexpectedly forwarded' + print('success') +except urllib.error.HTTPError as error: + body = error.read().decode('utf-8', errors='replace') + text = str(error.headers) + body + assert {denied} and error.code == 502, 'unexpected grant refusal' + for value in ['{ERROR_CANARY}', 'identity-secret-canary', 'stored-closure', '{INTERMEDIATE}', 'final-b', 'provider_authorization_required', 'workload_identity_rejected', 'authorization_url']: + assert value not in text, 'grant refusal exposed private diagnostics' + request_id = error.headers.get('x-openshell-request-id', '') + assert str(uuid.UUID(request_id)) == request_id, 'invalid refusal request ID' + print(request_id) +", + denied = if denied { "True" } else { "False" } + ); + let output = sandbox.exec(&["python3", "-c", &script]).await?; + no_secrets(&output)?; + Ok(output.trim().to_owned()) +} + +async fn snapshot(name: &str) -> Result { + let output = cli(&["sandbox", "get", name, "--output", "json"]).await?; + no_secrets(&output)?; + serde_json::from_str(&output).map_err(|_| "sandbox snapshot was invalid JSON".into()) +} + +async fn wait_cause(name: &str, request_id: Option<&str>) -> Result { + let deadline = tokio::time::Instant::now() + Duration::from_secs(60); + loop { + let value = snapshot(name).await?; + let cause = value + .get("last_token_grant_failure") + .ok_or("sandbox snapshot omitted grant failures")?; + if request_id.map_or_else( + || cause.is_null(), + |id| cause["failure"]["request_id"] == id, + ) { + return Ok(value); + } + if tokio::time::Instant::now() >= deadline { + return Err("grant observation did not settle".into()); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +fn check_cause( + value: &Value, + port: u16, + code: &str, + stage: &str, + recovery: &str, +) -> Result<(), String> { + let cause = &value["last_token_grant_failure"]; + let failure = &cause["failure"]; + if cause["destination_host"] != "host.openshell.internal" + || cause["destination_port"].as_u64() != Some(u64::from(port)) + || failure["code"] != code + || failure["stage"] != stage + || failure["recovery_action"] != recovery + { + return Err("trusted grant cause did not match the refused exchange".into()); + } + if !value["endpoint_statuses"] + .as_array() + .is_some_and(Vec::is_empty) + { + return Err("REST fixture unexpectedly depended on MCP endpoint status".into()); + } + Ok(()) +} + +async fn check_logs(sandbox: &SandboxGuard, request_id: &str) -> Result<(), String> { + let deadline = tokio::time::Instant::now() + Duration::from_secs(10); + loop { + let logs = cli(&[ + "logs", + &sandbox.name, + "--source", + "sandbox", + "--since", + "5m", + ]) + .await?; + no_secrets(&logs)?; + if logs + .lines() + .any(|line| line.contains(request_id) && line.contains("Token grant failed")) + { + break; + } + if tokio::time::Instant::now() >= deadline { + return Err("supervisor log omitted the refused request ID".into()); + } + tokio::time::sleep(Duration::from_millis(200)).await; + } + let gateway_log = + std::env::var_os("OPENSHELL_E2E_GATEWAY_LOG").ok_or("gateway log path is missing")?; + let gateway = + fs::read_to_string(gateway_log).map_err(|_| "could not read gateway fixture log")?; + no_secrets(&gateway)?; + if !gateway + .lines() + .any(|line| line.contains(request_id) && line.contains("provider_authorization_required")) + { + return Err("gateway log omitted the classified exchange".into()); + } + let denials = sandbox.exec(&["python3", "-c", "import urllib.request; print(urllib.request.urlopen('http://policy.local/v1/denials?last=20').read().decode())"]).await?; + no_secrets(&denials)?; + if !denials.contains(request_id) || denials.contains("provider_authorization_required") { + return Err("agent denial stream lost correlation or exposed classification".into()); + } + Ok(()) +} + +#[derive(Clone, PartialEq, prost::Message)] +struct GetSandbox { + #[prost(string, tag = "1")] + name: String, + #[prost(message, optional, tag = "2")] + workspace_scope: Option, +} +#[derive(Clone, PartialEq, prost::Message)] +struct Workspace { + #[prost(string, tag = "1")] + workspace: String, +} +#[derive(Clone, PartialEq, prost::Message)] +struct Empty {} + +async fn assert_untrusted_reader(name: &str, bearer: Option<&str>) -> Result<(), String> { + let gateway = std::env::var("OPENSHELL_GATEWAY").map_err(|_| "gateway name is missing")?; + let home = std::env::var_os("XDG_CONFIG_HOME").ok_or("gateway config directory is missing")?; + let directory = PathBuf::from(home).join("openshell/gateways").join(gateway); + let metadata: Value = serde_json::from_slice( + &fs::read(directory.join("metadata.json")).map_err(|_| "gateway metadata is missing")?, + ) + .map_err(|_| "gateway metadata is invalid")?; + let endpoint = metadata["gateway_endpoint"] + .as_str() + .ok_or("gateway endpoint is missing")?; + let ca = fs::read(directory.join("mtls/ca.crt")).map_err(|_| "gateway CA is missing")?; + let channel = GrpcEndpoint::from_shared(endpoint.to_owned()) + .map_err(|_| "invalid gateway endpoint")? + .tls_config(ClientTlsConfig::new().ca_certificate(Certificate::from_pem(ca))) + .map_err(|_| "invalid gateway TLS configuration")? + .connect() + .await + .map_err(|_| "could not connect untrusted reader over verified TLS")?; + let mut client = tonic::client::Grpc::new(channel); + client + .ready() + .await + .map_err(|_| "untrusted reader transport is not ready")?; + let mut request = Request::new(GetSandbox { + name: name.to_owned(), + workspace_scope: Some(Workspace { + workspace: "default".into(), + }), + }); + if let Some(token) = bearer { + request.metadata_mut().insert( + "authorization", + format!("Bearer {token}") + .parse() + .map_err(|_| "invalid fixture bearer")?, + ); + } + let result: Result, Status> = client + .unary( + request, + http::uri::PathAndQuery::from_static("/openshell.v1.OpenShell/GetSandbox"), + tonic_prost::ProstCodec::default(), + ) + .await; + match result { + Err(status) + if matches!( + status.code(), + tonic::Code::Unauthenticated | tonic::Code::PermissionDenied + ) => + { + Ok(()) + } + _ => Err("untrusted reader was not denied access to grant details".into()), + } +} + +async fn supervisor_token(engine: &ContainerEngine, name: &str) -> Result { + let mut list = Command::from(engine.command()); + let output = checked( + list.args([ + "ps", + "--quiet", + "--filter", + &format!("label=openshell.ai/sandbox-name={name}"), + "--filter", + "label=openshell.ai/isolation-role=supervisor", + ]), + "locate fixture supervisor", + ) + .await?; + let ids = output.lines().collect::>(); + let [id] = ids.as_slice() else { + return Err("fixture supervisor is ambiguous".into()); + }; + let directory = + TempDir::new().map_err(|_| "could not create private authentication fixture")?; + let path = directory.path().join("auth.json"); + let mut copy = Command::from(engine.command()); + checked( + copy.arg("cp") + .arg(format!("{id}:/.openshell/supervisor/auth.json")) + .arg(&path), + "copy private fixture supervisor authentication", + ) + .await?; + let value: Value = + serde_json::from_slice(&fs::read(&path).map_err(|_| "fixture authentication is missing")?) + .map_err(|_| "fixture authentication is invalid")?; + value["gateway_token"] + .as_str() + .map(str::to_owned) + .ok_or("fixture supervisor token is missing".into()) +} + +async fn failure_cases( + sandbox: &SandboxGuard, + port: u16, + service: &TokenService, + denied_identity: &AtomicBool, +) -> Result<(), String> { + for (mode, stage) in [ + (Mode::IntermediateAuthorization, "intermediate_exchange"), + (Mode::FinalAuthorization, "token_endpoint"), + ] { + *service.mode.lock().unwrap() = mode; + let request_id = probe(sandbox, port, true).await?; + let value = wait_cause(&sandbox.name, Some(&request_id)).await?; + check_cause( + &value, + port, + "provider_authorization_required", + stage, + "reauthorize", + )?; + if value["last_token_grant_failure"]["failure"]["upstream_http_status"] != 409 { + return Err("trusted cause lost the token service HTTP status".into()); + } + let hops = { + let requests = service.requests.lock().unwrap(); + requests + .iter() + .filter(|(_, id)| id == &request_id) + .map(|(final_hop, _)| *final_hop) + .collect::>() + }; + let expected = if stage == "intermediate_exchange" { + vec![false] + } else { + vec![false, true] + }; + if hops != expected { + return Err( + "actual token-service hops did not preserve the workload request ID".into(), + ); + } + if stage == "intermediate_exchange" { + check_logs(sandbox, &request_id).await?; + } + } + *service.mode.lock().unwrap() = Mode::Bare401; + let request_id = probe(sandbox, port, true).await?; + let value = wait_cause(&sandbox.name, Some(&request_id)).await?; + check_cause( + &value, + port, + "unclassified", + "token_endpoint", + "investigate", + )?; + let requests = service.requests.lock().unwrap().len(); + denied_identity.store(true, Ordering::SeqCst); + let request_id = probe(sandbox, port, true).await?; + let value = wait_cause(&sandbox.name, Some(&request_id)).await?; + denied_identity.store(false, Ordering::SeqCst); + check_cause( + &value, + port, + "workload_identity_rejected", + "workload_identity", + "fix_configuration", + )?; + if service.requests.lock().unwrap().len() != requests { + return Err("rejected workload identity reached the token service".into()); + } + Ok(()) +} + +async fn create_rest_fixture( + name: &str, + token_port: u16, + port: u16, +) -> Result { + let profile = write_route_profile(name, token_port, port, "closure"); + let contents = fs::read_to_string(profile.path()) + .map_err(|_| "fixture profile is missing")? + .replace("http://127.0.0.1:", "https://127.0.0.1:") + .replace("cache_ttl_seconds: 300", "cache_ttl_seconds: 0"); + fs::write(profile.path(), contents).map_err(|_| "could not set verified token endpoint")?; + run_cli(&[ + "profile", + "import", + "-f", + profile.path().to_str().ok_or("profile path is invalid")?, + ]) + .await?; + run_cli(&[ + "provider", + "create", + "--name", + name, + "--type", + name, + "--credential", + "subject_token=stored-closure", + ]) + .await?; + SandboxGuard::create_keep_with_args( + &["--name", name, "--provider", name], + &["sh", "-lc", &sandbox_script()], + "token-server-ready", + ) + .await +} + +async fn recovery_cases( + sandbox: &SandboxGuard, + port: u16, + service: &TokenService, +) -> Result<(), String> { + *service.mode.lock().unwrap() = Mode::Success; + if probe(sandbox, port, false).await? != "success" { + return Err("reauthorized request did not reach its destination".into()); + } + wait_cause(&sandbox.name, None).await?; + *service.mode.lock().unwrap() = Mode::FinalAuthorization; + let request_id = probe(sandbox, port, true).await?; + wait_cause(&sandbox.name, Some(&request_id)).await?; + run_cli(&[ + "provider", + "update", + &sandbox.name, + "--credential", + "subject_token=stored-closure-replacement", + ]) + .await?; + wait_cause(&sandbox.name, None).await?; + Ok(()) +} + +#[tokio::test] +async fn verified_grant_failures_reach_trusted_rest_status_and_recover() -> Result<(), String> { + let _fixture_guard = SPIFFE_FIXTURE_LOCK.lock().await; + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + let gateway_socket = PathBuf::from( + std::env::var("OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET") + .map_err(|_| "gateway Workload API fixture is missing")?, + ); + let provider_socket = PathBuf::from( + std::env::var("OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET") + .map_err(|_| "provider Workload API fixture is missing")?, + ); + let token_port = std::env::var("OPENSHELL_E2E_TOKEN_PORT").map_or_else( + |_| Ok(find_free_port()), + |value| { + value + .parse::() + .map_err(|_| "invalid fixed token fixture port") + }, + )?; + let mut trust = GatewayTrust::new().await?; + let gateway_spiffe = start_spiffe_workload_api( + &gateway_socket, + &format!("spiffe://{TRUST_DOMAIN}/openshell/gateway"), + ) + .await; + let deny_identity = Arc::new(AtomicBool::new(false)); + let provider_spiffe = start_controlled_spiffe_workload_api( + &provider_socket, + &format!("spiffe://{TRUST_DOMAIN}/openshell/sandbox/e2e"), + deny_identity.clone(), + ) + .await; + let port = find_free_port(); + let service = Arc::new(TokenService { + mode: Mutex::new(Mode::IntermediateAuthorization), + requests: Mutex::new(Vec::new()), + }); + let tokens = match token_server(trust.directory.path(), token_port, service.clone()).await { + Ok(tokens) => tokens, + Err(error) => { + trust.restore().await?; + return Err(error); + } + }; + let observed = Arc::new(Mutex::new(Vec::new())); + let observed_bytes = Arc::new(AtomicUsize::new(0)); + let target = start_route_target(port, observed.clone(), observed_bytes.clone()).await; + let name = format!("closure-{:08x}", rand::random::()); + let mut sandbox = None; + let mut other = None; + let result = async { + trust.install().await?; + sandbox = Some(create_rest_fixture(&name, token_port, port).await?); + let active = sandbox.as_ref().unwrap(); + failure_cases(active, port, &service, &deny_identity).await?; + if !observed.lock().unwrap().is_empty() || observed_bytes.load(Ordering::Relaxed) != 0 { + return Err("a refused grant forwarded protected request bytes".into()); + } + assert_untrusted_reader(&name, None).await?; + let bearer = supervisor_token(&trust.engine, &name).await?; + assert_untrusted_reader(&name, Some(&bearer)).await?; + let other_name = format!("other-{:08x}", rand::random::()); + other = Some( + SandboxGuard::create_keep_with_args( + &["--name", &other_name, "--no-auto-providers"], + &["sh", "-lc", &sandbox_script()], + "token-server-ready", + ) + .await?, + ); + assert_untrusted_reader(&other_name, Some(&bearer)).await?; + recovery_cases(active, port, &service).await?; + Ok::<(), String>(()) + } + .await; + if let Some(mut sandbox) = sandbox { + sandbox.cleanup().await; + } + if let Some(mut other) = other { + other.cleanup().await; + } + run_cli_ignore_error(&["sandbox", "delete", &name]).await; + run_cli_ignore_error(&["provider", "delete", &name, "--yes"]).await; + run_cli_ignore_error(&["profile", "delete", &name, "--yes"]).await; + let drained = target.drain().await; + tokens.shutdown().await; + provider_spiffe.shutdown().await; + gateway_spiffe.shutdown().await; + let restored = trust.restore().await; + result?; + drained?; + restored?; + let requests = observed.lock().unwrap(); + if requests.len() != 1 + || observed_bytes.load(Ordering::Relaxed) + != requests.iter().map(|request| request.bytes).sum::() + { + return Err("grant recovery reached an unexpected number of protected requests".into()); + } + Ok(()) +} diff --git a/proto/openshell.proto b/proto/openshell.proto index a4c32bcd42..f4186ea440 100644 --- a/proto/openshell.proto +++ b/proto/openshell.proto @@ -1203,6 +1203,10 @@ message SandboxStatus { google.protobuf.Timestamp next_restart_time = 15; // Time when the current main process became ready. google.protobuf.Timestamp main_process_started_time = 16; + // Latest refused token grant across all outbound HTTP protocols, including + // failures before a configured endpoint or credential could be selected. + // Matching grant success or a policy, provider, or supervisor reset clears it. + TokenGrantFailureObservation last_token_grant_failure = 17; } // User-facing sandbox condition derived from platform or gateway observations. @@ -3804,6 +3808,9 @@ message EndpointObservation { string endpoint_id = 1; // Latest result under the reported configuration and supervisor session. EndpointResult result = 2; + // Bounded cause of result, when the supervisor classified it. Each failure + // domain names the results it can explain. + EndpointFailure failure = 3; } // Complete endpoint status report for the caller's current configuration. @@ -3825,6 +3832,26 @@ message ReportEndpointStatusRequest { // are allowed when an inventory reset supersedes a frozen snapshot. Retrying // a report preserves its complete body and sequence for idempotent acknowledgement. uint64 report_sequence = 7; + // Complete latest grant state. Omission retains legacy compatibility; + // a present message with no last_failure explicitly clears the prior cause. + TokenGrantStatus token_grant_status = 8; +} + +// Latest token-grant state sent by the authenticated supervisor. +message TokenGrantStatus { + TokenGrantFailureObservation last_failure = 1; +} + +// A refused grant and its destination, without request paths or remote text. +// This describes the failed attempt, not the destination's current health. +message TokenGrantFailureObservation { + // Canonical observed host or IP address, bounded to 253 ASCII bytes. + string destination_host = 1; + // Observed destination port in the range 1 through 65535. + uint32 destination_port = 2; + // Bounded token_grant cause. Provider and credential may be absent when + // selection failed; the request ID still identifies the refused exchange. + EndpointFailure failure = 3; } // Empty acknowledgement for a persisted endpoint status report. @@ -3851,6 +3878,11 @@ message EndpointStatus { // time: still-valid evidence can be reaccepted after a reset. Identical // same-sequence retries do not advance it. Absent until a result is reported. google.protobuf.Timestamp last_reported_time = 106; + // Bounded cause of last_result, when the supervisor classified it. Each + // failure domain names the results it can explain. Replaced or cleared by + // every later observation, and cleared by the same resets that clear + // last_result. + EndpointFailure last_failure = 7; } // Durable provisioning attempt, independent of supervisor registration and polling. @@ -3911,3 +3943,40 @@ enum ServiceAuthorizationMode { // Forward one syntactically valid bearer Authorization header unchanged. SERVICE_AUTHORIZATION_MODE_BEARER_PASSTHROUGH = 2; } + +// Subsystem whose closed code set explains an endpoint failure. Each domain +// defines its codes, stages, recovery actions, and the endpoint results it +// can explain. Kept at the end of the file so adding it does not renumber +// existing generated descriptors. +enum EndpointFailureDomain { + ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED = 0; + // A dynamic token grant for a provider credential failed, so OpenShell + // refused the request before any of it reached the endpoint. Explains + // ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE. + ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT = 1; +} + +// Bounded cause of a failed endpoint observation, including failures before +// any request reaches the endpoint. No field carries remote +// text: code and stage come from the domain's closed sets, provider and +// credential_key are configured names, and request_id is an opaque random ID. +message EndpointFailure { + EndpointFailureDomain domain = 1; + // Stable lower_snake_case code from the domain's closed set, such as + // provider_authorization_required. + string code = 2; + // Who can resolve the failure; always the action documented for the code. + ProviderCredentialRefreshRecoveryAction recovery_action = 3; + // Provider whose credential failed. Empty when the failure happened before + // a credential was selected. + string provider = 4; + // Credential name within the provider. Never a credential value. + string credential_key = 5; + // Step that observed the failure, from the domain's closed set. + string stage = 6; + // HTTP status a token service returned, when one was received. + optional uint32 upstream_http_status = 7; + // Supervisor-minted UUID of the refused exchange. The workload receives the + // same value in the x-openshell-request-id response header. + string request_id = 8; +} diff --git a/sdk/go/docs/src/api/sandboxes.md b/sdk/go/docs/src/api/sandboxes.md index 6981adf65d..f94b294ce3 100644 --- a/sdk/go/docs/src/api/sandboxes.md +++ b/sdk/go/docs/src/api/sandboxes.md @@ -80,6 +80,35 @@ for _, endpoint := range sb.Status.EndpointStatuses { OpenShell observes real traffic passively and does not expire idle observations. Keep `LastReportedAt` visible when displaying a result: it records when the gateway accepted the observation, not when the request happened. Retained evidence can be accepted after a reset. `v1.EndpointNoObservedExchange` has an empty timestamp and retains the configured address, so the endpoint remains identifiable before traffic is observed or after evidence is invalidated. +`LastFailure` explains a `v1.EndpointCredentialUnavailable` result when the supervisor classified the cause, and is `nil` otherwise. Each later observation replaces or clears it. `Code` and `Stage` come from closed sets, `Provider` and `CredentialKey` name the configured credential, and no field carries remote text. `RequestID` matches the `x-openshell-request-id` header the workload received, so you can correlate the refused request. + +```go +for _, endpoint := range sb.Status.EndpointStatuses { + if f := endpoint.LastFailure; f != nil && f.Domain == v1.EndpointFailureDomainTokenGrant { + fmt.Printf("%s/%s: %s at %s (request %s)\n", + f.Provider, f.CredentialKey, f.Code, f.Stage, f.RequestID) + if f.RecoveryAction == v1.RefreshRecoveryActionReauthorize { + // Ask the user to replace the provider's OAuth grant. + } + } +} +``` + +### Token grant failures + +`sb.Status.LastTokenGrantFailure` gives the latest classified grant refusal for any destination, including REST requests without an MCP inventory entry. It carries `DestinationHost`, `DestinationPort`, and the same bounded `EndpointFailure` as endpoint status. + +```go +if observation := sb.Status.LastTokenGrantFailure; observation != nil && observation.Failure != nil { + f := observation.Failure + fmt.Printf("%s (port %d): %s, recovery %s, request %s\n", + observation.DestinationHost, observation.DestinationPort, + f.Code, f.RecoveryAction, f.RequestID) +} +``` + +Check `RequestID` against the refused response before acting: a newer failure can replace this record. A successful grant for the same destination, provider, and credential clears it; a failure before credential selection clears on the next successful grant to that destination. Policy, provider, and supervisor-session resets also clear it. `nil` means no retained cause. This is one latest observation, not a request history or a server health check. + ## List `List` constructs a lazy pager; `NextPage` fetches one page at a time. diff --git a/sdk/go/openshell/v1/fake/fake_test.go b/sdk/go/openshell/v1/fake/fake_test.go index cfc144edd1..d196ba0ed3 100644 --- a/sdk/go/openshell/v1/fake/fake_test.go +++ b/sdk/go/openshell/v1/fake/fake_test.go @@ -232,6 +232,75 @@ func TestFakeClient_SandboxEndpointStatuses_DeepCopy(t *testing.T) { assert.Equal(t, "2026-09-11T10:00:00Z", listed[0].Status.EndpointStatuses[0].LastReportedAt) } +func TestFakeClient_SandboxGrantFailures_DeepCopy(t *testing.T) { + fc := NewClient() + ctx := context.Background() + upstreamStatus := uint32(401) + sb := &types.Sandbox{ + Name: "grant-sandbox", + Status: types.SandboxStatus{EndpointStatuses: []types.EndpointStatus{{ + EndpointID: "endpoint-grant", + Host: "tools.example.test", + Ports: []uint32{443}, + Path: "/mcp", + LastResult: types.EndpointCredentialUnavailable, + LastFailure: &types.EndpointFailure{ + Domain: types.EndpointFailureDomainTokenGrant, + Code: "provider_authorization_required", + RecoveryAction: types.RefreshRecoveryActionReauthorize, + Provider: "github", + CredentialKey: "token", + Stage: "gateway_rpc", + UpstreamHTTPStatus: &upstreamStatus, + RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + }, + }}}, + } + sb.Status.LastTokenGrantFailure = &types.TokenGrantFailureObservation{ + DestinationHost: "api.example.test", + DestinationPort: 8443, + Failure: sb.Status.EndpointStatuses[0].LastFailure, + } + fc.AddSandbox("default", sb) + + // Neither the seed object nor returned snapshots may mutate the stored failure. + sb.Status.EndpointStatuses[0].LastFailure.Code = "seed_changed" + upstreamStatus = 500 + got, err := fc.Sandboxes().Get(ctx, "default", "grant-sandbox") + require.NoError(t, err) + failure := got.Status.EndpointStatuses[0].LastFailure + require.NotNil(t, failure) + assert.Equal(t, "provider_authorization_required", failure.Code) + require.NotNil(t, failure.UpstreamHTTPStatus) + assert.Equal(t, uint32(401), *failure.UpstreamHTTPStatus) + failure.Stage = "changed" + *failure.UpstreamHTTPStatus = 503 + observation := got.Status.LastTokenGrantFailure + require.NotNil(t, observation) + assert.Equal(t, "api.example.test", observation.DestinationHost) + require.NotNil(t, observation.Failure) + assert.Equal(t, "provider_authorization_required", observation.Failure.Code) + observation.DestinationHost = "changed.example.test" + observation.Failure.Code = "changed" + *observation.Failure.UpstreamHTTPStatus = 503 + + listed, err := fc.Sandboxes().ListAll(ctx, "default") + require.NoError(t, err) + require.Len(t, listed, 1) + stored := listed[0].Status.EndpointStatuses[0].LastFailure + require.NotNil(t, stored) + assert.Equal(t, "gateway_rpc", stored.Stage) + require.NotNil(t, stored.UpstreamHTTPStatus) + assert.Equal(t, uint32(401), *stored.UpstreamHTTPStatus) + storedObservation := listed[0].Status.LastTokenGrantFailure + require.NotNil(t, storedObservation) + assert.Equal(t, "api.example.test", storedObservation.DestinationHost) + assert.Equal(t, uint32(8443), storedObservation.DestinationPort) + require.NotNil(t, storedObservation.Failure) + assert.Equal(t, "provider_authorization_required", storedObservation.Failure.Code) + assert.Equal(t, uint32(401), *storedObservation.Failure.UpstreamHTTPStatus) +} + func TestFakeClient_AddProvider(t *testing.T) { fc := NewClient() ctx := context.Background() diff --git a/sdk/go/openshell/v1/fake/sandbox.go b/sdk/go/openshell/v1/fake/sandbox.go index 2932457c25..d5b591f312 100644 --- a/sdk/go/openshell/v1/fake/sandbox.go +++ b/sdk/go/openshell/v1/fake/sandbox.go @@ -237,10 +237,28 @@ func copySandboxStatus(s types.SandboxStatus) types.SandboxStatus { s.EndpointStatuses = slices.Clone(s.EndpointStatuses) for i := range s.EndpointStatuses { s.EndpointStatuses[i].Ports = slices.Clone(s.EndpointStatuses[i].Ports) + s.EndpointStatuses[i].LastFailure = copyEndpointFailure(s.EndpointStatuses[i].LastFailure) + } + if s.LastTokenGrantFailure != nil { + observation := *s.LastTokenGrantFailure + observation.Failure = copyEndpointFailure(observation.Failure) + s.LastTokenGrantFailure = &observation } return s } +func copyEndpointFailure(f *types.EndpointFailure) *types.EndpointFailure { + if f == nil { + return nil + } + cp := *f + if f.UpstreamHTTPStatus != nil { + status := *f.UpstreamHTTPStatus + cp.UpstreamHTTPStatus = &status + } + return &cp +} + // copyStringMap returns a shallow copy of a string-to-string map. func copyStringMap(m map[string]string) map[string]string { if m == nil { diff --git a/sdk/go/openshell/v1/internal/converter/coverage_test.go b/sdk/go/openshell/v1/internal/converter/coverage_test.go index 1c58c400b6..785e12c9c6 100644 --- a/sdk/go/openshell/v1/internal/converter/coverage_test.go +++ b/sdk/go/openshell/v1/internal/converter/coverage_test.go @@ -122,6 +122,7 @@ func TestConverterCoversAllProtoFields_SandboxStatus(t *testing.T) { "phase": true, "conditions": true, "endpoint_statuses": true, + "last_token_grant_failure": true, "current_policy_version": true, "exit_code": true, "configuration_admission": true, @@ -159,11 +160,36 @@ func TestConverterCoversAllProtoFields_EndpointStatus(t *testing.T) { "path": true, "last_result": true, "last_reported_time": true, + "last_failure": true, } assertAllFieldsCovered(t, (&pb.EndpointStatus{}).ProtoReflect().Descriptor(), handled, nil) } +func TestConverterCoversAllProtoFields_EndpointFailure(t *testing.T) { + handled := fieldSet{ + "domain": true, + "code": true, + "recovery_action": true, + "provider": true, + "credential_key": true, + "stage": true, + "upstream_http_status": true, + "request_id": true, + } + + assertAllFieldsCovered(t, (&pb.EndpointFailure{}).ProtoReflect().Descriptor(), handled, nil) +} + +func TestConverterCoversAllProtoFields_TokenGrantFailureObservation(t *testing.T) { + handled := fieldSet{ + "destination_host": true, + "destination_port": true, + "failure": true, + } + assertAllFieldsCovered(t, (&pb.TokenGrantFailureObservation{}).ProtoReflect().Descriptor(), handled, nil) +} + func TestConverterCoversAllProtoFields_SandboxPolicy(t *testing.T) { handled := fieldSet{ "version": true, diff --git a/sdk/go/openshell/v1/internal/converter/sandbox.go b/sdk/go/openshell/v1/internal/converter/sandbox.go index cacd18e964..e75eb41de3 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox.go @@ -122,8 +122,16 @@ func sandboxStatusFromProto(status *pb.SandboxStatus) types.SandboxStatus { Path: endpoint.GetPath(), LastResult: endpointResultFromProto(endpoint.GetLastResult()), LastReportedAt: TimestampStringFromProto(endpoint.GetLastReportedTime()), + LastFailure: endpointFailureFromProto(endpoint.GetLastFailure()), }) } + if observation := status.GetLastTokenGrantFailure(); observation != nil { + result.LastTokenGrantFailure = &types.TokenGrantFailureObservation{ + DestinationHost: observation.GetDestinationHost(), + DestinationPort: observation.GetDestinationPort(), + Failure: endpointFailureFromProto(observation.GetFailure()), + } + } result.ExitCode = CopyInt32Ptr(status.ExitCode) if admission := status.GetConfigurationAdmission(); admission != nil { state := types.ConfigurationAdmissionUnknown @@ -173,6 +181,32 @@ func endpointResultFromProto(result pb.EndpointResult) types.EndpointResult { } } +func endpointFailureFromProto(failure *pb.EndpointFailure) *types.EndpointFailure { + if failure == nil { + return nil + } + return &types.EndpointFailure{ + Domain: endpointFailureDomainFromProto(failure.GetDomain()), + Code: failure.GetCode(), + RecoveryAction: RefreshRecoveryActionFromProto(failure.GetRecoveryAction()), + Provider: failure.GetProvider(), + CredentialKey: failure.GetCredentialKey(), + Stage: failure.GetStage(), + UpstreamHTTPStatus: CopyUint32Ptr(failure.UpstreamHttpStatus), + RequestID: failure.GetRequestId(), + } +} + +func endpointFailureDomainFromProto(domain pb.EndpointFailureDomain) types.EndpointFailureDomain { + switch domain { + case pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT: + return types.EndpointFailureDomainTokenGrant + default: + // Unknown wire values must not select a closed code set. + return types.EndpointFailureDomainUnspecified + } +} + // SandboxPhaseFromProto converts a proto SandboxPhase to an SDK SandboxPhase. func SandboxPhaseFromProto(phase pb.SandboxPhase) types.SandboxPhase { switch phase { diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index ada0c6ee2e..5b6d7aaced 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -256,6 +256,148 @@ func TestSandboxFromProto_EndpointStatuses(t *testing.T) { assert.Equal(t, uint32(443), input.Status.EndpointStatuses[1].Ports[0]) } +func TestSandboxFromProto_EndpointLastFailure(t *testing.T) { + upstreamStatus := uint32(401) + reportedAt := timestamppb.New(time.Date(2026, 9, 11, 10, 0, 0, 0, time.UTC)) + input := &pb.Sandbox{Status: &pb.SandboxStatus{EndpointStatuses: []*pb.EndpointStatus{ + { + EndpointId: "endpoint-grant", Host: "tools.example.test", Ports: []uint32{443}, Path: "/mcp", + LastResult: pb.EndpointResult_ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE, + LastReportedTime: reportedAt, + LastFailure: &pb.EndpointFailure{ + Domain: pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, + Code: "provider_authorization_required", + RecoveryAction: pb.ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE, + Provider: "github", + CredentialKey: "token", + Stage: "gateway_rpc", + UpstreamHttpStatus: &upstreamStatus, + RequestId: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + }, + }, + { + EndpointId: "endpoint-plain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/other", + LastResult: pb.EndpointResult_ENDPOINT_RESULT_TRANSPORT_FAILED, + LastReportedTime: reportedAt, + }, + { + EndpointId: "endpoint-unknown-domain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/future", + LastResult: pb.EndpointResult_ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE, + LastFailure: &pb.EndpointFailure{ + Domain: pb.EndpointFailureDomain(99), + Code: "future_code", + Stage: "future_stage", + RequestId: "4c1d7a2e-0f3b-4e8a-b6a1-9f2c3d4e5f60", + }, + }, + }}} + + s := SandboxFromProto(input) + require.NotNil(t, s) + wantStatus := uint32(401) + require.Equal(t, []v1.EndpointStatus{ + { + EndpointID: "endpoint-grant", Host: "tools.example.test", Ports: []uint32{443}, Path: "/mcp", + LastResult: v1.EndpointCredentialUnavailable, LastReportedAt: "2026-09-11T10:00:00Z", + LastFailure: &v1.EndpointFailure{ + Domain: v1.EndpointFailureDomainTokenGrant, + Code: "provider_authorization_required", + RecoveryAction: v1.RefreshRecoveryActionReauthorize, + Provider: "github", + CredentialKey: "token", + Stage: "gateway_rpc", + UpstreamHTTPStatus: &wantStatus, + RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + }, + }, + { + EndpointID: "endpoint-plain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/other", + LastResult: v1.EndpointTransportFailed, LastReportedAt: "2026-09-11T10:00:00Z", + }, + { + EndpointID: "endpoint-unknown-domain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/future", + LastResult: v1.EndpointCredentialUnavailable, + LastFailure: &v1.EndpointFailure{ + Domain: v1.EndpointFailureDomainUnspecified, + Code: "future_code", + Stage: "future_stage", + RequestID: "4c1d7a2e-0f3b-4e8a-b6a1-9f2c3d4e5f60", + }, + }, + }, s.Status.EndpointStatuses) + assert.Nil(t, s.Status.EndpointStatuses[1].LastFailure) + assert.Nil(t, s.Status.EndpointStatuses[2].LastFailure.UpstreamHTTPStatus) + + // The SDK failure must not share the wire message or its optional status. + input.Status.EndpointStatuses[0].LastFailure.Code = "changed" + upstreamStatus = 500 + assert.Equal(t, "provider_authorization_required", s.Status.EndpointStatuses[0].LastFailure.Code) + assert.Equal(t, uint32(401), *s.Status.EndpointStatuses[0].LastFailure.UpstreamHTTPStatus) + *s.Status.EndpointStatuses[0].LastFailure.UpstreamHTTPStatus = 503 + assert.Equal(t, uint32(500), *input.Status.EndpointStatuses[0].LastFailure.UpstreamHttpStatus) +} + +func TestSandboxFromProto_TokenGrantFailureOutsideEndpointInventory(t *testing.T) { + upstreamStatus := uint32(409) + input := &pb.Sandbox{Status: &pb.SandboxStatus{ + LastTokenGrantFailure: &pb.TokenGrantFailureObservation{ + DestinationHost: "api.example.test", + DestinationPort: 8443, + Failure: &pb.EndpointFailure{ + Domain: pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, + Code: "provider_authorization_required", + RecoveryAction: pb.ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE, + Provider: "github", + CredentialKey: "token", + Stage: "token_endpoint", + UpstreamHttpStatus: &upstreamStatus, + RequestId: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + }, + }, + }} + converted := SandboxFromProto(input) + require.Empty(t, converted.Status.EndpointStatuses) + wantStatus := uint32(409) + require.Equal(t, &v1.TokenGrantFailureObservation{ + DestinationHost: "api.example.test", + DestinationPort: 8443, + Failure: &v1.EndpointFailure{ + Domain: v1.EndpointFailureDomainTokenGrant, + Code: "provider_authorization_required", + RecoveryAction: v1.RefreshRecoveryActionReauthorize, + Provider: "github", + CredentialKey: "token", + Stage: "token_endpoint", + UpstreamHTTPStatus: &wantStatus, + RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + }, + }, converted.Status.LastTokenGrantFailure) + input.Status.LastTokenGrantFailure.DestinationHost = "changed.example.test" + input.Status.LastTokenGrantFailure.Failure.Code = "changed" + upstreamStatus = 500 + assert.Equal(t, "api.example.test", converted.Status.LastTokenGrantFailure.DestinationHost) + assert.Equal(t, "provider_authorization_required", converted.Status.LastTokenGrantFailure.Failure.Code) + assert.Equal(t, uint32(409), *converted.Status.LastTokenGrantFailure.Failure.UpstreamHTTPStatus) + input.Status.LastTokenGrantFailure = nil + assert.Nil(t, SandboxFromProto(input).Status.LastTokenGrantFailure) +} + +func TestEndpointFailureDomainFromProto(t *testing.T) { + for _, tc := range []struct { + input pb.EndpointFailureDomain + want v1.EndpointFailureDomain + }{ + {pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED, v1.EndpointFailureDomainUnspecified}, + {pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, v1.EndpointFailureDomainTokenGrant}, + {pb.EndpointFailureDomain(99), v1.EndpointFailureDomainUnspecified}, + } { + t.Run(tc.input.String(), func(t *testing.T) { + assert.Equal(t, tc.want, endpointFailureDomainFromProto(tc.input)) + }) + } + assert.Nil(t, endpointFailureFromProto(nil)) +} + func TestEndpointResultFromProto(t *testing.T) { for _, tc := range []struct { input pb.EndpointResult diff --git a/sdk/go/openshell/v1/refresh.go b/sdk/go/openshell/v1/refresh.go index c676697fb4..2a8a949752 100644 --- a/sdk/go/openshell/v1/refresh.go +++ b/sdk/go/openshell/v1/refresh.go @@ -27,6 +27,18 @@ const ( RefreshStrategyGoogleServiceAccountJWT = types.RefreshStrategyGoogleServiceAccountJWT ) +// RefreshRecoveryAction describes the action required after a credential failure. +type RefreshRecoveryAction = types.RefreshRecoveryAction + +// RefreshRecoveryAction values. +const ( + RefreshRecoveryActionUnspecified = types.RefreshRecoveryActionUnspecified + RefreshRecoveryActionRetry = types.RefreshRecoveryActionRetry + RefreshRecoveryActionReauthorize = types.RefreshRecoveryActionReauthorize + RefreshRecoveryActionFixConfiguration = types.RefreshRecoveryActionFixConfiguration + RefreshRecoveryActionInvestigate = types.RefreshRecoveryActionInvestigate +) + // RefreshInterface defines operations for managing provider credential refresh. type RefreshInterface interface { GetStatus(ctx context.Context, workspace, provider, credentialKey string) ([]*RefreshStatus, error) diff --git a/sdk/go/openshell/v1/sandbox.go b/sdk/go/openshell/v1/sandbox.go index 02daad876b..7c558f209a 100644 --- a/sdk/go/openshell/v1/sandbox.go +++ b/sdk/go/openshell/v1/sandbox.go @@ -42,6 +42,21 @@ const ( EndpointUpstreamRejected = types.EndpointUpstreamRejected ) +// EndpointFailure explains a failed endpoint result with bounded values. +type EndpointFailure = types.EndpointFailure + +// TokenGrantFailureObservation records a refused grant's destination and cause. +type TokenGrantFailureObservation = types.TokenGrantFailureObservation + +// EndpointFailureDomain names the subsystem that explains an endpoint failure. +type EndpointFailureDomain = types.EndpointFailureDomain + +// EndpointFailureDomain values select the closed set that Code and Stage come from. +const ( + EndpointFailureDomainUnspecified = types.EndpointFailureDomainUnspecified + EndpointFailureDomainTokenGrant = types.EndpointFailureDomainTokenGrant +) + // AttachProviderResult holds the result of attaching a provider to a sandbox. type AttachProviderResult = types.AttachProviderResult diff --git a/sdk/go/openshell/v1/types/sandbox.go b/sdk/go/openshell/v1/types/sandbox.go index 8e57c14f8e..f1cedb595f 100644 --- a/sdk/go/openshell/v1/types/sandbox.go +++ b/sdk/go/openshell/v1/types/sandbox.go @@ -124,13 +124,28 @@ type SandboxStatus struct { ExitCode *int32 // EndpointStatuses describes configured external tool endpoints and their // last accepted network results, independently of sandbox readiness. - EndpointStatuses []EndpointStatus + EndpointStatuses []EndpointStatus + // LastTokenGrantFailure is the latest classified grant refusal for any + // destination, including requests outside the MCP endpoint inventory. + // A successful grant for the same destination and provider/credential clears + // it. Before credential selection, any successful grant to that destination + // clears it. Policy, provider, and session resets also clear it. + // Nil means no retained cause. + LastTokenGrantFailure *TokenGrantFailureObservation ConfigurationAdmission *SandboxConfigurationAdmission RestartCount uint32 NextRestartAtMs int64 MainProcessStartedAtMs int64 } +// TokenGrantFailureObservation records a refused grant's destination and cause. +// It describes the attempted call, not the remote server's current health. +type TokenGrantFailureObservation struct { + DestinationHost string + DestinationPort uint32 + Failure *EndpointFailure +} + // ConfigurationAdmissionState describes validation of an effective configuration. type ConfigurationAdmissionState string @@ -167,8 +182,55 @@ type EndpointStatus struct { // observation, not the request time. Retained evidence can be accepted after // a reset. NoObservedExchange has no report timestamp. LastReportedAt string + // LastFailure is the bounded cause of LastResult, when the supervisor + // classified it. Each failure domain names the results it explains; token + // grant failures accompany EndpointCredentialUnavailable. Each later + // observation replaces or clears it, and the resets that clear LastResult + // also clear it. Nil means no classified cause. + LastFailure *EndpointFailure +} + +// EndpointFailure explains why an observed exchange failed. It does not +// establish that the endpoint is unhealthy. No field carries remote text. +type EndpointFailure struct { + // Domain names the subsystem whose closed code set explains the failure. + Domain EndpointFailureDomain + // Code is a stable lower_snake_case code from the domain's closed set, + // such as provider_authorization_required. + Code string + // RecoveryAction is the action documented for Code. + RecoveryAction RefreshRecoveryAction + // Provider is the configured provider whose credential failed. It is empty + // when the failure happened before a credential was selected. + Provider string + // CredentialKey is the configured credential name within Provider, never a + // credential value. It is empty when Provider is empty. + CredentialKey string + // Stage is the step that observed the failure, from the domain's closed + // set, such as gateway_rpc. + Stage string + // UpstreamHTTPStatus is the HTTP status a token service returned. Nil means + // no status was received. + UpstreamHTTPStatus *uint32 + // RequestID is the supervisor-minted UUID of the refused exchange. The + // workload receives the same value in the x-openshell-request-id response + // header. + RequestID string } +// EndpointFailureDomain names the subsystem that explains an endpoint failure. +type EndpointFailureDomain string + +// EndpointFailureDomain values select the closed set that Code and Stage come from. +const ( + // EndpointFailureDomainUnspecified means the domain was absent or was not recognized. + EndpointFailureDomainUnspecified EndpointFailureDomain = "Unspecified" + // EndpointFailureDomainTokenGrant means a dynamic token grant for a provider + // credential failed, so OpenShell refused the request before any of it + // reached the endpoint. + EndpointFailureDomainTokenGrant EndpointFailureDomain = "TokenGrant" +) + // EndpointResult classifies the last accepted network result for a tool endpoint. type EndpointResult string diff --git a/sdk/go/proto/openshellv1/openshell.pb.go b/sdk/go/proto/openshellv1/openshell.pb.go index 8a3f7a78d1..68a550bdec 100644 --- a/sdk/go/proto/openshellv1/openshell.pb.go +++ b/sdk/go/proto/openshellv1/openshell.pb.go @@ -1251,6 +1251,59 @@ func (ServiceAuthorizationMode) EnumDescriptor() ([]byte, []int) { return file_openshell_proto_rawDescGZIP(), []int{19} } +// Subsystem whose closed code set explains an endpoint failure. Each domain +// defines its codes, stages, recovery actions, and the endpoint results it +// can explain. Kept at the end of the file so adding it does not renumber +// existing generated descriptors. +type EndpointFailureDomain int32 + +const ( + EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED EndpointFailureDomain = 0 + // A dynamic token grant for a provider credential failed, so OpenShell + // refused the request before any of it reached the endpoint. Explains + // ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE. + EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT EndpointFailureDomain = 1 +) + +// Enum value maps for EndpointFailureDomain. +var ( + EndpointFailureDomain_name = map[int32]string{ + 0: "ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED", + 1: "ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT", + } + EndpointFailureDomain_value = map[string]int32{ + "ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED": 0, + "ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT": 1, + } +) + +func (x EndpointFailureDomain) Enum() *EndpointFailureDomain { + p := new(EndpointFailureDomain) + *p = x + return p +} + +func (x EndpointFailureDomain) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (EndpointFailureDomain) Descriptor() protoreflect.EnumDescriptor { + return file_openshell_proto_enumTypes[20].Descriptor() +} + +func (EndpointFailureDomain) Type() protoreflect.EnumType { + return &file_openshell_proto_enumTypes[20] +} + +func (x EndpointFailureDomain) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use EndpointFailureDomain.Descriptor instead. +func (EndpointFailureDomain) EnumDescriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{20} +} + // IssueSandboxToken request. Empty body; identity is established by the // authentication credentials carried in the request headers (a projected // Kubernetes ServiceAccount JWT in the K8s driver path). @@ -3137,8 +3190,12 @@ type SandboxStatus struct { NextRestartTime *timestamppb.Timestamp `protobuf:"bytes,15,opt,name=next_restart_time,json=nextRestartTime,proto3" json:"next_restart_time,omitempty"` // Time when the current main process became ready. MainProcessStartedTime *timestamppb.Timestamp `protobuf:"bytes,16,opt,name=main_process_started_time,json=mainProcessStartedTime,proto3" json:"main_process_started_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Latest refused token grant across all outbound HTTP protocols, including + // failures before a configured endpoint or credential could be selected. + // Matching grant success or a policy, provider, or supervisor reset clears it. + LastTokenGrantFailure *TokenGrantFailureObservation `protobuf:"bytes,17,opt,name=last_token_grant_failure,json=lastTokenGrantFailure,proto3" json:"last_token_grant_failure,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SandboxStatus) Reset() { @@ -3276,6 +3333,13 @@ func (x *SandboxStatus) GetMainProcessStartedTime() *timestamppb.Timestamp { return nil } +func (x *SandboxStatus) GetLastTokenGrantFailure() *TokenGrantFailureObservation { + if x != nil { + return x.LastTokenGrantFailure + } + return nil +} + // User-facing sandbox condition derived from platform or gateway observations. type SandboxCondition struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -17708,7 +17772,10 @@ type EndpointObservation struct { // Stable identifier derived from the configured host, ports, and path. EndpointId string `protobuf:"bytes,1,opt,name=endpoint_id,json=endpointId,proto3" json:"endpoint_id,omitempty"` // Latest result under the reported configuration and supervisor session. - Result EndpointResult `protobuf:"varint,2,opt,name=result,proto3,enum=openshell.v1.EndpointResult" json:"result,omitempty"` + Result EndpointResult `protobuf:"varint,2,opt,name=result,proto3,enum=openshell.v1.EndpointResult" json:"result,omitempty"` + // Bounded cause of result, when the supervisor classified it. Each failure + // domain names the results it can explain. + Failure *EndpointFailure `protobuf:"bytes,3,opt,name=failure,proto3" json:"failure,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -17757,6 +17824,13 @@ func (x *EndpointObservation) GetResult() EndpointResult { return EndpointResult_ENDPOINT_RESULT_UNSPECIFIED } +func (x *EndpointObservation) GetFailure() *EndpointFailure { + if x != nil { + return x.Failure + } + return nil +} + // Complete endpoint status report for the caller's current configuration. type ReportEndpointStatusRequest struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -17777,8 +17851,11 @@ type ReportEndpointStatusRequest struct { // are allowed when an inventory reset supersedes a frozen snapshot. Retrying // a report preserves its complete body and sequence for idempotent acknowledgement. ReportSequence uint64 `protobuf:"varint,7,opt,name=report_sequence,json=reportSequence,proto3" json:"report_sequence,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Complete latest grant state. Omission retains legacy compatibility; + // a present message with no last_failure explicitly clears the prior cause. + TokenGrantStatus *TokenGrantStatus `protobuf:"bytes,8,opt,name=token_grant_status,json=tokenGrantStatus,proto3" json:"token_grant_status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ReportEndpointStatusRequest) Reset() { @@ -17860,6 +17937,124 @@ func (x *ReportEndpointStatusRequest) GetReportSequence() uint64 { return 0 } +func (x *ReportEndpointStatusRequest) GetTokenGrantStatus() *TokenGrantStatus { + if x != nil { + return x.TokenGrantStatus + } + return nil +} + +// Latest token-grant state sent by the authenticated supervisor. +type TokenGrantStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + LastFailure *TokenGrantFailureObservation `protobuf:"bytes,1,opt,name=last_failure,json=lastFailure,proto3" json:"last_failure,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TokenGrantStatus) Reset() { + *x = TokenGrantStatus{} + mi := &file_openshell_proto_msgTypes[232] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TokenGrantStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TokenGrantStatus) ProtoMessage() {} + +func (x *TokenGrantStatus) ProtoReflect() protoreflect.Message { + mi := &file_openshell_proto_msgTypes[232] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TokenGrantStatus.ProtoReflect.Descriptor instead. +func (*TokenGrantStatus) Descriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{232} +} + +func (x *TokenGrantStatus) GetLastFailure() *TokenGrantFailureObservation { + if x != nil { + return x.LastFailure + } + return nil +} + +// A refused grant and its destination, without request paths or remote text. +// This describes the failed attempt, not the destination's current health. +type TokenGrantFailureObservation struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Canonical observed host or IP address, bounded to 253 ASCII bytes. + DestinationHost string `protobuf:"bytes,1,opt,name=destination_host,json=destinationHost,proto3" json:"destination_host,omitempty"` + // Observed destination port in the range 1 through 65535. + DestinationPort uint32 `protobuf:"varint,2,opt,name=destination_port,json=destinationPort,proto3" json:"destination_port,omitempty"` + // Bounded token_grant cause. Provider and credential may be absent when + // selection failed; the request ID still identifies the refused exchange. + Failure *EndpointFailure `protobuf:"bytes,3,opt,name=failure,proto3" json:"failure,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TokenGrantFailureObservation) Reset() { + *x = TokenGrantFailureObservation{} + mi := &file_openshell_proto_msgTypes[233] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TokenGrantFailureObservation) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TokenGrantFailureObservation) ProtoMessage() {} + +func (x *TokenGrantFailureObservation) ProtoReflect() protoreflect.Message { + mi := &file_openshell_proto_msgTypes[233] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TokenGrantFailureObservation.ProtoReflect.Descriptor instead. +func (*TokenGrantFailureObservation) Descriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{233} +} + +func (x *TokenGrantFailureObservation) GetDestinationHost() string { + if x != nil { + return x.DestinationHost + } + return "" +} + +func (x *TokenGrantFailureObservation) GetDestinationPort() uint32 { + if x != nil { + return x.DestinationPort + } + return 0 +} + +func (x *TokenGrantFailureObservation) GetFailure() *EndpointFailure { + if x != nil { + return x.Failure + } + return nil +} + // Empty acknowledgement for a persisted endpoint status report. type ReportEndpointStatusResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -17869,7 +18064,7 @@ type ReportEndpointStatusResponse struct { func (x *ReportEndpointStatusResponse) Reset() { *x = ReportEndpointStatusResponse{} - mi := &file_openshell_proto_msgTypes[232] + mi := &file_openshell_proto_msgTypes[234] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -17881,7 +18076,7 @@ func (x *ReportEndpointStatusResponse) String() string { func (*ReportEndpointStatusResponse) ProtoMessage() {} func (x *ReportEndpointStatusResponse) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[232] + mi := &file_openshell_proto_msgTypes[234] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -17894,7 +18089,7 @@ func (x *ReportEndpointStatusResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ReportEndpointStatusResponse.ProtoReflect.Descriptor instead. func (*ReportEndpointStatusResponse) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{232} + return file_openshell_proto_rawDescGZIP(), []int{234} } // A configured endpoint and its last accepted network result in one record. @@ -17917,13 +18112,18 @@ type EndpointStatus struct { // time: still-valid evidence can be reaccepted after a reset. Identical // same-sequence retries do not advance it. Absent until a result is reported. LastReportedTime *timestamppb.Timestamp `protobuf:"bytes,106,opt,name=last_reported_time,json=lastReportedTime,proto3" json:"last_reported_time,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Bounded cause of last_result, when the supervisor classified it. Each + // failure domain names the results it can explain. Replaced or cleared by + // every later observation, and cleared by the same resets that clear + // last_result. + LastFailure *EndpointFailure `protobuf:"bytes,7,opt,name=last_failure,json=lastFailure,proto3" json:"last_failure,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EndpointStatus) Reset() { *x = EndpointStatus{} - mi := &file_openshell_proto_msgTypes[233] + mi := &file_openshell_proto_msgTypes[235] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -17935,7 +18135,7 @@ func (x *EndpointStatus) String() string { func (*EndpointStatus) ProtoMessage() {} func (x *EndpointStatus) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[233] + mi := &file_openshell_proto_msgTypes[235] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -17948,7 +18148,7 @@ func (x *EndpointStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use EndpointStatus.ProtoReflect.Descriptor instead. func (*EndpointStatus) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{233} + return file_openshell_proto_rawDescGZIP(), []int{235} } func (x *EndpointStatus) GetEndpointId() string { @@ -17993,6 +18193,13 @@ func (x *EndpointStatus) GetLastReportedTime() *timestamppb.Timestamp { return nil } +func (x *EndpointStatus) GetLastFailure() *EndpointFailure { + if x != nil { + return x.LastFailure + } + return nil +} + // Durable provisioning attempt, independent of supervisor registration and polling. type SandboxProvisioning struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -18036,7 +18243,7 @@ type SandboxProvisioning struct { func (x *SandboxProvisioning) Reset() { *x = SandboxProvisioning{} - mi := &file_openshell_proto_msgTypes[234] + mi := &file_openshell_proto_msgTypes[236] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -18048,7 +18255,7 @@ func (x *SandboxProvisioning) String() string { func (*SandboxProvisioning) ProtoMessage() {} func (x *SandboxProvisioning) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[234] + mi := &file_openshell_proto_msgTypes[236] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -18061,7 +18268,7 @@ func (x *SandboxProvisioning) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxProvisioning.ProtoReflect.Descriptor instead. func (*SandboxProvisioning) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{234} + return file_openshell_proto_rawDescGZIP(), []int{236} } func (x *SandboxProvisioning) GetAttemptId() string { @@ -18184,7 +18391,7 @@ type SandboxServiceExposure struct { func (x *SandboxServiceExposure) Reset() { *x = SandboxServiceExposure{} - mi := &file_openshell_proto_msgTypes[235] + mi := &file_openshell_proto_msgTypes[237] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -18196,7 +18403,7 @@ func (x *SandboxServiceExposure) String() string { func (*SandboxServiceExposure) ProtoMessage() {} func (x *SandboxServiceExposure) ProtoReflect() protoreflect.Message { - mi := &file_openshell_proto_msgTypes[235] + mi := &file_openshell_proto_msgTypes[237] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -18209,7 +18416,7 @@ func (x *SandboxServiceExposure) ProtoReflect() protoreflect.Message { // Deprecated: Use SandboxServiceExposure.ProtoReflect.Descriptor instead. func (*SandboxServiceExposure) Descriptor() ([]byte, []int) { - return file_openshell_proto_rawDescGZIP(), []int{235} + return file_openshell_proto_rawDescGZIP(), []int{237} } func (x *SandboxServiceExposure) GetService() string { @@ -18233,6 +18440,120 @@ func (x *SandboxServiceExposure) GetAuthorizationMode() ServiceAuthorizationMode return ServiceAuthorizationMode_SERVICE_AUTHORIZATION_MODE_UNSPECIFIED } +// Bounded cause of a failed endpoint observation, including failures before +// any request reaches the endpoint. No field carries remote +// text: code and stage come from the domain's closed sets, provider and +// credential_key are configured names, and request_id is an opaque random ID. +type EndpointFailure struct { + state protoimpl.MessageState `protogen:"open.v1"` + Domain EndpointFailureDomain `protobuf:"varint,1,opt,name=domain,proto3,enum=openshell.v1.EndpointFailureDomain" json:"domain,omitempty"` + // Stable lower_snake_case code from the domain's closed set, such as + // provider_authorization_required. + Code string `protobuf:"bytes,2,opt,name=code,proto3" json:"code,omitempty"` + // Who can resolve the failure; always the action documented for the code. + RecoveryAction ProviderCredentialRefreshRecoveryAction `protobuf:"varint,3,opt,name=recovery_action,json=recoveryAction,proto3,enum=openshell.v1.ProviderCredentialRefreshRecoveryAction" json:"recovery_action,omitempty"` + // Provider whose credential failed. Empty when the failure happened before + // a credential was selected. + Provider string `protobuf:"bytes,4,opt,name=provider,proto3" json:"provider,omitempty"` + // Credential name within the provider. Never a credential value. + CredentialKey string `protobuf:"bytes,5,opt,name=credential_key,json=credentialKey,proto3" json:"credential_key,omitempty"` + // Step that observed the failure, from the domain's closed set. + Stage string `protobuf:"bytes,6,opt,name=stage,proto3" json:"stage,omitempty"` + // HTTP status a token service returned, when one was received. + UpstreamHttpStatus *uint32 `protobuf:"varint,7,opt,name=upstream_http_status,json=upstreamHttpStatus,proto3,oneof" json:"upstream_http_status,omitempty"` + // Supervisor-minted UUID of the refused exchange. The workload receives the + // same value in the x-openshell-request-id response header. + RequestId string `protobuf:"bytes,8,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EndpointFailure) Reset() { + *x = EndpointFailure{} + mi := &file_openshell_proto_msgTypes[238] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EndpointFailure) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EndpointFailure) ProtoMessage() {} + +func (x *EndpointFailure) ProtoReflect() protoreflect.Message { + mi := &file_openshell_proto_msgTypes[238] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EndpointFailure.ProtoReflect.Descriptor instead. +func (*EndpointFailure) Descriptor() ([]byte, []int) { + return file_openshell_proto_rawDescGZIP(), []int{238} +} + +func (x *EndpointFailure) GetDomain() EndpointFailureDomain { + if x != nil { + return x.Domain + } + return EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED +} + +func (x *EndpointFailure) GetCode() string { + if x != nil { + return x.Code + } + return "" +} + +func (x *EndpointFailure) GetRecoveryAction() ProviderCredentialRefreshRecoveryAction { + if x != nil { + return x.RecoveryAction + } + return ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_UNSPECIFIED +} + +func (x *EndpointFailure) GetProvider() string { + if x != nil { + return x.Provider + } + return "" +} + +func (x *EndpointFailure) GetCredentialKey() string { + if x != nil { + return x.CredentialKey + } + return "" +} + +func (x *EndpointFailure) GetStage() string { + if x != nil { + return x.Stage + } + return "" +} + +func (x *EndpointFailure) GetUpstreamHttpStatus() uint32 { + if x != nil && x.UpstreamHttpStatus != nil { + return *x.UpstreamHttpStatus + } + return 0 +} + +func (x *EndpointFailure) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" +} + var File_openshell_proto protoreflect.FileDescriptor const file_openshell_proto_rawDesc = "" + @@ -18374,7 +18695,7 @@ const file_openshell_proto_rawDesc = "" + "\tmax_burst\x18\x02 \x01(\rR\bmaxBurst\"b\n" + "!SandboxWorkloadTemplateProvenance\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12)\n" + - "\x10resource_version\x18\x02 \x01(\tR\x0fresourceVersion\"\x8d\a\n" + + "\x10resource_version\x18\x02 \x01(\tR\x0fresourceVersion\"\xf2\a\n" + "\rSandboxStatus\x12\x1b\n" + "\tagent_pod\x18\x02 \x01(\tR\bagentPod\x12\x19\n" + "\bagent_fd\x18\x03 \x01(\tR\aagentFd\x12\x1d\n" + @@ -18394,7 +18715,8 @@ const file_openshell_proto_rawDesc = "" + "\fprovisioning\x18\r \x01(\v2!.openshell.v1.SandboxProvisioningR\fprovisioning\x12#\n" + "\rrestart_count\x18\x0e \x01(\rR\frestartCount\x12F\n" + "\x11next_restart_time\x18\x0f \x01(\v2\x1a.google.protobuf.TimestampR\x0fnextRestartTime\x12U\n" + - "\x19main_process_started_time\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\x16mainProcessStartedTimeB\f\n" + + "\x19main_process_started_time\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\x16mainProcessStartedTime\x12c\n" + + "\x18last_token_grant_failure\x18\x11 \x01(\v2*.openshell.v1.TokenGrantFailureObservationR\x15lastTokenGrantFailureB\f\n" + "\n" + "_exit_codeB\x1a\n" + "\x18_configuration_activated\"\xd1\x01\n" + @@ -19555,11 +19877,12 @@ const file_openshell_proto_rawDesc = "" + "\x1aExtensionServiceCredential\x12!\n" + "\fservice_name\x18\x01 \x01(\tR\vserviceName\x12\x1a\n" + "\x05token\x18\x02 \x01(\tB\x04\x88\xb5\x18\x01R\x05token\x12C\n" + - "\x0fexpiration_time\x18g \x01(\v2\x1a.google.protobuf.TimestampR\x0eexpirationTimeJ\x04\b\x03\x10\x04R\rexpires_at_ms\"l\n" + + "\x0fexpiration_time\x18g \x01(\v2\x1a.google.protobuf.TimestampR\x0eexpirationTimeJ\x04\b\x03\x10\x04R\rexpires_at_ms\"\xa5\x01\n" + "\x13EndpointObservation\x12\x1f\n" + "\vendpoint_id\x18\x01 \x01(\tR\n" + "endpointId\x124\n" + - "\x06result\x18\x02 \x01(\x0e2\x1c.openshell.v1.EndpointResultR\x06result\"\xe9\x02\n" + + "\x06result\x18\x02 \x01(\x0e2\x1c.openshell.v1.EndpointResultR\x06result\x127\n" + + "\afailure\x18\x03 \x01(\v2\x1d.openshell.v1.EndpointFailureR\afailure\"\xb7\x03\n" + "\x1bReportEndpointStatusRequest\x12\x1d\n" + "\n" + "sandbox_id\x18\x01 \x01(\tR\tsandboxId\x12\x1f\n" + @@ -19569,8 +19892,15 @@ const file_openshell_proto_rawDesc = "" + "\fobservations\x18\x04 \x03(\v2!.openshell.v1.EndpointObservationR\fobservations\x122\n" + "\x15observed_endpoint_ids\x18\x05 \x03(\tR\x13observedEndpointIds\x122\n" + "\x15supervisor_session_id\x18\x06 \x01(\tR\x13supervisorSessionId\x12'\n" + - "\x0freport_sequence\x18\a \x01(\x04R\x0ereportSequence\"\x1e\n" + - "\x1cReportEndpointStatusResponse\"\x90\x02\n" + + "\x0freport_sequence\x18\a \x01(\x04R\x0ereportSequence\x12L\n" + + "\x12token_grant_status\x18\b \x01(\v2\x1e.openshell.v1.TokenGrantStatusR\x10tokenGrantStatus\"a\n" + + "\x10TokenGrantStatus\x12M\n" + + "\flast_failure\x18\x01 \x01(\v2*.openshell.v1.TokenGrantFailureObservationR\vlastFailure\"\xad\x01\n" + + "\x1cTokenGrantFailureObservation\x12)\n" + + "\x10destination_host\x18\x01 \x01(\tR\x0fdestinationHost\x12)\n" + + "\x10destination_port\x18\x02 \x01(\rR\x0fdestinationPort\x127\n" + + "\afailure\x18\x03 \x01(\v2\x1d.openshell.v1.EndpointFailureR\afailure\"\x1e\n" + + "\x1cReportEndpointStatusResponse\"\xd2\x02\n" + "\x0eEndpointStatus\x12\x1f\n" + "\vendpoint_id\x18\x01 \x01(\tR\n" + "endpointId\x12\x12\n" + @@ -19579,7 +19909,8 @@ const file_openshell_proto_rawDesc = "" + "\x04path\x18\x04 \x01(\tR\x04path\x12=\n" + "\vlast_result\x18\x05 \x01(\x0e2\x1c.openshell.v1.EndpointResultR\n" + "lastResult\x12H\n" + - "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTimeJ\x04\b\x06\x10\aR\x10last_reported_at\"\xd5\a\n" + + "\x12last_reported_time\x18j \x01(\v2\x1a.google.protobuf.TimestampR\x10lastReportedTime\x12@\n" + + "\flast_failure\x18\a \x01(\v2\x1d.openshell.v1.EndpointFailureR\vlastFailureJ\x04\b\x06\x10\aR\x10last_reported_at\"\xd5\a\n" + "\x13SandboxProvisioning\x12\x1d\n" + "\n" + "attempt_id\x18\x01 \x01(\tR\tattemptId\x126\n" + @@ -19602,7 +19933,18 @@ const file_openshell_proto_rawDesc = "" + "\aservice\x18\x01 \x01(\tR\aservice\x12\x1f\n" + "\vtarget_port\x18\x02 \x01(\rR\n" + "targetPort\x12U\n" + - "\x12authorization_mode\x18\x03 \x01(\x0e2&.openshell.v1.ServiceAuthorizationModeR\x11authorizationMode*\xca\x01\n" + + "\x12authorization_mode\x18\x03 \x01(\x0e2&.openshell.v1.ServiceAuthorizationModeR\x11authorizationMode\"\x8a\x03\n" + + "\x0fEndpointFailure\x12;\n" + + "\x06domain\x18\x01 \x01(\x0e2#.openshell.v1.EndpointFailureDomainR\x06domain\x12\x12\n" + + "\x04code\x18\x02 \x01(\tR\x04code\x12^\n" + + "\x0frecovery_action\x18\x03 \x01(\x0e25.openshell.v1.ProviderCredentialRefreshRecoveryActionR\x0erecoveryAction\x12\x1a\n" + + "\bprovider\x18\x04 \x01(\tR\bprovider\x12%\n" + + "\x0ecredential_key\x18\x05 \x01(\tR\rcredentialKey\x12\x14\n" + + "\x05stage\x18\x06 \x01(\tR\x05stage\x125\n" + + "\x14upstream_http_status\x18\a \x01(\rH\x00R\x12upstreamHttpStatus\x88\x01\x01\x12\x1d\n" + + "\n" + + "request_id\x18\b \x01(\tR\trequestIdB\x17\n" + + "\x15_upstream_http_status*\xca\x01\n" + "\rExtensionKind\x12\x1e\n" + "\x1aEXTENSION_KIND_UNSPECIFIED\x10\x00\x12!\n" + "\x1dEXTENSION_KIND_COMPUTE_DRIVER\x10\x01\x12$\n" + @@ -19746,7 +20088,10 @@ const file_openshell_proto_rawDesc = "" + "\x18ServiceAuthorizationMode\x12*\n" + "&SERVICE_AUTHORIZATION_MODE_UNSPECIFIED\x10\x00\x12$\n" + " SERVICE_AUTHORIZATION_MODE_STRIP\x10\x01\x121\n" + - "-SERVICE_AUTHORIZATION_MODE_BEARER_PASSTHROUGH\x10\x022\xb5V\n" + + "-SERVICE_AUTHORIZATION_MODE_BEARER_PASSTHROUGH\x10\x02*i\n" + + "\x15EndpointFailureDomain\x12'\n" + + "#ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED\x10\x00\x12'\n" + + "#ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT\x10\x012\xb5V\n" + "\tOpenShell\x12Z\n" + "\x06Health\x12\x1b.openshell.v1.HealthRequest\x1a\x1c.openshell.v1.HealthResponse\"\x15\x82\xb5\x18\x11\n" + "\x0funauthenticated\x12i\n" + @@ -19935,8 +20280,8 @@ func file_openshell_proto_rawDescGZIP() []byte { return file_openshell_proto_rawDescData } -var file_openshell_proto_enumTypes = make([]protoimpl.EnumInfo, 20) -var file_openshell_proto_msgTypes = make([]protoimpl.MessageInfo, 260) +var file_openshell_proto_enumTypes = make([]protoimpl.EnumInfo, 21) +var file_openshell_proto_msgTypes = make([]protoimpl.MessageInfo, 263) var file_openshell_proto_goTypes = []any{ (ExtensionKind)(0), // 0: openshell.v1.ExtensionKind (SandboxPhase)(0), // 1: openshell.v1.SandboxPhase @@ -19958,790 +20303,802 @@ var file_openshell_proto_goTypes = []any{ (DeletionOutcome)(0), // 17: openshell.v1.DeletionOutcome (EndpointResult)(0), // 18: openshell.v1.EndpointResult (ServiceAuthorizationMode)(0), // 19: openshell.v1.ServiceAuthorizationMode - (*IssueSandboxTokenRequest)(nil), // 20: openshell.v1.IssueSandboxTokenRequest - (*IssueSandboxTokenResponse)(nil), // 21: openshell.v1.IssueSandboxTokenResponse - (*RefreshSandboxTokenRequest)(nil), // 22: openshell.v1.RefreshSandboxTokenRequest - (*RefreshSandboxTokenResponse)(nil), // 23: openshell.v1.RefreshSandboxTokenResponse - (*HealthRequest)(nil), // 24: openshell.v1.HealthRequest - (*HealthResponse)(nil), // 25: openshell.v1.HealthResponse - (*GetCurrentUserRequest)(nil), // 26: openshell.v1.GetCurrentUserRequest - (*GetCurrentUserResponse)(nil), // 27: openshell.v1.GetCurrentUserResponse - (*GetGatewayInfoRequest)(nil), // 28: openshell.v1.GetGatewayInfoRequest - (*GetGatewayInfoResponse)(nil), // 29: openshell.v1.GetGatewayInfoResponse - (*NegotiatedExtensionInfo)(nil), // 30: openshell.v1.NegotiatedExtensionInfo - (*ComputeDriverInfo)(nil), // 31: openshell.v1.ComputeDriverInfo - (*ComputeDriverCapabilities)(nil), // 32: openshell.v1.ComputeDriverCapabilities - (*ResourceCapabilities)(nil), // 33: openshell.v1.ResourceCapabilities - (*CpuResourceCapabilities)(nil), // 34: openshell.v1.CpuResourceCapabilities - (*MemoryResourceCapabilities)(nil), // 35: openshell.v1.MemoryResourceCapabilities - (*GpuResourceCapabilities)(nil), // 36: openshell.v1.GpuResourceCapabilities - (*Sandbox)(nil), // 37: openshell.v1.Sandbox - (*SandboxSpec)(nil), // 38: openshell.v1.SandboxSpec - (*ResourceRequirements)(nil), // 39: openshell.v1.ResourceRequirements - (*GpuResourceRequirements)(nil), // 40: openshell.v1.GpuResourceRequirements - (*SandboxTemplate)(nil), // 41: openshell.v1.SandboxTemplate - (*SandboxWorkloadTemplate)(nil), // 42: openshell.v1.SandboxWorkloadTemplate - (*SandboxWorkloadTemplateSpec)(nil), // 43: openshell.v1.SandboxWorkloadTemplateSpec - (*SandboxWorkloadConfig)(nil), // 44: openshell.v1.SandboxWorkloadConfig - (*SandboxResources)(nil), // 45: openshell.v1.SandboxResources - (*SandboxServiceLevel)(nil), // 46: openshell.v1.SandboxServiceLevel - (*SandboxStartup)(nil), // 47: openshell.v1.SandboxStartup - (*SandboxWorkloadTemplateProvenance)(nil), // 48: openshell.v1.SandboxWorkloadTemplateProvenance - (*SandboxStatus)(nil), // 49: openshell.v1.SandboxStatus - (*SandboxCondition)(nil), // 50: openshell.v1.SandboxCondition - (*PlatformEvent)(nil), // 51: openshell.v1.PlatformEvent - (*CreateSandboxRequest)(nil), // 52: openshell.v1.CreateSandboxRequest - (*CreateSandboxTemplateRequest)(nil), // 53: openshell.v1.CreateSandboxTemplateRequest - (*GetSandboxTemplateRequest)(nil), // 54: openshell.v1.GetSandboxTemplateRequest - (*ListSandboxTemplatesRequest)(nil), // 55: openshell.v1.ListSandboxTemplatesRequest - (*DeleteSandboxTemplateRequest)(nil), // 56: openshell.v1.DeleteSandboxTemplateRequest - (*SandboxTemplateResponse)(nil), // 57: openshell.v1.SandboxTemplateResponse - (*ListSandboxTemplatesResponse)(nil), // 58: openshell.v1.ListSandboxTemplatesResponse - (*DeleteSandboxTemplateResponse)(nil), // 59: openshell.v1.DeleteSandboxTemplateResponse - (*BeginRootfsTarStagingRequest)(nil), // 60: openshell.v1.BeginRootfsTarStagingRequest - (*BeginRootfsTarStagingResponse)(nil), // 61: openshell.v1.BeginRootfsTarStagingResponse - (*GetSandboxRequest)(nil), // 62: openshell.v1.GetSandboxRequest - (*ListSandboxesRequest)(nil), // 63: openshell.v1.ListSandboxesRequest - (*ListSandboxProvidersRequest)(nil), // 64: openshell.v1.ListSandboxProvidersRequest - (*AttachSandboxProviderRequest)(nil), // 65: openshell.v1.AttachSandboxProviderRequest - (*DetachSandboxProviderRequest)(nil), // 66: openshell.v1.DetachSandboxProviderRequest - (*DeleteSandboxRequest)(nil), // 67: openshell.v1.DeleteSandboxRequest - (*StopSandboxRequest)(nil), // 68: openshell.v1.StopSandboxRequest - (*StartSandboxRequest)(nil), // 69: openshell.v1.StartSandboxRequest - (*SandboxResponse)(nil), // 70: openshell.v1.SandboxResponse - (*ListSandboxesResponse)(nil), // 71: openshell.v1.ListSandboxesResponse - (*ListSandboxProvidersResponse)(nil), // 72: openshell.v1.ListSandboxProvidersResponse - (*AttachSandboxProviderResponse)(nil), // 73: openshell.v1.AttachSandboxProviderResponse - (*DetachSandboxProviderResponse)(nil), // 74: openshell.v1.DetachSandboxProviderResponse - (*ProviderDesiredIdentity)(nil), // 75: openshell.v1.ProviderDesiredIdentity - (*ConfigSnapshotRevision)(nil), // 76: openshell.v1.ConfigSnapshotRevision - (*SandboxConfigRevision)(nil), // 77: openshell.v1.SandboxConfigRevision - (*ConfigUpdateOperation)(nil), // 78: openshell.v1.ConfigUpdateOperation - (*ProviderMutationReceipt)(nil), // 79: openshell.v1.ProviderMutationReceipt - (*ProviderReadinessObservation)(nil), // 80: openshell.v1.ProviderReadinessObservation - (*ProviderReadinessStatus)(nil), // 81: openshell.v1.ProviderReadinessStatus - (*GetSandboxProviderStatusRequest)(nil), // 82: openshell.v1.GetSandboxProviderStatusRequest - (*GetSandboxProviderStatusResponse)(nil), // 83: openshell.v1.GetSandboxProviderStatusResponse - (*ReportProviderReadinessRequest)(nil), // 84: openshell.v1.ReportProviderReadinessRequest - (*ReportProviderReadinessResponse)(nil), // 85: openshell.v1.ReportProviderReadinessResponse - (*DeleteSandboxResponse)(nil), // 86: openshell.v1.DeleteSandboxResponse - (*CreateSshSessionRequest)(nil), // 87: openshell.v1.CreateSshSessionRequest - (*CreateSshSessionResponse)(nil), // 88: openshell.v1.CreateSshSessionResponse - (*ExposeServiceRequest)(nil), // 89: openshell.v1.ExposeServiceRequest - (*GetServiceRequest)(nil), // 90: openshell.v1.GetServiceRequest - (*ListServicesRequest)(nil), // 91: openshell.v1.ListServicesRequest - (*ListServicesResponse)(nil), // 92: openshell.v1.ListServicesResponse - (*DeleteServiceRequest)(nil), // 93: openshell.v1.DeleteServiceRequest - (*DeleteServiceResponse)(nil), // 94: openshell.v1.DeleteServiceResponse - (*ServiceEndpoint)(nil), // 95: openshell.v1.ServiceEndpoint - (*ServiceEndpointResponse)(nil), // 96: openshell.v1.ServiceEndpointResponse - (*RevokeSshSessionRequest)(nil), // 97: openshell.v1.RevokeSshSessionRequest - (*RevokeSshSessionResponse)(nil), // 98: openshell.v1.RevokeSshSessionResponse - (*ExecSandboxRequest)(nil), // 99: openshell.v1.ExecSandboxRequest - (*ExecSandboxStdout)(nil), // 100: openshell.v1.ExecSandboxStdout - (*ExecSandboxStderr)(nil), // 101: openshell.v1.ExecSandboxStderr - (*ExecSandboxExit)(nil), // 102: openshell.v1.ExecSandboxExit - (*ExecSandboxEvent)(nil), // 103: openshell.v1.ExecSandboxEvent - (*TcpForwardInit)(nil), // 104: openshell.v1.TcpForwardInit - (*TcpForwardFrame)(nil), // 105: openshell.v1.TcpForwardFrame - (*ExecSandboxInput)(nil), // 106: openshell.v1.ExecSandboxInput - (*ExecSandboxWindowResize)(nil), // 107: openshell.v1.ExecSandboxWindowResize - (*SshSession)(nil), // 108: openshell.v1.SshSession - (*WatchSandboxRequest)(nil), // 109: openshell.v1.WatchSandboxRequest - (*SandboxStreamEvent)(nil), // 110: openshell.v1.SandboxStreamEvent - (*SandboxLogLine)(nil), // 111: openshell.v1.SandboxLogLine - (*SandboxStreamWarning)(nil), // 112: openshell.v1.SandboxStreamWarning - (*CreateProviderRequest)(nil), // 113: openshell.v1.CreateProviderRequest - (*GetProviderRequest)(nil), // 114: openshell.v1.GetProviderRequest - (*GetProviderCredentialsRequest)(nil), // 115: openshell.v1.GetProviderCredentialsRequest - (*ProviderCredentialValue)(nil), // 116: openshell.v1.ProviderCredentialValue - (*GetProviderCredentialsResponse)(nil), // 117: openshell.v1.GetProviderCredentialsResponse - (*ListProvidersRequest)(nil), // 118: openshell.v1.ListProvidersRequest - (*UpdateProviderRequest)(nil), // 119: openshell.v1.UpdateProviderRequest - (*DeleteProviderRequest)(nil), // 120: openshell.v1.DeleteProviderRequest - (*ProviderResponse)(nil), // 121: openshell.v1.ProviderResponse - (*ListProvidersResponse)(nil), // 122: openshell.v1.ListProvidersResponse - (*ListProviderProfilesRequest)(nil), // 123: openshell.v1.ListProviderProfilesRequest - (*GetProviderProfileRequest)(nil), // 124: openshell.v1.GetProviderProfileRequest - (*ProviderProfileImportItem)(nil), // 125: openshell.v1.ProviderProfileImportItem - (*ProviderProfileDiagnostic)(nil), // 126: openshell.v1.ProviderProfileDiagnostic - (*ProviderCredentialTokenGrantAudienceOverride)(nil), // 127: openshell.v1.ProviderCredentialTokenGrantAudienceOverride - (*ProviderCredentialTokenGrantSubjectToken)(nil), // 128: openshell.v1.ProviderCredentialTokenGrantSubjectToken - (*ProviderCredentialTokenGrant)(nil), // 129: openshell.v1.ProviderCredentialTokenGrant - (*ProviderProfileCredential)(nil), // 130: openshell.v1.ProviderProfileCredential - (*ProviderCredentialRefreshMaterial)(nil), // 131: openshell.v1.ProviderCredentialRefreshMaterial - (*ProviderCredentialRefreshOutput)(nil), // 132: openshell.v1.ProviderCredentialRefreshOutput - (*ProviderCredentialRefresh)(nil), // 133: openshell.v1.ProviderCredentialRefresh - (*ProviderCredentialRefreshStatus)(nil), // 134: openshell.v1.ProviderCredentialRefreshStatus - (*ProviderProfileDiscovery)(nil), // 135: openshell.v1.ProviderProfileDiscovery - (*GetProviderRefreshStatusRequest)(nil), // 136: openshell.v1.GetProviderRefreshStatusRequest - (*GetProviderRefreshStatusResponse)(nil), // 137: openshell.v1.GetProviderRefreshStatusResponse - (*ConfigureProviderRefreshRequest)(nil), // 138: openshell.v1.ConfigureProviderRefreshRequest - (*ConfigureProviderRefreshResponse)(nil), // 139: openshell.v1.ConfigureProviderRefreshResponse - (*RotateProviderCredentialRequest)(nil), // 140: openshell.v1.RotateProviderCredentialRequest - (*RotateProviderCredentialResponse)(nil), // 141: openshell.v1.RotateProviderCredentialResponse - (*DeleteProviderRefreshRequest)(nil), // 142: openshell.v1.DeleteProviderRefreshRequest - (*DeleteProviderRefreshResponse)(nil), // 143: openshell.v1.DeleteProviderRefreshResponse - (*ProviderProfile)(nil), // 144: openshell.v1.ProviderProfile - (*ProviderProfileFile)(nil), // 145: openshell.v1.ProviderProfileFile - (*ProviderProfileResponse)(nil), // 146: openshell.v1.ProviderProfileResponse - (*ListProviderProfilesResponse)(nil), // 147: openshell.v1.ListProviderProfilesResponse - (*ImportProviderProfilesRequest)(nil), // 148: openshell.v1.ImportProviderProfilesRequest - (*ImportProviderProfilesResponse)(nil), // 149: openshell.v1.ImportProviderProfilesResponse - (*UpdateProviderProfilesRequest)(nil), // 150: openshell.v1.UpdateProviderProfilesRequest - (*UpdateProviderProfilesResponse)(nil), // 151: openshell.v1.UpdateProviderProfilesResponse - (*LintProviderProfilesRequest)(nil), // 152: openshell.v1.LintProviderProfilesRequest - (*LintProviderProfilesResponse)(nil), // 153: openshell.v1.LintProviderProfilesResponse - (*DeleteProviderResponse)(nil), // 154: openshell.v1.DeleteProviderResponse - (*DeleteProviderProfileRequest)(nil), // 155: openshell.v1.DeleteProviderProfileRequest - (*DeleteProviderProfileResponse)(nil), // 156: openshell.v1.DeleteProviderProfileResponse - (*GetSandboxProviderEnvironmentRequest)(nil), // 157: openshell.v1.GetSandboxProviderEnvironmentRequest - (*StaticCredentialEndpointBinding)(nil), // 158: openshell.v1.StaticCredentialEndpointBinding - (*StaticCredentialBinding)(nil), // 159: openshell.v1.StaticCredentialBinding - (*GetSandboxProviderEnvironmentResponse)(nil), // 160: openshell.v1.GetSandboxProviderEnvironmentResponse - (*ExchangeProviderSubjectTokenRequest)(nil), // 161: openshell.v1.ExchangeProviderSubjectTokenRequest - (*ExchangeProviderSubjectTokenResponse)(nil), // 162: openshell.v1.ExchangeProviderSubjectTokenResponse - (*UpdateConfigRequest)(nil), // 163: openshell.v1.UpdateConfigRequest - (*PolicyMergeOperation)(nil), // 164: openshell.v1.PolicyMergeOperation - (*AddNetworkRule)(nil), // 165: openshell.v1.AddNetworkRule - (*RemoveNetworkEndpoint)(nil), // 166: openshell.v1.RemoveNetworkEndpoint - (*RemoveNetworkRule)(nil), // 167: openshell.v1.RemoveNetworkRule - (*L7RuleTarget)(nil), // 168: openshell.v1.L7RuleTarget - (*AddDenyRules)(nil), // 169: openshell.v1.AddDenyRules - (*AddAllowRules)(nil), // 170: openshell.v1.AddAllowRules - (*RemoveNetworkBinary)(nil), // 171: openshell.v1.RemoveNetworkBinary - (*UpdateConfigResponse)(nil), // 172: openshell.v1.UpdateConfigResponse - (*GetSandboxPolicyStatusRequest)(nil), // 173: openshell.v1.GetSandboxPolicyStatusRequest - (*GetSandboxPolicyStatusResponse)(nil), // 174: openshell.v1.GetSandboxPolicyStatusResponse - (*ListSandboxPoliciesRequest)(nil), // 175: openshell.v1.ListSandboxPoliciesRequest - (*ListSandboxPoliciesResponse)(nil), // 176: openshell.v1.ListSandboxPoliciesResponse - (*ReportPolicyStatusRequest)(nil), // 177: openshell.v1.ReportPolicyStatusRequest - (*ReportPolicyStatusResponse)(nil), // 178: openshell.v1.ReportPolicyStatusResponse - (*SandboxConfigurationAdmission)(nil), // 179: openshell.v1.SandboxConfigurationAdmission - (*ReportSandboxConfigurationRequest)(nil), // 180: openshell.v1.ReportSandboxConfigurationRequest - (*ReportSandboxConfigurationResponse)(nil), // 181: openshell.v1.ReportSandboxConfigurationResponse - (*SandboxPolicyRevision)(nil), // 182: openshell.v1.SandboxPolicyRevision - (*GetSandboxLogsRequest)(nil), // 183: openshell.v1.GetSandboxLogsRequest - (*PushSandboxLogsRequest)(nil), // 184: openshell.v1.PushSandboxLogsRequest - (*PushSandboxLogsResponse)(nil), // 185: openshell.v1.PushSandboxLogsResponse - (*GetSandboxLogsResponse)(nil), // 186: openshell.v1.GetSandboxLogsResponse - (*SupervisorMessage)(nil), // 187: openshell.v1.SupervisorMessage - (*GatewayMessage)(nil), // 188: openshell.v1.GatewayMessage - (*SupervisorHello)(nil), // 189: openshell.v1.SupervisorHello - (*SessionAccepted)(nil), // 190: openshell.v1.SessionAccepted - (*SessionRejected)(nil), // 191: openshell.v1.SessionRejected - (*SessionRedirect)(nil), // 192: openshell.v1.SessionRedirect - (*SupervisorHeartbeat)(nil), // 193: openshell.v1.SupervisorHeartbeat - (*GatewayHeartbeat)(nil), // 194: openshell.v1.GatewayHeartbeat - (*ReportMainProcessExitRequest)(nil), // 195: openshell.v1.ReportMainProcessExitRequest - (*ReportMainProcessExitResponse)(nil), // 196: openshell.v1.ReportMainProcessExitResponse - (*FinalizeMainProcessExitRequest)(nil), // 197: openshell.v1.FinalizeMainProcessExitRequest - (*FinalizeMainProcessExitResponse)(nil), // 198: openshell.v1.FinalizeMainProcessExitResponse - (*RelayOpen)(nil), // 199: openshell.v1.RelayOpen - (*SshRelayTarget)(nil), // 200: openshell.v1.SshRelayTarget - (*TcpRelayTarget)(nil), // 201: openshell.v1.TcpRelayTarget - (*RelayInit)(nil), // 202: openshell.v1.RelayInit - (*RelayFrame)(nil), // 203: openshell.v1.RelayFrame - (*PeerRelayInit)(nil), // 204: openshell.v1.PeerRelayInit - (*PeerRelayFrame)(nil), // 205: openshell.v1.PeerRelayFrame - (*RelayOpenResult)(nil), // 206: openshell.v1.RelayOpenResult - (*RelayClose)(nil), // 207: openshell.v1.RelayClose - (*L7RequestSample)(nil), // 208: openshell.v1.L7RequestSample - (*DenialSummary)(nil), // 209: openshell.v1.DenialSummary - (*DenialGroupCount)(nil), // 210: openshell.v1.DenialGroupCount - (*NetworkActivitySummary)(nil), // 211: openshell.v1.NetworkActivitySummary - (*PolicyChunk)(nil), // 212: openshell.v1.PolicyChunk - (*DraftPolicyUpdate)(nil), // 213: openshell.v1.DraftPolicyUpdate - (*SubmitPolicyAnalysisRequest)(nil), // 214: openshell.v1.SubmitPolicyAnalysisRequest - (*SubmitPolicyAnalysisResponse)(nil), // 215: openshell.v1.SubmitPolicyAnalysisResponse - (*GetDraftPolicyRequest)(nil), // 216: openshell.v1.GetDraftPolicyRequest - (*GetDraftPolicyResponse)(nil), // 217: openshell.v1.GetDraftPolicyResponse - (*ApproveDraftChunkRequest)(nil), // 218: openshell.v1.ApproveDraftChunkRequest - (*ApproveDraftChunkResponse)(nil), // 219: openshell.v1.ApproveDraftChunkResponse - (*RejectDraftChunkRequest)(nil), // 220: openshell.v1.RejectDraftChunkRequest - (*RejectDraftChunkResponse)(nil), // 221: openshell.v1.RejectDraftChunkResponse - (*DraftChunkApproval)(nil), // 222: openshell.v1.DraftChunkApproval - (*ApproveAllDraftChunksRequest)(nil), // 223: openshell.v1.ApproveAllDraftChunksRequest - (*ApproveAllDraftChunksResponse)(nil), // 224: openshell.v1.ApproveAllDraftChunksResponse - (*EditDraftChunkRequest)(nil), // 225: openshell.v1.EditDraftChunkRequest - (*EditDraftChunkResponse)(nil), // 226: openshell.v1.EditDraftChunkResponse - (*UndoDraftChunkRequest)(nil), // 227: openshell.v1.UndoDraftChunkRequest - (*UndoDraftChunkResponse)(nil), // 228: openshell.v1.UndoDraftChunkResponse - (*ClearDraftChunksRequest)(nil), // 229: openshell.v1.ClearDraftChunksRequest - (*ClearDraftChunksResponse)(nil), // 230: openshell.v1.ClearDraftChunksResponse - (*GetDraftHistoryRequest)(nil), // 231: openshell.v1.GetDraftHistoryRequest - (*DraftHistoryEntry)(nil), // 232: openshell.v1.DraftHistoryEntry - (*GetDraftHistoryResponse)(nil), // 233: openshell.v1.GetDraftHistoryResponse - (*CreateWorkspaceRequest)(nil), // 234: openshell.v1.CreateWorkspaceRequest - (*CreateWorkspaceResponse)(nil), // 235: openshell.v1.CreateWorkspaceResponse - (*GetWorkspaceRequest)(nil), // 236: openshell.v1.GetWorkspaceRequest - (*GetWorkspaceResponse)(nil), // 237: openshell.v1.GetWorkspaceResponse - (*ListWorkspacesRequest)(nil), // 238: openshell.v1.ListWorkspacesRequest - (*ListWorkspacesResponse)(nil), // 239: openshell.v1.ListWorkspacesResponse - (*DeleteWorkspaceRequest)(nil), // 240: openshell.v1.DeleteWorkspaceRequest - (*DeleteWorkspaceResponse)(nil), // 241: openshell.v1.DeleteWorkspaceResponse - (*WorkspaceMember)(nil), // 242: openshell.v1.WorkspaceMember - (*AddWorkspaceMemberRequest)(nil), // 243: openshell.v1.AddWorkspaceMemberRequest - (*AddWorkspaceMemberResponse)(nil), // 244: openshell.v1.AddWorkspaceMemberResponse - (*RemoveWorkspaceMemberRequest)(nil), // 245: openshell.v1.RemoveWorkspaceMemberRequest - (*RemoveWorkspaceMemberResponse)(nil), // 246: openshell.v1.RemoveWorkspaceMemberResponse - (*ListWorkspaceMembersRequest)(nil), // 247: openshell.v1.ListWorkspaceMembersRequest - (*ListWorkspaceMembersResponse)(nil), // 248: openshell.v1.ListWorkspaceMembersResponse - (*ExtensionServiceCredential)(nil), // 249: openshell.v1.ExtensionServiceCredential - (*EndpointObservation)(nil), // 250: openshell.v1.EndpointObservation - (*ReportEndpointStatusRequest)(nil), // 251: openshell.v1.ReportEndpointStatusRequest - (*ReportEndpointStatusResponse)(nil), // 252: openshell.v1.ReportEndpointStatusResponse - (*EndpointStatus)(nil), // 253: openshell.v1.EndpointStatus - (*SandboxProvisioning)(nil), // 254: openshell.v1.SandboxProvisioning - (*SandboxServiceExposure)(nil), // 255: openshell.v1.SandboxServiceExposure - nil, // 256: openshell.v1.SandboxSpec.EnvironmentEntry - nil, // 257: openshell.v1.SandboxTemplate.LabelsEntry - nil, // 258: openshell.v1.SandboxTemplate.AnnotationsEntry - nil, // 259: openshell.v1.SandboxTemplate.EnvironmentEntry - nil, // 260: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry - nil, // 261: openshell.v1.PlatformEvent.MetadataEntry - nil, // 262: openshell.v1.CreateSandboxRequest.LabelsEntry - nil, // 263: openshell.v1.CreateSandboxRequest.AnnotationsEntry - nil, // 264: openshell.v1.SandboxResponse.ServiceUrlsEntry - nil, // 265: openshell.v1.ExecSandboxRequest.EnvironmentEntry - nil, // 266: openshell.v1.SandboxLogLine.FieldsEntry - nil, // 267: openshell.v1.GetProviderCredentialsResponse.CredentialsEntry - nil, // 268: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - nil, // 269: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - nil, // 270: openshell.v1.ProviderProfile.AnnotationsEntry - nil, // 271: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - nil, // 272: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - nil, // 273: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - nil, // 274: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry - nil, // 275: openshell.v1.GetSandboxProviderEnvironmentResponse.FilesEntry - nil, // 276: openshell.v1.UpdateConfigRequest.AnnotationsEntry - nil, // 277: openshell.v1.UpdateConfigResponse.AnnotationsEntry - nil, // 278: openshell.v1.SandboxPolicyRevision.ProvenanceEntry - nil, // 279: openshell.v1.CreateWorkspaceRequest.LabelsEntry - (*timestamppb.Timestamp)(nil), // 280: google.protobuf.Timestamp - (*datamodelv1.ObjectMeta)(nil), // 281: openshell.datamodel.v1.ObjectMeta - (*sandboxv1.SandboxPolicy)(nil), // 282: openshell.sandbox.v1.SandboxPolicy - (*structpb.Struct)(nil), // 283: google.protobuf.Struct - (*durationpb.Duration)(nil), // 284: google.protobuf.Duration - (*datamodelv1.WorkspaceSelector)(nil), // 285: openshell.datamodel.v1.WorkspaceSelector - (*datamodelv1.Provider)(nil), // 286: openshell.datamodel.v1.Provider - (sandboxv1.PolicySource)(0), // 287: openshell.sandbox.v1.PolicySource - (*sandboxv1.NetworkEndpoint)(nil), // 288: openshell.sandbox.v1.NetworkEndpoint - (*sandboxv1.NetworkBinary)(nil), // 289: openshell.sandbox.v1.NetworkBinary - (*sandboxv1.SettingValue)(nil), // 290: openshell.sandbox.v1.SettingValue - (*sandboxv1.NetworkPolicyRule)(nil), // 291: openshell.sandbox.v1.NetworkPolicyRule - (*sandboxv1.L7DenyRule)(nil), // 292: openshell.sandbox.v1.L7DenyRule - (*sandboxv1.L7Rule)(nil), // 293: openshell.sandbox.v1.L7Rule - (*datamodelv1.Workspace)(nil), // 294: openshell.datamodel.v1.Workspace - (*sandboxv1.GetSandboxConfigRequest)(nil), // 295: openshell.sandbox.v1.GetSandboxConfigRequest - (*sandboxv1.GetGatewayConfigRequest)(nil), // 296: openshell.sandbox.v1.GetGatewayConfigRequest - (*sandboxv1.GetSandboxConfigResponse)(nil), // 297: openshell.sandbox.v1.GetSandboxConfigResponse - (*sandboxv1.GetGatewayConfigResponse)(nil), // 298: openshell.sandbox.v1.GetGatewayConfigResponse + (EndpointFailureDomain)(0), // 20: openshell.v1.EndpointFailureDomain + (*IssueSandboxTokenRequest)(nil), // 21: openshell.v1.IssueSandboxTokenRequest + (*IssueSandboxTokenResponse)(nil), // 22: openshell.v1.IssueSandboxTokenResponse + (*RefreshSandboxTokenRequest)(nil), // 23: openshell.v1.RefreshSandboxTokenRequest + (*RefreshSandboxTokenResponse)(nil), // 24: openshell.v1.RefreshSandboxTokenResponse + (*HealthRequest)(nil), // 25: openshell.v1.HealthRequest + (*HealthResponse)(nil), // 26: openshell.v1.HealthResponse + (*GetCurrentUserRequest)(nil), // 27: openshell.v1.GetCurrentUserRequest + (*GetCurrentUserResponse)(nil), // 28: openshell.v1.GetCurrentUserResponse + (*GetGatewayInfoRequest)(nil), // 29: openshell.v1.GetGatewayInfoRequest + (*GetGatewayInfoResponse)(nil), // 30: openshell.v1.GetGatewayInfoResponse + (*NegotiatedExtensionInfo)(nil), // 31: openshell.v1.NegotiatedExtensionInfo + (*ComputeDriverInfo)(nil), // 32: openshell.v1.ComputeDriverInfo + (*ComputeDriverCapabilities)(nil), // 33: openshell.v1.ComputeDriverCapabilities + (*ResourceCapabilities)(nil), // 34: openshell.v1.ResourceCapabilities + (*CpuResourceCapabilities)(nil), // 35: openshell.v1.CpuResourceCapabilities + (*MemoryResourceCapabilities)(nil), // 36: openshell.v1.MemoryResourceCapabilities + (*GpuResourceCapabilities)(nil), // 37: openshell.v1.GpuResourceCapabilities + (*Sandbox)(nil), // 38: openshell.v1.Sandbox + (*SandboxSpec)(nil), // 39: openshell.v1.SandboxSpec + (*ResourceRequirements)(nil), // 40: openshell.v1.ResourceRequirements + (*GpuResourceRequirements)(nil), // 41: openshell.v1.GpuResourceRequirements + (*SandboxTemplate)(nil), // 42: openshell.v1.SandboxTemplate + (*SandboxWorkloadTemplate)(nil), // 43: openshell.v1.SandboxWorkloadTemplate + (*SandboxWorkloadTemplateSpec)(nil), // 44: openshell.v1.SandboxWorkloadTemplateSpec + (*SandboxWorkloadConfig)(nil), // 45: openshell.v1.SandboxWorkloadConfig + (*SandboxResources)(nil), // 46: openshell.v1.SandboxResources + (*SandboxServiceLevel)(nil), // 47: openshell.v1.SandboxServiceLevel + (*SandboxStartup)(nil), // 48: openshell.v1.SandboxStartup + (*SandboxWorkloadTemplateProvenance)(nil), // 49: openshell.v1.SandboxWorkloadTemplateProvenance + (*SandboxStatus)(nil), // 50: openshell.v1.SandboxStatus + (*SandboxCondition)(nil), // 51: openshell.v1.SandboxCondition + (*PlatformEvent)(nil), // 52: openshell.v1.PlatformEvent + (*CreateSandboxRequest)(nil), // 53: openshell.v1.CreateSandboxRequest + (*CreateSandboxTemplateRequest)(nil), // 54: openshell.v1.CreateSandboxTemplateRequest + (*GetSandboxTemplateRequest)(nil), // 55: openshell.v1.GetSandboxTemplateRequest + (*ListSandboxTemplatesRequest)(nil), // 56: openshell.v1.ListSandboxTemplatesRequest + (*DeleteSandboxTemplateRequest)(nil), // 57: openshell.v1.DeleteSandboxTemplateRequest + (*SandboxTemplateResponse)(nil), // 58: openshell.v1.SandboxTemplateResponse + (*ListSandboxTemplatesResponse)(nil), // 59: openshell.v1.ListSandboxTemplatesResponse + (*DeleteSandboxTemplateResponse)(nil), // 60: openshell.v1.DeleteSandboxTemplateResponse + (*BeginRootfsTarStagingRequest)(nil), // 61: openshell.v1.BeginRootfsTarStagingRequest + (*BeginRootfsTarStagingResponse)(nil), // 62: openshell.v1.BeginRootfsTarStagingResponse + (*GetSandboxRequest)(nil), // 63: openshell.v1.GetSandboxRequest + (*ListSandboxesRequest)(nil), // 64: openshell.v1.ListSandboxesRequest + (*ListSandboxProvidersRequest)(nil), // 65: openshell.v1.ListSandboxProvidersRequest + (*AttachSandboxProviderRequest)(nil), // 66: openshell.v1.AttachSandboxProviderRequest + (*DetachSandboxProviderRequest)(nil), // 67: openshell.v1.DetachSandboxProviderRequest + (*DeleteSandboxRequest)(nil), // 68: openshell.v1.DeleteSandboxRequest + (*StopSandboxRequest)(nil), // 69: openshell.v1.StopSandboxRequest + (*StartSandboxRequest)(nil), // 70: openshell.v1.StartSandboxRequest + (*SandboxResponse)(nil), // 71: openshell.v1.SandboxResponse + (*ListSandboxesResponse)(nil), // 72: openshell.v1.ListSandboxesResponse + (*ListSandboxProvidersResponse)(nil), // 73: openshell.v1.ListSandboxProvidersResponse + (*AttachSandboxProviderResponse)(nil), // 74: openshell.v1.AttachSandboxProviderResponse + (*DetachSandboxProviderResponse)(nil), // 75: openshell.v1.DetachSandboxProviderResponse + (*ProviderDesiredIdentity)(nil), // 76: openshell.v1.ProviderDesiredIdentity + (*ConfigSnapshotRevision)(nil), // 77: openshell.v1.ConfigSnapshotRevision + (*SandboxConfigRevision)(nil), // 78: openshell.v1.SandboxConfigRevision + (*ConfigUpdateOperation)(nil), // 79: openshell.v1.ConfigUpdateOperation + (*ProviderMutationReceipt)(nil), // 80: openshell.v1.ProviderMutationReceipt + (*ProviderReadinessObservation)(nil), // 81: openshell.v1.ProviderReadinessObservation + (*ProviderReadinessStatus)(nil), // 82: openshell.v1.ProviderReadinessStatus + (*GetSandboxProviderStatusRequest)(nil), // 83: openshell.v1.GetSandboxProviderStatusRequest + (*GetSandboxProviderStatusResponse)(nil), // 84: openshell.v1.GetSandboxProviderStatusResponse + (*ReportProviderReadinessRequest)(nil), // 85: openshell.v1.ReportProviderReadinessRequest + (*ReportProviderReadinessResponse)(nil), // 86: openshell.v1.ReportProviderReadinessResponse + (*DeleteSandboxResponse)(nil), // 87: openshell.v1.DeleteSandboxResponse + (*CreateSshSessionRequest)(nil), // 88: openshell.v1.CreateSshSessionRequest + (*CreateSshSessionResponse)(nil), // 89: openshell.v1.CreateSshSessionResponse + (*ExposeServiceRequest)(nil), // 90: openshell.v1.ExposeServiceRequest + (*GetServiceRequest)(nil), // 91: openshell.v1.GetServiceRequest + (*ListServicesRequest)(nil), // 92: openshell.v1.ListServicesRequest + (*ListServicesResponse)(nil), // 93: openshell.v1.ListServicesResponse + (*DeleteServiceRequest)(nil), // 94: openshell.v1.DeleteServiceRequest + (*DeleteServiceResponse)(nil), // 95: openshell.v1.DeleteServiceResponse + (*ServiceEndpoint)(nil), // 96: openshell.v1.ServiceEndpoint + (*ServiceEndpointResponse)(nil), // 97: openshell.v1.ServiceEndpointResponse + (*RevokeSshSessionRequest)(nil), // 98: openshell.v1.RevokeSshSessionRequest + (*RevokeSshSessionResponse)(nil), // 99: openshell.v1.RevokeSshSessionResponse + (*ExecSandboxRequest)(nil), // 100: openshell.v1.ExecSandboxRequest + (*ExecSandboxStdout)(nil), // 101: openshell.v1.ExecSandboxStdout + (*ExecSandboxStderr)(nil), // 102: openshell.v1.ExecSandboxStderr + (*ExecSandboxExit)(nil), // 103: openshell.v1.ExecSandboxExit + (*ExecSandboxEvent)(nil), // 104: openshell.v1.ExecSandboxEvent + (*TcpForwardInit)(nil), // 105: openshell.v1.TcpForwardInit + (*TcpForwardFrame)(nil), // 106: openshell.v1.TcpForwardFrame + (*ExecSandboxInput)(nil), // 107: openshell.v1.ExecSandboxInput + (*ExecSandboxWindowResize)(nil), // 108: openshell.v1.ExecSandboxWindowResize + (*SshSession)(nil), // 109: openshell.v1.SshSession + (*WatchSandboxRequest)(nil), // 110: openshell.v1.WatchSandboxRequest + (*SandboxStreamEvent)(nil), // 111: openshell.v1.SandboxStreamEvent + (*SandboxLogLine)(nil), // 112: openshell.v1.SandboxLogLine + (*SandboxStreamWarning)(nil), // 113: openshell.v1.SandboxStreamWarning + (*CreateProviderRequest)(nil), // 114: openshell.v1.CreateProviderRequest + (*GetProviderRequest)(nil), // 115: openshell.v1.GetProviderRequest + (*GetProviderCredentialsRequest)(nil), // 116: openshell.v1.GetProviderCredentialsRequest + (*ProviderCredentialValue)(nil), // 117: openshell.v1.ProviderCredentialValue + (*GetProviderCredentialsResponse)(nil), // 118: openshell.v1.GetProviderCredentialsResponse + (*ListProvidersRequest)(nil), // 119: openshell.v1.ListProvidersRequest + (*UpdateProviderRequest)(nil), // 120: openshell.v1.UpdateProviderRequest + (*DeleteProviderRequest)(nil), // 121: openshell.v1.DeleteProviderRequest + (*ProviderResponse)(nil), // 122: openshell.v1.ProviderResponse + (*ListProvidersResponse)(nil), // 123: openshell.v1.ListProvidersResponse + (*ListProviderProfilesRequest)(nil), // 124: openshell.v1.ListProviderProfilesRequest + (*GetProviderProfileRequest)(nil), // 125: openshell.v1.GetProviderProfileRequest + (*ProviderProfileImportItem)(nil), // 126: openshell.v1.ProviderProfileImportItem + (*ProviderProfileDiagnostic)(nil), // 127: openshell.v1.ProviderProfileDiagnostic + (*ProviderCredentialTokenGrantAudienceOverride)(nil), // 128: openshell.v1.ProviderCredentialTokenGrantAudienceOverride + (*ProviderCredentialTokenGrantSubjectToken)(nil), // 129: openshell.v1.ProviderCredentialTokenGrantSubjectToken + (*ProviderCredentialTokenGrant)(nil), // 130: openshell.v1.ProviderCredentialTokenGrant + (*ProviderProfileCredential)(nil), // 131: openshell.v1.ProviderProfileCredential + (*ProviderCredentialRefreshMaterial)(nil), // 132: openshell.v1.ProviderCredentialRefreshMaterial + (*ProviderCredentialRefreshOutput)(nil), // 133: openshell.v1.ProviderCredentialRefreshOutput + (*ProviderCredentialRefresh)(nil), // 134: openshell.v1.ProviderCredentialRefresh + (*ProviderCredentialRefreshStatus)(nil), // 135: openshell.v1.ProviderCredentialRefreshStatus + (*ProviderProfileDiscovery)(nil), // 136: openshell.v1.ProviderProfileDiscovery + (*GetProviderRefreshStatusRequest)(nil), // 137: openshell.v1.GetProviderRefreshStatusRequest + (*GetProviderRefreshStatusResponse)(nil), // 138: openshell.v1.GetProviderRefreshStatusResponse + (*ConfigureProviderRefreshRequest)(nil), // 139: openshell.v1.ConfigureProviderRefreshRequest + (*ConfigureProviderRefreshResponse)(nil), // 140: openshell.v1.ConfigureProviderRefreshResponse + (*RotateProviderCredentialRequest)(nil), // 141: openshell.v1.RotateProviderCredentialRequest + (*RotateProviderCredentialResponse)(nil), // 142: openshell.v1.RotateProviderCredentialResponse + (*DeleteProviderRefreshRequest)(nil), // 143: openshell.v1.DeleteProviderRefreshRequest + (*DeleteProviderRefreshResponse)(nil), // 144: openshell.v1.DeleteProviderRefreshResponse + (*ProviderProfile)(nil), // 145: openshell.v1.ProviderProfile + (*ProviderProfileFile)(nil), // 146: openshell.v1.ProviderProfileFile + (*ProviderProfileResponse)(nil), // 147: openshell.v1.ProviderProfileResponse + (*ListProviderProfilesResponse)(nil), // 148: openshell.v1.ListProviderProfilesResponse + (*ImportProviderProfilesRequest)(nil), // 149: openshell.v1.ImportProviderProfilesRequest + (*ImportProviderProfilesResponse)(nil), // 150: openshell.v1.ImportProviderProfilesResponse + (*UpdateProviderProfilesRequest)(nil), // 151: openshell.v1.UpdateProviderProfilesRequest + (*UpdateProviderProfilesResponse)(nil), // 152: openshell.v1.UpdateProviderProfilesResponse + (*LintProviderProfilesRequest)(nil), // 153: openshell.v1.LintProviderProfilesRequest + (*LintProviderProfilesResponse)(nil), // 154: openshell.v1.LintProviderProfilesResponse + (*DeleteProviderResponse)(nil), // 155: openshell.v1.DeleteProviderResponse + (*DeleteProviderProfileRequest)(nil), // 156: openshell.v1.DeleteProviderProfileRequest + (*DeleteProviderProfileResponse)(nil), // 157: openshell.v1.DeleteProviderProfileResponse + (*GetSandboxProviderEnvironmentRequest)(nil), // 158: openshell.v1.GetSandboxProviderEnvironmentRequest + (*StaticCredentialEndpointBinding)(nil), // 159: openshell.v1.StaticCredentialEndpointBinding + (*StaticCredentialBinding)(nil), // 160: openshell.v1.StaticCredentialBinding + (*GetSandboxProviderEnvironmentResponse)(nil), // 161: openshell.v1.GetSandboxProviderEnvironmentResponse + (*ExchangeProviderSubjectTokenRequest)(nil), // 162: openshell.v1.ExchangeProviderSubjectTokenRequest + (*ExchangeProviderSubjectTokenResponse)(nil), // 163: openshell.v1.ExchangeProviderSubjectTokenResponse + (*UpdateConfigRequest)(nil), // 164: openshell.v1.UpdateConfigRequest + (*PolicyMergeOperation)(nil), // 165: openshell.v1.PolicyMergeOperation + (*AddNetworkRule)(nil), // 166: openshell.v1.AddNetworkRule + (*RemoveNetworkEndpoint)(nil), // 167: openshell.v1.RemoveNetworkEndpoint + (*RemoveNetworkRule)(nil), // 168: openshell.v1.RemoveNetworkRule + (*L7RuleTarget)(nil), // 169: openshell.v1.L7RuleTarget + (*AddDenyRules)(nil), // 170: openshell.v1.AddDenyRules + (*AddAllowRules)(nil), // 171: openshell.v1.AddAllowRules + (*RemoveNetworkBinary)(nil), // 172: openshell.v1.RemoveNetworkBinary + (*UpdateConfigResponse)(nil), // 173: openshell.v1.UpdateConfigResponse + (*GetSandboxPolicyStatusRequest)(nil), // 174: openshell.v1.GetSandboxPolicyStatusRequest + (*GetSandboxPolicyStatusResponse)(nil), // 175: openshell.v1.GetSandboxPolicyStatusResponse + (*ListSandboxPoliciesRequest)(nil), // 176: openshell.v1.ListSandboxPoliciesRequest + (*ListSandboxPoliciesResponse)(nil), // 177: openshell.v1.ListSandboxPoliciesResponse + (*ReportPolicyStatusRequest)(nil), // 178: openshell.v1.ReportPolicyStatusRequest + (*ReportPolicyStatusResponse)(nil), // 179: openshell.v1.ReportPolicyStatusResponse + (*SandboxConfigurationAdmission)(nil), // 180: openshell.v1.SandboxConfigurationAdmission + (*ReportSandboxConfigurationRequest)(nil), // 181: openshell.v1.ReportSandboxConfigurationRequest + (*ReportSandboxConfigurationResponse)(nil), // 182: openshell.v1.ReportSandboxConfigurationResponse + (*SandboxPolicyRevision)(nil), // 183: openshell.v1.SandboxPolicyRevision + (*GetSandboxLogsRequest)(nil), // 184: openshell.v1.GetSandboxLogsRequest + (*PushSandboxLogsRequest)(nil), // 185: openshell.v1.PushSandboxLogsRequest + (*PushSandboxLogsResponse)(nil), // 186: openshell.v1.PushSandboxLogsResponse + (*GetSandboxLogsResponse)(nil), // 187: openshell.v1.GetSandboxLogsResponse + (*SupervisorMessage)(nil), // 188: openshell.v1.SupervisorMessage + (*GatewayMessage)(nil), // 189: openshell.v1.GatewayMessage + (*SupervisorHello)(nil), // 190: openshell.v1.SupervisorHello + (*SessionAccepted)(nil), // 191: openshell.v1.SessionAccepted + (*SessionRejected)(nil), // 192: openshell.v1.SessionRejected + (*SessionRedirect)(nil), // 193: openshell.v1.SessionRedirect + (*SupervisorHeartbeat)(nil), // 194: openshell.v1.SupervisorHeartbeat + (*GatewayHeartbeat)(nil), // 195: openshell.v1.GatewayHeartbeat + (*ReportMainProcessExitRequest)(nil), // 196: openshell.v1.ReportMainProcessExitRequest + (*ReportMainProcessExitResponse)(nil), // 197: openshell.v1.ReportMainProcessExitResponse + (*FinalizeMainProcessExitRequest)(nil), // 198: openshell.v1.FinalizeMainProcessExitRequest + (*FinalizeMainProcessExitResponse)(nil), // 199: openshell.v1.FinalizeMainProcessExitResponse + (*RelayOpen)(nil), // 200: openshell.v1.RelayOpen + (*SshRelayTarget)(nil), // 201: openshell.v1.SshRelayTarget + (*TcpRelayTarget)(nil), // 202: openshell.v1.TcpRelayTarget + (*RelayInit)(nil), // 203: openshell.v1.RelayInit + (*RelayFrame)(nil), // 204: openshell.v1.RelayFrame + (*PeerRelayInit)(nil), // 205: openshell.v1.PeerRelayInit + (*PeerRelayFrame)(nil), // 206: openshell.v1.PeerRelayFrame + (*RelayOpenResult)(nil), // 207: openshell.v1.RelayOpenResult + (*RelayClose)(nil), // 208: openshell.v1.RelayClose + (*L7RequestSample)(nil), // 209: openshell.v1.L7RequestSample + (*DenialSummary)(nil), // 210: openshell.v1.DenialSummary + (*DenialGroupCount)(nil), // 211: openshell.v1.DenialGroupCount + (*NetworkActivitySummary)(nil), // 212: openshell.v1.NetworkActivitySummary + (*PolicyChunk)(nil), // 213: openshell.v1.PolicyChunk + (*DraftPolicyUpdate)(nil), // 214: openshell.v1.DraftPolicyUpdate + (*SubmitPolicyAnalysisRequest)(nil), // 215: openshell.v1.SubmitPolicyAnalysisRequest + (*SubmitPolicyAnalysisResponse)(nil), // 216: openshell.v1.SubmitPolicyAnalysisResponse + (*GetDraftPolicyRequest)(nil), // 217: openshell.v1.GetDraftPolicyRequest + (*GetDraftPolicyResponse)(nil), // 218: openshell.v1.GetDraftPolicyResponse + (*ApproveDraftChunkRequest)(nil), // 219: openshell.v1.ApproveDraftChunkRequest + (*ApproveDraftChunkResponse)(nil), // 220: openshell.v1.ApproveDraftChunkResponse + (*RejectDraftChunkRequest)(nil), // 221: openshell.v1.RejectDraftChunkRequest + (*RejectDraftChunkResponse)(nil), // 222: openshell.v1.RejectDraftChunkResponse + (*DraftChunkApproval)(nil), // 223: openshell.v1.DraftChunkApproval + (*ApproveAllDraftChunksRequest)(nil), // 224: openshell.v1.ApproveAllDraftChunksRequest + (*ApproveAllDraftChunksResponse)(nil), // 225: openshell.v1.ApproveAllDraftChunksResponse + (*EditDraftChunkRequest)(nil), // 226: openshell.v1.EditDraftChunkRequest + (*EditDraftChunkResponse)(nil), // 227: openshell.v1.EditDraftChunkResponse + (*UndoDraftChunkRequest)(nil), // 228: openshell.v1.UndoDraftChunkRequest + (*UndoDraftChunkResponse)(nil), // 229: openshell.v1.UndoDraftChunkResponse + (*ClearDraftChunksRequest)(nil), // 230: openshell.v1.ClearDraftChunksRequest + (*ClearDraftChunksResponse)(nil), // 231: openshell.v1.ClearDraftChunksResponse + (*GetDraftHistoryRequest)(nil), // 232: openshell.v1.GetDraftHistoryRequest + (*DraftHistoryEntry)(nil), // 233: openshell.v1.DraftHistoryEntry + (*GetDraftHistoryResponse)(nil), // 234: openshell.v1.GetDraftHistoryResponse + (*CreateWorkspaceRequest)(nil), // 235: openshell.v1.CreateWorkspaceRequest + (*CreateWorkspaceResponse)(nil), // 236: openshell.v1.CreateWorkspaceResponse + (*GetWorkspaceRequest)(nil), // 237: openshell.v1.GetWorkspaceRequest + (*GetWorkspaceResponse)(nil), // 238: openshell.v1.GetWorkspaceResponse + (*ListWorkspacesRequest)(nil), // 239: openshell.v1.ListWorkspacesRequest + (*ListWorkspacesResponse)(nil), // 240: openshell.v1.ListWorkspacesResponse + (*DeleteWorkspaceRequest)(nil), // 241: openshell.v1.DeleteWorkspaceRequest + (*DeleteWorkspaceResponse)(nil), // 242: openshell.v1.DeleteWorkspaceResponse + (*WorkspaceMember)(nil), // 243: openshell.v1.WorkspaceMember + (*AddWorkspaceMemberRequest)(nil), // 244: openshell.v1.AddWorkspaceMemberRequest + (*AddWorkspaceMemberResponse)(nil), // 245: openshell.v1.AddWorkspaceMemberResponse + (*RemoveWorkspaceMemberRequest)(nil), // 246: openshell.v1.RemoveWorkspaceMemberRequest + (*RemoveWorkspaceMemberResponse)(nil), // 247: openshell.v1.RemoveWorkspaceMemberResponse + (*ListWorkspaceMembersRequest)(nil), // 248: openshell.v1.ListWorkspaceMembersRequest + (*ListWorkspaceMembersResponse)(nil), // 249: openshell.v1.ListWorkspaceMembersResponse + (*ExtensionServiceCredential)(nil), // 250: openshell.v1.ExtensionServiceCredential + (*EndpointObservation)(nil), // 251: openshell.v1.EndpointObservation + (*ReportEndpointStatusRequest)(nil), // 252: openshell.v1.ReportEndpointStatusRequest + (*TokenGrantStatus)(nil), // 253: openshell.v1.TokenGrantStatus + (*TokenGrantFailureObservation)(nil), // 254: openshell.v1.TokenGrantFailureObservation + (*ReportEndpointStatusResponse)(nil), // 255: openshell.v1.ReportEndpointStatusResponse + (*EndpointStatus)(nil), // 256: openshell.v1.EndpointStatus + (*SandboxProvisioning)(nil), // 257: openshell.v1.SandboxProvisioning + (*SandboxServiceExposure)(nil), // 258: openshell.v1.SandboxServiceExposure + (*EndpointFailure)(nil), // 259: openshell.v1.EndpointFailure + nil, // 260: openshell.v1.SandboxSpec.EnvironmentEntry + nil, // 261: openshell.v1.SandboxTemplate.LabelsEntry + nil, // 262: openshell.v1.SandboxTemplate.AnnotationsEntry + nil, // 263: openshell.v1.SandboxTemplate.EnvironmentEntry + nil, // 264: openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + nil, // 265: openshell.v1.PlatformEvent.MetadataEntry + nil, // 266: openshell.v1.CreateSandboxRequest.LabelsEntry + nil, // 267: openshell.v1.CreateSandboxRequest.AnnotationsEntry + nil, // 268: openshell.v1.SandboxResponse.ServiceUrlsEntry + nil, // 269: openshell.v1.ExecSandboxRequest.EnvironmentEntry + nil, // 270: openshell.v1.SandboxLogLine.FieldsEntry + nil, // 271: openshell.v1.GetProviderCredentialsResponse.CredentialsEntry + nil, // 272: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + nil, // 273: openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + nil, // 274: openshell.v1.ProviderProfile.AnnotationsEntry + nil, // 275: openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + nil, // 276: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + nil, // 277: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + nil, // 278: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + nil, // 279: openshell.v1.GetSandboxProviderEnvironmentResponse.FilesEntry + nil, // 280: openshell.v1.UpdateConfigRequest.AnnotationsEntry + nil, // 281: openshell.v1.UpdateConfigResponse.AnnotationsEntry + nil, // 282: openshell.v1.SandboxPolicyRevision.ProvenanceEntry + nil, // 283: openshell.v1.CreateWorkspaceRequest.LabelsEntry + (*timestamppb.Timestamp)(nil), // 284: google.protobuf.Timestamp + (*datamodelv1.ObjectMeta)(nil), // 285: openshell.datamodel.v1.ObjectMeta + (*sandboxv1.SandboxPolicy)(nil), // 286: openshell.sandbox.v1.SandboxPolicy + (*structpb.Struct)(nil), // 287: google.protobuf.Struct + (*durationpb.Duration)(nil), // 288: google.protobuf.Duration + (*datamodelv1.WorkspaceSelector)(nil), // 289: openshell.datamodel.v1.WorkspaceSelector + (*datamodelv1.Provider)(nil), // 290: openshell.datamodel.v1.Provider + (sandboxv1.PolicySource)(0), // 291: openshell.sandbox.v1.PolicySource + (*sandboxv1.NetworkEndpoint)(nil), // 292: openshell.sandbox.v1.NetworkEndpoint + (*sandboxv1.NetworkBinary)(nil), // 293: openshell.sandbox.v1.NetworkBinary + (*sandboxv1.SettingValue)(nil), // 294: openshell.sandbox.v1.SettingValue + (*sandboxv1.NetworkPolicyRule)(nil), // 295: openshell.sandbox.v1.NetworkPolicyRule + (*sandboxv1.L7DenyRule)(nil), // 296: openshell.sandbox.v1.L7DenyRule + (*sandboxv1.L7Rule)(nil), // 297: openshell.sandbox.v1.L7Rule + (*datamodelv1.Workspace)(nil), // 298: openshell.datamodel.v1.Workspace + (*sandboxv1.GetSandboxConfigRequest)(nil), // 299: openshell.sandbox.v1.GetSandboxConfigRequest + (*sandboxv1.GetGatewayConfigRequest)(nil), // 300: openshell.sandbox.v1.GetGatewayConfigRequest + (*sandboxv1.GetSandboxConfigResponse)(nil), // 301: openshell.sandbox.v1.GetSandboxConfigResponse + (*sandboxv1.GetGatewayConfigResponse)(nil), // 302: openshell.sandbox.v1.GetGatewayConfigResponse } var file_openshell_proto_depIdxs = []int32{ - 280, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 280, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp - 249, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential - 280, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp + 284, // 0: openshell.v1.IssueSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 284, // 1: openshell.v1.RefreshSandboxTokenResponse.expiration_time:type_name -> google.protobuf.Timestamp + 250, // 2: openshell.v1.RefreshSandboxTokenResponse.extension_credentials:type_name -> openshell.v1.ExtensionServiceCredential + 284, // 3: openshell.v1.RefreshSandboxTokenResponse.sandbox_expiration_time:type_name -> google.protobuf.Timestamp 13, // 4: openshell.v1.HealthResponse.status:type_name -> openshell.v1.ServiceStatus 13, // 5: openshell.v1.GetGatewayInfoResponse.status:type_name -> openshell.v1.ServiceStatus - 31, // 6: openshell.v1.GetGatewayInfoResponse.compute_drivers:type_name -> openshell.v1.ComputeDriverInfo - 30, // 7: openshell.v1.GetGatewayInfoResponse.extensions:type_name -> openshell.v1.NegotiatedExtensionInfo + 32, // 6: openshell.v1.GetGatewayInfoResponse.compute_drivers:type_name -> openshell.v1.ComputeDriverInfo + 31, // 7: openshell.v1.GetGatewayInfoResponse.extensions:type_name -> openshell.v1.NegotiatedExtensionInfo 0, // 8: openshell.v1.NegotiatedExtensionInfo.kind:type_name -> openshell.v1.ExtensionKind - 32, // 9: openshell.v1.ComputeDriverInfo.capabilities:type_name -> openshell.v1.ComputeDriverCapabilities - 33, // 10: openshell.v1.ComputeDriverCapabilities.resource_capabilities:type_name -> openshell.v1.ResourceCapabilities - 34, // 11: openshell.v1.ResourceCapabilities.cpu:type_name -> openshell.v1.CpuResourceCapabilities - 35, // 12: openshell.v1.ResourceCapabilities.memory:type_name -> openshell.v1.MemoryResourceCapabilities - 36, // 13: openshell.v1.ResourceCapabilities.gpu:type_name -> openshell.v1.GpuResourceCapabilities - 281, // 14: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 38, // 15: openshell.v1.Sandbox.spec:type_name -> openshell.v1.SandboxSpec - 49, // 16: openshell.v1.Sandbox.status:type_name -> openshell.v1.SandboxStatus - 48, // 17: openshell.v1.Sandbox.created_from_workload_template:type_name -> openshell.v1.SandboxWorkloadTemplateProvenance - 256, // 18: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry - 41, // 19: openshell.v1.SandboxSpec.template:type_name -> openshell.v1.SandboxTemplate - 282, // 20: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 39, // 21: openshell.v1.SandboxSpec.resource_requirements:type_name -> openshell.v1.ResourceRequirements + 33, // 9: openshell.v1.ComputeDriverInfo.capabilities:type_name -> openshell.v1.ComputeDriverCapabilities + 34, // 10: openshell.v1.ComputeDriverCapabilities.resource_capabilities:type_name -> openshell.v1.ResourceCapabilities + 35, // 11: openshell.v1.ResourceCapabilities.cpu:type_name -> openshell.v1.CpuResourceCapabilities + 36, // 12: openshell.v1.ResourceCapabilities.memory:type_name -> openshell.v1.MemoryResourceCapabilities + 37, // 13: openshell.v1.ResourceCapabilities.gpu:type_name -> openshell.v1.GpuResourceCapabilities + 285, // 14: openshell.v1.Sandbox.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 39, // 15: openshell.v1.Sandbox.spec:type_name -> openshell.v1.SandboxSpec + 50, // 16: openshell.v1.Sandbox.status:type_name -> openshell.v1.SandboxStatus + 49, // 17: openshell.v1.Sandbox.created_from_workload_template:type_name -> openshell.v1.SandboxWorkloadTemplateProvenance + 260, // 18: openshell.v1.SandboxSpec.environment:type_name -> openshell.v1.SandboxSpec.EnvironmentEntry + 42, // 19: openshell.v1.SandboxSpec.template:type_name -> openshell.v1.SandboxTemplate + 286, // 20: openshell.v1.SandboxSpec.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 40, // 21: openshell.v1.SandboxSpec.resource_requirements:type_name -> openshell.v1.ResourceRequirements 16, // 22: openshell.v1.SandboxSpec.restart_policy:type_name -> openshell.v1.SandboxRestartPolicy - 40, // 23: openshell.v1.ResourceRequirements.gpu:type_name -> openshell.v1.GpuResourceRequirements - 257, // 24: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry - 258, // 25: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry - 259, // 26: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry - 283, // 27: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct - 283, // 28: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct - 281, // 29: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 43, // 30: openshell.v1.SandboxWorkloadTemplate.spec:type_name -> openshell.v1.SandboxWorkloadTemplateSpec - 44, // 31: openshell.v1.SandboxWorkloadTemplateSpec.workload:type_name -> openshell.v1.SandboxWorkloadConfig - 283, // 32: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct - 46, // 33: openshell.v1.SandboxWorkloadTemplateSpec.desired_service_level:type_name -> openshell.v1.SandboxServiceLevel - 260, // 34: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry - 45, // 35: openshell.v1.SandboxWorkloadConfig.resources:type_name -> openshell.v1.SandboxResources - 40, // 36: openshell.v1.SandboxResources.gpu:type_name -> openshell.v1.GpuResourceRequirements - 47, // 37: openshell.v1.SandboxServiceLevel.startup:type_name -> openshell.v1.SandboxStartup - 284, // 38: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration - 50, // 39: openshell.v1.SandboxStatus.conditions:type_name -> openshell.v1.SandboxCondition + 41, // 23: openshell.v1.ResourceRequirements.gpu:type_name -> openshell.v1.GpuResourceRequirements + 261, // 24: openshell.v1.SandboxTemplate.labels:type_name -> openshell.v1.SandboxTemplate.LabelsEntry + 262, // 25: openshell.v1.SandboxTemplate.annotations:type_name -> openshell.v1.SandboxTemplate.AnnotationsEntry + 263, // 26: openshell.v1.SandboxTemplate.environment:type_name -> openshell.v1.SandboxTemplate.EnvironmentEntry + 287, // 27: openshell.v1.SandboxTemplate.resources:type_name -> google.protobuf.Struct + 287, // 28: openshell.v1.SandboxTemplate.driver_config:type_name -> google.protobuf.Struct + 285, // 29: openshell.v1.SandboxWorkloadTemplate.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 44, // 30: openshell.v1.SandboxWorkloadTemplate.spec:type_name -> openshell.v1.SandboxWorkloadTemplateSpec + 45, // 31: openshell.v1.SandboxWorkloadTemplateSpec.workload:type_name -> openshell.v1.SandboxWorkloadConfig + 287, // 32: openshell.v1.SandboxWorkloadTemplateSpec.driver_config:type_name -> google.protobuf.Struct + 47, // 33: openshell.v1.SandboxWorkloadTemplateSpec.desired_service_level:type_name -> openshell.v1.SandboxServiceLevel + 264, // 34: openshell.v1.SandboxWorkloadConfig.environment:type_name -> openshell.v1.SandboxWorkloadConfig.EnvironmentEntry + 46, // 35: openshell.v1.SandboxWorkloadConfig.resources:type_name -> openshell.v1.SandboxResources + 41, // 36: openshell.v1.SandboxResources.gpu:type_name -> openshell.v1.GpuResourceRequirements + 48, // 37: openshell.v1.SandboxServiceLevel.startup:type_name -> openshell.v1.SandboxStartup + 288, // 38: openshell.v1.SandboxStartup.ready_within:type_name -> google.protobuf.Duration + 51, // 39: openshell.v1.SandboxStatus.conditions:type_name -> openshell.v1.SandboxCondition 1, // 40: openshell.v1.SandboxStatus.phase:type_name -> openshell.v1.SandboxPhase - 253, // 41: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus - 179, // 42: openshell.v1.SandboxStatus.configuration_admission:type_name -> openshell.v1.SandboxConfigurationAdmission - 254, // 43: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning - 280, // 44: openshell.v1.SandboxStatus.next_restart_time:type_name -> google.protobuf.Timestamp - 280, // 45: openshell.v1.SandboxStatus.main_process_started_time:type_name -> google.protobuf.Timestamp - 280, // 46: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp - 280, // 47: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp - 261, // 48: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry - 285, // 49: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 38, // 50: openshell.v1.CreateSandboxRequest.spec:type_name -> openshell.v1.SandboxSpec - 262, // 51: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry - 263, // 52: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry - 255, // 53: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure - 285, // 54: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 42, // 55: openshell.v1.CreateSandboxTemplateRequest.template:type_name -> openshell.v1.SandboxWorkloadTemplate - 285, // 56: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 57: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 58: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 42, // 59: openshell.v1.SandboxTemplateResponse.template:type_name -> openshell.v1.SandboxWorkloadTemplate - 42, // 60: openshell.v1.ListSandboxTemplatesResponse.templates:type_name -> openshell.v1.SandboxWorkloadTemplate - 17, // 61: openshell.v1.DeleteSandboxTemplateResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 285, // 62: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 280, // 63: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp - 285, // 64: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 65: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 66: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 67: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 68: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 69: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 70: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 71: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 37, // 72: openshell.v1.SandboxResponse.sandbox:type_name -> openshell.v1.Sandbox - 264, // 73: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry - 37, // 74: openshell.v1.ListSandboxesResponse.sandboxes:type_name -> openshell.v1.Sandbox - 286, // 75: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider - 37, // 76: openshell.v1.AttachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox - 79, // 77: openshell.v1.AttachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 37, // 78: openshell.v1.DetachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox - 79, // 79: openshell.v1.DetachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 77, // 80: openshell.v1.ConfigSnapshotRevision.sandbox_config:type_name -> openshell.v1.SandboxConfigRevision - 75, // 81: openshell.v1.ConfigSnapshotRevision.provider_target:type_name -> openshell.v1.ProviderDesiredIdentity - 287, // 82: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource - 5, // 83: openshell.v1.ConfigUpdateOperation.component:type_name -> openshell.v1.ConfigComponent - 76, // 84: openshell.v1.ConfigUpdateOperation.target_revision:type_name -> openshell.v1.ConfigSnapshotRevision - 7, // 85: openshell.v1.ConfigUpdateOperation.state:type_name -> openshell.v1.ConfigUpdateOperationState - 6, // 86: openshell.v1.ConfigUpdateOperation.outcome:type_name -> openshell.v1.ConfigApplyOutcome - 280, // 87: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp - 280, // 88: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp - 280, // 89: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp - 2, // 90: openshell.v1.ProviderMutationReceipt.kind:type_name -> openshell.v1.ProviderMutationKind - 75, // 91: openshell.v1.ProviderMutationReceipt.desired:type_name -> openshell.v1.ProviderDesiredIdentity - 280, // 92: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp - 4, // 93: openshell.v1.ProviderReadinessObservation.reason:type_name -> openshell.v1.ProviderReadinessReason - 79, // 94: openshell.v1.ProviderReadinessStatus.receipt:type_name -> openshell.v1.ProviderMutationReceipt - 3, // 95: openshell.v1.ProviderReadinessStatus.state:type_name -> openshell.v1.ProviderReadinessState - 4, // 96: openshell.v1.ProviderReadinessStatus.reason:type_name -> openshell.v1.ProviderReadinessReason - 80, // 97: openshell.v1.ProviderReadinessStatus.observed:type_name -> openshell.v1.ProviderReadinessObservation - 280, // 98: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp - 280, // 99: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp - 78, // 100: openshell.v1.ProviderReadinessStatus.operation:type_name -> openshell.v1.ConfigUpdateOperation - 285, // 101: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 81, // 102: openshell.v1.GetSandboxProviderStatusResponse.status:type_name -> openshell.v1.ProviderReadinessStatus - 80, // 103: openshell.v1.ReportProviderReadinessRequest.observation:type_name -> openshell.v1.ProviderReadinessObservation - 284, // 104: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration - 284, // 105: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration - 17, // 106: openshell.v1.DeleteSandboxResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 285, // 107: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 280, // 108: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp - 285, // 109: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 19, // 110: openshell.v1.ExposeServiceRequest.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode - 285, // 111: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 112: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 96, // 113: openshell.v1.ListServicesResponse.services:type_name -> openshell.v1.ServiceEndpointResponse - 285, // 114: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 17, // 115: openshell.v1.DeleteServiceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 281, // 116: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 19, // 117: openshell.v1.ServiceEndpoint.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode - 95, // 118: openshell.v1.ServiceEndpointResponse.endpoint:type_name -> openshell.v1.ServiceEndpoint - 17, // 119: openshell.v1.RevokeSshSessionResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 285, // 120: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 265, // 121: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry - 284, // 122: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration - 100, // 123: openshell.v1.ExecSandboxEvent.stdout:type_name -> openshell.v1.ExecSandboxStdout - 101, // 124: openshell.v1.ExecSandboxEvent.stderr:type_name -> openshell.v1.ExecSandboxStderr - 102, // 125: openshell.v1.ExecSandboxEvent.exit:type_name -> openshell.v1.ExecSandboxExit - 200, // 126: openshell.v1.TcpForwardInit.ssh:type_name -> openshell.v1.SshRelayTarget - 201, // 127: openshell.v1.TcpForwardInit.tcp:type_name -> openshell.v1.TcpRelayTarget - 104, // 128: openshell.v1.TcpForwardFrame.init:type_name -> openshell.v1.TcpForwardInit - 99, // 129: openshell.v1.ExecSandboxInput.start:type_name -> openshell.v1.ExecSandboxRequest - 107, // 130: openshell.v1.ExecSandboxInput.resize:type_name -> openshell.v1.ExecSandboxWindowResize - 281, // 131: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 280, // 132: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp - 285, // 133: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 280, // 134: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp - 37, // 135: openshell.v1.SandboxStreamEvent.sandbox:type_name -> openshell.v1.Sandbox - 111, // 136: openshell.v1.SandboxStreamEvent.log:type_name -> openshell.v1.SandboxLogLine - 51, // 137: openshell.v1.SandboxStreamEvent.event:type_name -> openshell.v1.PlatformEvent - 112, // 138: openshell.v1.SandboxStreamEvent.warning:type_name -> openshell.v1.SandboxStreamWarning - 213, // 139: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate - 280, // 140: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp - 266, // 141: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry - 285, // 142: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 286, // 143: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 285, // 144: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 145: openshell.v1.GetProviderCredentialsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 284, // 146: openshell.v1.GetProviderCredentialsRequest.minimum_remaining_lifetime:type_name -> google.protobuf.Duration - 280, // 147: openshell.v1.ProviderCredentialValue.expiration_time:type_name -> google.protobuf.Timestamp - 267, // 148: openshell.v1.GetProviderCredentialsResponse.credentials:type_name -> openshell.v1.GetProviderCredentialsResponse.CredentialsEntry - 285, // 149: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 150: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 286, // 151: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider - 268, // 152: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry - 285, // 153: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 286, // 154: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider - 79, // 155: openshell.v1.ProviderResponse.target_receipts:type_name -> openshell.v1.ProviderMutationReceipt - 286, // 156: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider - 285, // 157: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 158: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 144, // 159: openshell.v1.ProviderProfileImportItem.profile:type_name -> openshell.v1.ProviderProfile - 284, // 160: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration - 127, // 161: openshell.v1.ProviderCredentialTokenGrant.audience_overrides:type_name -> openshell.v1.ProviderCredentialTokenGrantAudienceOverride - 8, // 162: openshell.v1.ProviderCredentialTokenGrant.grant_type:type_name -> openshell.v1.ProviderCredentialTokenGrantType - 128, // 163: openshell.v1.ProviderCredentialTokenGrant.subject_token:type_name -> openshell.v1.ProviderCredentialTokenGrantSubjectToken - 133, // 164: openshell.v1.ProviderProfileCredential.refresh:type_name -> openshell.v1.ProviderCredentialRefresh - 129, // 165: openshell.v1.ProviderProfileCredential.token_grant:type_name -> openshell.v1.ProviderCredentialTokenGrant - 9, // 166: openshell.v1.ProviderCredentialRefresh.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 284, // 167: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration - 284, // 168: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration - 131, // 169: openshell.v1.ProviderCredentialRefresh.material:type_name -> openshell.v1.ProviderCredentialRefreshMaterial - 132, // 170: openshell.v1.ProviderCredentialRefresh.additional_outputs:type_name -> openshell.v1.ProviderCredentialRefreshOutput - 9, // 171: openshell.v1.ProviderCredentialRefreshStatus.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 280, // 172: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp - 280, // 173: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp - 280, // 174: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp - 15, // 175: openshell.v1.ProviderCredentialRefreshStatus.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction - 280, // 176: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp - 285, // 177: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 134, // 178: openshell.v1.GetProviderRefreshStatusResponse.credentials:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 285, // 179: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 9, // 180: openshell.v1.ConfigureProviderRefreshRequest.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy - 269, // 181: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry - 280, // 182: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp - 134, // 183: openshell.v1.ConfigureProviderRefreshResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 285, // 184: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 134, // 185: openshell.v1.RotateProviderCredentialResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus - 285, // 186: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 17, // 187: openshell.v1.DeleteProviderRefreshResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 10, // 188: openshell.v1.ProviderProfile.category:type_name -> openshell.v1.ProviderProfileCategory - 130, // 189: openshell.v1.ProviderProfile.credentials:type_name -> openshell.v1.ProviderProfileCredential - 288, // 190: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint - 289, // 191: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 135, // 192: openshell.v1.ProviderProfile.discovery:type_name -> openshell.v1.ProviderProfileDiscovery - 270, // 193: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry - 145, // 194: openshell.v1.ProviderProfile.files:type_name -> openshell.v1.ProviderProfileFile - 144, // 195: openshell.v1.ProviderProfileResponse.profile:type_name -> openshell.v1.ProviderProfile - 144, // 196: openshell.v1.ListProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 285, // 197: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 125, // 198: openshell.v1.ImportProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem - 126, // 199: openshell.v1.ImportProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 144, // 200: openshell.v1.ImportProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile - 285, // 201: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 125, // 202: openshell.v1.UpdateProviderProfilesRequest.profile:type_name -> openshell.v1.ProviderProfileImportItem - 126, // 203: openshell.v1.UpdateProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 144, // 204: openshell.v1.UpdateProviderProfilesResponse.profile:type_name -> openshell.v1.ProviderProfile - 285, // 205: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 125, // 206: openshell.v1.LintProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem - 126, // 207: openshell.v1.LintProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic - 17, // 208: openshell.v1.DeleteProviderResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 285, // 209: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 17, // 210: openshell.v1.DeleteProviderProfileResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 158, // 211: openshell.v1.StaticCredentialBinding.endpoints:type_name -> openshell.v1.StaticCredentialEndpointBinding - 271, // 212: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry - 272, // 213: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry - 273, // 214: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry - 274, // 215: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry - 4, // 216: openshell.v1.GetSandboxProviderEnvironmentResponse.readiness_reason:type_name -> openshell.v1.ProviderReadinessReason - 275, // 217: openshell.v1.GetSandboxProviderEnvironmentResponse.files:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.FilesEntry - 284, // 218: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration - 285, // 219: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 282, // 220: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 290, // 221: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue - 164, // 222: openshell.v1.UpdateConfigRequest.merge_operations:type_name -> openshell.v1.PolicyMergeOperation - 276, // 223: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry - 165, // 224: openshell.v1.PolicyMergeOperation.add_rule:type_name -> openshell.v1.AddNetworkRule - 166, // 225: openshell.v1.PolicyMergeOperation.remove_endpoint:type_name -> openshell.v1.RemoveNetworkEndpoint - 167, // 226: openshell.v1.PolicyMergeOperation.remove_rule:type_name -> openshell.v1.RemoveNetworkRule - 169, // 227: openshell.v1.PolicyMergeOperation.add_deny_rules:type_name -> openshell.v1.AddDenyRules - 170, // 228: openshell.v1.PolicyMergeOperation.add_allow_rules:type_name -> openshell.v1.AddAllowRules - 171, // 229: openshell.v1.PolicyMergeOperation.remove_binary:type_name -> openshell.v1.RemoveNetworkBinary - 291, // 230: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 289, // 231: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary - 292, // 232: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule - 168, // 233: openshell.v1.AddDenyRules.target:type_name -> openshell.v1.L7RuleTarget - 293, // 234: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule - 168, // 235: openshell.v1.AddAllowRules.target:type_name -> openshell.v1.L7RuleTarget - 277, // 236: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry - 285, // 237: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 182, // 238: openshell.v1.GetSandboxPolicyStatusResponse.revision:type_name -> openshell.v1.SandboxPolicyRevision - 285, // 239: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 182, // 240: openshell.v1.ListSandboxPoliciesResponse.revisions:type_name -> openshell.v1.SandboxPolicyRevision - 12, // 241: openshell.v1.ReportPolicyStatusRequest.status:type_name -> openshell.v1.PolicyStatus - 11, // 242: openshell.v1.SandboxConfigurationAdmission.state:type_name -> openshell.v1.ConfigurationAdmissionState - 179, // 243: openshell.v1.ReportSandboxConfigurationRequest.admission:type_name -> openshell.v1.SandboxConfigurationAdmission - 12, // 244: openshell.v1.SandboxPolicyRevision.status:type_name -> openshell.v1.PolicyStatus - 280, // 245: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp - 280, // 246: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp - 282, // 247: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 278, // 248: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry - 285, // 249: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 280, // 250: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp - 111, // 251: openshell.v1.PushSandboxLogsRequest.logs:type_name -> openshell.v1.SandboxLogLine - 111, // 252: openshell.v1.GetSandboxLogsResponse.logs:type_name -> openshell.v1.SandboxLogLine - 189, // 253: openshell.v1.SupervisorMessage.hello:type_name -> openshell.v1.SupervisorHello - 193, // 254: openshell.v1.SupervisorMessage.heartbeat:type_name -> openshell.v1.SupervisorHeartbeat - 206, // 255: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult - 207, // 256: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose - 190, // 257: openshell.v1.GatewayMessage.session_accepted:type_name -> openshell.v1.SessionAccepted - 191, // 258: openshell.v1.GatewayMessage.session_rejected:type_name -> openshell.v1.SessionRejected - 194, // 259: openshell.v1.GatewayMessage.heartbeat:type_name -> openshell.v1.GatewayHeartbeat - 199, // 260: openshell.v1.GatewayMessage.relay_open:type_name -> openshell.v1.RelayOpen - 207, // 261: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose - 192, // 262: openshell.v1.GatewayMessage.session_redirect:type_name -> openshell.v1.SessionRedirect - 284, // 263: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration - 200, // 264: openshell.v1.RelayOpen.ssh:type_name -> openshell.v1.SshRelayTarget - 201, // 265: openshell.v1.RelayOpen.tcp:type_name -> openshell.v1.TcpRelayTarget - 202, // 266: openshell.v1.RelayFrame.init:type_name -> openshell.v1.RelayInit - 199, // 267: openshell.v1.PeerRelayInit.relay_open:type_name -> openshell.v1.RelayOpen - 204, // 268: openshell.v1.PeerRelayFrame.init:type_name -> openshell.v1.PeerRelayInit - 280, // 269: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp - 280, // 270: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp - 208, // 271: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample - 210, // 272: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount - 291, // 273: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 280, // 274: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp - 280, // 275: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp - 280, // 276: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp - 280, // 277: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp - 282, // 278: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 282, // 279: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy - 285, // 280: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 209, // 281: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary - 212, // 282: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk - 211, // 283: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary - 285, // 284: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 212, // 285: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk - 280, // 286: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp - 285, // 287: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 288: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 289: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 222, // 290: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval - 285, // 291: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 291, // 292: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule - 285, // 293: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 294: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 285, // 295: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 280, // 296: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp - 232, // 297: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry - 279, // 298: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry - 294, // 299: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 294, // 300: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace - 294, // 301: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace - 17, // 302: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 281, // 303: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta - 14, // 304: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole - 285, // 305: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 14, // 306: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole - 242, // 307: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember - 285, // 308: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 17, // 309: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome - 285, // 310: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector - 242, // 311: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember - 280, // 312: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp - 18, // 313: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult - 250, // 314: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation - 18, // 315: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult - 280, // 316: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp - 280, // 317: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp - 280, // 318: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp - 280, // 319: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp - 280, // 320: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp - 280, // 321: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp - 280, // 322: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp - 280, // 323: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp - 280, // 324: openshell.v1.SandboxProvisioning.preparation_deadline:type_name -> google.protobuf.Timestamp - 280, // 325: openshell.v1.SandboxProvisioning.admission_start_time:type_name -> google.protobuf.Timestamp - 19, // 326: openshell.v1.SandboxServiceExposure.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode - 116, // 327: openshell.v1.GetProviderCredentialsResponse.CredentialsEntry.value:type_name -> openshell.v1.ProviderCredentialValue - 280, // 328: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 280, // 329: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp - 130, // 330: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential - 159, // 331: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding - 24, // 332: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest - 26, // 333: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest - 28, // 334: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest - 52, // 335: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest - 60, // 336: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest - 62, // 337: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest - 63, // 338: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest - 53, // 339: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest - 54, // 340: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest - 55, // 341: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest - 56, // 342: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest - 64, // 343: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest - 65, // 344: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest - 66, // 345: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest - 82, // 346: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 67, // 347: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest - 68, // 348: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest - 69, // 349: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest - 87, // 350: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest - 89, // 351: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest - 90, // 352: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest - 91, // 353: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest - 93, // 354: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest - 97, // 355: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest - 99, // 356: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest - 105, // 357: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame - 106, // 358: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput - 113, // 359: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest - 114, // 360: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest - 115, // 361: openshell.v1.OpenShell.GetProviderCredentials:input_type -> openshell.v1.GetProviderCredentialsRequest - 118, // 362: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest - 123, // 363: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest - 124, // 364: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest - 148, // 365: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest - 150, // 366: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest - 152, // 367: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest - 119, // 368: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest - 136, // 369: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest - 138, // 370: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest - 140, // 371: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest - 142, // 372: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest - 120, // 373: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest - 155, // 374: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest - 295, // 375: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest - 296, // 376: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest - 163, // 377: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest - 173, // 378: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest - 175, // 379: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest - 177, // 380: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest - 251, // 381: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 84, // 382: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 180, // 383: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest - 157, // 384: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest - 161, // 385: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest - 183, // 386: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest - 184, // 387: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest - 187, // 388: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage - 195, // 389: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest - 197, // 390: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest - 203, // 391: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame - 205, // 392: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame - 84, // 393: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest - 251, // 394: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest - 82, // 395: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest - 109, // 396: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest - 214, // 397: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest - 216, // 398: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest - 218, // 399: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest - 220, // 400: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest - 223, // 401: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest - 225, // 402: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest - 227, // 403: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest - 229, // 404: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest - 231, // 405: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest - 20, // 406: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest - 22, // 407: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest - 234, // 408: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest - 236, // 409: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest - 238, // 410: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest - 240, // 411: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest - 243, // 412: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest - 245, // 413: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest - 247, // 414: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest - 25, // 415: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse - 27, // 416: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse - 29, // 417: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse - 70, // 418: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse - 61, // 419: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse - 70, // 420: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse - 71, // 421: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse - 57, // 422: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 57, // 423: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse - 58, // 424: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse - 59, // 425: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse - 72, // 426: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse - 73, // 427: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse - 74, // 428: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse - 83, // 429: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 86, // 430: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse - 70, // 431: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse - 70, // 432: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse - 88, // 433: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse - 96, // 434: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse - 96, // 435: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse - 92, // 436: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse - 94, // 437: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse - 98, // 438: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse - 103, // 439: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent - 105, // 440: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame - 103, // 441: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent - 121, // 442: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse - 121, // 443: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse - 117, // 444: openshell.v1.OpenShell.GetProviderCredentials:output_type -> openshell.v1.GetProviderCredentialsResponse - 122, // 445: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse - 147, // 446: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse - 146, // 447: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse - 149, // 448: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse - 151, // 449: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse - 153, // 450: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse - 121, // 451: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse - 137, // 452: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse - 139, // 453: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse - 141, // 454: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse - 143, // 455: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse - 154, // 456: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse - 156, // 457: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse - 297, // 458: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse - 298, // 459: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse - 172, // 460: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse - 174, // 461: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse - 176, // 462: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse - 178, // 463: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse - 252, // 464: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 85, // 465: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 181, // 466: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse - 160, // 467: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse - 162, // 468: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse - 186, // 469: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse - 185, // 470: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse - 188, // 471: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage - 196, // 472: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse - 198, // 473: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse - 203, // 474: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame - 205, // 475: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame - 85, // 476: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse - 252, // 477: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse - 83, // 478: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse - 110, // 479: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent - 215, // 480: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse - 217, // 481: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse - 219, // 482: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse - 221, // 483: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse - 224, // 484: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse - 226, // 485: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse - 228, // 486: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse - 230, // 487: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse - 233, // 488: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse - 21, // 489: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse - 23, // 490: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse - 235, // 491: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse - 237, // 492: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse - 239, // 493: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse - 241, // 494: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse - 244, // 495: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse - 246, // 496: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse - 248, // 497: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse - 415, // [415:498] is the sub-list for method output_type - 332, // [332:415] is the sub-list for method input_type - 332, // [332:332] is the sub-list for extension type_name - 332, // [332:332] is the sub-list for extension extendee - 0, // [0:332] is the sub-list for field type_name + 256, // 41: openshell.v1.SandboxStatus.endpoint_statuses:type_name -> openshell.v1.EndpointStatus + 180, // 42: openshell.v1.SandboxStatus.configuration_admission:type_name -> openshell.v1.SandboxConfigurationAdmission + 257, // 43: openshell.v1.SandboxStatus.provisioning:type_name -> openshell.v1.SandboxProvisioning + 284, // 44: openshell.v1.SandboxStatus.next_restart_time:type_name -> google.protobuf.Timestamp + 284, // 45: openshell.v1.SandboxStatus.main_process_started_time:type_name -> google.protobuf.Timestamp + 254, // 46: openshell.v1.SandboxStatus.last_token_grant_failure:type_name -> openshell.v1.TokenGrantFailureObservation + 284, // 47: openshell.v1.SandboxCondition.transition_time:type_name -> google.protobuf.Timestamp + 284, // 48: openshell.v1.PlatformEvent.event_time:type_name -> google.protobuf.Timestamp + 265, // 49: openshell.v1.PlatformEvent.metadata:type_name -> openshell.v1.PlatformEvent.MetadataEntry + 289, // 50: openshell.v1.CreateSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 39, // 51: openshell.v1.CreateSandboxRequest.spec:type_name -> openshell.v1.SandboxSpec + 266, // 52: openshell.v1.CreateSandboxRequest.labels:type_name -> openshell.v1.CreateSandboxRequest.LabelsEntry + 267, // 53: openshell.v1.CreateSandboxRequest.annotations:type_name -> openshell.v1.CreateSandboxRequest.AnnotationsEntry + 258, // 54: openshell.v1.CreateSandboxRequest.service_exposures:type_name -> openshell.v1.SandboxServiceExposure + 289, // 55: openshell.v1.CreateSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 43, // 56: openshell.v1.CreateSandboxTemplateRequest.template:type_name -> openshell.v1.SandboxWorkloadTemplate + 289, // 57: openshell.v1.GetSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 58: openshell.v1.ListSandboxTemplatesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 59: openshell.v1.DeleteSandboxTemplateRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 43, // 60: openshell.v1.SandboxTemplateResponse.template:type_name -> openshell.v1.SandboxWorkloadTemplate + 43, // 61: openshell.v1.ListSandboxTemplatesResponse.templates:type_name -> openshell.v1.SandboxWorkloadTemplate + 17, // 62: openshell.v1.DeleteSandboxTemplateResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 289, // 63: openshell.v1.BeginRootfsTarStagingRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 284, // 64: openshell.v1.BeginRootfsTarStagingResponse.expiration_time:type_name -> google.protobuf.Timestamp + 289, // 65: openshell.v1.GetSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 66: openshell.v1.ListSandboxesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 67: openshell.v1.ListSandboxProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 68: openshell.v1.AttachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 69: openshell.v1.DetachSandboxProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 70: openshell.v1.DeleteSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 71: openshell.v1.StopSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 72: openshell.v1.StartSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 38, // 73: openshell.v1.SandboxResponse.sandbox:type_name -> openshell.v1.Sandbox + 268, // 74: openshell.v1.SandboxResponse.service_urls:type_name -> openshell.v1.SandboxResponse.ServiceUrlsEntry + 38, // 75: openshell.v1.ListSandboxesResponse.sandboxes:type_name -> openshell.v1.Sandbox + 290, // 76: openshell.v1.ListSandboxProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 38, // 77: openshell.v1.AttachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox + 80, // 78: openshell.v1.AttachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 38, // 79: openshell.v1.DetachSandboxProviderResponse.sandbox:type_name -> openshell.v1.Sandbox + 80, // 80: openshell.v1.DetachSandboxProviderResponse.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 78, // 81: openshell.v1.ConfigSnapshotRevision.sandbox_config:type_name -> openshell.v1.SandboxConfigRevision + 76, // 82: openshell.v1.ConfigSnapshotRevision.provider_target:type_name -> openshell.v1.ProviderDesiredIdentity + 291, // 83: openshell.v1.SandboxConfigRevision.policy_source:type_name -> openshell.sandbox.v1.PolicySource + 5, // 84: openshell.v1.ConfigUpdateOperation.component:type_name -> openshell.v1.ConfigComponent + 77, // 85: openshell.v1.ConfigUpdateOperation.target_revision:type_name -> openshell.v1.ConfigSnapshotRevision + 7, // 86: openshell.v1.ConfigUpdateOperation.state:type_name -> openshell.v1.ConfigUpdateOperationState + 6, // 87: openshell.v1.ConfigUpdateOperation.outcome:type_name -> openshell.v1.ConfigApplyOutcome + 284, // 88: openshell.v1.ConfigUpdateOperation.created_time:type_name -> google.protobuf.Timestamp + 284, // 89: openshell.v1.ConfigUpdateOperation.updated_time:type_name -> google.protobuf.Timestamp + 284, // 90: openshell.v1.ConfigUpdateOperation.completed_time:type_name -> google.protobuf.Timestamp + 2, // 91: openshell.v1.ProviderMutationReceipt.kind:type_name -> openshell.v1.ProviderMutationKind + 76, // 92: openshell.v1.ProviderMutationReceipt.desired:type_name -> openshell.v1.ProviderDesiredIdentity + 284, // 93: openshell.v1.ProviderMutationReceipt.persisted_time:type_name -> google.protobuf.Timestamp + 4, // 94: openshell.v1.ProviderReadinessObservation.reason:type_name -> openshell.v1.ProviderReadinessReason + 80, // 95: openshell.v1.ProviderReadinessStatus.receipt:type_name -> openshell.v1.ProviderMutationReceipt + 3, // 96: openshell.v1.ProviderReadinessStatus.state:type_name -> openshell.v1.ProviderReadinessState + 4, // 97: openshell.v1.ProviderReadinessStatus.reason:type_name -> openshell.v1.ProviderReadinessReason + 81, // 98: openshell.v1.ProviderReadinessStatus.observed:type_name -> openshell.v1.ProviderReadinessObservation + 284, // 99: openshell.v1.ProviderReadinessStatus.observed_time:type_name -> google.protobuf.Timestamp + 284, // 100: openshell.v1.ProviderReadinessStatus.evaluated_time:type_name -> google.protobuf.Timestamp + 79, // 101: openshell.v1.ProviderReadinessStatus.operation:type_name -> openshell.v1.ConfigUpdateOperation + 289, // 102: openshell.v1.GetSandboxProviderStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 82, // 103: openshell.v1.GetSandboxProviderStatusResponse.status:type_name -> openshell.v1.ProviderReadinessStatus + 81, // 104: openshell.v1.ReportProviderReadinessRequest.observation:type_name -> openshell.v1.ProviderReadinessObservation + 288, // 105: openshell.v1.ReportProviderReadinessResponse.report_interval:type_name -> google.protobuf.Duration + 288, // 106: openshell.v1.ReportProviderReadinessResponse.observation_ttl:type_name -> google.protobuf.Duration + 17, // 107: openshell.v1.DeleteSandboxResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 289, // 108: openshell.v1.CreateSshSessionRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 284, // 109: openshell.v1.CreateSshSessionResponse.expiration_time:type_name -> google.protobuf.Timestamp + 289, // 110: openshell.v1.ExposeServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 19, // 111: openshell.v1.ExposeServiceRequest.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode + 289, // 112: openshell.v1.GetServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 113: openshell.v1.ListServicesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 97, // 114: openshell.v1.ListServicesResponse.services:type_name -> openshell.v1.ServiceEndpointResponse + 289, // 115: openshell.v1.DeleteServiceRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 116: openshell.v1.DeleteServiceResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 285, // 117: openshell.v1.ServiceEndpoint.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 19, // 118: openshell.v1.ServiceEndpoint.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode + 96, // 119: openshell.v1.ServiceEndpointResponse.endpoint:type_name -> openshell.v1.ServiceEndpoint + 17, // 120: openshell.v1.RevokeSshSessionResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 289, // 121: openshell.v1.ExecSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 269, // 122: openshell.v1.ExecSandboxRequest.environment:type_name -> openshell.v1.ExecSandboxRequest.EnvironmentEntry + 288, // 123: openshell.v1.ExecSandboxRequest.execution_timeout:type_name -> google.protobuf.Duration + 101, // 124: openshell.v1.ExecSandboxEvent.stdout:type_name -> openshell.v1.ExecSandboxStdout + 102, // 125: openshell.v1.ExecSandboxEvent.stderr:type_name -> openshell.v1.ExecSandboxStderr + 103, // 126: openshell.v1.ExecSandboxEvent.exit:type_name -> openshell.v1.ExecSandboxExit + 201, // 127: openshell.v1.TcpForwardInit.ssh:type_name -> openshell.v1.SshRelayTarget + 202, // 128: openshell.v1.TcpForwardInit.tcp:type_name -> openshell.v1.TcpRelayTarget + 105, // 129: openshell.v1.TcpForwardFrame.init:type_name -> openshell.v1.TcpForwardInit + 100, // 130: openshell.v1.ExecSandboxInput.start:type_name -> openshell.v1.ExecSandboxRequest + 108, // 131: openshell.v1.ExecSandboxInput.resize:type_name -> openshell.v1.ExecSandboxWindowResize + 285, // 132: openshell.v1.SshSession.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 284, // 133: openshell.v1.SshSession.expiration_time:type_name -> google.protobuf.Timestamp + 289, // 134: openshell.v1.WatchSandboxRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 284, // 135: openshell.v1.WatchSandboxRequest.since_time:type_name -> google.protobuf.Timestamp + 38, // 136: openshell.v1.SandboxStreamEvent.sandbox:type_name -> openshell.v1.Sandbox + 112, // 137: openshell.v1.SandboxStreamEvent.log:type_name -> openshell.v1.SandboxLogLine + 52, // 138: openshell.v1.SandboxStreamEvent.event:type_name -> openshell.v1.PlatformEvent + 113, // 139: openshell.v1.SandboxStreamEvent.warning:type_name -> openshell.v1.SandboxStreamWarning + 214, // 140: openshell.v1.SandboxStreamEvent.draft_policy_update:type_name -> openshell.v1.DraftPolicyUpdate + 284, // 141: openshell.v1.SandboxLogLine.event_time:type_name -> google.protobuf.Timestamp + 270, // 142: openshell.v1.SandboxLogLine.fields:type_name -> openshell.v1.SandboxLogLine.FieldsEntry + 289, // 143: openshell.v1.CreateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 290, // 144: openshell.v1.CreateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 289, // 145: openshell.v1.GetProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 146: openshell.v1.GetProviderCredentialsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 288, // 147: openshell.v1.GetProviderCredentialsRequest.minimum_remaining_lifetime:type_name -> google.protobuf.Duration + 284, // 148: openshell.v1.ProviderCredentialValue.expiration_time:type_name -> google.protobuf.Timestamp + 271, // 149: openshell.v1.GetProviderCredentialsResponse.credentials:type_name -> openshell.v1.GetProviderCredentialsResponse.CredentialsEntry + 289, // 150: openshell.v1.ListProvidersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 151: openshell.v1.UpdateProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 290, // 152: openshell.v1.UpdateProviderRequest.provider:type_name -> openshell.datamodel.v1.Provider + 272, // 153: openshell.v1.UpdateProviderRequest.credential_expiration_times:type_name -> openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry + 289, // 154: openshell.v1.DeleteProviderRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 290, // 155: openshell.v1.ProviderResponse.provider:type_name -> openshell.datamodel.v1.Provider + 80, // 156: openshell.v1.ProviderResponse.target_receipts:type_name -> openshell.v1.ProviderMutationReceipt + 290, // 157: openshell.v1.ListProvidersResponse.providers:type_name -> openshell.datamodel.v1.Provider + 289, // 158: openshell.v1.ListProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 159: openshell.v1.GetProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 145, // 160: openshell.v1.ProviderProfileImportItem.profile:type_name -> openshell.v1.ProviderProfile + 288, // 161: openshell.v1.ProviderCredentialTokenGrant.cache_ttl:type_name -> google.protobuf.Duration + 128, // 162: openshell.v1.ProviderCredentialTokenGrant.audience_overrides:type_name -> openshell.v1.ProviderCredentialTokenGrantAudienceOverride + 8, // 163: openshell.v1.ProviderCredentialTokenGrant.grant_type:type_name -> openshell.v1.ProviderCredentialTokenGrantType + 129, // 164: openshell.v1.ProviderCredentialTokenGrant.subject_token:type_name -> openshell.v1.ProviderCredentialTokenGrantSubjectToken + 134, // 165: openshell.v1.ProviderProfileCredential.refresh:type_name -> openshell.v1.ProviderCredentialRefresh + 130, // 166: openshell.v1.ProviderProfileCredential.token_grant:type_name -> openshell.v1.ProviderCredentialTokenGrant + 9, // 167: openshell.v1.ProviderCredentialRefresh.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 288, // 168: openshell.v1.ProviderCredentialRefresh.refresh_before:type_name -> google.protobuf.Duration + 288, // 169: openshell.v1.ProviderCredentialRefresh.max_lifetime:type_name -> google.protobuf.Duration + 132, // 170: openshell.v1.ProviderCredentialRefresh.material:type_name -> openshell.v1.ProviderCredentialRefreshMaterial + 133, // 171: openshell.v1.ProviderCredentialRefresh.additional_outputs:type_name -> openshell.v1.ProviderCredentialRefreshOutput + 9, // 172: openshell.v1.ProviderCredentialRefreshStatus.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 284, // 173: openshell.v1.ProviderCredentialRefreshStatus.expiration_time:type_name -> google.protobuf.Timestamp + 284, // 174: openshell.v1.ProviderCredentialRefreshStatus.next_refresh_time:type_name -> google.protobuf.Timestamp + 284, // 175: openshell.v1.ProviderCredentialRefreshStatus.last_refresh_time:type_name -> google.protobuf.Timestamp + 15, // 176: openshell.v1.ProviderCredentialRefreshStatus.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction + 284, // 177: openshell.v1.ProviderCredentialRefreshStatus.last_error_time:type_name -> google.protobuf.Timestamp + 289, // 178: openshell.v1.GetProviderRefreshStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 135, // 179: openshell.v1.GetProviderRefreshStatusResponse.credentials:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 289, // 180: openshell.v1.ConfigureProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 9, // 181: openshell.v1.ConfigureProviderRefreshRequest.strategy:type_name -> openshell.v1.ProviderCredentialRefreshStrategy + 273, // 182: openshell.v1.ConfigureProviderRefreshRequest.material:type_name -> openshell.v1.ConfigureProviderRefreshRequest.MaterialEntry + 284, // 183: openshell.v1.ConfigureProviderRefreshRequest.expiration_time:type_name -> google.protobuf.Timestamp + 135, // 184: openshell.v1.ConfigureProviderRefreshResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 289, // 185: openshell.v1.RotateProviderCredentialRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 135, // 186: openshell.v1.RotateProviderCredentialResponse.status:type_name -> openshell.v1.ProviderCredentialRefreshStatus + 289, // 187: openshell.v1.DeleteProviderRefreshRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 188: openshell.v1.DeleteProviderRefreshResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 10, // 189: openshell.v1.ProviderProfile.category:type_name -> openshell.v1.ProviderProfileCategory + 131, // 190: openshell.v1.ProviderProfile.credentials:type_name -> openshell.v1.ProviderProfileCredential + 292, // 191: openshell.v1.ProviderProfile.endpoints:type_name -> openshell.sandbox.v1.NetworkEndpoint + 293, // 192: openshell.v1.ProviderProfile.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 136, // 193: openshell.v1.ProviderProfile.discovery:type_name -> openshell.v1.ProviderProfileDiscovery + 274, // 194: openshell.v1.ProviderProfile.annotations:type_name -> openshell.v1.ProviderProfile.AnnotationsEntry + 146, // 195: openshell.v1.ProviderProfile.files:type_name -> openshell.v1.ProviderProfileFile + 145, // 196: openshell.v1.ProviderProfileResponse.profile:type_name -> openshell.v1.ProviderProfile + 145, // 197: openshell.v1.ListProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile + 289, // 198: openshell.v1.ImportProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 126, // 199: openshell.v1.ImportProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem + 127, // 200: openshell.v1.ImportProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 145, // 201: openshell.v1.ImportProviderProfilesResponse.profiles:type_name -> openshell.v1.ProviderProfile + 289, // 202: openshell.v1.UpdateProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 126, // 203: openshell.v1.UpdateProviderProfilesRequest.profile:type_name -> openshell.v1.ProviderProfileImportItem + 127, // 204: openshell.v1.UpdateProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 145, // 205: openshell.v1.UpdateProviderProfilesResponse.profile:type_name -> openshell.v1.ProviderProfile + 289, // 206: openshell.v1.LintProviderProfilesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 126, // 207: openshell.v1.LintProviderProfilesRequest.profiles:type_name -> openshell.v1.ProviderProfileImportItem + 127, // 208: openshell.v1.LintProviderProfilesResponse.diagnostics:type_name -> openshell.v1.ProviderProfileDiagnostic + 17, // 209: openshell.v1.DeleteProviderResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 289, // 210: openshell.v1.DeleteProviderProfileRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 211: openshell.v1.DeleteProviderProfileResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 159, // 212: openshell.v1.StaticCredentialBinding.endpoints:type_name -> openshell.v1.StaticCredentialEndpointBinding + 275, // 213: openshell.v1.GetSandboxProviderEnvironmentResponse.environment:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.EnvironmentEntry + 276, // 214: openshell.v1.GetSandboxProviderEnvironmentResponse.credential_expiration_times:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry + 277, // 215: openshell.v1.GetSandboxProviderEnvironmentResponse.dynamic_credentials:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry + 278, // 216: openshell.v1.GetSandboxProviderEnvironmentResponse.static_credential_bindings:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry + 4, // 217: openshell.v1.GetSandboxProviderEnvironmentResponse.readiness_reason:type_name -> openshell.v1.ProviderReadinessReason + 279, // 218: openshell.v1.GetSandboxProviderEnvironmentResponse.files:type_name -> openshell.v1.GetSandboxProviderEnvironmentResponse.FilesEntry + 288, // 219: openshell.v1.ExchangeProviderSubjectTokenResponse.expires_after:type_name -> google.protobuf.Duration + 289, // 220: openshell.v1.UpdateConfigRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 286, // 221: openshell.v1.UpdateConfigRequest.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 294, // 222: openshell.v1.UpdateConfigRequest.setting_value:type_name -> openshell.sandbox.v1.SettingValue + 165, // 223: openshell.v1.UpdateConfigRequest.merge_operations:type_name -> openshell.v1.PolicyMergeOperation + 280, // 224: openshell.v1.UpdateConfigRequest.annotations:type_name -> openshell.v1.UpdateConfigRequest.AnnotationsEntry + 166, // 225: openshell.v1.PolicyMergeOperation.add_rule:type_name -> openshell.v1.AddNetworkRule + 167, // 226: openshell.v1.PolicyMergeOperation.remove_endpoint:type_name -> openshell.v1.RemoveNetworkEndpoint + 168, // 227: openshell.v1.PolicyMergeOperation.remove_rule:type_name -> openshell.v1.RemoveNetworkRule + 170, // 228: openshell.v1.PolicyMergeOperation.add_deny_rules:type_name -> openshell.v1.AddDenyRules + 171, // 229: openshell.v1.PolicyMergeOperation.add_allow_rules:type_name -> openshell.v1.AddAllowRules + 172, // 230: openshell.v1.PolicyMergeOperation.remove_binary:type_name -> openshell.v1.RemoveNetworkBinary + 295, // 231: openshell.v1.AddNetworkRule.rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 293, // 232: openshell.v1.L7RuleTarget.binaries:type_name -> openshell.sandbox.v1.NetworkBinary + 296, // 233: openshell.v1.AddDenyRules.deny_rules:type_name -> openshell.sandbox.v1.L7DenyRule + 169, // 234: openshell.v1.AddDenyRules.target:type_name -> openshell.v1.L7RuleTarget + 297, // 235: openshell.v1.AddAllowRules.rules:type_name -> openshell.sandbox.v1.L7Rule + 169, // 236: openshell.v1.AddAllowRules.target:type_name -> openshell.v1.L7RuleTarget + 281, // 237: openshell.v1.UpdateConfigResponse.annotations:type_name -> openshell.v1.UpdateConfigResponse.AnnotationsEntry + 289, // 238: openshell.v1.GetSandboxPolicyStatusRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 183, // 239: openshell.v1.GetSandboxPolicyStatusResponse.revision:type_name -> openshell.v1.SandboxPolicyRevision + 289, // 240: openshell.v1.ListSandboxPoliciesRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 183, // 241: openshell.v1.ListSandboxPoliciesResponse.revisions:type_name -> openshell.v1.SandboxPolicyRevision + 12, // 242: openshell.v1.ReportPolicyStatusRequest.status:type_name -> openshell.v1.PolicyStatus + 11, // 243: openshell.v1.SandboxConfigurationAdmission.state:type_name -> openshell.v1.ConfigurationAdmissionState + 180, // 244: openshell.v1.ReportSandboxConfigurationRequest.admission:type_name -> openshell.v1.SandboxConfigurationAdmission + 12, // 245: openshell.v1.SandboxPolicyRevision.status:type_name -> openshell.v1.PolicyStatus + 284, // 246: openshell.v1.SandboxPolicyRevision.created_time:type_name -> google.protobuf.Timestamp + 284, // 247: openshell.v1.SandboxPolicyRevision.loaded_time:type_name -> google.protobuf.Timestamp + 286, // 248: openshell.v1.SandboxPolicyRevision.policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 282, // 249: openshell.v1.SandboxPolicyRevision.provenance:type_name -> openshell.v1.SandboxPolicyRevision.ProvenanceEntry + 289, // 250: openshell.v1.GetSandboxLogsRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 284, // 251: openshell.v1.GetSandboxLogsRequest.since_time:type_name -> google.protobuf.Timestamp + 112, // 252: openshell.v1.PushSandboxLogsRequest.logs:type_name -> openshell.v1.SandboxLogLine + 112, // 253: openshell.v1.GetSandboxLogsResponse.logs:type_name -> openshell.v1.SandboxLogLine + 190, // 254: openshell.v1.SupervisorMessage.hello:type_name -> openshell.v1.SupervisorHello + 194, // 255: openshell.v1.SupervisorMessage.heartbeat:type_name -> openshell.v1.SupervisorHeartbeat + 207, // 256: openshell.v1.SupervisorMessage.relay_open_result:type_name -> openshell.v1.RelayOpenResult + 208, // 257: openshell.v1.SupervisorMessage.relay_close:type_name -> openshell.v1.RelayClose + 191, // 258: openshell.v1.GatewayMessage.session_accepted:type_name -> openshell.v1.SessionAccepted + 192, // 259: openshell.v1.GatewayMessage.session_rejected:type_name -> openshell.v1.SessionRejected + 195, // 260: openshell.v1.GatewayMessage.heartbeat:type_name -> openshell.v1.GatewayHeartbeat + 200, // 261: openshell.v1.GatewayMessage.relay_open:type_name -> openshell.v1.RelayOpen + 208, // 262: openshell.v1.GatewayMessage.relay_close:type_name -> openshell.v1.RelayClose + 193, // 263: openshell.v1.GatewayMessage.session_redirect:type_name -> openshell.v1.SessionRedirect + 288, // 264: openshell.v1.SessionAccepted.heartbeat_interval:type_name -> google.protobuf.Duration + 201, // 265: openshell.v1.RelayOpen.ssh:type_name -> openshell.v1.SshRelayTarget + 202, // 266: openshell.v1.RelayOpen.tcp:type_name -> openshell.v1.TcpRelayTarget + 203, // 267: openshell.v1.RelayFrame.init:type_name -> openshell.v1.RelayInit + 200, // 268: openshell.v1.PeerRelayInit.relay_open:type_name -> openshell.v1.RelayOpen + 205, // 269: openshell.v1.PeerRelayFrame.init:type_name -> openshell.v1.PeerRelayInit + 284, // 270: openshell.v1.DenialSummary.first_seen_time:type_name -> google.protobuf.Timestamp + 284, // 271: openshell.v1.DenialSummary.last_seen_time:type_name -> google.protobuf.Timestamp + 209, // 272: openshell.v1.DenialSummary.l7_request_samples:type_name -> openshell.v1.L7RequestSample + 211, // 273: openshell.v1.NetworkActivitySummary.denials_by_group:type_name -> openshell.v1.DenialGroupCount + 295, // 274: openshell.v1.PolicyChunk.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 284, // 275: openshell.v1.PolicyChunk.created_time:type_name -> google.protobuf.Timestamp + 284, // 276: openshell.v1.PolicyChunk.decided_time:type_name -> google.protobuf.Timestamp + 284, // 277: openshell.v1.PolicyChunk.first_seen_time:type_name -> google.protobuf.Timestamp + 284, // 278: openshell.v1.PolicyChunk.last_seen_time:type_name -> google.protobuf.Timestamp + 286, // 279: openshell.v1.PolicyChunk.current_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 286, // 280: openshell.v1.PolicyChunk.candidate_effective_policy:type_name -> openshell.sandbox.v1.SandboxPolicy + 289, // 281: openshell.v1.SubmitPolicyAnalysisRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 210, // 282: openshell.v1.SubmitPolicyAnalysisRequest.summaries:type_name -> openshell.v1.DenialSummary + 213, // 283: openshell.v1.SubmitPolicyAnalysisRequest.proposed_chunks:type_name -> openshell.v1.PolicyChunk + 212, // 284: openshell.v1.SubmitPolicyAnalysisRequest.network_activity_summaries:type_name -> openshell.v1.NetworkActivitySummary + 289, // 285: openshell.v1.GetDraftPolicyRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 213, // 286: openshell.v1.GetDraftPolicyResponse.chunks:type_name -> openshell.v1.PolicyChunk + 284, // 287: openshell.v1.GetDraftPolicyResponse.last_analyzed_time:type_name -> google.protobuf.Timestamp + 289, // 288: openshell.v1.ApproveDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 289: openshell.v1.RejectDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 290: openshell.v1.ApproveAllDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 223, // 291: openshell.v1.ApproveAllDraftChunksRequest.approvals:type_name -> openshell.v1.DraftChunkApproval + 289, // 292: openshell.v1.EditDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 295, // 293: openshell.v1.EditDraftChunkRequest.proposed_rule:type_name -> openshell.sandbox.v1.NetworkPolicyRule + 289, // 294: openshell.v1.UndoDraftChunkRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 295: openshell.v1.ClearDraftChunksRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 289, // 296: openshell.v1.GetDraftHistoryRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 284, // 297: openshell.v1.DraftHistoryEntry.event_time:type_name -> google.protobuf.Timestamp + 233, // 298: openshell.v1.GetDraftHistoryResponse.entries:type_name -> openshell.v1.DraftHistoryEntry + 283, // 299: openshell.v1.CreateWorkspaceRequest.labels:type_name -> openshell.v1.CreateWorkspaceRequest.LabelsEntry + 298, // 300: openshell.v1.CreateWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 298, // 301: openshell.v1.GetWorkspaceResponse.workspace:type_name -> openshell.datamodel.v1.Workspace + 298, // 302: openshell.v1.ListWorkspacesResponse.workspaces:type_name -> openshell.datamodel.v1.Workspace + 17, // 303: openshell.v1.DeleteWorkspaceResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 285, // 304: openshell.v1.WorkspaceMember.metadata:type_name -> openshell.datamodel.v1.ObjectMeta + 14, // 305: openshell.v1.WorkspaceMember.role:type_name -> openshell.v1.WorkspaceRole + 289, // 306: openshell.v1.AddWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 14, // 307: openshell.v1.AddWorkspaceMemberRequest.role:type_name -> openshell.v1.WorkspaceRole + 243, // 308: openshell.v1.AddWorkspaceMemberResponse.member:type_name -> openshell.v1.WorkspaceMember + 289, // 309: openshell.v1.RemoveWorkspaceMemberRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 17, // 310: openshell.v1.RemoveWorkspaceMemberResponse.outcome:type_name -> openshell.v1.DeletionOutcome + 289, // 311: openshell.v1.ListWorkspaceMembersRequest.workspace_scope:type_name -> openshell.datamodel.v1.WorkspaceSelector + 243, // 312: openshell.v1.ListWorkspaceMembersResponse.members:type_name -> openshell.v1.WorkspaceMember + 284, // 313: openshell.v1.ExtensionServiceCredential.expiration_time:type_name -> google.protobuf.Timestamp + 18, // 314: openshell.v1.EndpointObservation.result:type_name -> openshell.v1.EndpointResult + 259, // 315: openshell.v1.EndpointObservation.failure:type_name -> openshell.v1.EndpointFailure + 251, // 316: openshell.v1.ReportEndpointStatusRequest.observations:type_name -> openshell.v1.EndpointObservation + 253, // 317: openshell.v1.ReportEndpointStatusRequest.token_grant_status:type_name -> openshell.v1.TokenGrantStatus + 254, // 318: openshell.v1.TokenGrantStatus.last_failure:type_name -> openshell.v1.TokenGrantFailureObservation + 259, // 319: openshell.v1.TokenGrantFailureObservation.failure:type_name -> openshell.v1.EndpointFailure + 18, // 320: openshell.v1.EndpointStatus.last_result:type_name -> openshell.v1.EndpointResult + 284, // 321: openshell.v1.EndpointStatus.last_reported_time:type_name -> google.protobuf.Timestamp + 259, // 322: openshell.v1.EndpointStatus.last_failure:type_name -> openshell.v1.EndpointFailure + 284, // 323: openshell.v1.SandboxProvisioning.configuration_change_time:type_name -> google.protobuf.Timestamp + 284, // 324: openshell.v1.SandboxProvisioning.first_rejection_time:type_name -> google.protobuf.Timestamp + 284, // 325: openshell.v1.SandboxProvisioning.deadline:type_name -> google.protobuf.Timestamp + 284, // 326: openshell.v1.SandboxProvisioning.timeout_time:type_name -> google.protobuf.Timestamp + 284, // 327: openshell.v1.SandboxProvisioning.cleanup_completed_time:type_name -> google.protobuf.Timestamp + 284, // 328: openshell.v1.SandboxProvisioning.cleanup_retry_time:type_name -> google.protobuf.Timestamp + 284, // 329: openshell.v1.SandboxProvisioning.attachment_change_time:type_name -> google.protobuf.Timestamp + 284, // 330: openshell.v1.SandboxProvisioning.preparation_deadline:type_name -> google.protobuf.Timestamp + 284, // 331: openshell.v1.SandboxProvisioning.admission_start_time:type_name -> google.protobuf.Timestamp + 19, // 332: openshell.v1.SandboxServiceExposure.authorization_mode:type_name -> openshell.v1.ServiceAuthorizationMode + 20, // 333: openshell.v1.EndpointFailure.domain:type_name -> openshell.v1.EndpointFailureDomain + 15, // 334: openshell.v1.EndpointFailure.recovery_action:type_name -> openshell.v1.ProviderCredentialRefreshRecoveryAction + 117, // 335: openshell.v1.GetProviderCredentialsResponse.CredentialsEntry.value:type_name -> openshell.v1.ProviderCredentialValue + 284, // 336: openshell.v1.UpdateProviderRequest.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 284, // 337: openshell.v1.GetSandboxProviderEnvironmentResponse.CredentialExpirationTimesEntry.value:type_name -> google.protobuf.Timestamp + 131, // 338: openshell.v1.GetSandboxProviderEnvironmentResponse.DynamicCredentialsEntry.value:type_name -> openshell.v1.ProviderProfileCredential + 160, // 339: openshell.v1.GetSandboxProviderEnvironmentResponse.StaticCredentialBindingsEntry.value:type_name -> openshell.v1.StaticCredentialBinding + 25, // 340: openshell.v1.OpenShell.Health:input_type -> openshell.v1.HealthRequest + 27, // 341: openshell.v1.OpenShell.GetCurrentUser:input_type -> openshell.v1.GetCurrentUserRequest + 29, // 342: openshell.v1.OpenShell.GetGatewayInfo:input_type -> openshell.v1.GetGatewayInfoRequest + 53, // 343: openshell.v1.OpenShell.CreateSandbox:input_type -> openshell.v1.CreateSandboxRequest + 61, // 344: openshell.v1.OpenShell.BeginRootfsTarStaging:input_type -> openshell.v1.BeginRootfsTarStagingRequest + 63, // 345: openshell.v1.OpenShell.GetSandbox:input_type -> openshell.v1.GetSandboxRequest + 64, // 346: openshell.v1.OpenShell.ListSandboxes:input_type -> openshell.v1.ListSandboxesRequest + 54, // 347: openshell.v1.OpenShell.CreateSandboxTemplate:input_type -> openshell.v1.CreateSandboxTemplateRequest + 55, // 348: openshell.v1.OpenShell.GetSandboxTemplate:input_type -> openshell.v1.GetSandboxTemplateRequest + 56, // 349: openshell.v1.OpenShell.ListSandboxTemplates:input_type -> openshell.v1.ListSandboxTemplatesRequest + 57, // 350: openshell.v1.OpenShell.DeleteSandboxTemplate:input_type -> openshell.v1.DeleteSandboxTemplateRequest + 65, // 351: openshell.v1.OpenShell.ListSandboxProviders:input_type -> openshell.v1.ListSandboxProvidersRequest + 66, // 352: openshell.v1.OpenShell.AttachSandboxProvider:input_type -> openshell.v1.AttachSandboxProviderRequest + 67, // 353: openshell.v1.OpenShell.DetachSandboxProvider:input_type -> openshell.v1.DetachSandboxProviderRequest + 83, // 354: openshell.v1.OpenShell.GetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 68, // 355: openshell.v1.OpenShell.DeleteSandbox:input_type -> openshell.v1.DeleteSandboxRequest + 69, // 356: openshell.v1.OpenShell.StopSandbox:input_type -> openshell.v1.StopSandboxRequest + 70, // 357: openshell.v1.OpenShell.StartSandbox:input_type -> openshell.v1.StartSandboxRequest + 88, // 358: openshell.v1.OpenShell.CreateSshSession:input_type -> openshell.v1.CreateSshSessionRequest + 90, // 359: openshell.v1.OpenShell.ExposeService:input_type -> openshell.v1.ExposeServiceRequest + 91, // 360: openshell.v1.OpenShell.GetService:input_type -> openshell.v1.GetServiceRequest + 92, // 361: openshell.v1.OpenShell.ListServices:input_type -> openshell.v1.ListServicesRequest + 94, // 362: openshell.v1.OpenShell.DeleteService:input_type -> openshell.v1.DeleteServiceRequest + 98, // 363: openshell.v1.OpenShell.RevokeSshSession:input_type -> openshell.v1.RevokeSshSessionRequest + 100, // 364: openshell.v1.OpenShell.ExecSandbox:input_type -> openshell.v1.ExecSandboxRequest + 106, // 365: openshell.v1.OpenShell.ForwardTcp:input_type -> openshell.v1.TcpForwardFrame + 107, // 366: openshell.v1.OpenShell.ExecSandboxInteractive:input_type -> openshell.v1.ExecSandboxInput + 114, // 367: openshell.v1.OpenShell.CreateProvider:input_type -> openshell.v1.CreateProviderRequest + 115, // 368: openshell.v1.OpenShell.GetProvider:input_type -> openshell.v1.GetProviderRequest + 116, // 369: openshell.v1.OpenShell.GetProviderCredentials:input_type -> openshell.v1.GetProviderCredentialsRequest + 119, // 370: openshell.v1.OpenShell.ListProviders:input_type -> openshell.v1.ListProvidersRequest + 124, // 371: openshell.v1.OpenShell.ListProviderProfiles:input_type -> openshell.v1.ListProviderProfilesRequest + 125, // 372: openshell.v1.OpenShell.GetProviderProfile:input_type -> openshell.v1.GetProviderProfileRequest + 149, // 373: openshell.v1.OpenShell.ImportProviderProfiles:input_type -> openshell.v1.ImportProviderProfilesRequest + 151, // 374: openshell.v1.OpenShell.UpdateProviderProfiles:input_type -> openshell.v1.UpdateProviderProfilesRequest + 153, // 375: openshell.v1.OpenShell.LintProviderProfiles:input_type -> openshell.v1.LintProviderProfilesRequest + 120, // 376: openshell.v1.OpenShell.UpdateProvider:input_type -> openshell.v1.UpdateProviderRequest + 137, // 377: openshell.v1.OpenShell.GetProviderRefreshStatus:input_type -> openshell.v1.GetProviderRefreshStatusRequest + 139, // 378: openshell.v1.OpenShell.ConfigureProviderRefresh:input_type -> openshell.v1.ConfigureProviderRefreshRequest + 141, // 379: openshell.v1.OpenShell.RotateProviderCredential:input_type -> openshell.v1.RotateProviderCredentialRequest + 143, // 380: openshell.v1.OpenShell.DeleteProviderRefresh:input_type -> openshell.v1.DeleteProviderRefreshRequest + 121, // 381: openshell.v1.OpenShell.DeleteProvider:input_type -> openshell.v1.DeleteProviderRequest + 156, // 382: openshell.v1.OpenShell.DeleteProviderProfile:input_type -> openshell.v1.DeleteProviderProfileRequest + 299, // 383: openshell.v1.OpenShell.GetSandboxConfig:input_type -> openshell.sandbox.v1.GetSandboxConfigRequest + 300, // 384: openshell.v1.OpenShell.GetGatewayConfig:input_type -> openshell.sandbox.v1.GetGatewayConfigRequest + 164, // 385: openshell.v1.OpenShell.UpdateConfig:input_type -> openshell.v1.UpdateConfigRequest + 174, // 386: openshell.v1.OpenShell.GetSandboxPolicyStatus:input_type -> openshell.v1.GetSandboxPolicyStatusRequest + 176, // 387: openshell.v1.OpenShell.ListSandboxPolicies:input_type -> openshell.v1.ListSandboxPoliciesRequest + 178, // 388: openshell.v1.OpenShell.ReportPolicyStatus:input_type -> openshell.v1.ReportPolicyStatusRequest + 252, // 389: openshell.v1.OpenShell.ReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 85, // 390: openshell.v1.OpenShell.ReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 181, // 391: openshell.v1.OpenShell.ReportSandboxConfiguration:input_type -> openshell.v1.ReportSandboxConfigurationRequest + 158, // 392: openshell.v1.OpenShell.GetSandboxProviderEnvironment:input_type -> openshell.v1.GetSandboxProviderEnvironmentRequest + 162, // 393: openshell.v1.OpenShell.ExchangeProviderSubjectToken:input_type -> openshell.v1.ExchangeProviderSubjectTokenRequest + 184, // 394: openshell.v1.OpenShell.GetSandboxLogs:input_type -> openshell.v1.GetSandboxLogsRequest + 185, // 395: openshell.v1.OpenShell.PushSandboxLogs:input_type -> openshell.v1.PushSandboxLogsRequest + 188, // 396: openshell.v1.OpenShell.ConnectSupervisor:input_type -> openshell.v1.SupervisorMessage + 196, // 397: openshell.v1.OpenShell.ReportMainProcessExit:input_type -> openshell.v1.ReportMainProcessExitRequest + 198, // 398: openshell.v1.OpenShell.FinalizeMainProcessExit:input_type -> openshell.v1.FinalizeMainProcessExitRequest + 204, // 399: openshell.v1.OpenShell.RelayStream:input_type -> openshell.v1.RelayFrame + 206, // 400: openshell.v1.OpenShell.PeerRelay:input_type -> openshell.v1.PeerRelayFrame + 85, // 401: openshell.v1.OpenShell.PeerReportProviderReadiness:input_type -> openshell.v1.ReportProviderReadinessRequest + 252, // 402: openshell.v1.OpenShell.PeerReportEndpointStatus:input_type -> openshell.v1.ReportEndpointStatusRequest + 83, // 403: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:input_type -> openshell.v1.GetSandboxProviderStatusRequest + 110, // 404: openshell.v1.OpenShell.WatchSandbox:input_type -> openshell.v1.WatchSandboxRequest + 215, // 405: openshell.v1.OpenShell.SubmitPolicyAnalysis:input_type -> openshell.v1.SubmitPolicyAnalysisRequest + 217, // 406: openshell.v1.OpenShell.GetDraftPolicy:input_type -> openshell.v1.GetDraftPolicyRequest + 219, // 407: openshell.v1.OpenShell.ApproveDraftChunk:input_type -> openshell.v1.ApproveDraftChunkRequest + 221, // 408: openshell.v1.OpenShell.RejectDraftChunk:input_type -> openshell.v1.RejectDraftChunkRequest + 224, // 409: openshell.v1.OpenShell.ApproveAllDraftChunks:input_type -> openshell.v1.ApproveAllDraftChunksRequest + 226, // 410: openshell.v1.OpenShell.EditDraftChunk:input_type -> openshell.v1.EditDraftChunkRequest + 228, // 411: openshell.v1.OpenShell.UndoDraftChunk:input_type -> openshell.v1.UndoDraftChunkRequest + 230, // 412: openshell.v1.OpenShell.ClearDraftChunks:input_type -> openshell.v1.ClearDraftChunksRequest + 232, // 413: openshell.v1.OpenShell.GetDraftHistory:input_type -> openshell.v1.GetDraftHistoryRequest + 21, // 414: openshell.v1.OpenShell.IssueSandboxToken:input_type -> openshell.v1.IssueSandboxTokenRequest + 23, // 415: openshell.v1.OpenShell.RefreshSandboxToken:input_type -> openshell.v1.RefreshSandboxTokenRequest + 235, // 416: openshell.v1.OpenShell.CreateWorkspace:input_type -> openshell.v1.CreateWorkspaceRequest + 237, // 417: openshell.v1.OpenShell.GetWorkspace:input_type -> openshell.v1.GetWorkspaceRequest + 239, // 418: openshell.v1.OpenShell.ListWorkspaces:input_type -> openshell.v1.ListWorkspacesRequest + 241, // 419: openshell.v1.OpenShell.DeleteWorkspace:input_type -> openshell.v1.DeleteWorkspaceRequest + 244, // 420: openshell.v1.OpenShell.AddWorkspaceMember:input_type -> openshell.v1.AddWorkspaceMemberRequest + 246, // 421: openshell.v1.OpenShell.RemoveWorkspaceMember:input_type -> openshell.v1.RemoveWorkspaceMemberRequest + 248, // 422: openshell.v1.OpenShell.ListWorkspaceMembers:input_type -> openshell.v1.ListWorkspaceMembersRequest + 26, // 423: openshell.v1.OpenShell.Health:output_type -> openshell.v1.HealthResponse + 28, // 424: openshell.v1.OpenShell.GetCurrentUser:output_type -> openshell.v1.GetCurrentUserResponse + 30, // 425: openshell.v1.OpenShell.GetGatewayInfo:output_type -> openshell.v1.GetGatewayInfoResponse + 71, // 426: openshell.v1.OpenShell.CreateSandbox:output_type -> openshell.v1.SandboxResponse + 62, // 427: openshell.v1.OpenShell.BeginRootfsTarStaging:output_type -> openshell.v1.BeginRootfsTarStagingResponse + 71, // 428: openshell.v1.OpenShell.GetSandbox:output_type -> openshell.v1.SandboxResponse + 72, // 429: openshell.v1.OpenShell.ListSandboxes:output_type -> openshell.v1.ListSandboxesResponse + 58, // 430: openshell.v1.OpenShell.CreateSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 58, // 431: openshell.v1.OpenShell.GetSandboxTemplate:output_type -> openshell.v1.SandboxTemplateResponse + 59, // 432: openshell.v1.OpenShell.ListSandboxTemplates:output_type -> openshell.v1.ListSandboxTemplatesResponse + 60, // 433: openshell.v1.OpenShell.DeleteSandboxTemplate:output_type -> openshell.v1.DeleteSandboxTemplateResponse + 73, // 434: openshell.v1.OpenShell.ListSandboxProviders:output_type -> openshell.v1.ListSandboxProvidersResponse + 74, // 435: openshell.v1.OpenShell.AttachSandboxProvider:output_type -> openshell.v1.AttachSandboxProviderResponse + 75, // 436: openshell.v1.OpenShell.DetachSandboxProvider:output_type -> openshell.v1.DetachSandboxProviderResponse + 84, // 437: openshell.v1.OpenShell.GetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 87, // 438: openshell.v1.OpenShell.DeleteSandbox:output_type -> openshell.v1.DeleteSandboxResponse + 71, // 439: openshell.v1.OpenShell.StopSandbox:output_type -> openshell.v1.SandboxResponse + 71, // 440: openshell.v1.OpenShell.StartSandbox:output_type -> openshell.v1.SandboxResponse + 89, // 441: openshell.v1.OpenShell.CreateSshSession:output_type -> openshell.v1.CreateSshSessionResponse + 97, // 442: openshell.v1.OpenShell.ExposeService:output_type -> openshell.v1.ServiceEndpointResponse + 97, // 443: openshell.v1.OpenShell.GetService:output_type -> openshell.v1.ServiceEndpointResponse + 93, // 444: openshell.v1.OpenShell.ListServices:output_type -> openshell.v1.ListServicesResponse + 95, // 445: openshell.v1.OpenShell.DeleteService:output_type -> openshell.v1.DeleteServiceResponse + 99, // 446: openshell.v1.OpenShell.RevokeSshSession:output_type -> openshell.v1.RevokeSshSessionResponse + 104, // 447: openshell.v1.OpenShell.ExecSandbox:output_type -> openshell.v1.ExecSandboxEvent + 106, // 448: openshell.v1.OpenShell.ForwardTcp:output_type -> openshell.v1.TcpForwardFrame + 104, // 449: openshell.v1.OpenShell.ExecSandboxInteractive:output_type -> openshell.v1.ExecSandboxEvent + 122, // 450: openshell.v1.OpenShell.CreateProvider:output_type -> openshell.v1.ProviderResponse + 122, // 451: openshell.v1.OpenShell.GetProvider:output_type -> openshell.v1.ProviderResponse + 118, // 452: openshell.v1.OpenShell.GetProviderCredentials:output_type -> openshell.v1.GetProviderCredentialsResponse + 123, // 453: openshell.v1.OpenShell.ListProviders:output_type -> openshell.v1.ListProvidersResponse + 148, // 454: openshell.v1.OpenShell.ListProviderProfiles:output_type -> openshell.v1.ListProviderProfilesResponse + 147, // 455: openshell.v1.OpenShell.GetProviderProfile:output_type -> openshell.v1.ProviderProfileResponse + 150, // 456: openshell.v1.OpenShell.ImportProviderProfiles:output_type -> openshell.v1.ImportProviderProfilesResponse + 152, // 457: openshell.v1.OpenShell.UpdateProviderProfiles:output_type -> openshell.v1.UpdateProviderProfilesResponse + 154, // 458: openshell.v1.OpenShell.LintProviderProfiles:output_type -> openshell.v1.LintProviderProfilesResponse + 122, // 459: openshell.v1.OpenShell.UpdateProvider:output_type -> openshell.v1.ProviderResponse + 138, // 460: openshell.v1.OpenShell.GetProviderRefreshStatus:output_type -> openshell.v1.GetProviderRefreshStatusResponse + 140, // 461: openshell.v1.OpenShell.ConfigureProviderRefresh:output_type -> openshell.v1.ConfigureProviderRefreshResponse + 142, // 462: openshell.v1.OpenShell.RotateProviderCredential:output_type -> openshell.v1.RotateProviderCredentialResponse + 144, // 463: openshell.v1.OpenShell.DeleteProviderRefresh:output_type -> openshell.v1.DeleteProviderRefreshResponse + 155, // 464: openshell.v1.OpenShell.DeleteProvider:output_type -> openshell.v1.DeleteProviderResponse + 157, // 465: openshell.v1.OpenShell.DeleteProviderProfile:output_type -> openshell.v1.DeleteProviderProfileResponse + 301, // 466: openshell.v1.OpenShell.GetSandboxConfig:output_type -> openshell.sandbox.v1.GetSandboxConfigResponse + 302, // 467: openshell.v1.OpenShell.GetGatewayConfig:output_type -> openshell.sandbox.v1.GetGatewayConfigResponse + 173, // 468: openshell.v1.OpenShell.UpdateConfig:output_type -> openshell.v1.UpdateConfigResponse + 175, // 469: openshell.v1.OpenShell.GetSandboxPolicyStatus:output_type -> openshell.v1.GetSandboxPolicyStatusResponse + 177, // 470: openshell.v1.OpenShell.ListSandboxPolicies:output_type -> openshell.v1.ListSandboxPoliciesResponse + 179, // 471: openshell.v1.OpenShell.ReportPolicyStatus:output_type -> openshell.v1.ReportPolicyStatusResponse + 255, // 472: openshell.v1.OpenShell.ReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 86, // 473: openshell.v1.OpenShell.ReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 182, // 474: openshell.v1.OpenShell.ReportSandboxConfiguration:output_type -> openshell.v1.ReportSandboxConfigurationResponse + 161, // 475: openshell.v1.OpenShell.GetSandboxProviderEnvironment:output_type -> openshell.v1.GetSandboxProviderEnvironmentResponse + 163, // 476: openshell.v1.OpenShell.ExchangeProviderSubjectToken:output_type -> openshell.v1.ExchangeProviderSubjectTokenResponse + 187, // 477: openshell.v1.OpenShell.GetSandboxLogs:output_type -> openshell.v1.GetSandboxLogsResponse + 186, // 478: openshell.v1.OpenShell.PushSandboxLogs:output_type -> openshell.v1.PushSandboxLogsResponse + 189, // 479: openshell.v1.OpenShell.ConnectSupervisor:output_type -> openshell.v1.GatewayMessage + 197, // 480: openshell.v1.OpenShell.ReportMainProcessExit:output_type -> openshell.v1.ReportMainProcessExitResponse + 199, // 481: openshell.v1.OpenShell.FinalizeMainProcessExit:output_type -> openshell.v1.FinalizeMainProcessExitResponse + 204, // 482: openshell.v1.OpenShell.RelayStream:output_type -> openshell.v1.RelayFrame + 206, // 483: openshell.v1.OpenShell.PeerRelay:output_type -> openshell.v1.PeerRelayFrame + 86, // 484: openshell.v1.OpenShell.PeerReportProviderReadiness:output_type -> openshell.v1.ReportProviderReadinessResponse + 255, // 485: openshell.v1.OpenShell.PeerReportEndpointStatus:output_type -> openshell.v1.ReportEndpointStatusResponse + 84, // 486: openshell.v1.OpenShell.PeerGetSandboxProviderStatus:output_type -> openshell.v1.GetSandboxProviderStatusResponse + 111, // 487: openshell.v1.OpenShell.WatchSandbox:output_type -> openshell.v1.SandboxStreamEvent + 216, // 488: openshell.v1.OpenShell.SubmitPolicyAnalysis:output_type -> openshell.v1.SubmitPolicyAnalysisResponse + 218, // 489: openshell.v1.OpenShell.GetDraftPolicy:output_type -> openshell.v1.GetDraftPolicyResponse + 220, // 490: openshell.v1.OpenShell.ApproveDraftChunk:output_type -> openshell.v1.ApproveDraftChunkResponse + 222, // 491: openshell.v1.OpenShell.RejectDraftChunk:output_type -> openshell.v1.RejectDraftChunkResponse + 225, // 492: openshell.v1.OpenShell.ApproveAllDraftChunks:output_type -> openshell.v1.ApproveAllDraftChunksResponse + 227, // 493: openshell.v1.OpenShell.EditDraftChunk:output_type -> openshell.v1.EditDraftChunkResponse + 229, // 494: openshell.v1.OpenShell.UndoDraftChunk:output_type -> openshell.v1.UndoDraftChunkResponse + 231, // 495: openshell.v1.OpenShell.ClearDraftChunks:output_type -> openshell.v1.ClearDraftChunksResponse + 234, // 496: openshell.v1.OpenShell.GetDraftHistory:output_type -> openshell.v1.GetDraftHistoryResponse + 22, // 497: openshell.v1.OpenShell.IssueSandboxToken:output_type -> openshell.v1.IssueSandboxTokenResponse + 24, // 498: openshell.v1.OpenShell.RefreshSandboxToken:output_type -> openshell.v1.RefreshSandboxTokenResponse + 236, // 499: openshell.v1.OpenShell.CreateWorkspace:output_type -> openshell.v1.CreateWorkspaceResponse + 238, // 500: openshell.v1.OpenShell.GetWorkspace:output_type -> openshell.v1.GetWorkspaceResponse + 240, // 501: openshell.v1.OpenShell.ListWorkspaces:output_type -> openshell.v1.ListWorkspacesResponse + 242, // 502: openshell.v1.OpenShell.DeleteWorkspace:output_type -> openshell.v1.DeleteWorkspaceResponse + 245, // 503: openshell.v1.OpenShell.AddWorkspaceMember:output_type -> openshell.v1.AddWorkspaceMemberResponse + 247, // 504: openshell.v1.OpenShell.RemoveWorkspaceMember:output_type -> openshell.v1.RemoveWorkspaceMemberResponse + 249, // 505: openshell.v1.OpenShell.ListWorkspaceMembers:output_type -> openshell.v1.ListWorkspaceMembersResponse + 423, // [423:506] is the sub-list for method output_type + 340, // [340:423] is the sub-list for method input_type + 340, // [340:340] is the sub-list for extension type_name + 340, // [340:340] is the sub-list for extension extendee + 0, // [0:340] is the sub-list for field type_name } func init() { file_openshell_proto_init() } @@ -20817,13 +21174,14 @@ func file_openshell_proto_init() { (*PeerRelayFrame_Init)(nil), (*PeerRelayFrame_Data)(nil), } + file_openshell_proto_msgTypes[238].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_openshell_proto_rawDesc), len(file_openshell_proto_rawDesc)), - NumEnums: 20, - NumMessages: 260, + NumEnums: 21, + NumMessages: 263, NumExtensions: 0, NumServices: 1, }, diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index ad331a2bda..c7435c3f8e 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -307,7 +307,7 @@ Common findings: - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. - SSH host-key startup errors: use matching gateway, compute-driver, and supervisor releases. The gateway retains each sandbox's SSH key in its configured credential store and sends it only through the supervisor bootstrap bundle. Check credential-driver availability and compare the public `host_key_fingerprint` from sandbox JSON output; never print the bootstrap bundle or private key. A missing stored key for a sandbox with a fingerprint is an error, not permission to replace its identity. See the [sandbox SSH identity documentation](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/overview). -- Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. If several paths share a host and port, a failure before the path is known remains in logs. Verify the actual operation when current tool availability matters. +- Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. For token grant failures on any HTTP protocol, including failures before endpoint selection, inspect `last_token_grant_failure` in JSON output. It contains the bounded destination and cause, survives unrelated successful traffic, and clears on matching grant success or a policy, provider, or supervisor reset. Verify the actual operation when current tool availability matters. A `CredentialUnavailable` result from a failed dynamic token grant also carries `last_failure` with a closed-set `code`, `stage`, `recovery_action`, the provider and credential names, and the `request_id` that the workload received in its `x-openshell-request-id` response header. Search `openshell logs ` for that ID: the supervisor's `Token grant failed` line carries the code and stage for every endpoint, and the gateway's `dynamic token grant failed` line appears when the gateway hop failed. - On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages can mean host networking is disabled. Enable host networking in Docker Desktop, ensure Enhanced Container Isolation is disabled, and verify the From 78e587e577149ef321c79ee539f58a61d066af99 Mon Sep 17 00:00:00 2001 From: Shiju Date: Mon, 12 Oct 2026 23:53:33 +0530 Subject: [PATCH 02/13] test(providers): align policy metadata and TLS lockfile Signed-off-by: Shiju --- .../openshell-supervisor-network/src/opa.rs | 25 +++++++++++-------- tests/suites/features/Cargo.lock | 1 + 2 files changed, 16 insertions(+), 10 deletions(-) diff --git a/crates/openshell-supervisor-network/src/opa.rs b/crates/openshell-supervisor-network/src/opa.rs index f3d12d8569..2d1a73a949 100644 --- a/crates/openshell-supervisor-network/src/opa.rs +++ b/crates/openshell-supervisor-network/src/opa.rs @@ -3489,17 +3489,15 @@ mod tests { assert!(yaml_config.get("endpoint_id").is_none()); assert!(yaml_config.get("policy_hash").is_none()); let mut expected = yaml_config.clone(); + // Grant reporting binds every protocol to the canonical policy. Derive + // this metadata independently and compare all remaining configuration. + expected["policy_hash"] = deterministic_policy_hash(canonical_policy).into(); + assert_eq!(proto_config["policy_hash"], expected["policy_hash"]); if is_mcp_protocol(&endpoint.protocol) { - // Gateway MCP observations carry canonical endpoint/policy identity. - // Derive only that metadata independently; compare every remaining - // configuration field without filtering actual runtime output. expected["endpoint_id"] = openshell_core::endpoint_status::endpoint_id(endpoint).into(); - expected["policy_hash"] = deterministic_policy_hash(canonical_policy).into(); assert_eq!(proto_config["endpoint_id"], expected["endpoint_id"]); - assert_eq!(proto_config["policy_hash"], expected["policy_hash"]); } else { assert!(proto_config.get("endpoint_id").is_none()); - assert!(proto_config.get("policy_hash").is_none()); } assert_eq!( &expected, proto_config, @@ -3867,10 +3865,17 @@ network_policies: ancestors: vec![], cmdline_paths: vec![], }; - assert_eq!( - yaml_engine.query_endpoint_config(&input).unwrap(), - proto_engine.query_endpoint_config(&input).unwrap() - ); + let yaml_config = yaml_engine.query_endpoint_config(&input).unwrap(); + let proto_config = proto_engine.query_endpoint_config(&input).unwrap(); + match (&yaml_config, &proto_config) { + (Some(yaml_config), Some(proto_config)) => assert_endpoint_config_parity( + &serde_json::from_str(&yaml_config.to_json_str().unwrap()).unwrap(), + &serde_json::from_str(&proto_config.to_json_str().unwrap()).unwrap(), + &proto, + &proto.network_policies["parity"].endpoints[0], + ), + _ => assert_eq!(yaml_config, proto_config), + } for engine in [&yaml_engine, &proto_engine] { assert!(matches!( engine.evaluate_network_action(&input).unwrap(), diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index b92ccd05f5..b096620b70 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1586,6 +1586,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", From a6f32bb0d273ba653e98b6de14d47b881fd4e420 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 00:31:24 +0530 Subject: [PATCH 03/13] test(providers): fix trusted grant recovery fixture Signed-off-by: Shiju --- e2e/rust/tests/provider_token_exchange/closure.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/e2e/rust/tests/provider_token_exchange/closure.rs b/e2e/rust/tests/provider_token_exchange/closure.rs index 4709a03767..0b2fd6bd15 100644 --- a/e2e/rust/tests/provider_token_exchange/closure.rs +++ b/e2e/rust/tests/provider_token_exchange/closure.rs @@ -116,7 +116,7 @@ impl GatewayTrust { ) .map_err(|_| "could not stage public fixture CA")?; fs::write(context.join("Dockerfile"), format!( - "FROM {base_image} AS original\nFROM {E2E_WORKLOAD_IMAGE} AS bundle\nUSER 0\nCOPY --from=original /etc/ssl/certs/ca-certificates.crt /tmp/roots.crt\nCOPY fixture-ca.crt /tmp/fixture-ca.crt\nRUN [\"/usr/bin/python3\", \"-c\", \"from pathlib import Path; p=Path('/tmp/roots.crt'); p.write_bytes(p.read_bytes()+b'\\n'+Path('/tmp/fixture-ca.crt').read_bytes())\"]\nFROM {base_image}\nCOPY --from=bundle /tmp/roots.crt /etc/ssl/certs/ca-certificates.crt\n" + "FROM {base_image} AS original\nFROM {E2E_WORKLOAD_IMAGE} AS bundle\nUSER 0\nCOPY --from=original /etc/ssl/certs/ca-certificates.crt /tmp/roots.crt\nCOPY fixture-ca.crt /tmp/fixture-ca.crt\nRUN [\"/usr/bin/python3\", \"-c\", \"from pathlib import Path; p=Path('/tmp/roots.crt'); p.write_bytes(p.read_bytes()+Path('/tmp/fixture-ca.crt').read_bytes())\"]\nFROM {base_image}\nCOPY --from=bundle /tmp/roots.crt /etc/ssl/certs/ca-certificates.crt\n" )).map_err(|_| "could not write supervisor fixture Dockerfile")?; let engine = ContainerEngine::from_env()?; let image = format!( @@ -125,7 +125,14 @@ impl GatewayTrust { ); let mut build = Command::from(engine.command()); build.args(["build", "--tag", &image]).arg(&context); - checked(&mut build, "build trusted supervisor fixture").await?; + build.kill_on_drop(true); + let status = tokio::time::timeout(Duration::from_secs(300), build.status()) + .await + .map_err(|_| "build trusted supervisor fixture timed out")? + .map_err(|_| "build trusted supervisor fixture could not start")?; + if !status.success() { + return Err("build trusted supervisor fixture failed".into()); + } Ok(Self { directory, engine, @@ -648,7 +655,7 @@ async fn create_rest_fixture( let contents = fs::read_to_string(profile.path()) .map_err(|_| "fixture profile is missing")? .replace("http://127.0.0.1:", "https://127.0.0.1:") - .replace("cache_ttl_seconds: 300", "cache_ttl_seconds: 0"); + .replace("cache_ttl_seconds: 300", "cache_ttl: \"0s\""); fs::write(profile.path(), contents).map_err(|_| "could not set verified token endpoint")?; run_cli(&[ "profile", From 0174d6b7be4b96bd8fb93aed39a733601dbbda97 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 01:01:03 +0530 Subject: [PATCH 04/13] test(providers): enable denial API in recovery fixture Signed-off-by: Shiju --- .../tests/provider_token_exchange/closure.rs | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/e2e/rust/tests/provider_token_exchange/closure.rs b/e2e/rust/tests/provider_token_exchange/closure.rs index 0b2fd6bd15..02d1963977 100644 --- a/e2e/rust/tests/provider_token_exchange/closure.rs +++ b/e2e/rust/tests/provider_token_exchange/closure.rs @@ -675,12 +675,40 @@ async fn create_rest_fixture( "subject_token=stored-closure", ]) .await?; - SandboxGuard::create_keep_with_args( + let sandbox = SandboxGuard::create_keep_with_args( &["--name", name, "--provider", name], &["sh", "-lc", &sandbox_script()], "token-server-ready", ) - .await + .await?; + cli(&[ + "settings", + "set", + name, + "--key", + "agent_policy_proposals_enabled", + "--value", + "true", + ]) + .await?; + sandbox + .exec(&[ + "python3", + "-c", + r"import time, urllib.request, urllib.error +deadline = time.monotonic() + 60 +while True: + try: + with urllib.request.urlopen('http://policy.local/v1/denials?last=1', timeout=2): + break + except urllib.error.HTTPError as error: + if error.code != 404 or time.monotonic() >= deadline: + raise + time.sleep(0.2) +", + ]) + .await?; + Ok(sandbox) } async fn recovery_cases( From 61e186ab358feeaa17b4153b8d9a82d25c3035b3 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 12:57:25 +0530 Subject: [PATCH 05/13] refactor(providers): simplify token-grant failure reporting Remove duplication and dead paths from the token-grant failure reporting without changing classification, the workload-visible 502 response, or the wire format. - Build the gRPC failure envelope with tonic-types, store the typed failure in EndpointFailure, and share one request-ID rule and one endpoint-status authority check. - Merge the gateway exchange client wrapper, drop forwarding validators, and name the request field when a reported cause is rejected. - Inject fallback credentials only from live provider state and delete the unread per-connection credential copy. Keep one 502 writer per path and restore the base token-cache handling. - Check the token endpoint after the cache lookup, still before any credential acquisition. - Document 429 and 5xx classification as implemented, and drop tests that repeat retained assertions through the same code path. Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 13 +- crates/openshell-core/src/endpoint_failure.rs | 82 +--- crates/openshell-core/src/endpoint_status.rs | 62 +-- crates/openshell-core/src/grpc_client.rs | 57 +-- crates/openshell-core/src/oauth.rs | 161 ++------ .../openshell-core/src/token_grant_failure.rs | 87 ++--- crates/openshell-server/src/grpc/policy.rs | 3 +- .../src/grpc/policy/endpoint_status.rs | 11 +- .../src/grpc/policy/endpoint_status_tests.rs | 77 +--- crates/openshell-server/src/grpc/provider.rs | 276 +++++--------- .../src/supervisor_session.rs | 17 +- .../src/l7/mod.rs | 10 +- .../src/l7/relay.rs | 102 ++--- .../l7/relay/token_grant_ownership_tests.rs | 40 +- .../src/l7/token_grant_injection.rs | 360 +++++------------- .../token_grant_injection/multiple_grants.rs | 17 +- .../openshell-supervisor-network/src/proxy.rs | 156 ++------ .../src/proxy/relay.rs | 14 +- .../src/proxy/tests/compatibility.rs | 1 - .../src/proxy/tests/token_grants.rs | 1 - .../src/token_grant.rs | 160 +++----- docs/how-it-works/providers/profiles.mdx | 13 +- docs/observability/logging.mdx | 2 +- .../tests/provider_token_exchange/closure.rs | 52 +-- .../v1/internal/converter/sandbox_test.go | 22 -- skills/debug-openshell-cluster/SKILL.md | 2 +- 26 files changed, 511 insertions(+), 1287 deletions(-) diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 37e66d1725..58b83e0288 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -3185,15 +3185,6 @@ fn endpoint_failure_domain_name(domain: i32) -> &'static str { } } -fn endpoint_failure_recovery( - failure: &openshell_core::proto::EndpointFailure, -) -> openshell_core::proto::ProviderCredentialRefreshRecoveryAction { - openshell_core::proto::ProviderCredentialRefreshRecoveryAction::try_from( - failure.recovery_action, - ) - .unwrap_or(openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Unspecified) -} - fn endpoint_failure_recovery_name( action: openshell_core::proto::ProviderCredentialRefreshRecoveryAction, ) -> &'static str { @@ -3210,7 +3201,7 @@ fn endpoint_failure_to_json(failure: &openshell_core::proto::EndpointFailure) -> serde_json::json!({ "domain": endpoint_failure_domain_name(failure.domain), "code": failure.code, - "recovery_action": endpoint_failure_recovery_name(endpoint_failure_recovery(failure)), + "recovery_action": endpoint_failure_recovery_name(failure.recovery_action()), "provider": failure.provider, "credential_key": failure.credential_key, "stage": failure.stage, @@ -3225,7 +3216,7 @@ fn endpoint_failure_display_lines(failure: &openshell_core::proto::EndpointFailu .upstream_http_status .map(|status| format!(", upstream HTTP {status}")) .unwrap_or_default(); - let recovery = match endpoint_failure_recovery(failure) { + let recovery = match failure.recovery_action() { Recovery::Retry => "Retry later; the failure can be temporary.", Recovery::Reauthorize => "Reauthorize the provider credential.", Recovery::FixConfiguration => { diff --git a/crates/openshell-core/src/endpoint_failure.rs b/crates/openshell-core/src/endpoint_failure.rs index dba48aaa07..abd7034732 100644 --- a/crates/openshell-core/src/endpoint_failure.rs +++ b/crates/openshell-core/src/endpoint_failure.rs @@ -3,27 +3,19 @@ //! Bounded causes for failed endpoint results. //! -//! A cause holds a code and a stage from the closed sets of its domain, the -//! recovery action documented for the code, configured provider and -//! credential names, an optional HTTP status, and the supervisor's request ID. //! The supervisor builds causes from typed failures and the gateway validates -//! reported causes against the same sets here, so neither side can accept a -//! code the other does not define. Each domain also names the endpoint results -//! it can explain. No cause carries remote text. -//! -//! A new producer adds a domain variant and one validation function in this -//! module that checks its result, code and stage sets, and code-to-recovery -//! mapping. Names, status, and the request ID are checked the same way for -//! every domain. +//! reported causes against the same closed sets here, so neither side can +//! accept a code the other does not define. Each domain also names the endpoint +//! results it can explain. No cause carries remote text. -use crate::proto::{self, ProviderCredentialRefreshRecoveryAction}; +use crate::proto; use crate::token_grant_failure::{TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage}; /// Longest provider or credential name a reported cause may carry. /// /// A reported name is also limited to printable ASCII, so status renderers /// cannot be given control, bidirectional, or format characters. -pub const MAX_ENDPOINT_FAILURE_NAME_BYTES: usize = 253; +const MAX_ENDPOINT_FAILURE_NAME_BYTES: usize = 253; /// The bounded cause of one failed endpoint observation. /// @@ -31,13 +23,9 @@ pub const MAX_ENDPOINT_FAILURE_NAME_BYTES: usize = 253; /// endpoint is unhealthy. A token grant can fail before the request reaches it. #[derive(Clone, Debug, Eq, PartialEq)] pub struct EndpointFailure { - domain: proto::EndpointFailureDomain, - code: &'static str, - stage: &'static str, - recovery: ProviderCredentialRefreshRecoveryAction, + failure: TokenGrantFailure, provider: String, credential_key: String, - upstream_http_status: Option, request_id: String, } @@ -64,13 +52,9 @@ impl EndpointFailure { ("", "") }; Self { - domain: proto::EndpointFailureDomain::TokenGrant, - code: failure.code(), - stage: failure.stage().code(), - recovery: failure.recovery(), + failure, provider: provider.to_string(), credential_key: credential_key.to_string(), - upstream_http_status: failure.upstream_status(), request_id: request_id.to_string(), } } @@ -79,13 +63,13 @@ impl EndpointFailure { #[must_use] pub fn to_proto(&self) -> proto::EndpointFailure { proto::EndpointFailure { - domain: self.domain.into(), - code: self.code.to_string(), - recovery_action: self.recovery.into(), + domain: proto::EndpointFailureDomain::TokenGrant.into(), + code: self.failure.code().to_string(), + recovery_action: self.failure.recovery().into(), provider: self.provider.clone(), credential_key: self.credential_key.clone(), - stage: self.stage.to_string(), - upstream_http_status: self.upstream_http_status.map(u32::from), + stage: self.failure.stage().code().to_string(), + upstream_http_status: self.failure.upstream_status().map(u32::from), request_id: self.request_id.clone(), } } @@ -161,7 +145,7 @@ pub fn validate_endpoint_failure( { return Err(EndpointFailureValidationError::UpstreamStatus); } - if !is_canonical_uuid(&failure.request_id) { + if !is_canonical_request_id(&failure.request_id) { return Err(EndpointFailureValidationError::RequestId); } Ok(()) @@ -190,13 +174,15 @@ fn valid_name(name: &str) -> bool { && name.bytes().all(|byte| byte.is_ascii_graphic()) } -fn is_canonical_uuid(value: &str) -> bool { +/// Accept only a lowercase hyphenated UUID, the request-ID form causes and gateway logs share. +pub fn is_canonical_request_id(value: &str) -> bool { uuid::Uuid::parse_str(value).is_ok_and(|parsed| parsed.hyphenated().to_string() == value) } #[cfg(test)] mod tests { use super::*; + use crate::proto::ProviderCredentialRefreshRecoveryAction; const REQUEST_ID: &str = "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9"; const CREDENTIAL_UNAVAILABLE: proto::EndpointResult = @@ -237,38 +223,6 @@ mod tests { ); } - #[test] - fn every_token_grant_classification_validates() { - for kind in [ - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantFailureKind::WorkloadIdentityRejected, - TokenGrantFailureKind::Configuration, - TokenGrantFailureKind::Timeout, - TokenGrantFailureKind::Unavailable, - TokenGrantFailureKind::Unclassified, - ] { - for stage in [ - TokenGrantStage::Configuration, - TokenGrantStage::WorkloadIdentity, - TokenGrantStage::GatewayRpc, - TokenGrantStage::IntermediateExchange, - TokenGrantStage::TokenEndpoint, - TokenGrantStage::Injection, - ] { - let failure = EndpointFailure::token_grant( - TokenGrantFailure::new(kind, stage), - "", - "", - REQUEST_ID, - ); - assert_eq!( - validate_endpoint_failure(&failure.to_proto(), CREDENTIAL_UNAVAILABLE), - Ok(()) - ); - } - } - } - type Mutation = fn(&mut proto::EndpointFailure); #[test] @@ -288,10 +242,6 @@ mod tests { "{result:?}" ); } - assert_eq!( - validate_endpoint_failure(&reported(), CREDENTIAL_UNAVAILABLE), - Ok(()) - ); } #[test] diff --git a/crates/openshell-core/src/endpoint_status.rs b/crates/openshell-core/src/endpoint_status.rs index 4cb604cf2b..d878f4f53a 100644 --- a/crates/openshell-core/src/endpoint_status.rs +++ b/crates/openshell-core/src/endpoint_status.rs @@ -195,9 +195,7 @@ impl EndpointObservationContext { #[must_use] pub fn matches_config(&self, policy_hash: &str, provider_env_revision: Option) -> bool { self.inventory.config_version.policy_hash == policy_hash - && provider_env_revision.is_none_or(|revision| { - revision == self.inventory.config_version.provider_env_revision - }) + && self.matches_provider_revision(provider_env_revision) } /// Check a provider snapshot when forwarding has no L7 endpoint selection. @@ -347,11 +345,7 @@ impl EndpointObservationSender { policy_hash: &str, provider_env_revision: Option, ) -> Option { - let config = &context.inventory.config_version; - if config.policy_hash != policy_hash - || provider_env_revision - .is_some_and(|revision| revision != config.provider_env_revision) - { + if !context.matches_config(policy_hash, provider_env_revision) { // Runtime activation and inventory publication are separate. Reject // observations during that gap instead of assigning another policy's result. return None; @@ -366,7 +360,7 @@ impl EndpointObservationSender { return None; } Some(EndpointObservationHandle { - config_version: config.clone(), + config_version: context.inventory.config_version.clone(), endpoint_id, inventory_generation: context.inventory.generation.clone(), authority_generation: context.authority_generation.clone(), @@ -592,17 +586,11 @@ impl EndpointStatusTracker { inventory_generation, authority_generation, } => { - if self.config_version.as_ref() != Some(&config_version) - || !self - .inventory_generation - .as_ref() - .is_some_and(|current| current.matches(&inventory_generation)) - || !self - .current - .borrow() - .authority_generation - .matches(&authority_generation) - { + if !self.is_current( + &config_version, + &inventory_generation, + &authority_generation, + ) { // Stale evidence must not change results or freshness. return false; } @@ -621,17 +609,11 @@ impl EndpointStatusTracker { destination_port, result, } => { - if self.config_version.as_ref() != Some(&context.inventory.config_version) - || !self - .inventory_generation - .as_ref() - .is_some_and(|generation| generation.matches(&context.inventory.generation)) - || !self - .current - .borrow() - .authority_generation - .matches(&context.authority_generation) - { + if !self.is_current( + &context.inventory.config_version, + &context.inventory.generation, + &context.authority_generation, + ) { return false; } match result { @@ -751,6 +733,24 @@ impl EndpointStatusTracker { .collect(); self.config_version = Some(config_version); } + + fn is_current( + &self, + config_version: &EndpointConfigVersion, + inventory_generation: &EndpointObservationGeneration, + authority_generation: &EndpointObservationGeneration, + ) -> bool { + self.config_version.as_ref() == Some(config_version) + && self + .inventory_generation + .as_ref() + .is_some_and(|current| current.matches(inventory_generation)) + && self + .current + .borrow() + .authority_generation + .matches(authority_generation) + } } #[cfg(test)] diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index 8b07457086..f32e1fe78d 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -1370,33 +1370,9 @@ pub async fn exchange_provider_subject_token( request_id: Option<&str>, ) -> std::result::Result { let status = ClientSpanStatus::current(); - status.finish( - exchange_provider_subject_token_inner( - endpoint, - sandbox_id, - provider, - credential_key, - supervisor_jwt_svid, - request_id, - ) - .await, - ) -} - -async fn exchange_provider_subject_token_inner( - endpoint: &str, - sandbox_id: &str, - provider: &str, - credential_key: &str, - supervisor_jwt_svid: &str, - request_id: Option<&str>, -) -> std::result::Result { let endpoint = token_grant_gateway_endpoint(endpoint)?; if sandbox_id.is_empty() || provider.is_empty() || credential_key.is_empty() { - return Err(TokenGrantFailure::new( - TokenGrantFailureKind::Configuration, - TokenGrantStage::GatewayRpc, - )); + return Err(gateway_grant_failure(TokenGrantFailureKind::Configuration)); } if supervisor_jwt_svid.is_empty() { return Err(TokenGrantFailure::new( @@ -1426,7 +1402,9 @@ async fn exchange_provider_subject_token_inner( .await .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))? .map_err(|status| provider_subject_token_exchange_status(&status, &endpoint))?; - provider_subject_token_exchange_result(response.into_inner()) + status.finish(provider_subject_token_exchange_result( + response.into_inner(), + )) } /// Attach the exchange's request ID as `x-request-id` gRPC metadata. @@ -1448,16 +1426,7 @@ fn gateway_grant_failure(kind: TokenGrantFailureKind) -> TokenGrantFailure { } /// Validate a grant gateway authority before acquiring a workload assertion. -/// -/// Rejects endpoint components that cannot be used by the gRPC transport and -/// might otherwise carry sensitive values into connection diagnostics. -pub fn validate_token_grant_gateway_endpoint( - endpoint: &str, -) -> std::result::Result<(), TokenGrantFailure> { - token_grant_gateway_endpoint(endpoint).map(|_| ()) -} - -fn token_grant_gateway_endpoint( +pub fn token_grant_gateway_endpoint( endpoint: &str, ) -> std::result::Result { let endpoint = url::Url::parse(endpoint) @@ -1558,21 +1527,6 @@ mod token_grant_exchange_tests { } } - #[test] - fn gateway_error_text_is_never_retained() { - use std::error::Error; - - let status = Status::failed_precondition( - "SYNTHETIC_GATEWAY_SECRET https://user:secret@token.example/path?token=secret", - ); - let https = url::Url::parse("https://gateway.example:8080/").unwrap(); - let failure = provider_subject_token_exchange_status(&status, &https); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(failure.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); - assert!(failure.source().is_none()); - } - #[test] fn grant_gateway_endpoint_rejects_sensitive_components() { for endpoint in [ @@ -1788,7 +1742,6 @@ pub struct ProviderSubjectTokenExchangeResult { pub access_token: String, /// Remaining lifetime in seconds, or zero when the gateway omitted it. pub expires_in: i64, - /// Token type declared by the service. pub token_type: String, } diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index cc4046bb23..bd7a35538f 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -18,10 +18,8 @@ use crate::token_grant_failure::{ REQUEST_ID_HEADER, TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, }; -/// JWT bearer client-assertion type used when the caller supplies no override. pub const DEFAULT_CLIENT_ASSERTION_TYPE: &str = "urn:ietf:params:oauth:client-assertion-type:jwt-bearer"; -/// RFC 8693 access-token type used for unspecified exchange token types. pub const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_token"; const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; const ERROR_CONTRACT: &str = "openshell.token-grant.v1"; @@ -160,11 +158,8 @@ async fn post_oauth_token_request( pub struct TokenGrantParams<'a> { /// Secret JWT client assertion sent only to the configured endpoint. pub client_assertion: &'a str, - /// Assertion format; an empty value selects the JWT bearer default. pub client_assertion_type: &'a str, - /// Optional resource audience carried in the grant form. pub audience: &'a str, - /// Optional scopes carried in the grant form. pub scopes: &'a [String], } @@ -200,17 +195,12 @@ pub async fn post_oauth_token_grant( pub struct TokenExchangeParams<'a> { /// Secret JWT client assertion sent only to the configured endpoint. pub client_assertion: &'a str, - /// Assertion format; an empty value selects the JWT bearer default. pub client_assertion_type: &'a str, /// Secret subject credential to exchange for a resource token. pub subject_token: &'a str, - /// Subject format; an empty value selects the access-token default. pub subject_token_type: &'a str, - /// Optional resource audience carried in the exchange form. pub audience: &'a str, - /// Optional scopes carried in the exchange form. pub scopes: &'a [String], - /// Requested result format; an empty value selects the access-token default. pub requested_token_type: &'a str, } @@ -248,7 +238,6 @@ pub async fn post_oauth_token_exchange( .await } -/// Select the JWT bearer assertion type when no nonblank override is supplied. pub fn effective_client_assertion_type(client_assertion_type: &str) -> &str { if client_assertion_type.trim().is_empty() { DEFAULT_CLIENT_ASSERTION_TYPE @@ -257,7 +246,6 @@ pub fn effective_client_assertion_type(client_assertion_type: &str) -> &str { } } -/// Select the RFC 8693 access-token type when no nonblank override is supplied. pub fn effective_token_type(token_type: &str) -> &str { if token_type.trim().is_empty() { ACCESS_TOKEN_TYPE @@ -266,7 +254,11 @@ pub fn effective_token_type(token_type: &str) -> &str { } } -fn parse_token_endpoint_url(token_endpoint: &str) -> Result { +/// Validate the configured endpoint before acquiring any workload credentials. +/// +/// HTTPS is accepted; plain HTTP is limited to loopback and Kubernetes service +/// hosts and never authenticates a precise token-service error classification. +pub fn parse_token_endpoint_url(token_endpoint: &str) -> Result { let invalid = TokenGrantFailure::new( TokenGrantFailureKind::Configuration, TokenGrantStage::Configuration, @@ -278,14 +270,6 @@ fn parse_token_endpoint_url(token_endpoint: &str) -> Result Result<(), TokenGrantFailure> { - parse_token_endpoint_url(token_endpoint).map(|_| ()) -} - fn token_endpoint_transport_allowed(url: &reqwest::Url) -> bool { match url.scheme() { "https" => true, @@ -685,78 +669,41 @@ mod tests { } #[tokio::test] - async fn verified_https_signal_is_preserved_for_both_grants() { - let cases = [ - ( - "409 Conflict", - "provider_authorization_required", - TokenGrantFailureKind::ProviderAuthorizationRequired, - ), - ( - "401 Unauthorized", - "workload_identity_rejected", - TokenGrantFailureKind::WorkloadIdentityRejected, - ), - ( - "403 Forbidden", - "workload_identity_rejected", - TokenGrantFailureKind::WorkloadIdentityRejected, - ), - ( - "400 Bad Request", - "configuration_error", - TokenGrantFailureKind::Configuration, - ), - ]; - for (status, code, expected) in cases { - for exchange in [false, true] { - let body = format!( - r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# - ); - let (endpoint, root, request) = tls_token_endpoint_once(status, &body).await; - let client = trusted_test_client(root); - let failure = if exchange { - post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None).await - } else { - post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None).await - } - .expect_err("adopted failure"); - let request = request - .await - .expect("TLS fixture") - .expect("authenticated request"); - assert_eq!(request.form.contains_key("subject_token"), exchange); - assert_eq!(failure.kind(), expected); - assert_eq!(failure.stage(), TokenGrantStage::TokenEndpoint); - assert_generic_failure(failure); - } - } + async fn verified_https_signal_is_preserved() { + let body = format!( + r#"{{"error":"provider_authorization_required","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# + ); + let (endpoint, root, request) = tls_token_endpoint_once("409 Conflict", &body).await; + let client = trusted_test_client(root); + let failure = post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None) + .await + .expect_err("adopted failure"); + let request = request + .await + .expect("TLS fixture") + .expect("authenticated request"); + assert!(request.form.contains_key("subject_token")); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::ProviderAuthorizationRequired + ); + assert_eq!(failure.stage(), TokenGrantStage::TokenEndpoint); + assert_generic_failure(failure); } #[tokio::test] - async fn plain_http_cannot_assert_adopted_failure_for_either_grant() { - for (status, code) in [ - ("409 Conflict", "provider_authorization_required"), - ("401 Unauthorized", "workload_identity_rejected"), - ("400 Bad Request", "configuration_error"), - ] { - for exchange in [false, true] { - let body = format!( - r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# - ); - let (endpoint, request) = token_endpoint_once(status, &body).await; - let client = test_client(); - let failure = if exchange { - post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None).await - } else { - post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None).await - } + async fn plain_http_cannot_assert_adopted_failure() { + let body = format!( + r#"{{"error":"provider_authorization_required","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# + ); + let (endpoint, request) = token_endpoint_once("409 Conflict", &body).await; + let failure = + post_oauth_token_exchange(&test_client(), &endpoint, &exchange_params(), None) + .await .expect_err("unauthenticated failure"); - request.await.expect("HTTP fixture"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_generic_failure(failure); - } - } + request.await.expect("HTTP fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_generic_failure(failure); } #[tokio::test] @@ -800,6 +747,7 @@ mod tests { .expect_err("safe endpoint failure"); request.await.expect("HTTP fixture"); assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.upstream_status(), Some(401)); assert_generic_failure(failure); } for endpoint in [ @@ -819,6 +767,7 @@ mod tests { async fn malformed_success_fields_are_generic_and_never_return_a_token() { for body in [ "{}", + r#"{"access_token":42"#, r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":null}"#, r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","expires_in":"DURATION_SECRET_CANARY"}"#, r#"{"access_token":"TOKEN_SECRET_CANARY","access_token":"SECOND_SECRET_CANARY","token_type":"Bearer"}"#, @@ -1018,25 +967,6 @@ mod tests { assert_eq!(err.upstream_status(), Some(302)); } - #[tokio::test] - async fn discards_oauth_error_descriptions_and_unrecognized_codes() { - let (endpoint, request) = token_endpoint_once( - "401 Unauthorized", - r#"{"error":"invalid_client","error_description":"bad assertion"}"#, - ) - .await; - let client = test_client(); - - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) - .await - .expect_err("should fail on OAuth error"); - let _ = request.await; - - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(err.upstream_status(), Some(401)); - } - #[tokio::test] async fn does_not_echo_unstructured_error_body() { let (endpoint, request) = token_endpoint_once( @@ -1077,21 +1007,6 @@ mod tests { } } - #[tokio::test] - async fn reports_malformed_success_json() { - let (endpoint, request) = token_endpoint_once("200 OK", r#"{"access_token":42"#).await; - let client = test_client(); - - let err = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) - .await - .expect_err("should fail on malformed JSON"); - let _ = request.await; - - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(err.upstream_status(), Some(200)); - } - #[tokio::test] async fn token_exchange_posts_rfc8693_form_fields() { let (endpoint, request) = token_endpoint_once( diff --git a/crates/openshell-core/src/token_grant_failure.rs b/crates/openshell-core/src/token_grant_failure.rs index 5038218bb7..3fdbc3109c 100644 --- a/crates/openshell-core/src/token_grant_failure.rs +++ b/crates/openshell-core/src/token_grant_failure.rs @@ -13,7 +13,7 @@ use prost::Message; use tonic::{Code, Status}; use crate::proto::ProviderCredentialRefreshRecoveryAction; -use crate::rpc_error::ERROR_DOMAIN; +use crate::rpc_error::{ERROR_DOMAIN, ErrorDetails, StatusExt}; /// Header that carries the supervisor-minted request ID of an HTTP exchange. /// @@ -24,7 +24,6 @@ pub const REQUEST_ID_HEADER: &str = "x-openshell-request-id"; const FAILURE_MESSAGE: &str = "dynamic token grant failed"; const FAILURE_REASON: &str = "TOKEN_GRANT_FAILURE"; -const ERROR_INFO_TYPE_URL: &str = "type.googleapis.com/google.rpc.ErrorInfo"; const MAX_DETAILS_BYTES: usize = 1024; /// An allowlisted cause of a dynamic token-grant failure. @@ -45,6 +44,15 @@ pub enum TokenGrantFailureKind { } impl TokenGrantFailureKind { + pub(crate) const ALL: [Self; 6] = [ + Self::ProviderAuthorizationRequired, + Self::WorkloadIdentityRejected, + Self::Configuration, + Self::Timeout, + Self::Unavailable, + Self::Unclassified, + ]; + const fn code(self) -> &'static str { match self { Self::ProviderAuthorizationRequired => "provider_authorization_required", @@ -57,15 +65,7 @@ impl TokenGrantFailureKind { } pub(crate) fn from_code(code: &str) -> Option { - match code { - "provider_authorization_required" => Some(Self::ProviderAuthorizationRequired), - "workload_identity_rejected" => Some(Self::WorkloadIdentityRejected), - "configuration_error" => Some(Self::Configuration), - "timeout" => Some(Self::Timeout), - "unavailable" => Some(Self::Unavailable), - "unclassified" => Some(Self::Unclassified), - _ => None, - } + Self::ALL.into_iter().find(|kind| kind.code() == code) } pub(crate) const fn recovery(self) -> ProviderCredentialRefreshRecoveryAction { @@ -110,6 +110,15 @@ pub enum TokenGrantStage { } impl TokenGrantStage { + pub(crate) const ALL: [Self; 6] = [ + Self::Configuration, + Self::WorkloadIdentity, + Self::GatewayRpc, + Self::IntermediateExchange, + Self::TokenEndpoint, + Self::Injection, + ]; + /// Return the stable stage code for trusted local consumers. pub const fn code(self) -> &'static str { match self { @@ -123,15 +132,7 @@ impl TokenGrantStage { } pub(crate) fn from_code(code: &str) -> Option { - match code { - "configuration" => Some(Self::Configuration), - "workload_identity" => Some(Self::WorkloadIdentity), - "gateway_rpc" => Some(Self::GatewayRpc), - "intermediate_exchange" => Some(Self::IntermediateExchange), - "token_endpoint" => Some(Self::TokenEndpoint), - "injection" => Some(Self::Injection), - _ => None, - } + Self::ALL.into_iter().find(|stage| stage.code() == code) } } @@ -218,20 +219,11 @@ impl TokenGrantFailure { if let Some(status) = self.upstream_status { metadata.insert("upstream_http_status".to_owned(), status.to_string()); } - let info = tonic_types::pb::ErrorInfo { - reason: FAILURE_REASON.to_owned(), - domain: ERROR_DOMAIN.to_owned(), - metadata, - }; - let envelope = tonic_types::pb::Status { - code: code as i32, - message: FAILURE_MESSAGE.to_owned(), - details: vec![prost_types::Any { - type_url: ERROR_INFO_TYPE_URL.to_owned(), - value: info.encode_to_vec(), - }], - }; - Status::with_details(code, FAILURE_MESSAGE, envelope.encode_to_vec().into()) + Status::with_error_details( + code, + FAILURE_MESSAGE, + ErrorDetails::with_error_info(FAILURE_REASON, ERROR_DOMAIN, metadata), + ) } /// Decode strictly bounded internal details, discarding all other diagnostics. @@ -263,7 +255,7 @@ impl TokenGrantFailure { let [detail] = envelope.details.as_slice() else { return None; }; - if detail.type_url != ERROR_INFO_TYPE_URL { + if detail.type_url != tonic_types::ErrorInfo::TYPE_URL { return None; } let info = tonic_types::pb::ErrorInfo::decode(detail.value.as_slice()).ok()?; @@ -320,27 +312,10 @@ impl miette::Diagnostic for TokenGrantFailure {} mod tests { use super::*; - const KINDS: [TokenGrantFailureKind; 6] = [ - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantFailureKind::WorkloadIdentityRejected, - TokenGrantFailureKind::Configuration, - TokenGrantFailureKind::Timeout, - TokenGrantFailureKind::Unavailable, - TokenGrantFailureKind::Unclassified, - ]; - const STAGES: [TokenGrantStage; 6] = [ - TokenGrantStage::Configuration, - TokenGrantStage::WorkloadIdentity, - TokenGrantStage::GatewayRpc, - TokenGrantStage::IntermediateExchange, - TokenGrantStage::TokenEndpoint, - TokenGrantStage::Injection, - ]; - #[test] fn every_kind_has_a_documented_code_and_recovery_action() { use ProviderCredentialRefreshRecoveryAction as Recovery; - let table = KINDS.map(|kind| { + let table = TokenGrantFailureKind::ALL.map(|kind| { let failure = TokenGrantFailure::new(kind, TokenGrantStage::TokenEndpoint); (failure.code(), failure.recovery()) }); @@ -437,8 +412,8 @@ mod tests { #[test] fn all_classifications_and_stages_round_trip_without_printable_detail() { - for kind in KINDS { - for stage in STAGES { + for kind in TokenGrantFailureKind::ALL { + for stage in TokenGrantStage::ALL { for upstream in [None, Some(100), Some(409), Some(599)] { let mut failure = TokenGrantFailure::new(kind, stage); if let Some(status) = upstream { @@ -468,7 +443,7 @@ mod tests { Code::Unavailable, Code::Unknown, ]; - for (kind, code) in KINDS.into_iter().zip(expected) { + for (kind, code) in TokenGrantFailureKind::ALL.into_iter().zip(expected) { let failure = TokenGrantFailure::new(kind, TokenGrantStage::GatewayRpc); assert_eq!(failure.to_status().code(), code); for override_code in [ diff --git a/crates/openshell-server/src/grpc/policy.rs b/crates/openshell-server/src/grpc/policy.rs index 0ebdb8c51d..824eab50ec 100644 --- a/crates/openshell-server/src/grpc/policy.rs +++ b/crates/openshell-server/src/grpc/policy.rs @@ -20,7 +20,8 @@ pub(super) use endpoint_status::{ handle_peer_report_endpoint_status, handle_report_endpoint_status, }; pub use endpoint_status::{ - invalidate_endpoint_status_on_startup, reset_endpoint_status_for_supervisor_session, + invalidate_endpoint_status_on_startup, invalidate_endpoint_status_without_session, + reset_endpoint_status_for_supervisor_session, retry_endpoint_status_after_supervisor_disconnect, }; diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status.rs b/crates/openshell-server/src/grpc/policy/endpoint_status.rs index 211457f727..41be3eca1d 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status.rs @@ -580,7 +580,7 @@ async fn has_fresh_shared_owner( .map_err(|error| Status::unavailable(format!("resolve supervisor owner failed: {error}"))) } -fn invalidate_endpoint_status_without_session(sandbox: &mut Sandbox) { +pub fn invalidate_endpoint_status_without_session(sandbox: &mut Sandbox) { let Some(status) = sandbox.status.as_mut() else { return; }; @@ -647,7 +647,7 @@ fn validate_endpoint_snapshot( )); } if let Some(failure) = endpoint.failure.as_ref() { - validate_reported_failure(result, failure, attached_providers)?; + validate_reported_failure("observations.failure", result, failure, attached_providers)?; } if validated .insert( @@ -700,13 +700,13 @@ fn validate_endpoint_snapshot( /// The credential name is checked for shape only. It is a name the sandbox's /// supervisor received with its provider environment, never a credential value. fn validate_reported_failure( + field: &str, result: EndpointResult, failure: &openshell_core::proto::EndpointFailure, attached_providers: &[String], ) -> Result<(), Status> { - validate_endpoint_failure(failure, result).map_err(|invalid| { - openshell_core::rpc_error::invalid_argument("observations.failure", invalid.message()) - })?; + validate_endpoint_failure(failure, result) + .map_err(|invalid| openshell_core::rpc_error::invalid_argument(field, invalid.message()))?; if !failure.provider.is_empty() && !attached_providers.contains(&failure.provider) { return Err(openshell_core::rpc_error::failed_precondition( "ENDPOINT_FAILURE_PROVIDER_NOT_ATTACHED", @@ -744,6 +744,7 @@ fn validate_token_grant_status( )); } validate_reported_failure( + "token_grant_status.last_failure.failure", EndpointResult::CredentialUnavailable, failure, attached_providers, diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs index c034f04b78..e2f78ea1a5 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs @@ -850,6 +850,8 @@ async fn rest_grant_status_fixture( #[test] fn sandbox_grant_status_validates_destination_cause_and_provider() { + use openshell_core::rpc_error::StatusExt; + let attached = ["github".to_string()]; let validate = |failure| { validate_token_grant_status( @@ -864,33 +866,26 @@ fn sandbox_grant_status_validates_destination_cause_and_provider() { observation.destination_host = host.to_string(); validate(observation).expect("bounded DNS and IP destinations are accepted"); } - for host in [ - "", - "API.EXAMPLE.COM", - "https://SECRET.example/token", - ":", - "bad\nSECRET", - ] { + for host in ["API.EXAMPLE.COM", "bad\nSECRET"] { let mut observation = sandbox_grant_failure(); observation.destination_host = host.to_string(); let error = validate(observation).unwrap_err(); assert_eq!(error.code(), Code::InvalidArgument); assert!(!error.message().contains("SECRET")); } - for port in [0, 65536] { - let mut observation = sandbox_grant_failure(); - observation.destination_port = port; - assert_eq!( - validate(observation).unwrap_err().code(), - Code::InvalidArgument - ); - } let mut missing_cause = sandbox_grant_failure(); missing_cause.failure = None; assert_eq!( validate(missing_cause).unwrap_err().code(), Code::InvalidArgument ); + let mut unknown_code = sandbox_grant_failure(); + unknown_code.failure.as_mut().unwrap().code = "bogus".to_string(); + let details = validate(unknown_code).unwrap_err().get_error_details(); + assert_eq!( + details.bad_request().unwrap().field_violations[0].field, + "token_grant_status.last_failure.failure" + ); let mut unknown_domain = sandbox_grant_failure(); unknown_domain.failure.as_mut().unwrap().domain = 99; assert_eq!( @@ -926,20 +921,12 @@ async fn rest_grant_failure_is_retrievable_retained_and_explicitly_cleared() { ) .await .expect("REST failure is reported without MCP observations"); - let first = stored_sandbox(&state, sandbox_id).await; assert_eq!( public_status(&state, sandbox_id) .await .last_token_grant_failure, Some(Box::new(sandbox_grant_failure())) ); - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), - ) - .await - .expect("identical retry"); - assert_eq!(stored_sandbox(&state, sandbox_id).await, first); let mut changed_retry = report.clone(); changed_retry .token_grant_status @@ -983,16 +970,6 @@ async fn rest_grant_failure_is_retrievable_retained_and_explicitly_cleared() { .last_token_grant_failure .is_none() ); - assert_eq!( - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id) - ) - .await - .unwrap_err() - .code(), - Code::FailedPrecondition - ); assert_eq!( handle_report_endpoint_status(&state, with_sandbox(Request::new(report), "other-sandbox")) .await @@ -1029,7 +1006,6 @@ async fn rest_grant_failure_resets_on_provider_and_policy_changes() { let sandbox = stored_sandbox(&state, sandbox_id).await; let context = active_endpoint_context(&state, &sandbox).await.unwrap(); assert_ne!(context.provider_env_revision, report.provider_env_revision); - let old_report = report.clone(); report.report_sequence = 2; report.policy_hash = context.policy_hash; report.provider_env_revision = context.provider_env_revision; @@ -1046,13 +1022,6 @@ async fn rest_grant_failure_resets_on_provider_and_policy_changes() { .last_token_grant_failure .is_none() ); - assert_eq!( - handle_report_endpoint_status(&state, with_sandbox(Request::new(old_report), sandbox_id)) - .await - .unwrap_err() - .code(), - Code::FailedPrecondition - ); report.report_sequence = 3; report.token_grant_status = Some(TokenGrantStatus { last_failure: Some(sandbox_grant_failure()), @@ -1120,7 +1089,6 @@ async fn rest_grant_failure_resets_on_session_loss_replacement_and_startup() { .supervisor_sessions .initialize_endpoint_status_authority(sandbox_id, session) ); - let predecessor = report.clone(); report.supervisor_session_id = session.to_string(); handle_report_endpoint_status( &state, @@ -1128,16 +1096,6 @@ async fn rest_grant_failure_resets_on_session_loss_replacement_and_startup() { ) .await .unwrap(); - assert_eq!( - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(predecessor), sandbox_id) - ) - .await - .unwrap_err() - .code(), - Code::PermissionDenied - ); } state .supervisor_sessions @@ -1211,21 +1169,6 @@ fn endpoint_snapshot_rejects_failures_outside_the_contract() { "endpoint failure domain does not explain this endpoint result" ); - let mut unknown_code = reauthorize_failure(); - unknown_code.code = "issuer said: token SECRET expired".to_string(); - let error = invalid(EndpointResult::CredentialUnavailable, unknown_code); - assert_eq!(error.code(), Code::InvalidArgument); - assert!(!error.message().contains("SECRET")); - - let mut mismatched = reauthorize_failure(); - mismatched.recovery_action = - openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Retry as i32; - let error = invalid(EndpointResult::CredentialUnavailable, mismatched); - assert_eq!( - error.message(), - "endpoint failure recovery action does not match its code" - ); - let mut foreign = reauthorize_failure(); foreign.provider = "other-team".to_string(); let error = invalid(EndpointResult::CredentialUnavailable, foreign); diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index e3aeda2edf..31bc77cb4c 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -2484,8 +2484,8 @@ use openshell_core::proto::{ UpdateProviderProfilesRequest, UpdateProviderProfilesResponse, UpdateProviderRequest, }; use openshell_core::spiffe::{ - JwtSvidParseError, SpiffeJwtClaims, parse_unverified_jwt_svid_claims, - trust_domain as spiffe_trust_domain, workload_api_endpoint, + SpiffeJwtClaims, parse_unverified_jwt_svid_claims, trust_domain as spiffe_trust_domain, + workload_api_endpoint, }; use openshell_providers::{ CredentialRefreshProfile, ProfileValidationDiagnostic, ProviderTypeProfile, @@ -4085,9 +4085,12 @@ fn sanitize_token_grant_status(status: Status) -> Status { /// The supervisor sends the ID of the HTTP exchange that needs the grant. Any /// other value is not echoed into logs or a token-service request. fn canonical_request_id(metadata: &tonic::metadata::MetadataMap) -> Option { - let value = metadata.get("x-request-id")?.to_str().ok()?; - let parsed = uuid::Uuid::parse_str(value).ok()?; - (parsed.hyphenated().to_string() == value).then(|| value.to_string()) + metadata + .get("x-request-id")? + .to_str() + .ok() + .filter(|value| openshell_core::endpoint_failure::is_canonical_request_id(value)) + .map(str::to_owned) } /// Log a grant failure for operators with its bounded classification. @@ -4123,16 +4126,26 @@ fn token_grant_configuration_status(code: tonic::Code) -> Status { .to_status_with_code(code) } -fn token_grant_identity_status(kind: TokenGrantFailureKind, code: tonic::Code) -> Status { - TokenGrantFailure::new(kind, TokenGrantStage::WorkloadIdentity).to_status_with_code(code) +fn token_grant_store_status() -> Status { + TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::GatewayRpc, + ) + .to_status_with_code(tonic::Code::Internal) +} + +fn token_grant_identity_rejected(code: tonic::Code) -> Status { + TokenGrantFailure::new( + TokenGrantFailureKind::WorkloadIdentityRejected, + TokenGrantStage::WorkloadIdentity, + ) + .to_status_with_code(code) } fn token_grant_workload_api_status(error: spiffe::WorkloadApiError, code: tonic::Code) -> Status { use spiffe::WorkloadApiError; let kind = match error { - WorkloadApiError::MissingEndpointSocket - | WorkloadApiError::NotUnicodeEndpointSocket(_) - | WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, + WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, WorkloadApiError::NoIdentityIssued | WorkloadApiError::PermissionDenied(_) => { TokenGrantFailureKind::WorkloadIdentityRejected } @@ -4149,7 +4162,7 @@ fn token_grant_workload_api_status(error: spiffe::WorkloadApiError, code: tonic: } _ => TokenGrantFailureKind::Unclassified, }; - token_grant_identity_status(kind, code) + TokenGrantFailure::new(kind, TokenGrantStage::WorkloadIdentity).to_status_with_code(code) } fn token_grant_dependency_status(status: Status) -> Status { @@ -4203,13 +4216,7 @@ async fn exchange_provider_subject_token_inner( .store .get_message::(&req.sandbox_id) .await - .map_err(|_| { - TokenGrantFailure::new( - TokenGrantFailureKind::Unavailable, - TokenGrantStage::GatewayRpc, - ) - .to_status_with_code(tonic::Code::Internal) - })? + .map_err(|_| token_grant_store_status())? .ok_or_else(|| token_grant_configuration_status(tonic::Code::NotFound))?; let workspace = sandbox.object_workspace().to_string(); let spec = sandbox @@ -4230,13 +4237,7 @@ async fn exchange_provider_subject_token_inner( .store .get_message_by_name::(&workspace, &req.provider) .await - .map_err(|_| { - TokenGrantFailure::new( - TokenGrantFailureKind::Unavailable, - TokenGrantStage::GatewayRpc, - ) - .to_status_with_code(tonic::Code::Internal) - })? + .map_err(|_| token_grant_store_status())? .ok_or_else(|| token_grant_configuration_status(tonic::Code::NotFound))?; let catalog = state .provider_profile_sources @@ -4369,7 +4370,7 @@ fn validate_intermediate_grant_configuration( provider: &Provider, subject_credential: &str, ) -> Result<(Option, Option), Status> { - oauth::validate_token_endpoint(&grant.token_endpoint).map_err(TokenGrantFailure::to_status)?; + oauth::parse_token_endpoint_url(&grant.token_endpoint).map_err(TokenGrantFailure::to_status)?; let cache_ttl = grant .cache_ttl .as_ref() @@ -4566,21 +4567,12 @@ fn validate_gateway_jwt_svid_claims( claims: &SpiffeJwtClaims, expected_audience: &str, ) -> Result<(), Status> { - if !claims.aud.contains(expected_audience) { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::FailedPrecondition, - )); - } - if spiffe_trust_domain(&claims.sub).is_none() { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::FailedPrecondition, - )); - } - if claims.exp > 0 && claims.exp.saturating_mul(1000) <= crate::persistence::current_time_ms() { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, + if !claims.aud.contains(expected_audience) + || spiffe_trust_domain(&claims.sub).is_none() + || (claims.exp > 0 + && claims.exp.saturating_mul(1000) <= crate::persistence::current_time_ms()) + { + return Err(token_grant_identity_rejected( tonic::Code::FailedPrecondition, )); } @@ -4594,34 +4586,17 @@ async fn validate_supervisor_jwt_svid( ) -> Result { let unverified = parse_unverified_spiffe_claims(token)?; if unverified.iss != gateway_claims.iss { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - )); + return Err(token_grant_identity_rejected(tonic::Code::PermissionDenied)); } if !unverified.aud.contains(expected_audience) { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - )); + return Err(token_grant_identity_rejected(tonic::Code::PermissionDenied)); } - let supervisor_trust_domain = spiffe_trust_domain(&unverified.sub).ok_or_else(|| { - token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - ) - })?; - let gateway_trust_domain = spiffe_trust_domain(&gateway_claims.sub).ok_or_else(|| { - token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::FailedPrecondition, - ) - })?; + let supervisor_trust_domain = spiffe_trust_domain(&unverified.sub) + .ok_or_else(|| token_grant_identity_rejected(tonic::Code::PermissionDenied))?; + let gateway_trust_domain = spiffe_trust_domain(&gateway_claims.sub) + .ok_or_else(|| token_grant_identity_rejected(tonic::Code::FailedPrecondition))?; if supervisor_trust_domain != gateway_trust_domain { - return Err(token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - )); + return Err(token_grant_identity_rejected(tonic::Code::PermissionDenied)); } let socket_path = std::env::var(GATEWAY_SPIFFE_WORKLOAD_API_SOCKET) @@ -4636,24 +4611,14 @@ async fn validate_supervisor_jwt_svid( .fetch_jwt_bundles() .await .map_err(|error| token_grant_workload_api_status(error, tonic::Code::Internal))?; - spiffe::JwtSvid::parse_and_validate(token, &bundles, &[expected_audience]).map_err(|_| { - token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - ) - })?; + spiffe::JwtSvid::parse_and_validate(token, &bundles, &[expected_audience]) + .map_err(|_| token_grant_identity_rejected(tonic::Code::PermissionDenied))?; Ok(unverified) } fn parse_unverified_spiffe_claims(token: &str) -> Result { - parse_unverified_jwt_svid_claims(token).map_err(jwt_svid_parse_error_status) -} - -fn jwt_svid_parse_error_status(_error: JwtSvidParseError) -> Status { - token_grant_identity_status( - TokenGrantFailureKind::WorkloadIdentityRejected, - tonic::Code::PermissionDenied, - ) + parse_unverified_jwt_svid_claims(token) + .map_err(|_| token_grant_identity_rejected(tonic::Code::PermissionDenied)) } async fn perform_intermediate_token_exchange( @@ -5720,21 +5685,15 @@ mod tests { #[test] fn strict_typed_failure_survives_final_boundary() { - for kind in [ + let failure = TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantFailureKind::WorkloadIdentityRejected, - TokenGrantFailureKind::Configuration, - TokenGrantFailureKind::Timeout, - TokenGrantFailureKind::Unavailable, - TokenGrantFailureKind::Unclassified, - ] { - let failure = TokenGrantFailure::new(kind, TokenGrantStage::IntermediateExchange) - .with_upstream_status(409); - let raw = failure.to_status(); - let code = raw.code(); - let status = sanitize_token_grant_status(raw); - assert_eq!(assert_generic(&status, code), failure); - } + TokenGrantStage::IntermediateExchange, + ) + .with_upstream_status(409); + let raw = failure.to_status(); + let code = raw.code(); + let status = sanitize_token_grant_status(raw); + assert_eq!(assert_generic(&status, code), failure); } #[test] @@ -5844,15 +5803,8 @@ mod tests { } #[tokio::test] - async fn handler_missing_sandbox_and_unattached_provider_are_generic() { + async fn handler_unattached_provider_is_generic() { let state = test_server_state_without_provider_profiles().await; - let missing = handle_exchange_provider_subject_token( - &state, - sandbox_request(request_body(), "grant-sandbox"), - ) - .await - .unwrap_err(); - assert_generic(&missing, Code::NotFound); state .store .put_message(&Sandbox { @@ -5937,15 +5889,6 @@ mod tests { .credential_expiration_times .insert("expired-sibling".to_string(), ts(0)); let (output, capture) = capture_grant_warnings(); - let resolved = credentials - .resolve_provider_handles(&provider, crate::persistence::current_time_ms()) - .await - .unwrap(); - assert_eq!( - resolved.values.get("subject").map(String::as_str), - Some(SECRET) - ); - assert!(!resolved.values.contains_key("expired-sibling")); let subject = resolve_subject_token_credential(&credentials, &provider, "subject") .await .unwrap(); @@ -5993,53 +5936,38 @@ mod tests { } #[test] - fn workload_api_status_preserves_recovery_and_discards_diagnostics() { - use openshell_core::proto::ProviderCredentialRefreshRecoveryAction as Recovery; + fn workload_api_status_classifies_and_discards_diagnostics() { use spiffe::WorkloadApiError; - for code in [Code::FailedPrecondition, Code::Internal] { - for (error, kind, recovery) in [ - ( - WorkloadApiError::from(Status::permission_denied(SECRET)), - TokenGrantFailureKind::WorkloadIdentityRejected, - Recovery::FixConfiguration, - ), - ( - WorkloadApiError::NoIdentityIssued, - TokenGrantFailureKind::WorkloadIdentityRejected, - Recovery::FixConfiguration, - ), - ( - WorkloadApiError::from(Status::deadline_exceeded(SECRET)), - TokenGrantFailureKind::Timeout, - Recovery::Retry, - ), - ( - WorkloadApiError::from(Status::unavailable(SECRET)), - TokenGrantFailureKind::Unavailable, - Recovery::Retry, - ), - ( - WorkloadApiError::from(Status::invalid_argument(SECRET)), - TokenGrantFailureKind::Configuration, - Recovery::FixConfiguration, - ), - ( - WorkloadApiError::MissingEndpointSocket, - TokenGrantFailureKind::Configuration, - Recovery::FixConfiguration, - ), - ( - WorkloadApiError::HintNotFound(SECRET.to_string()), - TokenGrantFailureKind::Unclassified, - Recovery::Investigate, - ), - ] { - let status = token_grant_workload_api_status(error, code); - let failure = assert_generic(&status, code); - assert_eq!(failure.kind(), kind); - assert_eq!(failure.recovery(), recovery); - assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); - } + for (error, kind) in [ + ( + WorkloadApiError::from(Status::permission_denied(SECRET)), + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + WorkloadApiError::NoIdentityIssued, + TokenGrantFailureKind::WorkloadIdentityRejected, + ), + ( + WorkloadApiError::from(Status::deadline_exceeded(SECRET)), + TokenGrantFailureKind::Timeout, + ), + ( + WorkloadApiError::from(Status::unavailable(SECRET)), + TokenGrantFailureKind::Unavailable, + ), + ( + WorkloadApiError::from(Status::invalid_argument(SECRET)), + TokenGrantFailureKind::Configuration, + ), + ( + WorkloadApiError::HintNotFound(SECRET.to_string()), + TokenGrantFailureKind::Unclassified, + ), + ] { + let status = token_grant_workload_api_status(error, Code::FailedPrecondition); + let failure = assert_generic(&status, Code::FailedPrecondition); + assert_eq!(failure.kind(), kind); + assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); } } @@ -6145,29 +6073,19 @@ mod tests { #[tokio::test] async fn intermediate_http_failure_keeps_stage_and_status_without_body() { - for (http_status, expected_kind, adopted_extension) in [ - (409, TokenGrantFailureKind::Unclassified, true), - (503, TokenGrantFailureKind::Unavailable, false), - (503, TokenGrantFailureKind::Unclassified, true), - ] { - let body = if adopted_extension { - format!( - "{{\"error\":\"provider_authorization_required\",\"openshell_error_contract\":\"openshell.token-grant.v1\",\"error_description\":\"{SECRET}\"}}" - ) - } else { - format!("{{\"error\":\"{SECRET}\",\"error_description\":\"{SECRET}\"}}") - }; - let (endpoint, task) = token_endpoint_response(http_status, body).await; - let result = - perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) - .await; - task.await.unwrap(); - let status = result.unwrap_err(); - let failure = assert_generic(&status, status.code()); - assert_eq!(failure.kind(), expected_kind); - assert_eq!(failure.stage(), TokenGrantStage::IntermediateExchange); - assert_eq!(failure.upstream_status(), Some(http_status)); - } + let body = format!( + "{{\"error\":\"provider_authorization_required\",\"openshell_error_contract\":\"openshell.token-grant.v1\",\"error_description\":\"{SECRET}\"}}" + ); + let (endpoint, task) = token_endpoint_response(409, body).await; + let result = + perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) + .await; + task.await.unwrap(); + let status = result.unwrap_err(); + let failure = assert_generic(&status, status.code()); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.stage(), TokenGrantStage::IntermediateExchange); + assert_eq!(failure.upstream_status(), Some(409)); } #[tokio::test] diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index 77bcc89aba..bf3e0db1a3 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -679,17 +679,7 @@ impl SupervisorSessionRegistry { if has_authority || remote_authority { return; } - let Some(status) = sandbox.status.as_mut() else { - return; - }; - // Status without its live observation authority is unknown. Keep the - // configured address so a caller can still identify each endpoint. - status.last_token_grant_failure = None; - for endpoint in &mut status.endpoint_statuses { - endpoint.last_result = openshell_core::proto::EndpointResult::NoObservedExchange as i32; - endpoint.last_reported_time = None; - endpoint.last_failure = None; - } + crate::grpc::policy::invalidate_endpoint_status_without_session(sandbox); } /// Return the active supervisor session identifier for gateway-owned @@ -3311,11 +3301,6 @@ mod tests { assert_eq!(projected_status.conditions, vec![ready.clone()]); assert!(projected_status.last_token_grant_failure.is_none()); - let mut rest_only = sandbox.clone(); - rest_only.status.as_mut().unwrap().endpoint_statuses.clear(); - registry.project_endpoint_status(&mut rest_only, false); - assert!(rest_only.status.unwrap().last_token_grant_failure.is_none()); - let mut remotely_owned = sandbox.clone(); registry.project_endpoint_status(&mut remotely_owned, true); let remote_status = remotely_owned.status.expect("status"); diff --git a/crates/openshell-supervisor-network/src/l7/mod.rs b/crates/openshell-supervisor-network/src/l7/mod.rs index 21abccd511..3f26114998 100644 --- a/crates/openshell-supervisor-network/src/l7/mod.rs +++ b/crates/openshell-supervisor-network/src/l7/mod.rs @@ -250,17 +250,9 @@ impl EndpointObserver { if !self.uses_provider_credentials { return; } - let (provider, credential_key) = refusal.credential().unwrap_or_default(); self.observe_with_failure( EndpointResult::CredentialUnavailable, - Some( - openshell_core::endpoint_failure::EndpointFailure::token_grant( - refusal.failure(), - provider, - credential_key, - request_id, - ), - ), + Some(refusal.endpoint_failure(request_id)), ); } diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index 5bbaf8e50b..fdf96ffbe1 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -78,14 +78,6 @@ pub struct L7EvalContext { pub(crate) body_classifier: Option>, /// Anonymous activity counter channel. pub(crate) activity_tx: Option, - /// Dynamic credentials (token grants) keyed by endpoint-bound provider metadata. - pub(crate) dynamic_credentials: Option< - Arc< - std::sync::RwLock< - std::collections::HashMap, - >, - >, - >, /// Dynamic token grant resolver for endpoint-bound credentials. pub(crate) token_grant_resolver: Option>, @@ -143,8 +135,8 @@ fn credential_generation_guard( } /// Resolve a grant from the provider installation and policy owners admitted -/// for this request. Provider refresh is independent of tunnel lifetime: the -/// connection's original dynamic map cannot authorize a later request. +/// for this request. Provider refresh is independent of tunnel lifetime: each +/// request reads the live installation. /// /// A refused grant returns a [`TokenGrantInjectionError`](crate::l7::token_grant_injection::TokenGrantInjectionError) /// inside the report, so callers can tell it apart from other failures. @@ -699,10 +691,10 @@ where ) { observer.observe_token_grant_refusal(refusal, request_id); } - write_token_grant_failure_response(client, request_id).await?; + write_bad_gateway_response(client, Some(request_id)).await?; } else { warn!(error = %error, "Token grant failed before forwarding"); - write_bad_gateway_response(client).await?; + write_bad_gateway_response(client, None).await?; } return Ok(None); } @@ -1491,7 +1483,7 @@ where Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client).await?; + write_bad_gateway_response(client, None).await?; return Ok(()); } }; @@ -2040,6 +2032,7 @@ where let observation_context = ctx .endpoint_observation_tx .as_ref() + .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); // Parse one HTTP request from client let req = match provider.parse_request(client).await { @@ -2237,7 +2230,7 @@ where Ok(preflight) => preflight, Err(error) => { warn!(error = %error, "WebSocket middleware preflight failed"); - write_bad_gateway_response(client).await?; + write_bad_gateway_response(client, None).await?; return Ok(()); } }; @@ -2734,6 +2727,7 @@ where let observation_context = ctx .endpoint_observation_tx .as_ref() + .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); // Validate the head, including body framing, before deciding whether // this endpoint can inspect the requested protocol. Upgrade refusal @@ -3514,6 +3508,7 @@ where // `allow_encoded_slash` opt-in applies. let provider = crate::l7::rest::RestProvider::default(); let mut request_count: u64 = 0; + let mut grant_ctx = ctx.clone(); loop { if close_if_stale(generation_guard, ctx) { return Ok(()); @@ -3522,6 +3517,7 @@ where let observation_context = ctx .endpoint_observation_tx .as_ref() + .filter(|_| ctx.token_grant_resolver.is_some()) .and_then(EndpointObservationSender::capture); // Read next request from client. let req = match provider.parse_request(client).await { @@ -3648,7 +3644,6 @@ where req }; - let mut grant_ctx = ctx.clone(); let Ok(req_with_auth) = crate::l7::token_grant_injection::inject_if_needed_captured( req, &mut grant_ctx, @@ -3659,7 +3654,7 @@ where else { // The refusal event names the credential and request ID. // The sandbox-level grant record does not require an MCP endpoint identity. - write_token_grant_failure_response(client, &request_id).await?; + write_bad_gateway_response(client, Some(&request_id)).await?; return Ok(()); }; let ctx = &grant_ctx; @@ -3718,27 +3713,20 @@ where Ok(()) } -async fn write_bad_gateway_response(client: &mut W) -> Result<()> -where - W: AsyncWrite + Unpin, -{ - let response = b"HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; - client.write_all(response).await.into_diagnostic()?; - client.flush().await.into_diagnostic()?; - Ok(()) -} - -/// Refuse a request whose dynamic token grant failed. -/// -/// The response is the generic empty `502` plus the exchange's request ID, so -/// the caller can quote it to an operator. It never carries the cause. -async fn write_token_grant_failure_response(client: &mut W, request_id: &str) -> Result<()> +/// Write the generic empty `502`. A refused token grant passes its request ID, +/// so the caller can quote it to an operator; the response never carries the cause. +async fn write_bad_gateway_response(client: &mut W, request_id: Option<&str>) -> Result<()> where W: AsyncWrite + Unpin, { + let request_id = request_id.map_or_else(String::new, |request_id| { + format!( + "{}: {request_id}\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ) + }); let response = format!( - "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n{}: {request_id}\r\nConnection: close\r\n\r\n", - openshell_core::token_grant_failure::REQUEST_ID_HEADER, + "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n{request_id}Connection: close\r\n\r\n" ); client .write_all(response.as_bytes()) @@ -3771,8 +3759,11 @@ mod tests { } /// A refused grant returns the generic empty 502 plus a request ID, and no - /// part of the response reveals the classification. - fn assert_grant_refusal_response(response: &str) { + /// part of the response reveals the classification of `failure`. + pub(super) fn assert_grant_refusal_response( + response: &str, + failure: TokenGrantFailure, + ) -> String { assert!( response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), "{response}" @@ -3788,10 +3779,18 @@ mod tests { assert!(uuid::Uuid::parse_str(request_id).is_ok(), "{request_id}"); // The random ID can contain any digits; check everything else. let without_id = response.replace(request_id, ""); - let failure = relay_grant_failure(); - for classification in [failure.code(), failure.stage().code(), "409"] { + let status = failure.upstream_status().map(|status| status.to_string()); + let recovery = failure + .recovery() + .as_str_name() + .trim_start_matches("PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_"); + for classification in [failure.code(), failure.stage().code(), recovery] + .into_iter() + .chain(status.as_deref()) + { assert!(!without_id.contains(classification), "{response}"); } + request_id.to_string() } #[tokio::test] @@ -4480,7 +4479,6 @@ network_policies: ancestors: vec![], cmdline_paths: vec![], secret_resolver: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() @@ -4514,7 +4512,6 @@ network_policies: ) } }; - ctx.dynamic_credentials = Some(fixture.dynamic_credentials()); ctx.provider_credentials = Some(fixture.provider_credentials()); ctx.token_grant_resolver = Some(fixture.resolver()); @@ -4623,7 +4620,7 @@ network_policies: ancestors: vec![], cmdline_paths: vec![], secret_resolver: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -4850,7 +4847,6 @@ network_policies: request_default_port: Some(8080), policy_name: "rest_api".into(), binary_path: "/usr/bin/curl".into(), - dynamic_credentials: Some(fixture.dynamic_credentials()), provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() @@ -4967,7 +4963,7 @@ network_policies: ) } }; - ctx.dynamic_credentials = Some(fixture.dynamic_credentials()); + ctx.provider_credentials = Some(fixture.provider_credentials()); ctx.token_grant_resolver = Some(fixture.resolver()); (generation_guard, ctx, fixture) @@ -5372,7 +5368,7 @@ network_policies: if identity_result.is_ok() { assert!(response.contains("204 No Content")); } else { - assert_grant_refusal_response(&response); + assert_grant_refusal_response(&response, relay_grant_failure()); } assert!(!response.contains("service-token")); drop(app); @@ -5500,7 +5496,10 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); + assert_grant_refusal_response( + &String::from_utf8_lossy(&client_response[..n]), + relay_grant_failure(), + ); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -5620,7 +5619,10 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); + assert_grant_refusal_response( + &String::from_utf8_lossy(&client_response[..n]), + relay_grant_failure(), + ); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -8042,7 +8044,10 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); + assert_grant_refusal_response( + &String::from_utf8_lossy(&client_response[..n]), + relay_grant_failure(), + ); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -8159,7 +8164,10 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response(&String::from_utf8_lossy(&client_response[..n])); + assert_grant_refusal_response( + &String::from_utf8_lossy(&client_response[..n]), + relay_grant_failure(), + ); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 5fd18eb00d..033c4a5965 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -20,8 +20,8 @@ use std::collections::HashMap; use std::future::Future; use std::path::PathBuf; use std::pin::Pin; +use std::sync::Mutex; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Mutex, RwLock}; use std::task::{Context, Poll}; use std::time::Duration; use tokio::io::{AsyncReadExt, DuplexStream, ReadBuf}; @@ -290,23 +290,6 @@ impl Fixture { .engine .clone_engine_for_tunnel(self.generation) .expect("tunnel policy snapshot"); - let snapshot = self.state.snapshot(); - // Mirror the proxy cache identity. Same-revision replacements remain - // distinct because the installation ID precedes the provider segment. - let dynamic_credentials = snapshot - .dynamic_credentials - .iter() - .map(|(key, credential)| { - let (selector, owner) = key.rsplit_once('\t').expect("endpoint-bound fixture key"); - ( - format!( - "{selector}\trev:{}\tinstallation:{}\t{owner}", - snapshot.revision, snapshot.installation_id - ), - credential.clone(), - ) - }) - .collect(); let ctx = L7EvalContext { host: HOST.into(), port: PORT, @@ -314,9 +297,6 @@ impl Fixture { policy_name: "fixture".into(), binary_path: "/usr/bin/curl".into(), provider_credentials: Some(self.state.clone()), - // Match the initial tunnel snapshot while retaining live provider - // state. Refresh tests require requests to use the live ownership. - dynamic_credentials: Some(Arc::new(RwLock::new(dynamic_credentials))), token_grant_resolver: Some(resolver), endpoint_observation_tx: self.observation.clone(), ..Default::default() @@ -767,17 +747,13 @@ async fn refused_grant( } fn refusal_request_id(response: &str) -> String { - assert!(response.starts_with("HTTP/1.1 502"), "{response}"); - let request_id = response - .lines() - .find_map(|line| line.strip_prefix("x-openshell-request-id: ")) - .expect("refusal carries the request ID") - .to_string(); - let without_id = response.replace(&request_id, ""); - for classification in ["unavailable", "token_endpoint", "RETRY"] { - assert!(!without_id.contains(classification), "{response}"); - } - request_id + tests::assert_grant_refusal_response( + response, + openshell_core::token_grant_failure::TokenGrantFailure::new( + openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, + openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + ), + ) } #[tokio::test] diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 85bfaf4638..19942d18e9 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -95,7 +95,6 @@ pub fn default_resolver() -> Arc { /// binding, absent when the failure happened before a binding was selected. /// Display and Debug print only the fixed grant failure message, so callers /// can log this error without classifying it. -#[derive(Clone, PartialEq, Eq)] pub struct TokenGrantInjectionError { failure: TokenGrantFailure, credential: Option, @@ -125,6 +124,12 @@ impl TokenGrantInjectionError { pub fn credential(&self) -> Option<(&str, &str)> { self.credential.as_deref()?.split_once(':') } + + /// Return the bounded status cause of this refusal for one request. + pub fn endpoint_failure(&self, request_id: &str) -> EndpointFailure { + let (provider, credential) = self.credential().unwrap_or_default(); + EndpointFailure::token_grant(self.failure, provider, credential, request_id) + } } impl fmt::Display for TokenGrantInjectionError { @@ -175,44 +180,18 @@ pub async fn inject_if_needed_captured( request_id: &str, observation_context: Option<&EndpointObservationContext>, ) -> Result { - let request_path = request_path(&req); // A connection can sit idle across a refresh before its first request. - // Select and pin the live installation, not its accept-time credential map. - let snapshot = ctx + // Select and pin the live installation. + let Some(snapshot) = ctx .provider_credentials .as_ref() - .map(ProviderCredentialState::snapshot); - if let Some(snapshot) = &snapshot { - ctx.provider_credential_revision = Some(snapshot.revision); - ctx.provider_credential_installation_id = Some(snapshot.installation_id.clone()); - } - let selected = if let Some(snapshot) = snapshot { - token_grant_candidates(&snapshot.dynamic_credentials, ctx, request_path, None) - .and_then(select_token_grants) - .map(|selected| { - selected - .into_iter() - .map(|(key, credential)| (snapshot.scoped_key(&key), credential)) - .collect() - }) - } else { - ctx.dynamic_credentials.as_ref().map_or_else( - || Ok(Vec::new()), - |dynamic_credentials| { - dynamic_credentials - .read() - .map_err(|_| TokenGrantInjectionError::new(credential_lock_failure())) - .and_then(|credentials| { - select_token_grants(token_grant_candidates( - &credentials, - ctx, - request_path, - None, - )?) - }) - }, - ) + .map(ProviderCredentialState::snapshot) + else { + return Ok(req); }; + ctx.provider_credential_revision = Some(snapshot.revision); + ctx.provider_credential_installation_id = Some(snapshot.installation_id.clone()); + let selected = select_scoped(&snapshot, ctx, request_path(&req.target), None); inject_or_report(req, ctx, selected, request_id, observation_context).await } @@ -232,7 +211,7 @@ pub(super) async fn inject_for_admitted_owners( request_id: &str, observation_context: Option<&EndpointObservationContext>, ) -> Result { - let request_path = request_path(&req); + let request_path = request_path(&req.target); let unowned = snapshot .dynamic_credentials .iter() @@ -250,23 +229,29 @@ pub(super) async fn inject_for_admitted_owners( key, )) } else { - token_grant_candidates( - &snapshot.dynamic_credentials, - ctx, - request_path, - Some(admitted_owners), - ) - .and_then(select_token_grants) - .map(|selected| { - selected - .into_iter() - .map(|(key, cred)| (snapshot.scoped_key(&key), cred)) - .collect() - }) + select_scoped(snapshot, ctx, request_path, Some(admitted_owners)) }; inject_or_report(req, ctx, selected, request_id, observation_context).await } +fn select_scoped( + snapshot: &ProviderCredentialSnapshot, + ctx: &L7EvalContext, + request_path: &str, + admitted_owners: Option<&HashSet>, +) -> Result, TokenGrantInjectionError> { + let candidates = token_grant_candidates( + &snapshot.dynamic_credentials, + ctx, + request_path, + admitted_owners, + )?; + Ok(select_token_grants(candidates)? + .into_iter() + .map(|(key, credential)| (snapshot.scoped_key(&key), credential)) + .collect()) +} + /// Inject the selected grants, or record why the request was refused. /// /// Every refusal emits one generic event that carries the request ID. The @@ -283,7 +268,7 @@ async fn inject_or_report( return Ok(req); } let method = req.action.clone(); - let request_path = request_path(&req).to_string(); + let request_path = request_path(&req.target).to_string(); let observation_context = observation_context .filter(|context| context.matches_provider_revision(ctx.provider_credential_revision)); let successful_credentials: Vec<_> = if observation_context.is_some() { @@ -315,14 +300,11 @@ async fn inject_or_report( { match &result { Err(error) => { - let (provider, credential) = error.credential().unwrap_or(("", "")); - let failure = - EndpointFailure::token_grant(error.failure(), provider, credential, request_id); let _ = sender.try_observe_token_grant( context.clone(), &ctx.host, ctx.port, - Err(failure), + Err(error.endpoint_failure(request_id)), ); } Ok(_) => { @@ -357,7 +339,7 @@ fn report_refused_grant( .credential .as_deref() .map_or_else(|| "a dynamic credential".to_string(), ocsf_message_field); - let failure = error.failure; + let failure = error.failure(); warn!( host = %ctx.host, port = ctx.port, @@ -400,7 +382,6 @@ async fn inject_selected( credentials: Vec<(String, ProviderProfileCredential)>, request_id: &str, ) -> Result { - let request_path = request_path(&req).to_string(); let resolver = ctx .token_grant_resolver .as_ref() @@ -440,7 +421,7 @@ async fn inject_selected( .severity(SeverityId::Informational) .http_request(HttpRequest::new( &req.action, - OcsfUrl::new("http", &ctx.host, &request_path, ctx.port), + OcsfUrl::new("http", &ctx.host, request_path(&req.target), ctx.port), )) .dst_endpoint(Endpoint::from_domain(&ctx.host, ctx.port)) .message(format!( @@ -457,15 +438,8 @@ async fn inject_selected( Ok(L7Request { raw_header, ..req }) } -fn request_path(req: &L7Request) -> &str { - req.target.split('?').next().unwrap_or(req.target.as_str()) -} - -fn credential_lock_failure() -> TokenGrantFailure { - TokenGrantFailure::new( - TokenGrantFailureKind::Unavailable, - TokenGrantStage::Injection, - ) +fn request_path(target: &str) -> &str { + target.split('?').next().unwrap_or(target) } fn injection_configuration_failure() -> TokenGrantFailure { @@ -1102,6 +1076,10 @@ mod tests { use super::*; use crate::l7::provider::{BodyLength, L7Request}; use crate::l7::token_grant_injection::test_support::TokenGrantTestFixture; + use openshell_core::endpoint_status::{ + EndpointConfigVersion, EndpointStatusReceiver, EndpointStatusTracker, + endpoint_status_channel, + }; const REQUEST_ID: &str = "6a1f0e2d-3c4b-4a59-8e7f-0123456789ab"; @@ -1344,7 +1322,7 @@ mod tests { ancestors: vec![], cmdline_paths: vec![], secret_resolver: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -1383,7 +1361,7 @@ mod tests { cmdline_paths: vec![], secret_resolver: None, activity_tx: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -1422,7 +1400,7 @@ mod tests { ancestors: vec![], cmdline_paths: vec![], secret_resolver: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -1454,24 +1432,14 @@ mod tests { "api.example.com\t443\t/v1/**\tprovider:access_token", failure, ); - let ctx = L7EvalContext { - host: "api.example.com".into(), - port: 443, - dynamic_credentials: Some(fixture.dynamic_credentials()), - token_grant_resolver: Some(fixture.resolver()), - ..Default::default() - }; - let req = L7Request { - action: "GET".to_string(), - target: "/v1/projects".to_string(), - query_params: HashMap::new(), - raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), - body_length: BodyLength::None, - }; - let err = inject_if_needed(req, &ctx, REQUEST_ID) - .await - .expect_err("failed grant refuses the request"); + let err = inject_if_needed( + grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), + &grant_context(&fixture), + REQUEST_ID, + ) + .await + .expect_err("failed grant refuses the request"); assert_eq!(err.failure(), failure); assert_eq!(err.credential(), Some(("provider", "access_token"))); @@ -1484,7 +1452,7 @@ mod tests { L7EvalContext { host: "api.example.com".into(), port: 443, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() } @@ -1524,142 +1492,56 @@ mod tests { } #[tokio::test] - async fn poisoned_credential_snapshot_fails_closed() { - use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + async fn missing_resolver_refuses_without_naming_a_credential() { let fixture = TokenGrantTestFixture::success( "api.example.com\t443\t/v1/**\tprovider:access_token", "grant-token", ); - let credentials = fixture.dynamic_credentials(); - assert!( - std::thread::spawn(move || { - let _guard = credentials.write().expect("lock fixture credentials"); - panic!("poison fixture credentials"); - }) - .join() - .is_err() - ); - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset( - EndpointConfigVersion { - policy_hash: "policy".into(), - provider_env_revision: 1, - }, - Vec::new(), - ) - .await - .unwrap(); - assert!(tracker.apply(receiver.recv().await.unwrap())); + let (sender, mut receiver, mut tracker, _) = installed_status(1).await; let mut ctx = grant_context(&fixture); + ctx.token_grant_resolver = None; ctx.endpoint_observation_tx = Some(sender); + let err = inject_if_needed( grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), &ctx, REQUEST_ID, ) .await - .expect_err("a poisoned snapshot cannot prove no grant applies"); - assert_eq!(err.failure(), credential_lock_failure()); + .expect_err("a grant without a resolver refuses the request"); + + assert_eq!(err.failure(), injection_configuration_failure()); assert_eq!(err.credential(), None); fixture.assert_no_requests(); - assert!(tracker.apply(receiver.recv().await.unwrap())); - let observation = tracker + assert!(tracker.apply(receiver.try_recv().unwrap())); + let failure = tracker .snapshot() .unwrap() .last_token_grant_failure + .unwrap() + .failure .unwrap(); - let failure = observation.failure.unwrap(); - assert_eq!(failure.code, "unavailable"); + assert_eq!(failure.code, injection_configuration_failure().code()); assert_eq!(failure.request_id, REQUEST_ID); assert!(failure.provider.is_empty()); assert!(failure.credential_key.is_empty()); } - #[tokio::test] - async fn missing_resolver_refuses_without_naming_a_credential() { - let fixture = TokenGrantTestFixture::success( - "api.example.com\t443\t/v1/**\tprovider:access_token", - "grant-token", - ); - let ctx = L7EvalContext { - host: "api.example.com".into(), - port: 443, - dynamic_credentials: Some(fixture.dynamic_credentials()), - token_grant_resolver: None, - ..Default::default() - }; - let req = L7Request { - action: "GET".to_string(), - target: "/v1/projects".to_string(), - query_params: HashMap::new(), - raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), - body_length: BodyLength::None, - }; - - let err = inject_if_needed(req, &ctx, REQUEST_ID) - .await - .expect_err("a grant without a resolver refuses the request"); - - assert_eq!(err.failure(), injection_configuration_failure()); - assert_eq!(err.credential(), None); - fixture.assert_no_requests(); - } - - fn refreshed_grant_context( - accepted_key: &str, - current: &TokenGrantTestFixture, - ) -> L7EvalContext { - let accepted = TokenGrantTestFixture::success(accepted_key, "stale-token"); - let accepted_snapshot = ProviderCredentialState::from_environment( - 0, - HashMap::new(), - HashMap::new(), - accepted.dynamic_credentials().read().unwrap().clone(), - ) - .snapshot(); - let live_state = current.provider_credentials(); - // A socket retains scoped credentials from accept time. Register that - // old key too, so selecting it succeeds instead of failing in the fake. - current.add_credential( - &accepted_snapshot.scoped_key(accepted_key), - accepted_snapshot.dynamic_credentials[accepted_key].clone(), - Ok("stale-token"), - ); - let mut ctx = grant_context(current); - ctx.dynamic_credentials = Some(Arc::new(std::sync::RwLock::new( - accepted_snapshot - .dynamic_credentials - .iter() - .map(|(key, credential)| (accepted_snapshot.scoped_key(key), credential.clone())) - .collect(), - ))); - ctx.provider_credentials = Some(live_state); - ctx - } - - #[tokio::test] - async fn fallback_reports_live_credential_after_accept_time_refresh() { - use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; - const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; - let failure = TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::TokenEndpoint, - ); - let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); - let mut ctx = refreshed_grant_context( - "api.example.com\t443\t/v1/**\told-provider:access_token", - &fixture, - ); - let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); + async fn installed_status( + revision: u64, + ) -> ( + EndpointObservationSender, + EndpointStatusReceiver, + EndpointStatusTracker, + EndpointObservationContext, + ) { let (sender, mut receiver) = endpoint_status_channel(); let mut tracker = receiver.tracker(); sender .reset( EndpointConfigVersion { policy_hash: "policy".into(), - provider_env_revision: snapshot.revision, + provider_env_revision: revision, }, Vec::new(), ) @@ -1667,6 +1549,21 @@ mod tests { .unwrap(); assert!(tracker.apply(receiver.try_recv().unwrap())); let authority = sender.capture().unwrap(); + (sender, receiver, tracker, authority) + } + + #[tokio::test] + async fn fallback_reports_live_credential_refusal() { + const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; + let failure = TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, + ); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let mut ctx = grant_context(&fixture); + let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); + let (sender, mut receiver, mut tracker, authority) = + installed_status(snapshot.revision).await; ctx.endpoint_observation_tx = Some(sender); let error = inject_if_needed_captured( @@ -1676,7 +1573,7 @@ mod tests { Some(&authority), ) .await - .expect_err("the live credential refuses, even though the accepted one succeeds"); + .expect_err("the live credential refuses"); assert_eq!(error.failure(), failure); assert_eq!( error.credential(), @@ -1703,60 +1600,20 @@ mod tests { } #[tokio::test] - async fn fallback_cannot_clear_live_failure_with_stale_credential_success() { - use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; + async fn fallback_live_credential_success_clears_its_failure() { const CURRENT_KEY: &str = "api.example.com\t443\t/v1/projects\tprovider:access_token"; let failure = TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::TokenEndpoint, ); let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); - let mut ctx = refreshed_grant_context( - "api.example.com\t443\t/v1/**\tprovider:access_token", - &fixture, - ); + let mut ctx = grant_context(&fixture); let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset( - EndpointConfigVersion { - policy_hash: "policy".into(), - provider_env_revision: snapshot.revision, - }, - Vec::new(), - ) - .await - .unwrap(); - assert!(tracker.apply(receiver.try_recv().unwrap())); - let authority = sender.capture().unwrap(); + let (sender, mut receiver, mut tracker, authority) = + installed_status(snapshot.revision).await; ctx.endpoint_observation_tx = Some(sender); - // Establish B's failure through the production entry point with B's map. - let mut fresh = ctx.clone(); - fresh.dynamic_credentials = Some(Arc::new(std::sync::RwLock::new( - snapshot.dynamic_credentials.clone(), - ))); let raw_header = b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"; - inject_if_needed_captured( - grant_request(raw_header), - &mut fresh, - REQUEST_ID, - Some(&authority), - ) - .await - .expect_err("current grant first records its failure"); - assert!(tracker.apply(receiver.try_recv().unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_some() - ); - - // A has the same credential identity, so its old success would clear B - // if fallback injection used the socket's copied map after the refresh. let error = inject_if_needed_captured( grant_request(raw_header), &mut ctx, @@ -1764,7 +1621,7 @@ mod tests { Some(&authority), ) .await - .expect_err("the stale successful credential cannot replace the live refusal"); + .expect_err("current grant first records its failure"); assert_eq!(error.failure(), failure); assert!(tracker.apply(receiver.try_recv().unwrap())); let observation = tracker @@ -1792,7 +1649,7 @@ mod tests { .unwrap() .contains("Authorization: Bearer current-token\r\n") ); - fixture.assert_requested_keys(&[CURRENT_KEY, CURRENT_KEY, CURRENT_KEY]); + fixture.assert_requested_keys(&[CURRENT_KEY, CURRENT_KEY]); assert!(tracker.apply(receiver.try_recv().unwrap())); assert!( tracker @@ -1805,31 +1662,14 @@ mod tests { #[tokio::test] async fn fallback_rejects_telemetry_authority_captured_before_provider_refresh() { - use openshell_core::endpoint_status::{EndpointConfigVersion, endpoint_status_channel}; const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; let failure = TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::TokenEndpoint, ); let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); - let mut ctx = refreshed_grant_context( - "api.example.com\t443\t/v1/**\told-provider:access_token", - &fixture, - ); - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset( - EndpointConfigVersion { - policy_hash: "policy".into(), - provider_env_revision: 0, - }, - Vec::new(), - ) - .await - .unwrap(); - assert!(tracker.apply(receiver.try_recv().unwrap())); - let accepted_authority = sender.capture().unwrap(); + let mut ctx = grant_context(&fixture); + let (sender, mut receiver, mut tracker, accepted_authority) = installed_status(0).await; sender .reset( EndpointConfigVersion { diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs index 4dc0346f47..0eb8741252 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs @@ -44,7 +44,7 @@ fn fixture( let ctx = L7EvalContext { host: "api.example.com".into(), port: 443, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -53,7 +53,7 @@ fn fixture( #[tokio::test] async fn malformed_endpoint_key_fails_before_acquisition() { - let (fixture, ctx) = fixture(Ok("identity-token")); + let (fixture, mut ctx) = fixture(Ok("identity-token")); { let credentials = fixture.dynamic_credentials(); let mut credentials = credentials.write().unwrap(); @@ -63,6 +63,7 @@ async fn malformed_endpoint_key_fails_before_acquisition() { credential, ); } + ctx.provider_credentials = Some(fixture.provider_credentials()); assert!(inject_if_needed(request(), &ctx, REQUEST_ID).await.is_err()); fixture.assert_no_requests(); } @@ -97,11 +98,12 @@ async fn injects_independent_grants_and_replaces_all_protected_headers() { #[tokio::test] async fn chooses_specific_binding_independently_for_each_header() { - let (fixture, ctx) = fixture(Ok("identity-token")); + let (fixture, mut ctx) = fixture(Ok("identity-token")); let specific_key = "api.example.com\t443\t/v1/projects\trev:7\tother:service"; let mut specific = fixture.dynamic_credentials().read().unwrap()[SERVICE].clone(); specific.token_grant.as_mut().unwrap().audience = "projects-only".into(); fixture.add_credential(specific_key, specific, Ok("specific-service")); + ctx.provider_credentials = Some(fixture.provider_credentials()); let rewritten = inject_if_needed(request(), &ctx, REQUEST_ID).await.unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); assert!(bytes.contains("Authorization: Bearer specific-service\r\n")); @@ -111,11 +113,12 @@ async fn chooses_specific_binding_independently_for_each_header() { #[tokio::test] async fn audience_override_changes_only_its_own_credential() { - let (fixture, ctx) = fixture(Ok("identity-token")); + let (fixture, mut ctx) = fixture(Ok("identity-token")); let override_key = "api.example.com\t443\t/v1/projects\trev:7\tprovider:identity"; let mut identity = fixture.dynamic_credentials().read().unwrap()[IDENTITY].clone(); identity.token_grant.as_mut().unwrap().audience = "project-identity".into(); fixture.add_credential(override_key, identity.clone(), Ok("project-identity-token")); + ctx.provider_credentials = Some(fixture.provider_credentials()); let rewritten = inject_if_needed(request(), &ctx, REQUEST_ID).await.unwrap(); let bytes = String::from_utf8(rewritten.raw_header).unwrap(); assert!(bytes.contains("Authorization: Bearer service-token\r\n")); @@ -127,7 +130,7 @@ async fn audience_override_changes_only_its_own_credential() { #[tokio::test] async fn rejects_tied_credentials_for_same_header_before_acquisition() { for header in ["Authorization", " authorization ", ""] { - let (fixture, ctx) = fixture(Ok("identity-token")); + let (fixture, mut ctx) = fixture(Ok("identity-token")); let mut collision = fixture.dynamic_credentials().read().unwrap()[SERVICE].clone(); collision.name = "collision".into(); collision.header_name = header.into(); @@ -136,6 +139,7 @@ async fn rejects_tied_credentials_for_same_header_before_acquisition() { collision, Ok("collision-token"), ); + ctx.provider_credentials = Some(fixture.provider_credentials()); let error = inject_if_needed(request(), &ctx, REQUEST_ID) .await .unwrap_err(); @@ -148,10 +152,11 @@ async fn rejects_tied_credentials_for_same_header_before_acquisition() { #[tokio::test] async fn rejects_invalid_second_header_before_acquisition() { - let (fixture, ctx) = fixture(Ok("identity-token")); + let (fixture, mut ctx) = fixture(Ok("identity-token")); let mut identity = fixture.dynamic_credentials().read().unwrap()[IDENTITY].clone(); identity.header_name = "Content-Length".into(); fixture.add_credential(IDENTITY, identity, Ok("identity-token")); + ctx.provider_credentials = Some(fixture.provider_credentials()); assert!(inject_if_needed(request(), &ctx, REQUEST_ID).await.is_err()); fixture.assert_no_requests(); } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index 4fd73af354..bb4f3146c6 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -30,7 +30,7 @@ use openshell_core::net::{ set_tcp_nodelay_best_effort, }; use openshell_core::policy::ProxyPolicy; -use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; +use openshell_core::provider_credentials::ProviderCredentialState; use openshell_core::secrets::{self, SecretResolver, rewrite_header_line_checked}; #[cfg(test)] use openshell_isolation_interface::contract::ExecutableIdentity as ContractExecutableIdentity; @@ -214,16 +214,6 @@ pub(crate) const HOST_GATEWAY_ALIASES: &[&str] = &[ "host.docker.internal", ]; -fn revision_scoped_dynamic_credentials( - snapshot: &ProviderCredentialSnapshot, -) -> std::collections::HashMap { - snapshot - .dynamic_credentials - .iter() - .map(|(key, credential)| (snapshot.scoped_key(key), credential.clone())) - .collect() -} - /// Cloud instance metadata IPs that are NEVER exempted from SSRF blocking, /// even when they coincidentally match a host-gateway alias resolution. /// This list covers the well-known IMDS endpoints across major cloud providers. @@ -493,11 +483,6 @@ impl ProxyHandle { let resolver = provider_credentials .as_ref() .and_then(ProviderCredentialState::resolver); - let dynamic_credentials = provider_credentials.as_ref().map(|state| { - Arc::new(std::sync::RwLock::new(revision_scoped_dynamic_credentials( - &state.snapshot(), - ))) - }); let dtx = denial_tx.clone(); let atx = activity_tx.clone(); let endpoint_observations = endpoint_observation_tx.clone(); @@ -520,7 +505,6 @@ impl ProxyHandle { up_proxy, credentials, resolver, - dynamic_credentials, dtx, atx, endpoint_observations, @@ -1231,7 +1215,6 @@ async fn handle_transparent_tcp_connection( &decision, None, None, - None, agent_proposals, // The transparent TCP path carries no PolicyLocalContext, so no // workspace is available here; matches the CONNECT path default when @@ -2313,13 +2296,6 @@ async fn handle_tcp_connection( upstream_proxy: Arc>, provider_credentials: Option, secret_resolver: Option>, - dynamic_credentials: Option< - Arc< - std::sync::RwLock< - std::collections::HashMap, - >, - >, - >, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -2343,7 +2319,6 @@ async fn handle_tcp_connection( upstream_proxy, provider_credentials, secret_resolver, - dynamic_credentials, denial_tx, activity_tx, endpoint_observation_tx, @@ -2408,13 +2383,6 @@ async fn handle_mediated_connection( upstream_proxy: Arc>, provider_credentials: Option, secret_resolver: Option>, - dynamic_credentials: Option< - Arc< - std::sync::RwLock< - std::collections::HashMap, - >, - >, - >, denial_tx: Option>, activity_tx: Option, endpoint_observation_tx: Option, @@ -2581,7 +2549,6 @@ async fn handle_mediated_connection( trusted_host_gateway, provider_credentials, secret_resolver, - dynamic_credentials, denial_tx.as_ref(), activity_tx.as_ref(), endpoint_observation_tx, @@ -3000,7 +2967,6 @@ async fn handle_mediated_connection( &decision, provider_credentials, secret_resolver.clone(), - dynamic_credentials.clone(), agent_proposals, workspace, relay::RelaySignals { @@ -5285,13 +5251,6 @@ async fn handle_forward_proxy( trusted_host_gateway: Arc>, provider_credentials: Option, secret_resolver: Option>, - dynamic_credentials: Option< - Arc< - std::sync::RwLock< - std::collections::HashMap, - >, - >, - >, denial_tx: Option<&mpsc::UnboundedSender>, activity_tx: Option<&ActivitySender>, endpoint_observation_tx: Option, @@ -5623,7 +5582,6 @@ async fn handle_forward_proxy( &decision, provider_credentials, secret_resolver.clone(), - dynamic_credentials.clone(), agent_proposals, workspace, relay::RelaySignals { @@ -6854,26 +6812,25 @@ fn build_json_error_response(status: u16, status_text: &str, error: &str, detail /// Build the forward proxy's refusal for a failed dynamic token grant. /// -/// The JSON body matches every other grant refusal. The request ID header lets -/// the caller quote the exchange to an operator without learning its cause. +/// This is the generic grant refusal plus a request ID header, which lets the +/// caller quote the exchange to an operator without learning its cause. fn build_token_grant_failure_response(request_id: &str) -> Vec { - let body = serde_json::json!({ - "error": "token_grant_failed", - "detail": "dynamic token grant failed", - }) - .to_string(); - format!( - "HTTP/1.1 502 Bad Gateway\r\n\ - Content-Type: application/json\r\n\ - Content-Length: {}\r\n\ - {}: {request_id}\r\n\ - Connection: close\r\n\ - \r\n\ - {body}", - body.len(), - openshell_core::token_grant_failure::REQUEST_ID_HEADER, - ) - .into_bytes() + let generic = build_json_error_response( + 502, + "Bad Gateway", + "token_grant_failed", + "dynamic token grant failed", + ); + String::from_utf8_lossy(&generic) + .replacen( + "\r\nConnection: close\r\n", + &format!( + "\r\n{}: {request_id}\r\nConnection: close\r\n", + openshell_core::token_grant_failure::REQUEST_ID_HEADER + ), + 1, + ) + .into_bytes() } fn build_json_error_response_with_reason( @@ -7634,7 +7591,6 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, None, None, - None, )) .await .unwrap(); @@ -8054,7 +8010,6 @@ process: { run_as_user: sandbox, run_as_group: sandbox } None, None, None, - None, )); workload .write_all(b"GET /v1/policy/current HTTP/1.1\r\nHost: policy.local\r\nConnection: close\r\n\r\n") @@ -8182,7 +8137,6 @@ network_policies: None, None, None, - None, )); let request = tokio::time::timeout(std::time::Duration::from_secs(5), request_rx) .await @@ -8756,7 +8710,6 @@ network_policies: {} None, None, None, - None, )) .await .expect("malformed request should be handled"); @@ -8994,7 +8947,6 @@ network_policies: None, None, None, - None, )), )) .await @@ -9126,7 +9078,6 @@ network_policies: None, None, None, - None, )), ) .await @@ -9245,7 +9196,6 @@ network_policies: None, None, None, - None, )), ) .await @@ -9373,7 +9323,6 @@ network_policies: None, None, None, - None, )), ) .await @@ -9516,7 +9465,6 @@ network_policies: None, None, None, - None, )) .await }); @@ -10296,30 +10244,6 @@ network_policies: ); } - #[test] - fn revision_scoped_dynamic_credentials_preserves_endpoint_selector_and_adds_revision() { - let mut dynamic_credentials = std::collections::HashMap::new(); - dynamic_credentials.insert( - "api.example.test\t443\t/v1/**\tprovider:access_token".to_string(), - openshell_core::proto::ProviderProfileCredential { - name: "access_token".to_string(), - ..Default::default() - }, - ); - let snapshot = ProviderCredentialSnapshot { - installation_id: "fixture-installation".to_string(), - revision: 42, - child_env: std::collections::HashMap::new(), - dynamic_credentials, - }; - - let scoped = revision_scoped_dynamic_credentials(&snapshot); - - assert!( - scoped.contains_key("api.example.test\t443\t/v1/**\trev:42\tinstallation:fixture-installation\tprovider:access_token") - ); - } - #[test] fn connect_activity_is_skipped_when_l7_will_count_the_request() { let (tx, mut rx) = mpsc::channel(4); @@ -11020,29 +10944,11 @@ network_policies: const FORWARD_REQUEST_ID: &str = "9b8a7f6e-5d4c-4b3a-9f8e-7d6c5b4a3f2e"; #[test] - fn forward_grant_refusal_keeps_its_body_and_adds_only_the_request_id() { - let response = String::from_utf8(build_token_grant_failure_response(FORWARD_REQUEST_ID)) - .expect("UTF-8 response"); - let (head, body) = response.split_once("\r\n\r\n").expect("header terminator"); - let generic = String::from_utf8(build_json_error_response( - 502, - "Bad Gateway", - "token_grant_failed", - "dynamic token grant failed", - )) - .expect("UTF-8 response"); - let (generic_head, generic_body) = - generic.split_once("\r\n\r\n").expect("header terminator"); - assert_eq!(body, generic_body, "the refusal body is unchanged"); - let mut added = head - .lines() - .filter(|line| !generic_head.lines().any(|generic| generic == *line)) - .collect::>(); - added.sort_unstable(); - assert_eq!( - added, - [format!("x-openshell-request-id: {FORWARD_REQUEST_ID}").as_str()] - ); + fn forward_grant_refusal_adds_the_request_id_header() { + let response = build_token_grant_failure_response(FORWARD_REQUEST_ID); + let header = + format!("\r\nx-openshell-request-id: {FORWARD_REQUEST_ID}\r\nConnection: close\r\n"); + assert!(String::from_utf8_lossy(&response).contains(&header)); } fn forward_grant_failure() -> openshell_core::token_grant_failure::TokenGrantFailure { @@ -11073,7 +10979,7 @@ network_policies: ancestors: vec![], cmdline_paths: vec![], secret_resolver: None, - dynamic_credentials: Some(fixture.dynamic_credentials()), + provider_credentials: Some(fixture.provider_credentials()), token_grant_resolver: Some(fixture.resolver()), ..Default::default() }; @@ -11093,7 +10999,7 @@ network_policies: let (mut ctx, _) = forward_token_grant_context(Ok("unused-token")); let provider_key = "api.example.test\t8080\t/v1/**\tprovider:access_token"; let fixture = forward_token_grant_fixture(provider_key, resolver_response, true); - ctx.dynamic_credentials = Some(fixture.dynamic_credentials()); + ctx.provider_credentials = Some(fixture.provider_credentials()); ctx.token_grant_resolver = Some(fixture.resolver()); (ctx, fixture) @@ -13384,7 +13290,7 @@ network_policies: let (mut ctx, fixture) = forward_token_exchange_context(Err(forward_grant_failure())); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - let err = inject_token_grant_for_forward_request( + inject_token_grant_for_forward_request( "GET", "/v1/projects", raw, @@ -13396,12 +13302,6 @@ network_policies: .await .expect_err("forward token exchange failure should stop request rewriting"); - let refusal = err - .downcast_ref::() - .expect("typed grant refusal"); - assert_eq!(refusal.failure(), forward_grant_failure()); - assert_eq!(refusal.credential(), Some(("provider", "access_token"))); - assert_eq!(err.to_string(), "dynamic token grant failed"); fixture.assert_one_token_exchange_request( "api.example.test\t8080\t/v1/**\tprovider:access_token", ); @@ -14537,7 +14437,6 @@ network_policies: Arc::new(None), // upstream_proxy None, // provider_credentials None, // secret_resolver - None, // dynamic_credentials Some(denial_tx), // denial_tx — positive allow/deny signal None, // activity_tx None, // endpoint_observation_tx @@ -14607,7 +14506,6 @@ network_policies: Arc::new(None), None, None, - None, Some(denial_tx), None, None, diff --git a/crates/openshell-supervisor-network/src/proxy/relay.rs b/crates/openshell-supervisor-network/src/proxy/relay.rs index 54c93586cb..363ef8868d 100644 --- a/crates/openshell-supervisor-network/src/proxy/relay.rs +++ b/crates/openshell-supervisor-network/src/proxy/relay.rs @@ -9,14 +9,10 @@ use crate::opa::{NetworkAction, OpaEngine, PolicyGenerationGuard, TunnelPolicyEn use miette::{IntoDiagnostic, Result}; use openshell_core::activity::ActivitySender; use openshell_core::endpoint_status::EndpointObservationSender; -use openshell_core::proto::ProviderProfileCredential; use openshell_core::secrets::SecretResolver; -use std::collections::HashMap; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncWrite}; -type DynamicCredentials = Arc>>; - enum PreparedHttpPolicy { Inspect { configs: Vec, @@ -51,7 +47,6 @@ pub(super) fn http_context( decision: &EgressDecision, provider_credentials: Option, secret_resolver: Option>, - dynamic_credentials: Option, agent_proposals: openshell_core::proposals::AgentProposals, workspace: String, signals: RelaySignals, @@ -63,6 +58,9 @@ pub(super) fn http_context( .is_none() .then_some(secret_resolver) .flatten(); + let token_grant_resolver = provider_credentials + .as_ref() + .map(|_| crate::l7::token_grant_injection::default_resolver()); let policy_name = match &decision.action { NetworkAction::Allow { matched_policy } => matched_policy.clone().unwrap_or_default(), NetworkAction::Deny { .. } => String::new(), @@ -94,10 +92,7 @@ pub(super) fn http_context( provider_credential_installation_id: None, body_classifier: None, activity_tx: signals.activity, - dynamic_credentials: dynamic_credentials.clone(), - token_grant_resolver: dynamic_credentials - .as_ref() - .map(|_| crate::l7::token_grant_injection::default_resolver()), + token_grant_resolver, agent_proposals, workspace, endpoint_observation_tx: signals.endpoint_observation, @@ -397,7 +392,6 @@ mod tests { provider_credential_installation_id: None, body_classifier: None, activity_tx: None, - dynamic_credentials: None, token_grant_resolver: None, agent_proposals: openshell_core::proposals::AgentProposals::default(), workspace: String::new(), diff --git a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs index a039cfbc14..010cb466c5 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/compatibility.rs @@ -548,7 +548,6 @@ network_policies: None, None, None, - None, )) .await .unwrap(); diff --git a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs index 13e3938fb6..0d26319749 100644 --- a/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs +++ b/crates/openshell-supervisor-network/src/proxy/tests/token_grants.rs @@ -44,7 +44,6 @@ fn forward_inspection_fixture() -> ( Ok("grant-token"), false, ); - ctx.dynamic_credentials = Some(fixture.dynamic_credentials()); ctx.provider_credentials = Some(fixture.provider_credentials()); ctx.token_grant_resolver = Some(fixture.resolver()); (config, engine, ctx, fixture) diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index c0ce14798f..a1a2ca6635 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -77,32 +77,28 @@ impl TokenCache { } /// Get a cached token if it exists and is not expired. - fn get(&self, provider_name: &str) -> Result, TokenGrantFailure> { + fn get(&self, provider_name: &str) -> Option { let now_ms = current_time_ms(); - let tokens = self.tokens.read().map_err(|_| cache_failure())?; - Ok(tokens - .get(provider_name) - .filter(|cached| cached.expires_at_ms > now_ms) - .map(|cached| cached.access_token.clone())) + let tokens = self.tokens.read().ok()?; + let cached = tokens.get(provider_name)?; + if cached.expires_at_ms > now_ms { + Some(cached.access_token.clone()) + } else { + None + } } /// Store a token with expiration time. - fn set( - &self, - provider_name: String, - access_token: String, - expires_at_ms: i64, - ) -> Result<(), TokenGrantFailure> { - // Poisoning invalidates cache integrity; never silently bypass it. - let mut tokens = self.tokens.write().map_err(|_| cache_failure())?; - tokens.insert( - provider_name, - CachedToken { - access_token, - expires_at_ms, - }, - ); - Ok(()) + fn set(&self, provider_name: String, access_token: String, expires_at_ms: i64) { + if let Ok(mut tokens) = self.tokens.write() { + tokens.insert( + provider_name, + CachedToken { + access_token, + expires_at_ms, + }, + ); + } } } @@ -154,20 +150,6 @@ pub async fn obtain_provider_token( ) -> Result { let grant_type = ProviderCredentialTokenGrantType::try_from(request.grant_type) .map_err(|_| configuration_failure())?; - // Resolve local exchange prerequisites before fetching any workload credential. - let exchange_context = if grant_type == ProviderCredentialTokenGrantType::TokenExchange { - let (provider, credential_key) = parse_provider_credential_key(request.provider_name)?; - let endpoint = supervisor_gateway_endpoint_from_env()?; - openshell_core::grpc_client::validate_token_grant_gateway_endpoint(&endpoint)?; - Some(( - provider, - credential_key, - endpoint, - supervisor_sandbox_id_from_env()?, - )) - } else { - None - }; obtain_provider_token_with_grant( ObtainProviderTokenInput { cache: &TOKEN_CACHE, @@ -182,13 +164,29 @@ pub async fn obtain_provider_token( requested_token_type: request.requested_token_type, }, |jwt_audience| async move { + // Resolve local exchange prerequisites before fetching any workload credential. + let exchange_context = match grant_type { + ProviderCredentialTokenGrantType::ClientCredentials + | ProviderCredentialTokenGrantType::Unspecified => None, + ProviderCredentialTokenGrantType::TokenExchange => { + let (provider, credential_key) = + parse_provider_credential_key(request.provider_name)?; + let endpoint = supervisor_gateway_endpoint_from_env()?; + openshell_core::grpc_client::token_grant_gateway_endpoint(&endpoint)?; + Some(( + provider, + credential_key, + endpoint, + supervisor_sandbox_id_from_env()?, + )) + } + }; // Fetch JWT-SVID with authorization server as audience // For RFC 7523, the JWT assertion's aud claim identifies the issuer/realm let jwt_svid = fetch_jwt_svid_for_token_grant(&jwt_audience).await?; - match grant_type { - ProviderCredentialTokenGrantType::ClientCredentials - | ProviderCredentialTokenGrantType::Unspecified => { + match exchange_context { + None => { // Perform OAuth2 JWT client assertion grant. The audience // parameter in the token request specifies the resource server. perform_token_grant( @@ -201,9 +199,7 @@ pub async fn obtain_provider_token( ) .await } - ProviderCredentialTokenGrantType::TokenExchange => { - let (provider, credential_key, endpoint, sandbox_id) = - exchange_context.ok_or_else(configuration_failure)?; + Some((provider, credential_key, endpoint, sandbox_id)) => { let intermediate = openshell_core::grpc_client::exchange_provider_subject_token( &endpoint, @@ -259,14 +255,6 @@ where F: FnOnce(String) -> Fut, Fut: Future>, { - // Even a cache hit cannot authorize an invalid endpoint configuration. - oauth::validate_token_endpoint(input.token_endpoint)?; - if input - .cache_ttl_override - .is_some_and(|ttl| i64::try_from(ttl.as_millis()).is_err()) - { - return Err(configuration_failure()); - } let jwt_audience = effective_jwt_svid_audience(input.token_endpoint, input.jwt_svid_audience); let cache_key = token_cache_key(TokenCacheKeyInput { provider_name: input.provider_name, @@ -280,11 +268,18 @@ where }); if input.cache_ttl_override != Some(Duration::ZERO) - && let Some(cached) = input.cache.get(&cache_key)? + && let Some(cached) = input.cache.get(&cache_key) { return Ok(cached); } + oauth::parse_token_endpoint_url(input.token_endpoint)?; + if input + .cache_ttl_override + .is_some_and(|ttl| i64::try_from(ttl.as_millis()).is_err()) + { + return Err(configuration_failure()); + } let token_response = grant(jwt_audience).await?; validate_access_token(&token_response.access_token)?; @@ -296,7 +291,7 @@ where cache_key, token_response.access_token.clone(), expires_at_ms, - )?; + ); } Ok(token_response.access_token) @@ -335,9 +330,7 @@ fn workload_api_failure(error: WorkloadApiError) -> TokenGrantFailure { // Classify the local Workload API's typed rejection without retaining its // endpoint, response text, JWT parsing details, or transport source. let kind = match error { - WorkloadApiError::MissingEndpointSocket - | WorkloadApiError::NotUnicodeEndpointSocket(_) - | WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, + WorkloadApiError::Endpoint(_) => TokenGrantFailureKind::Configuration, WorkloadApiError::NoIdentityIssued | WorkloadApiError::PermissionDenied(_) => { TokenGrantFailureKind::WorkloadIdentityRejected } @@ -364,13 +357,6 @@ fn configuration_failure() -> TokenGrantFailure { ) } -fn cache_failure() -> TokenGrantFailure { - TokenGrantFailure::new( - TokenGrantFailureKind::Unavailable, - TokenGrantStage::Configuration, - ) -} - fn provider_spiffe_workload_api_socket_from_env() -> Result { std::env::var(sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET) .ok() @@ -976,10 +962,6 @@ mod tests { WorkloadApiError::NoIdentityIssued, TokenGrantFailureKind::WorkloadIdentityRejected, ), - ( - WorkloadApiError::MissingEndpointSocket, - TokenGrantFailureKind::Configuration, - ), ( WorkloadApiError::HintNotFound("secret-jwt-canary".to_string()), TokenGrantFailureKind::Unclassified, @@ -1052,36 +1034,6 @@ mod tests { } } - #[tokio::test] - async fn poisoned_token_cache_fails_before_credential_acquisition() { - let cache = TokenCache::new(); - let tokens = cache.tokens.clone(); - assert!( - std::thread::spawn(move || { - let _guard = tokens.write().expect("lock fixture cache"); - panic!("poison fixture cache"); - }) - .join() - .is_err() - ); - let grant_calls = Arc::new(AtomicUsize::new(0)); - let failure = obtain_counted_test_token(CountedTokenGrantInput { - cache: &cache, - provider_name: "provider:access_token", - token_endpoint: "https://auth.example.com/token", - jwt_svid_audience: "", - audience: "", - scopes: &[], - cache_ttl_override: None, - expires_in: 60, - grant_calls: grant_calls.clone(), - }) - .await - .expect_err("poisoned cache must fail closed"); - assert_eq!(failure, cache_failure()); - assert_eq!(grant_calls.load(Ordering::SeqCst), 0); - } - #[test] fn test_derive_issuer_from_keycloak_token_endpoint() { let token_endpoint = @@ -1295,9 +1247,7 @@ mod tests { grant_type: ProviderCredentialTokenGrantType::ClientCredentials, requested_token_type: ACCESS_TOKEN_TYPE, }); - cache - .set(identity_cache_key, "expired".into(), current_time_ms() - 1) - .expect("test cache accepts an expired entry"); + cache.set(identity_cache_key, "expired".into(), current_time_ms() - 1); let service_token = obtain_token_without_grant_call( &cache, service, @@ -1436,13 +1386,11 @@ mod tests { grant_type: ProviderCredentialTokenGrantType::ClientCredentials, requested_token_type: ACCESS_TOKEN_TYPE, }); - cache - .set( - cache_key, - "expired-token".to_string(), - current_time_ms() - 1, - ) - .expect("cache should accept expired fixture"); + cache.set( + cache_key, + "expired-token".to_string(), + current_time_ms() - 1, + ); let token = obtain_counted_test_token(CountedTokenGrantInput { cache: &cache, diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 43ebbcd3f7..f7b3ddb40a 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -660,7 +660,7 @@ The gateway only brokers an intermediate token for a sandbox principal, and only ### Token Grant Failures -When a token grant fails, OpenShell refuses the protected request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header. L7 relays return an empty body, and the forward proxy returns the JSON body `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`. The response, the supervisor's OCSF events, and `policy.local` denials never say why the grant failed, because the sandboxed agent can read them. Users authorized to read the sandbox can retrieve the cause from `Sandbox.status.last_token_grant_failure` and `openshell logs`. +When a token grant fails, OpenShell refuses the protected request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header and the body described in [Proxy Error Responses](/observability/logging#proxy-error-responses). The response, the supervisor's OCSF events, and `policy.local` denials never say why the grant failed, because the sandboxed agent can read them. Users authorized to read the sandbox can retrieve the cause from `Sandbox.status.last_token_grant_failure` and `openshell logs`. The request ID is a random UUID that the supervisor mints for each HTTP exchange. It names the same exchange everywhere: @@ -682,11 +682,12 @@ OpenShell classifies each failure into one of these codes. The recovery action u The stage names the step that observed the failure: `configuration`, `workload_identity`, `gateway_rpc`, `intermediate_exchange`, `token_endpoint`, or `injection`. OpenShell maps token-service responses as follows: -- HTTP 429 and 5xx, and connection failures, are `unavailable`. A timeout, including a slow response body, is `timeout`. -- A bare HTTP 401, 403, or 409 is `unclassified`, with its status recorded. OpenShell never infers `provider_authorization_required` from a 401. +- A response body over 64 KiB is `unclassified`, whatever its HTTP status. +- HTTP 429 and 5xx responses are `unavailable` unless they carry a recognized extension code, described below. Failures to connect, including TLS certificate or hostname verification failures, are also `unavailable`, but a connection that fails after it is established is `unclassified`. A timeout, including a slow response body, is `timeout`. +- Without a recognized extension code, any other failed HTTP status is `unclassified`, including 401, 403, and 409. OpenShell never infers `provider_authorization_required` from a 401. - For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these causes only over its verified TLS connection to the gateway. -OpenShell keeps only the code, stage, and HTTP status. It discards response descriptions, headers, endpoint diagnostics, and error chains. Success and failure response bodies are limited to 64 KiB. A successful response must contain a nonempty header-compatible `access_token`. +OpenShell keeps only the code, stage, and HTTP status. It discards response descriptions, headers, endpoint diagnostics, and error chains. A successful response without a nonempty header-compatible `access_token` is `unclassified`. A token service can also state a cause by adopting the `openshell.token-grant.v1` error extension. For example, a service that requires the user to authorize a provider returns HTTP 409 with: @@ -703,9 +704,9 @@ OpenShell accepts this cause only from the configured endpoint over verified HTT - HTTP 401 or 403 with `workload_identity_rejected`. - HTTP 400 with `configuration_error`. -Each response also requires the exact `openshell_error_contract` marker. Unknown versions, inconsistent status and code pairs, malformed responses, and signals received over plain HTTP are `unclassified`, except that HTTP 429 and 5xx stay `unavailable`. OpenShell does not infer identity rejection from generic OAuth errors such as `invalid_client` or `invalid_grant`, which can have more than one cause. +Each response also requires the exact `openshell_error_contract` marker. OpenShell classifies a response with an unknown contract version or an unrecognized code by its HTTP status alone. A recognized code with a different HTTP status, or one received over plain HTTP, is `unclassified`. OpenShell does not infer identity rejection from generic OAuth errors such as `invalid_client` or `invalid_grant`, which can have more than one cause. -The latest classified grant refusal is available as `last_token_grant_failure` for every destination, including REST requests and failures before an HTTP path is known. Its `destination_host` and `destination_port` identify the attempted call, and `failure` contains the bounded cause. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) for retrieval and clearing rules. Find correlated events in `openshell logs`: the supervisor's `Token grant failed` line or, when the gateway hop failed, the gateway's `dynamic token grant failed` line. The token service receives the ID with every request. The no-bytes guarantee excludes connection setup: an inspected CONNECT path can already have completed an upstream TLS handshake before a grant fails. +Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) for how to read `last_token_grant_failure` and when it clears. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. The no-bytes guarantee excludes connection setup: an inspected CONNECT path can already have completed an upstream TLS handshake before a grant fails. ## Provider Instances diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 755c0d14a2..95ccf8d35f 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -232,7 +232,7 @@ The `error` field is a short machine-readable code (`policy_denied`, `middleware For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When the policy advisor is enabled, the body also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware` and can include a validated `reason_code`. A fail-closed runtime failure uses `middleware_failed` with platform-owned text. Both middleware responses omit `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. -A failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) returns `502 Bad Gateway` with an `x-openshell-request-id` header. The forward proxy returns `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`; L7 relays return an empty body. The response does not say why the grant failed. Give the request ID to someone who can read the sandbox. They can find the cause in `last_token_grant_failure` from `openshell sandbox get --output json` or in `openshell logs`, on the supervisor's `Token grant failed` line or, when the gateway hop failed, the gateway's `dynamic token grant failed` line. +A failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) returns `502 Bad Gateway` with an `x-openshell-request-id` header. The forward proxy returns `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`; L7 relays return an empty body. The response does not say why the grant failed. Give the request ID to someone who can read the sandbox. ## Filesystem Sandbox Logs diff --git a/e2e/rust/tests/provider_token_exchange/closure.rs b/e2e/rust/tests/provider_token_exchange/closure.rs index 02d1963977..191aeca7d1 100644 --- a/e2e/rust/tests/provider_token_exchange/closure.rs +++ b/e2e/rust/tests/provider_token_exchange/closure.rs @@ -4,6 +4,7 @@ //! Exercise trusted grant failure retrieval through a real REST workload. use super::*; +use openshell_e2e::harness::cli::wait_for_healthy; use openshell_e2e::harness::container::ContainerEngine; use openshell_e2e::harness::gateway::ManagedGateway; use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; @@ -53,7 +54,6 @@ struct GatewayTrust { directory: TempDir, engine: ContainerEngine, image: String, - health_port: u16, changed: bool, } @@ -81,9 +81,6 @@ impl GatewayTrust { .map(|pair| pair[1]) .ok_or("gateway argument is missing") }; - let health_port = argument("--health-port")? - .parse() - .map_err(|_| "gateway health port is invalid")?; let config = fs::read_to_string(argument("--config")?) .map_err(|_| "could not read fixture gateway configuration")?; let base_image = std::env::var("OPENSHELL_SUPERVISOR_IMAGE") @@ -137,7 +134,6 @@ impl GatewayTrust { directory, engine, image, - health_port, changed: false, }) } @@ -149,35 +145,7 @@ impl GatewayTrust { gateway.set_native_ca_bundle(self.directory.path().join("host-roots.crt")); gateway.stop()?; gateway.start()?; - self.wait_ready().await - } - - async fn wait_ready(&self) -> Result<(), String> { - let url = format!("http://127.0.0.1:{}/healthz", self.health_port); - let deadline = tokio::time::Instant::now() + Duration::from_secs(120); - loop { - if checked( - Command::new("curl").args([ - "--noproxy", - "*", - "--silent", - "--fail", - "--max-time", - "2", - &url, - ]), - "gateway health probe", - ) - .await - .is_ok() - { - return Ok(()); - } - if tokio::time::Instant::now() >= deadline { - return Err("trusted fixture gateway did not become healthy".into()); - } - tokio::time::sleep(Duration::from_millis(200)).await; - } + wait_for_healthy(Duration::from_secs(120)).await } async fn restore(&mut self) -> Result<(), String> { @@ -185,7 +153,7 @@ impl GatewayTrust { let gateway = ManagedGateway::from_env()?.ok_or("managed gateway is missing")?; gateway.stop()?; gateway.start()?; - self.wait_ready().await?; + wait_for_healthy(Duration::from_secs(120)).await?; self.changed = false; } let mut remove = Command::from(self.engine.command()); @@ -786,7 +754,6 @@ async fn verified_grant_failures_reach_trusted_rest_status_and_recover() -> Resu let target = start_route_target(port, observed.clone(), observed_bytes.clone()).await; let name = format!("closure-{:08x}", rand::random::()); let mut sandbox = None; - let mut other = None; let result = async { trust.install().await?; sandbox = Some(create_rest_fixture(&name, token_port, port).await?); @@ -798,16 +765,6 @@ async fn verified_grant_failures_reach_trusted_rest_status_and_recover() -> Resu assert_untrusted_reader(&name, None).await?; let bearer = supervisor_token(&trust.engine, &name).await?; assert_untrusted_reader(&name, Some(&bearer)).await?; - let other_name = format!("other-{:08x}", rand::random::()); - other = Some( - SandboxGuard::create_keep_with_args( - &["--name", &other_name, "--no-auto-providers"], - &["sh", "-lc", &sandbox_script()], - "token-server-ready", - ) - .await?, - ); - assert_untrusted_reader(&other_name, Some(&bearer)).await?; recovery_cases(active, port, &service).await?; Ok::<(), String>(()) } @@ -815,9 +772,6 @@ async fn verified_grant_failures_reach_trusted_rest_status_and_recover() -> Resu if let Some(mut sandbox) = sandbox { sandbox.cleanup().await; } - if let Some(mut other) = other { - other.cleanup().await; - } run_cli_ignore_error(&["sandbox", "delete", &name]).await; run_cli_ignore_error(&["provider", "delete", &name, "--yes"]).await; run_cli_ignore_error(&["profile", "delete", &name, "--yes"]).await; diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index 5b6d7aaced..dce84bc302 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -280,16 +280,6 @@ func TestSandboxFromProto_EndpointLastFailure(t *testing.T) { LastResult: pb.EndpointResult_ENDPOINT_RESULT_TRANSPORT_FAILED, LastReportedTime: reportedAt, }, - { - EndpointId: "endpoint-unknown-domain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/future", - LastResult: pb.EndpointResult_ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE, - LastFailure: &pb.EndpointFailure{ - Domain: pb.EndpointFailureDomain(99), - Code: "future_code", - Stage: "future_stage", - RequestId: "4c1d7a2e-0f3b-4e8a-b6a1-9f2c3d4e5f60", - }, - }, }}} s := SandboxFromProto(input) @@ -314,19 +304,7 @@ func TestSandboxFromProto_EndpointLastFailure(t *testing.T) { EndpointID: "endpoint-plain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/other", LastResult: v1.EndpointTransportFailed, LastReportedAt: "2026-09-11T10:00:00Z", }, - { - EndpointID: "endpoint-unknown-domain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/future", - LastResult: v1.EndpointCredentialUnavailable, - LastFailure: &v1.EndpointFailure{ - Domain: v1.EndpointFailureDomainUnspecified, - Code: "future_code", - Stage: "future_stage", - RequestID: "4c1d7a2e-0f3b-4e8a-b6a1-9f2c3d4e5f60", - }, - }, }, s.Status.EndpointStatuses) - assert.Nil(t, s.Status.EndpointStatuses[1].LastFailure) - assert.Nil(t, s.Status.EndpointStatuses[2].LastFailure.UpstreamHTTPStatus) // The SDK failure must not share the wire message or its optional status. input.Status.EndpointStatuses[0].LastFailure.Code = "changed" diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index c7435c3f8e..d4eb8f56af 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -307,7 +307,7 @@ Common findings: - Docker driver cannot initialize because it cannot find `openshell-sandbox`: verify the sibling binary next to `openshell-gateway`, or that the configured `sandbox_runtime_image` contains `/openshell-sandbox`. - Sandbox never registers: check gateway logs and the supervisor's gateway endpoint. - SSH host-key startup errors: use matching gateway, compute-driver, and supervisor releases. The gateway retains each sandbox's SSH key in its configured credential store and sends it only through the supervisor bootstrap bundle. Check credential-driver availability and compare the public `host_key_fingerprint` from sandbox JSON output; never print the bootstrap bundle or private key. A missing stored key for a sandbox with a fingerprint is an error, not permission to replace its identity. See the [sandbox SSH identity documentation](https://docs.nvidia.com/openshell/latest/how-it-works/sandboxes/overview). -- Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. For token grant failures on any HTTP protocol, including failures before endpoint selection, inspect `last_token_grant_failure` in JSON output. It contains the bounded destination and cause, survives unrelated successful traffic, and clears on matching grant success or a policy, provider, or supervisor reset. Verify the actual operation when current tool availability matters. A `CredentialUnavailable` result from a failed dynamic token grant also carries `last_failure` with a closed-set `code`, `stage`, `recovery_action`, the provider and credential names, and the `request_id` that the workload received in its `x-openshell-request-id` response header. Search `openshell logs ` for that ID: the supervisor's `Token grant failed` line carries the code and stage for every endpoint, and the gateway's `dynamic token grant failed` line appears when the gateway hop failed. +- Calls to an external tool server fail while the sandbox is Ready: inspect `Tool server connections` in `openshell sandbox get `. For configured MCP-over-HTTP endpoints, JSON output exposes each address together with `last_result` and `last_reported_at` in `endpoint_statuses`. Select the endpoint by host, path, and ports, then check the reported failure boundary. `last_reported_at` records gateway acceptance time and can advance when retained evidence is accepted after a reset. Results do not expire or prove current availability; `HttpResponseReceived` can still contain a tool error. For token grant failures on any HTTP protocol, including failures before endpoint selection, inspect `last_token_grant_failure` in JSON output. It contains the bounded destination and cause, survives unrelated successful traffic, and clears on matching grant success or a policy, provider, or supervisor reset. If several paths share a host and port, other failures before the path is known, such as TLS failures, remain only in logs. A `CredentialUnavailable` result from a failed dynamic token grant also carries `last_failure` with a closed-set `code`, `stage`, `recovery_action`, the provider and credential names, and the `request_id` that the workload received in its `x-openshell-request-id` response header. Search `openshell logs ` for that ID: the supervisor's `Token grant failed` line carries the code and stage for every endpoint, and the gateway's `dynamic token grant failed` line appears when the gateway hop failed. Verify the actual operation when current tool availability matters. - On Docker Desktop, repeated `Policy fetch failed after 5 attempts` messages can mean host networking is disabled. Enable host networking in Docker Desktop, ensure Enhanced Container Isolation is disabled, and verify the From 40cafa8727c663512a13b4b63d1e781e6caade54 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 18:26:22 +0530 Subject: [PATCH 06/13] refactor(providers): trim token-grant failure tests and supervisor API Cut the token-grant failure tests to one test per distinct behavior across core, gateway, supervisor, CLI and Go, and replace the 794-line closure E2E with status assertions in the existing route-recovery E2E: the refused grant appears in sandbox status with the response's request ID, code, stage and upstream status, and the matching grant's later success clears it. - Pin the live provider snapshot with one helper used by inspected and fallback forwarding, and inject selector-matched grants through inject_for_selectors, which borrows the pinned context. Move the test-only wrapper into test_support. - Drop the 35-second wrappers around the gateway grant RPC. The shared channel bounds the connection attempt and the gateway bounds its token-service calls. - Log the token service's OAuth error and error_description, without control characters and limited to 256 characters each, with the request ID in the gateway or supervisor log. Status and the workload response still carry only the bounded cause. - Keep one check that an untrusted token-service certificate cannot assert a cause. Signed-off-by: Shiju --- crates/openshell-cli/src/run.rs | 158 ++-- crates/openshell-core/src/endpoint_failure.rs | 114 +-- crates/openshell-core/src/endpoint_status.rs | 375 +++------ crates/openshell-core/src/grpc_client.rs | 118 +-- crates/openshell-core/src/oauth.rs | 478 +++-------- .../openshell-core/src/token_grant_failure.rs | 254 ++---- crates/openshell-server/src/compute/mod.rs | 17 +- .../src/grpc/policy/endpoint_status_tests.rs | 471 ++++------- crates/openshell-server/src/grpc/provider.rs | 523 ++---------- .../src/supervisor_session.rs | 16 +- .../src/l7/relay.rs | 94 +-- .../l7/relay/token_grant_ownership_tests.rs | 181 ++-- .../src/l7/token_grant_injection.rs | 384 +++------ .../token_grant_injection/multiple_grants.rs | 38 +- .../openshell-supervisor-network/src/proxy.rs | 100 +-- .../src/token_grant.rs | 84 +- docs/how-it-works/providers/profiles.mdx | 2 +- e2e/rust/Cargo.lock | 1 - e2e/rust/Cargo.toml | 2 +- e2e/rust/src/harness/gateway.rs | 11 - e2e/rust/tests/provider_token_exchange.rs | 100 ++- .../tests/provider_token_exchange/closure.rs | 794 ------------------ sdk/go/openshell/v1/fake/fake_test.go | 70 +- .../v1/internal/converter/sandbox_test.go | 145 +--- 24 files changed, 1111 insertions(+), 3419 deletions(-) delete mode 100644 e2e/rust/tests/provider_token_exchange/closure.rs diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index 58b83e0288..9bd3db70c8 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -8852,140 +8852,82 @@ mod tests { } #[test] - fn endpoint_status_explains_a_classified_credential_failure() { - let failure = openshell_core::proto::EndpointFailure { - domain: openshell_core::proto::EndpointFailureDomain::TokenGrant as i32, + fn token_grant_failure_renders_in_text_and_structured_output() { + use openshell_core::proto::{ + EndpointFailure, EndpointFailureDomain, ProviderCredentialRefreshRecoveryAction, + TokenGrantFailureObservation, + }; + let failure = EndpointFailure { + domain: EndpointFailureDomain::TokenGrant.into(), code: "provider_authorization_required".to_string(), - recovery_action: - openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Reauthorize as i32, + recovery_action: ProviderCredentialRefreshRecoveryAction::Reauthorize.into(), provider: "github".to_string(), credential_key: "access_token".to_string(), - stage: "gateway_rpc".to_string(), + stage: "token_endpoint".to_string(), upstream_http_status: Some(409), request_id: "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908".to_string(), }; let endpoint = EndpointStatus { - endpoint_id: "endpoint:example".to_string(), host: "tools.example.test".to_string(), - ports: vec![443], - path: "/mcp".to_string(), - last_result: EndpointResult::CredentialUnavailable as i32, - last_reported_time: Some("2026-09-05T11:01:00Z".parse().unwrap()), - last_failure: Some(failure), - }; - - assert_eq!( - super::endpoint_status_display_lines(&endpoint), - vec![ - "tools.example.test (ports: 443; path: /mcp)", - "Last result: Required OpenShell-managed credential was unavailable.", - "Cause: provider_authorization_required (token grant, stage gateway_rpc, upstream HTTP 409)", - "Credential: github/access_token", - "Recovery: Reauthorize the provider credential.", - "Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", - "Reported at (gateway acceptance): 2026-09-05T11:01:00Z", - ] - ); - let sandbox = Sandbox { - status: Some(SandboxStatus { - endpoint_statuses: vec![endpoint], - ..Default::default() - }), + last_failure: Some(failure.clone()), ..Default::default() }; - let json = super::sandbox_to_json(&sandbox); - let yaml: serde_json::Value = - serde_yml::from_str(&serde_yml::to_string(&json).unwrap()).unwrap(); - for output in [json, yaml] { - assert_eq!( - output["endpoint_statuses"][0]["last_failure"], - serde_json::json!({ - "domain": "token_grant", - "code": "provider_authorization_required", - "recovery_action": "reauthorize", - "provider": "github", - "credential_key": "access_token", - "stage": "gateway_rpc", - "upstream_http_status": 409, - "request_id": "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", - }) - ); - } - } - - #[test] - fn sandbox_reports_rest_token_grant_failure_without_mcp_inventory() { - let observation = openshell_core::proto::TokenGrantFailureObservation { + let observation = TokenGrantFailureObservation { destination_host: "api.example.test".to_string(), destination_port: 8443, - failure: Some(openshell_core::proto::EndpointFailure { - domain: openshell_core::proto::EndpointFailureDomain::TokenGrant.into(), - code: "provider_authorization_required".to_string(), - recovery_action: - openshell_core::proto::ProviderCredentialRefreshRecoveryAction::Reauthorize - .into(), - provider: "github".to_string(), - credential_key: "access_token".to_string(), - stage: "token_endpoint".to_string(), - upstream_http_status: Some(409), - request_id: "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908".to_string(), - }), + failure: Some(failure), }; + let cause = [ + "Cause: provider_authorization_required (token grant, stage token_endpoint, upstream HTTP 409)", + "Credential: github/access_token", + "Recovery: Reauthorize the provider credential.", + "Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + ]; + // MCP endpoint status places the cause between the result and report time. + let endpoint_lines = super::endpoint_status_display_lines(&endpoint); + assert_eq!(endpoint_lines[2..6], cause); + assert!(endpoint_lines[6].starts_with("Reported at")); assert_eq!( super::token_grant_failure_display_lines(&observation), - vec![ - "Destination: api.example.test (port 8443)", - "Cause: provider_authorization_required (token grant, stage token_endpoint, upstream HTTP 409)", - "Credential: github/access_token", - "Recovery: Reauthorize the provider credential.", - "Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", - ] + [&["Destination: api.example.test (port 8443)"][..], &cause].concat() ); - let sandbox = Sandbox { + + let output = super::sandbox_to_json(&Sandbox { status: Some(SandboxStatus { - phase: SandboxPhase::Ready.into(), + endpoint_statuses: vec![endpoint], last_token_grant_failure: Some(Box::new(observation)), ..Default::default() }), ..Default::default() - }; - let json = super::sandbox_to_json(&sandbox); - let yaml: serde_json::Value = - serde_yml::from_str(&serde_yml::to_string(&json).unwrap()).unwrap(); - for output in [json, yaml] { - assert_eq!(output["phase"], "Ready"); - assert_eq!(output["endpoint_statuses"], serde_json::json!([])); - assert_eq!( - output["last_token_grant_failure"], - serde_json::json!({ - "destination_host": "api.example.test", - "destination_port": 8443, - "failure": { - "domain": "token_grant", - "code": "provider_authorization_required", - "recovery_action": "reauthorize", - "provider": "github", - "credential_key": "access_token", - "stage": "token_endpoint", - "upstream_http_status": 409, - "request_id": "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", - }, - }) - ); - } + }); + let failure_json = serde_json::json!({ + "domain": "token_grant", + "code": "provider_authorization_required", + "recovery_action": "reauthorize", + "provider": "github", + "credential_key": "access_token", + "stage": "token_endpoint", + "upstream_http_status": 409, + "request_id": "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", + }); + assert_eq!(output["endpoint_statuses"][0]["last_failure"], failure_json); + assert_eq!( + output["last_token_grant_failure"], + serde_json::json!({ + "destination_host": "api.example.test", + "destination_port": 8443, + "failure": failure_json, + }) + ); assert!(super::sandbox_to_json(&Sandbox::default())["last_token_grant_failure"].is_null()); - } - #[test] - fn endpoint_failure_json_names_an_unspecified_recovery() { - let failure = openshell_core::proto::EndpointFailure { + let unknown = super::endpoint_failure_to_json(&EndpointFailure { domain: 99, recovery_action: 99, ..Default::default() - }; - let json = super::endpoint_failure_to_json(&failure); - assert_eq!(json["domain"], "unknown"); - assert_eq!(json["recovery_action"], "unspecified"); + }); + assert_eq!(unknown["domain"], "unknown"); + assert_eq!(unknown["recovery_action"], "unspecified"); } #[test] diff --git a/crates/openshell-core/src/endpoint_failure.rs b/crates/openshell-core/src/endpoint_failure.rs index abd7034732..8826ce67d5 100644 --- a/crates/openshell-core/src/endpoint_failure.rs +++ b/crates/openshell-core/src/endpoint_failure.rs @@ -188,20 +188,24 @@ mod tests { const CREDENTIAL_UNAVAILABLE: proto::EndpointResult = proto::EndpointResult::CredentialUnavailable; - fn reported() -> proto::EndpointFailure { + fn reported_with(provider: &str, credential_key: &str) -> proto::EndpointFailure { EndpointFailure::token_grant( TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::GatewayRpc, ) .with_upstream_status(409), - "github", - "access_token", + provider, + credential_key, REQUEST_ID, ) .to_proto() } + fn reported() -> proto::EndpointFailure { + reported_with("github", "access_token") + } + #[test] fn token_grant_failure_reports_its_code_stage_recovery_and_names() { assert_eq!( @@ -217,53 +221,16 @@ mod tests { request_id: REQUEST_ID.into(), } ); - assert_eq!( - validate_endpoint_failure(&reported(), CREDENTIAL_UNAVAILABLE), - Ok(()) - ); } - type Mutation = fn(&mut proto::EndpointFailure); - #[test] - fn token_grant_causes_explain_only_credential_failures() { - for result in [ - proto::EndpointResult::Unspecified, - proto::EndpointResult::NoObservedExchange, - proto::EndpointResult::HttpResponseReceived, - proto::EndpointResult::PolicyDenied, - proto::EndpointResult::TlsFailed, - proto::EndpointResult::TransportFailed, - proto::EndpointResult::UpstreamRejected, - ] { - assert_eq!( - validate_endpoint_failure(&reported(), result), - Err(EndpointFailureValidationError::Result), - "{result:?}" - ); - } - } - - #[test] - fn constructed_causes_always_pass_gateway_validation() { + fn constructed_causes_drop_unreportable_names() { let oversized = "k".repeat(MAX_ENDPOINT_FAILURE_NAME_BYTES + 1); for (provider, credential_key) in [ ("github", oversized.as_str()), ("git\u{202e}hub", "access_token"), - ("github", "access token"), - ("github", ""), - ("", "access_token"), ] { - let failure = EndpointFailure::token_grant( - TokenGrantFailure::new( - TokenGrantFailureKind::Configuration, - TokenGrantStage::Injection, - ), - provider, - credential_key, - REQUEST_ID, - ) - .to_proto(); + let failure = reported_with(provider, credential_key); assert_eq!( validate_endpoint_failure(&failure, CREDENTIAL_UNAVAILABLE), Ok(()) @@ -273,59 +240,28 @@ mod tests { } #[test] - fn rejects_values_outside_the_closed_sets() { - let cases: [(Mutation, EndpointFailureValidationError); 15] = [ - (|f| f.domain = 0, EndpointFailureValidationError::Domain), - (|f| f.domain = 99, EndpointFailureValidationError::Domain), - ( - |f| f.code = "reauthorization_required".into(), - EndpointFailureValidationError::Code, - ), - ( - |f| f.code = "Provider_Authorization_Required".into(), - EndpointFailureValidationError::Code, - ), - ( - |f| f.stage = "token service said no".into(), - EndpointFailureValidationError::Stage, - ), + fn rejects_each_invalid_field() { + use EndpointFailureValidationError as E; + assert_eq!( + validate_endpoint_failure(&reported(), proto::EndpointResult::PolicyDenied), + Err(E::Result) + ); + let cases: [(fn(&mut proto::EndpointFailure), E); 9] = [ + (|f| f.domain = 99, E::Domain), + (|f| f.code = "reauthorization_required".into(), E::Code), + (|f| f.stage = "token service said no".into(), E::Stage), ( |f| f.recovery_action = ProviderCredentialRefreshRecoveryAction::Retry.into(), - EndpointFailureValidationError::Recovery, - ), - (|f| f.provider.clear(), EndpointFailureValidationError::Name), - ( - |f| f.credential_key.clear(), - EndpointFailureValidationError::Name, - ), - ( - |f| f.provider = "git\nhub".into(), - EndpointFailureValidationError::Name, - ), - ( - |f| f.provider = "git\u{202e}hub".into(), - EndpointFailureValidationError::Name, - ), - ( - |f| f.credential_key = "k".repeat(MAX_ENDPOINT_FAILURE_NAME_BYTES + 1), - EndpointFailureValidationError::Name, - ), - ( - |f| f.upstream_http_status = Some(600), - EndpointFailureValidationError::UpstreamStatus, - ), - ( - |f| f.upstream_http_status = Some(99), - EndpointFailureValidationError::UpstreamStatus, + E::Recovery, ), + (|f| f.credential_key.clear(), E::Name), + (|f| f.provider = "git\nhub".into(), E::Name), + (|f| f.upstream_http_status = Some(600), E::UpstreamStatus), ( |f| f.request_id = REQUEST_ID.to_ascii_uppercase(), - EndpointFailureValidationError::RequestId, - ), - ( - |f| f.request_id.clear(), - EndpointFailureValidationError::RequestId, + E::RequestId, ), + (|f| f.request_id.clear(), E::RequestId), ]; for (mutate, expected) in cases { let mut failure = reported(); diff --git a/crates/openshell-core/src/endpoint_status.rs b/crates/openshell-core/src/endpoint_status.rs index d878f4f53a..9e4d28375b 100644 --- a/crates/openshell-core/src/endpoint_status.rs +++ b/crates/openshell-core/src/endpoint_status.rs @@ -896,34 +896,17 @@ mod tests { #[test] fn failure_travels_with_its_result_and_clears_with_it() { - use crate::token_grant_failure::{ - TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, - }; - let failure = EndpointFailure::token_grant( - TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::GatewayRpc, - ), - "github", - "access_token", - "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", - ); + let failure = grant_refusal("github", "access_token"); let latest = |tracker: &EndpointStatusTracker| { - tracker - .snapshot() - .expect("inventory installed") - .endpoints - .into_iter() - .find(|endpoint| endpoint.endpoint_id == "endpoint:credentialed") - .expect("credentialed endpoint") + tracker.snapshot().expect("inventory installed").endpoints[0].clone() }; - let fail = |tracker: &EndpointStatusTracker, version| { + let observe_result = |tracker: &EndpointStatusTracker, result, failure| { observe_with_failure( tracker, - version, + config_version("policy-a", 1), "endpoint:credentialed", - EndpointResult::CredentialUnavailable, - Some(failure.clone()), + result, + failure, ) }; let mut tracker = endpoint_status_channel().1.tracker(); @@ -932,39 +915,23 @@ mod tests { config_version: config_version("policy-a", 1), endpoints: inventory(), }); + let unavailable = EndpointResult::CredentialUnavailable; - assert!(tracker.apply(fail(&tracker, config_version("policy-a", 1)))); - assert_eq!(latest(&tracker).failure.as_ref(), Some(&failure)); + assert!(tracker.apply(observe_result(&tracker, unavailable, Some(failure.clone())))); + assert_eq!(latest(&tracker).failure, Some(failure.clone())); // A later success replaces the result and its cause together. - tracker.apply(observe( + tracker.apply(observe_result( &tracker, - config_version("policy-a", 1), - "endpoint:credentialed", EndpointResult::HttpResponseReceived, + None, )); assert_eq!(latest(&tracker).failure, None); // A replacement supervisor session forgets the cause. - tracker.apply(fail(&tracker, config_version("policy-a", 1))); + tracker.apply(observe_result(&tracker, unavailable, Some(failure))); assert!(tracker.clear_observations()); assert_eq!(latest(&tracker).failure, None); - - // A provider revision, such as a reauthorization, clears credentialed causes. - tracker.apply(fail(&tracker, config_version("policy-a", 1))); - tracker.apply(EndpointStatusCommand::Reset { - inventory_generation: EndpointObservationGeneration::new(), - config_version: config_version("policy-a", 2), - endpoints: inventory(), - }); - assert_eq!( - latest(&tracker), - EndpointObservation { - endpoint_id: "endpoint:credentialed".to_string(), - result: EndpointResult::NoObservedExchange, - failure: None, - } - ); } #[test] @@ -1450,246 +1417,146 @@ mod tests { ) } + struct GrantStatus { + sender: EndpointObservationSender, + receiver: EndpointStatusReceiver, + tracker: EndpointStatusTracker, + } + + impl GrantStatus { + async fn new() -> Self { + let (sender, receiver) = endpoint_status_channel(); + let tracker = receiver.tracker(); + let mut status = Self { + sender, + receiver, + tracker, + }; + status.reset(config_version("policy-a", 1)).await; + status + } + + async fn reset(&mut self, version: EndpointConfigVersion) { + self.sender.reset(version, Vec::new()).await.unwrap(); + assert!(self.tracker.apply(self.receiver.recv().await.unwrap())); + } + + /// Enqueue a grant outcome and return whether the tracker applied it. + async fn observe( + &mut self, + context: &EndpointObservationContext, + host: &str, + port: u16, + result: Result<(&str, &str), EndpointFailure>, + ) -> bool { + let result = result.map(|(provider, key)| (provider.into(), key.into())); + assert!( + self.sender + .try_observe_token_grant(context.clone(), host, port, result) + ); + self.tracker.apply(self.receiver.recv().await.unwrap()) + } + + fn last_failure(&self) -> Option { + self.tracker.snapshot().unwrap().last_token_grant_failure + } + } + #[tokio::test] async fn grant_failure_without_mcp_inventory_clears_only_on_matching_success() { - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset(config_version("policy-a", 1), Vec::new()) - .await - .unwrap(); - assert!(tracker.apply(receiver.recv().await.unwrap())); - let context = sender.capture().unwrap(); + let mut status = GrantStatus::new().await; + let context = status.sender.capture().unwrap(); let failure = grant_refusal("github", "token"); - assert!(sender.try_observe_token_grant( - context.clone(), - "API.EXAMPLE.TEST", - 8443, - Err(failure.clone()), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!( + status + .observe(&context, "API.EXAMPLE.TEST", 8443, Err(failure.clone())) + .await + ); let expected = crate::proto::TokenGrantFailureObservation { destination_host: "api.example.test".into(), destination_port: 8443, failure: Some(failure.to_proto()), }; - assert!(tracker.snapshot().unwrap().endpoints.is_empty()); - assert_eq!( - tracker.snapshot().unwrap().last_token_grant_failure, - Some(expected.clone()) - ); - for (host, port, provider, credential) in [ - ("other.example.test", 8443, "github", "token"), - ("api.example.test", 443, "github", "token"), - ("api.example.test", 8443, "other", "token"), - ("api.example.test", 8443, "github", "other"), - ] { - assert!(sender.try_observe_token_grant( - context.clone(), - host, - port, - Ok((provider.into(), credential.into())), - )); - assert!(!tracker.apply(receiver.recv().await.unwrap())); - assert_eq!( - tracker.snapshot().unwrap().last_token_grant_failure, - Some(expected.clone()) - ); - } - assert!(sender.try_observe_token_grant( + assert!(status.tracker.snapshot().unwrap().endpoints.is_empty()); + assert_eq!(status.last_failure(), Some(expected.clone())); + // An invalid destination drops only the telemetry. + assert!(!status.sender.try_observe_token_grant( context.clone(), - "api.example.test", - 8443, - Ok(("github".into(), "token".into())), + "https://api.example.test/", + 443, + Err(failure.clone()), )); - assert!(tracker.apply(receiver.recv().await.unwrap())); + assert!(status.receiver.try_recv().is_err()); + // A success for another destination or credential leaves the failure. + for (host, port, credential) in [ + ("other.example.test", 8443, ("github", "token")), + ("api.example.test", 443, ("github", "token")), + ("api.example.test", 8443, ("github", "other")), + ] { + assert!(!status.observe(&context, host, port, Ok(credential)).await); + assert_eq!(status.last_failure(), Some(expected.clone())); + } + let matching = Ok(("github", "token")); assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() + status + .observe(&context, "api.example.test", 8443, matching) + .await ); + assert_eq!(status.last_failure(), None); - assert!(sender.try_observe_token_grant( - context.clone(), - "api.example.test", - 8443, - Err(grant_refusal("", "")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); - assert!(sender.try_observe_token_grant( - context, - "api.example.test", - 8443, - Ok(("selected".into(), "token".into())), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); + // A failure before credential selection clears on any success there. + let unselected = Err(grant_refusal("", "")); assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() + status + .observe(&context, "api.example.test", 8443, unselected) + .await ); + assert!( + status + .observe(&context, "api.example.test", 8443, Ok(("other", "key"))) + .await + ); + assert_eq!(status.last_failure(), None); } #[tokio::test] - async fn grant_context_stays_retired_across_configuration_and_authority_cycles() { - for replacement in [config_version("policy-b", 1), config_version("policy-a", 2)] { - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset(config_version("policy-a", 1), Vec::new()) + async fn configuration_change_clears_grant_failure_and_retires_its_context() { + let mut status = GrantStatus::new().await; + let retired = status.sender.capture().unwrap(); + let refusal = || Err(grant_refusal("github", "token")); + assert!( + status + .observe(&retired, "api.example.test", 443, refusal()) .await - .unwrap(); - assert!(tracker.apply(receiver.recv().await.unwrap())); - let retired = sender.capture().unwrap(); - assert!(sender.try_observe_token_grant( - retired.clone(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); - for version in [replacement, config_version("policy-a", 1)] { - sender.reset(version, Vec::new()).await.unwrap(); - assert!(tracker.apply(receiver.recv().await.unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() - ); - } - assert!(sender.try_observe_token_grant( - retired, - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(!tracker.apply(receiver.recv().await.unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() - ); - let retired = sender.capture().unwrap(); - assert!(sender.try_observe_token_grant( - retired.clone(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); - assert!(tracker.clear_observations()); - assert!(sender.try_observe_token_grant( - retired, - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(!tracker.apply(receiver.recv().await.unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() - ); - assert!(sender.try_observe_token_grant( - sender.capture().unwrap(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_some() - ); - } + ); + // Returning to the same configuration must not revive the old context. + status.reset(config_version("policy-b", 1)).await; + status.reset(config_version("policy-a", 1)).await; + assert_eq!(status.last_failure(), None); + assert!( + !status + .observe(&retired, "api.example.test", 443, refusal()) + .await + ); + assert_eq!(status.last_failure(), None); } - #[tokio::test] - async fn invalid_destinations_and_full_grant_queue_drop_only_telemetry() { - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset(config_version("policy-a", 1), Vec::new()) - .await - .unwrap(); - assert!(tracker.apply(receiver.recv().await.unwrap())); - let context = sender.capture().unwrap(); + #[test] + fn grant_destination_accepts_only_bounded_hosts_and_ports() { let oversized = "a".repeat(254); for (host, port) in [ ("", 443), ("api.example.test", 0), - ("api\n.example.test", 443), + ("api.example.test", 65536), ("api example.test", 443), ("https://api.example.test/path?token=secret", 443), - ("user@api.example.test", 443), ("api\u{202e}.example.test", 443), (oversized.as_str(), 443), ] { - assert!(!sender.try_observe_token_grant( - context.clone(), - host, - port, - Err(grant_refusal("github", "token")), - )); + assert!(!valid_token_grant_destination(host, port), "{host}:{port}"); } - assert!(receiver.try_recv().is_err()); - assert!(!valid_token_grant_destination("api.example.test", 65536)); - let longest = [ - "a".repeat(63), - "b".repeat(63), - "c".repeat(63), - "d".repeat(61), - ] - .join("."); - for (host, port) in [(longest.as_str(), u16::MAX), ("::1", 443)] { - assert!(sender.try_observe_token_grant( - context.clone(), - host, - port, - Err(grant_refusal("github", "token")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); + for host in ["api.example.test", "::1", "[::1]"] { + assert!(valid_token_grant_destination(host, 443), "{host}"); } - for _ in 0..ENDPOINT_STATUS_CHANNEL_CAPACITY { - assert!(sender.try_observe_token_grant( - context.clone(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - } - assert!(!sender.try_observe_token_grant( - context.clone(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - assert!(tracker.apply(receiver.recv().await.unwrap())); - assert!(sender.try_observe_token_grant( - context.clone(), - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); - drop(receiver); - assert!(!sender.try_observe_token_grant( - context, - "api.example.test", - 443, - Err(grant_refusal("github", "token")), - )); } } diff --git a/crates/openshell-core/src/grpc_client.rs b/crates/openshell-core/src/grpc_client.rs index f32e1fe78d..ce76bc3d7b 100644 --- a/crates/openshell-core/src/grpc_client.rs +++ b/crates/openshell-core/src/grpc_client.rs @@ -1382,9 +1382,10 @@ pub async fn exchange_provider_subject_token( } // TLS setup errors can contain paths and transport errors can contain URLs. // Discard those sources at the grant boundary; never diagnose from prose. - let mut client = tokio::time::timeout(Duration::from_secs(35), connect(endpoint.as_str())) + // The shared channel bounds the connection attempt; the gateway bounds its + // token-service calls, and a gateway deadline arrives as DeadlineExceeded. + let mut client = connect(endpoint.as_str()) .await - .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))? .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Unclassified))?; let request = provider_subject_token_exchange_request( ExchangeProviderSubjectTokenRequest { @@ -1395,13 +1396,10 @@ pub async fn exchange_provider_subject_token( }, request_id, ); - let response = tokio::time::timeout( - Duration::from_secs(35), - client.exchange_provider_subject_token(request), - ) - .await - .map_err(|_| gateway_grant_failure(TokenGrantFailureKind::Timeout))? - .map_err(|status| provider_subject_token_exchange_status(&status, &endpoint))?; + let response = client + .exchange_provider_subject_token(request) + .await + .map_err(|status| provider_subject_token_exchange_status(&status, &endpoint))?; status.finish(provider_subject_token_exchange_result( response.into_inner(), )) @@ -1489,41 +1487,33 @@ mod token_grant_exchange_tests { let failure = TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::IntermediateExchange, - ) - .with_upstream_status(409); + ); let status = failure.to_status(); - let https = url::Url::parse("https://gateway.example:8080/").unwrap(); - let http = url::Url::parse("http://gateway.example:8080/").unwrap(); + let endpoint = |value: &str| url::Url::parse(value).unwrap(); assert_eq!( - provider_subject_token_exchange_status(&status, &https), + provider_subject_token_exchange_status(&status, &endpoint("https://gateway:8080/")), failure ); - let downgraded = provider_subject_token_exchange_status(&status, &http); - assert_eq!(downgraded.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(downgraded.stage(), TokenGrantStage::GatewayRpc); - assert_eq!(downgraded.upstream_status(), None); + assert_eq!( + provider_subject_token_exchange_status(&status, &endpoint("http://gateway:8080/")), + gateway_grant_failure(TokenGrantFailureKind::Unclassified) + ); } #[test] fn exchange_request_carries_only_a_valid_request_id() { - let body = || ExchangeProviderSubjectTokenRequest { - sandbox_id: "sandbox".into(), - provider: "provider".into(), - credential_key: "credential".into(), - supervisor_jwt_svid: "svid".into(), - }; let request_id = "5c4b3a29-1807-4f6e-8d5c-4b3a29180716"; - let request = provider_subject_token_exchange_request(body(), Some(request_id)); - assert_eq!( - request - .metadata() - .get("x-request-id") - .and_then(|value| value.to_str().ok()), - Some(request_id) - ); - for absent in [None, Some("line\nbreak")] { - let request = provider_subject_token_exchange_request(body(), absent); - assert!(request.metadata().get("x-request-id").is_none()); + for (value, expected) in [ + (Some(request_id), Some(request_id)), + (Some("line\nbreak"), None), + (None, None), + ] { + let request = provider_subject_token_exchange_request( + ExchangeProviderSubjectTokenRequest::default(), + value, + ); + let carried = request.metadata().get("x-request-id"); + assert_eq!(carried.and_then(|value| value.to_str().ok()), expected); } } @@ -1532,63 +1522,29 @@ mod token_grant_exchange_tests { for endpoint in [ "https://user:SECRET@localhost/", "https://localhost/?token=SECRET", - "https://localhost/#SECRET", "https://localhost/SECRET", "file:///SECRET", - "not an endpoint SECRET", ] { let failure = token_grant_gateway_endpoint(endpoint).unwrap_err(); assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); } - assert_eq!( - token_grant_gateway_endpoint("https://localhost:8443") - .unwrap() - .scheme(), - "https" - ); - assert_eq!( - token_grant_gateway_endpoint("http://localhost:8080") - .unwrap() - .scheme(), - "http" - ); - } - - #[test] - fn malformed_gateway_duration_discards_token_and_diagnostic() { - let failure = provider_subject_token_exchange_result( - crate::proto::ExchangeProviderSubjectTokenResponse { - access_token: "SYNTHETIC_ACCESS_SECRET".into(), - token_type: "Bearer".into(), - expires_after: Some(prost_types::Duration { - seconds: 1, - nanos: -1, - }), - }, - ) - .err() - .expect("invalid duration is rejected"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); + assert!(token_grant_gateway_endpoint("https://localhost:8443").is_ok()); } #[test] - fn successful_gateway_response_preserves_cache_expiration() { - let result = provider_subject_token_exchange_result( - crate::proto::ExchangeProviderSubjectTokenResponse { - access_token: "valid-token".into(), - token_type: "Bearer".into(), - expires_after: Some(prost_types::Duration { - seconds: 120, - nanos: 0, - }), - }, - ) - .expect("valid response"); + fn gateway_response_keeps_expiration_and_rejects_invalid_duration() { + let response = |seconds, nanos| crate::proto::ExchangeProviderSubjectTokenResponse { + access_token: "valid-token".into(), + token_type: "Bearer".into(), + expires_after: Some(prost_types::Duration { seconds, nanos }), + }; + let result = provider_subject_token_exchange_result(response(120, 0)).unwrap(); assert_eq!(result.access_token, "valid-token"); assert_eq!(result.expires_in, 120); - assert_eq!(result.token_type, "Bearer"); + let failure = provider_subject_token_exchange_result(response(1, -1)) + .err() + .expect("invalid duration is rejected"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); } } diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index bd7a35538f..7fa2119321 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -24,6 +24,7 @@ pub const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_tok const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; const ERROR_CONTRACT: &str = "openshell.token-grant.v1"; const MAX_TOKEN_RESPONSE_BYTES: usize = 64 * 1024; +const MAX_LOGGED_ERROR_CHARS: usize = 256; /// HTTP client that authenticates HTTPS endpoints and never redirects assertions. /// @@ -95,6 +96,7 @@ struct RawTokenResponse { #[derive(Deserialize)] struct OAuthErrorResponse { error: Option, + error_description: Option, openshell_error_contract: Option, } @@ -138,6 +140,7 @@ async fn post_oauth_token_request( let status = response.status(); let body = read_bounded_body(response).await?; if !status.is_success() { + log_token_service_error(status, &body, request_id); return Err(classify_http_failure(status, &body, authenticated)); } @@ -375,6 +378,31 @@ async fn read_bounded_body(mut response: reqwest::Response) -> Result, T Ok(body) } +/// Record the token service's `OAuth` error text for operators. +/// +/// Only the gateway and supervisor logs receive it; the workload cannot read +/// either, and status reports and responses keep only the bounded failure. +fn log_token_service_error(status: reqwest::StatusCode, body: &[u8], request_id: Option<&str>) { + let Ok(response) = serde_json::from_slice::(body) else { + return; + }; + let printable = |value: Option| -> String { + value + .unwrap_or_default() + .chars() + .filter(|character| !character.is_control()) + .take(MAX_LOGGED_ERROR_CHARS) + .collect() + }; + tracing::warn!( + request_id = request_id.unwrap_or("none"), + upstream_http_status = status.as_u16(), + error = %printable(response.error), + error_description = %printable(response.error_description), + "token service rejected the grant" + ); +} + fn classify_http_failure( status: reqwest::StatusCode, body: &[u8], @@ -418,7 +446,6 @@ fn classify_http_failure( mod tests { use super::*; use std::collections::HashMap; - use std::error::Error; use std::sync::Arc; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio::net::TcpListener; @@ -512,8 +539,8 @@ mod tests { body: &str, ) -> ( String, - reqwest::Certificate, - tokio::task::JoinHandle>, + TokenGrantClient, + tokio::task::JoinHandle, ) { let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); let mut ca_params = rcgen::CertificateParams::new(Vec::::new()).expect("CA params"); @@ -540,32 +567,22 @@ mod tests { let body = body.to_string(); let task = tokio::spawn(async move { let (socket, _) = listener.accept().await.expect("TLS accept"); - // A rejected trust root/hostname must fail during the handshake, - // before the server can receive the secret-bearing grant request. - let Ok(mut stream) = acceptor.accept(socket).await else { - return None; - }; + let mut stream = acceptor.accept(socket).await.expect("TLS handshake"); let request = capture_request(&mut stream).await; send_response(&mut stream, &status, &body).await; - Some(request) + request }); + let client = TokenGrantClient::builder() + .add_root_certificate(reqwest::Certificate::from_der(ca.der()).expect("CA")) + .build() + .expect("trusted test client"); ( format!("https://localhost:{port}/token"), - reqwest::Certificate::from_der(ca.der()).expect("trusted CA"), + TokenGrantClient { client }, task, ) } - fn trusted_test_client(root: reqwest::Certificate) -> TokenGrantClient { - TokenGrantClient { - client: TokenGrantClient::builder() - .timeout(Duration::from_secs(5)) - .add_root_certificate(root) - .build() - .expect("trusted test client"), - } - } - async fn token_endpoint_redirect_once(location: &str) -> (String, tokio::task::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0") .await @@ -638,153 +655,43 @@ mod tests { } } - fn exchange_params() -> TokenExchangeParams<'static> { - TokenExchangeParams { - client_assertion: "ASSERTION_SECRET_CANARY", - client_assertion_type: "", - subject_token: "SUBJECT_SECRET_CANARY", - subject_token_type: "", - audience: "", - scopes: &[], - requested_token_type: "", - } - } - - fn assert_generic_failure(failure: TokenGrantFailure) { - assert_eq!(failure.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); - assert!(failure.source().is_none()); - let report = miette::Report::new(failure); - let rendered = format!("{report:?}"); - for forbidden in [ - "CANARY", - "provider_authorization_required", - "workload_identity_rejected", - ] { - assert!( - !rendered.contains(forbidden), - "diagnostics must stay generic" - ); - } + fn adopted_failure_body(code: &str) -> String { + format!(r#"{{"error":"{code}","openshell_error_contract":"{ERROR_CONTRACT}"}}"#) } #[tokio::test] - async fn verified_https_signal_is_preserved() { - let body = format!( - r#"{{"error":"provider_authorization_required","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# - ); - let (endpoint, root, request) = tls_token_endpoint_once("409 Conflict", &body).await; - let client = trusted_test_client(root); - let failure = post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None) + async fn adopted_signal_requires_verified_https() { + let body = adopted_failure_body("provider_authorization_required"); + let (endpoint, client, request) = tls_token_endpoint_once("409 Conflict", &body).await; + let failure = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("adopted failure"); - let request = request - .await - .expect("TLS fixture") - .expect("authenticated request"); - assert!(request.form.contains_key("subject_token")); + request.await.expect("TLS fixture"); assert_eq!( failure.kind(), TokenGrantFailureKind::ProviderAuthorizationRequired ); - assert_eq!(failure.stage(), TokenGrantStage::TokenEndpoint); - assert_generic_failure(failure); - } - #[tokio::test] - async fn plain_http_cannot_assert_adopted_failure() { - let body = format!( - r#"{{"error":"provider_authorization_required","openshell_error_contract":"{ERROR_CONTRACT}","error_description":"DESCRIPTION_SECRET_CANARY"}}"# - ); + // An untrusted certificate never reaches the body: the client refuses + // the handshake, so the service cannot assert any cause. + let (endpoint, _, request) = tls_token_endpoint_once("409 Conflict", &body).await; + let failure = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect_err("untrusted certificate"); + request.abort(); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unavailable); + assert_eq!(failure.upstream_status(), None); + + // Plain HTTP carries no service identity, so the same body is unclassified. let (endpoint, request) = token_endpoint_once("409 Conflict", &body).await; let failure = - post_oauth_token_exchange(&test_client(), &endpoint, &exchange_params(), None) + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) .await .expect_err("unauthenticated failure"); request.await.expect("HTTP fixture"); assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_generic_failure(failure); - } - - #[tokio::test] - async fn tls_requires_a_trusted_certificate_and_matching_hostname() { - for trust_certificate in [false, true] { - let body = r#"{"error":"provider_authorization_required","openshell_error_contract":"openshell.token-grant.v1"}"#; - let (endpoint, root, request) = tls_token_endpoint_once("409 Conflict", body).await; - let (client, endpoint) = if trust_certificate { - ( - trusted_test_client(root), - endpoint.replace("localhost", "127.0.0.1"), - ) - } else { - (test_client(), endpoint) - }; - let failure = post_oauth_token_exchange(&client, &endpoint, &exchange_params(), None) - .await - .expect_err("TLS must authenticate the configured server"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unavailable); - assert_eq!(failure.upstream_status(), None); - assert!(request.await.expect("TLS fixture").is_none()); - assert_generic_failure(failure); - } - } - - #[tokio::test] - async fn secret_canaries_in_errors_and_endpoint_components_never_survive() { - for body in [ - r#"{"error":"ERROR_SECRET_CANARY","error_description":"DESCRIPTION_SECRET_CANARY"}"#, - r#"{"error":"invalid_client","error_description":"ASSERTION_SECRET_CANARY TOKEN_SECRET_CANARY"}"#, - "RAW_BODY_SECRET_CANARY", - ] { - let (endpoint, request) = token_endpoint_once("401 Unauthorized", body).await; - let endpoint = endpoint.replace( - "http://", - "http://USER_SECRET_CANARY:PASSWORD_SECRET_CANARY@", - ) + "?credential=QUERY_SECRET_CANARY#FRAGMENT_SECRET_CANARY"; - let failure = - post_oauth_token_exchange(&test_client(), &endpoint, &exchange_params(), None) - .await - .expect_err("safe endpoint failure"); - request.await.expect("HTTP fixture"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(failure.upstream_status(), Some(401)); - assert_generic_failure(failure); - } - for endpoint in [ - "not a URL ENDPOINT_SECRET_CANARY", - "ftp://USER_SECRET_CANARY:PASSWORD_SECRET_CANARY@localhost/token?QUERY_SECRET_CANARY", - ] { - let failure = - post_oauth_token_grant(&test_client(), endpoint, &empty_grant_params(), None) - .await - .expect_err("invalid endpoint"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); - assert_generic_failure(failure); - } - } - - #[tokio::test] - async fn malformed_success_fields_are_generic_and_never_return_a_token() { - for body in [ - "{}", - r#"{"access_token":42"#, - r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":null}"#, - r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","expires_in":"DURATION_SECRET_CANARY"}"#, - r#"{"access_token":"TOKEN_SECRET_CANARY","access_token":"SECOND_SECRET_CANARY","token_type":"Bearer"}"#, - r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","token_type":"Bearer"}"#, - r#"{"access_token":"TOKEN_SECRET_CANARY","token_type":"Bearer","expires_in":1,"expires_in":1}"#, - r#"{"access_token":"","token_type":"Bearer"}"#, - ] { - let (endpoint, request) = token_endpoint_once("200 OK", body).await; - let failure = - post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) - .await - .expect_err("malformed success"); - request.await.expect("HTTP fixture"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(failure.upstream_status(), Some(200)); - assert_generic_failure(failure); - } + assert_eq!(failure.upstream_status(), Some(409)); } #[test] @@ -798,7 +705,7 @@ mod tests { } #[tokio::test] - async fn request_and_body_timeouts_drop_sources_and_keep_observed_status() { + async fn request_and_body_timeouts_keep_observed_status() { for after_headers in [false, true] { let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); let address = listener.local_addr().expect("addr"); @@ -820,7 +727,7 @@ mod tests { .build() .expect("timeout client"), }; - let endpoint = format!("http://{address}/token?QUERY_SECRET_CANARY"); + let endpoint = format!("http://{address}/token"); let failure = post_oauth_token_grant(&client, &endpoint, &empty_grant_params(), None) .await .expect_err("bounded timeout"); @@ -828,83 +735,47 @@ mod tests { server.await.expect("timeout fixture"); assert_eq!(failure.kind(), TokenGrantFailureKind::Timeout); assert_eq!(failure.upstream_status(), after_headers.then_some(200)); - assert_generic_failure(failure); } } - async fn raw_token_endpoint_once(response: Vec) -> (String, tokio::task::JoinHandle<()>) { - let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); - let address = listener.local_addr().expect("addr"); - let server = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.expect("accept"); - capture_request(&mut stream).await; - // Bound rejection can close the reader before an oversized fixture - // finishes writing. That is the behavior this fixture exercises. - let _ = stream.write_all(&response).await; - }); - (format!("http://{address}/token"), server) - } - #[tokio::test] - async fn response_body_bound_applies_to_success_errors_and_chunked_bodies() { + async fn response_body_is_bounded_for_declared_and_chunked_lengths() { let body = "X".repeat(MAX_TOKEN_RESPONSE_BYTES + 1); - for status in ["200 OK", "409 Conflict", "503 Service Unavailable"] { - for chunked in [false, true] { - let response = if chunked { - format!( - "HTTP/1.1 {status}\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n{:x}\r\n{body}\r\n0\r\n\r\n", - body.len() - ) - } else { - // The oversized declared length must suffice for rejection, - // without waiting for a body the server does not provide. - format!( - "HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ) - }; - let (endpoint, server) = raw_token_endpoint_once(response.into_bytes()).await; - let failure = - post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) - .await - .expect_err("bounded response"); - server.await.expect("bound fixture"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert!(failure.upstream_status().is_some()); - assert_generic_failure(failure); - } + for response in [ + // The declared length alone must suffice for rejection. + format!( + "HTTP/1.1 503 Service Unavailable\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ), + format!( + "HTTP/1.1 503 Service Unavailable\r\nTransfer-Encoding: chunked\r\nConnection: close\r\n\r\n{:x}\r\n{body}\r\n0\r\n\r\n", + body.len() + ), + ] { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let endpoint = format!("http://{}/token", listener.local_addr().expect("addr")); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + capture_request(&mut stream).await; + // Rejection can close the reader before the fixture finishes. + let _ = stream.write_all(response.as_bytes()).await; + }); + let failure = + post_oauth_token_grant(&test_client(), &endpoint, &empty_grant_params(), None) + .await + .expect_err("bounded response"); + server.await.expect("bound fixture"); + assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); + assert_eq!(failure.upstream_status(), Some(503)); } - } - #[tokio::test] - async fn exact_body_bound_preserves_success_and_default_expiration() { - let mut body = r#"{"access_token":"access-token","token_type":"Bearer"}"#.to_string(); - body.extend(std::iter::repeat_n( - ' ', - MAX_TOKEN_RESPONSE_BYTES - body.len(), - )); + let token = r#"{"access_token":"access-token"}"#; + let body = format!("{token: TokenGrantFailure { TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, @@ -363,131 +343,85 @@ mod tests { } fn assert_rejected(status: &Status) { - let failure = TokenGrantFailure::from_status(status); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); - assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); - assert_eq!(failure.upstream_status(), None); - assert_eq!(failure.to_string(), FAILURE_MESSAGE); - assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); - assert!(std::error::Error::source(&failure).is_none()); + assert_eq!( + TokenGrantFailure::from_status(status), + TokenGrantFailure::new( + TokenGrantFailureKind::Unclassified, + TokenGrantStage::GatewayRpc + ) + ); } #[test] - fn protobuf_normalization_still_validates_the_classification() { - let original = sample().to_status(); - let mut value = envelope(&original); - // Generated decoders ignore unknown wire fields. Only the validated - // classification survives; peer diagnostics never enter the failure. - value.details[0].value.extend([32, 1]); + fn every_kind_maps_to_code_recovery_and_grpc_status() { + use ProviderCredentialRefreshRecoveryAction as Recovery; + let table = TokenGrantFailureKind::ALL.map(|kind| { + let failure = TokenGrantFailure::new(kind, TokenGrantStage::TokenEndpoint); + ( + failure.code(), + failure.recovery(), + failure.to_status().code(), + ) + }); assert_eq!( - TokenGrantFailure::from_status(&replace_envelope(&original, &value)), - sample() + table, + [ + ( + "provider_authorization_required", + Recovery::Reauthorize, + Code::FailedPrecondition + ), + ( + "workload_identity_rejected", + Recovery::FixConfiguration, + Code::PermissionDenied + ), + ( + "configuration_error", + Recovery::FixConfiguration, + Code::FailedPrecondition + ), + ("timeout", Recovery::Retry, Code::DeadlineExceeded), + ("unavailable", Recovery::Retry, Code::Unavailable), + ("unclassified", Recovery::Investigate, Code::Unknown), + ] ); - - // Protobuf map entries use the last value. Check that a replacement - // still passes the closed code set instead of bypassing validation. - for (code, expected) in [ - ("timeout", Some(TokenGrantFailureKind::Timeout)), - ("SYNTHETIC_SECRET", None), - ] { - let replacement = tonic_types::pb::ErrorInfo { - metadata: HashMap::from([("code".to_owned(), code.to_owned())]), - ..Default::default() - }; - let mut value = envelope(&original); - value.details[0].value.extend(replacement.encode_to_vec()); - let status = replace_envelope(&original, &value); - if let Some(kind) = expected { - let failure = TokenGrantFailure::from_status(&status); - assert_eq!( - failure, - TokenGrantFailure::new(kind, sample().stage()).with_upstream_status(409) - ); - assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); - } else { - assert_rejected(&status); - } - } } #[test] - fn all_classifications_and_stages_round_trip_without_printable_detail() { + fn status_round_trips_classifications_without_printable_detail() { for kind in TokenGrantFailureKind::ALL { for stage in TokenGrantStage::ALL { - for upstream in [None, Some(100), Some(409), Some(599)] { - let mut failure = TokenGrantFailure::new(kind, stage); - if let Some(status) = upstream { - failure = failure.with_upstream_status(status); - } - let status = failure.to_status(); - assert!(status.details().len() <= MAX_DETAILS_BYTES); - assert_eq!(status.message(), FAILURE_MESSAGE); - assert_eq!(TokenGrantFailure::from_status(&status), failure); - assert_eq!(failure.to_string(), FAILURE_MESSAGE); - assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); - assert_eq!(format!("{failure:#?}"), FAILURE_MESSAGE); - assert!(std::error::Error::source(&failure).is_none()); - assert_eq!(miette::Report::new(failure).to_string(), FAILURE_MESSAGE); - } - } - } - } - - #[test] - fn status_mapping_preserves_guard_codes_and_never_encodes_success() { - let expected = [ - Code::FailedPrecondition, - Code::PermissionDenied, - Code::FailedPrecondition, - Code::DeadlineExceeded, - Code::Unavailable, - Code::Unknown, - ]; - for (kind, code) in TokenGrantFailureKind::ALL.into_iter().zip(expected) { - let failure = TokenGrantFailure::new(kind, TokenGrantStage::GatewayRpc); - assert_eq!(failure.to_status().code(), code); - for override_code in [ - Code::PermissionDenied, - Code::Unauthenticated, - Code::NotFound, - ] { - let status = failure.to_status_with_code(override_code); - assert_eq!(status.code(), override_code); - assert_eq!(TokenGrantFailure::from_status(&status), failure); + let failure = TokenGrantFailure::new(kind, stage); + assert_eq!( + TokenGrantFailure::from_status(&failure.to_status()), + failure + ); } } - assert_eq!(sample().to_status_with_code(Code::Ok).code(), Code::Unknown); - } - - #[test] - fn status_bounds_and_stage_changes_preserve_only_allowlisted_values() { - for status in [0, 99, 600, u16::MAX] { - assert_eq!( - sample().with_upstream_status(status).upstream_status(), - None - ); + let failure = sample(); + let guarded = failure.to_status_with_code(Code::Unauthenticated); + assert_eq!(guarded.code(), Code::Unauthenticated); + assert_eq!(TokenGrantFailure::from_status(&guarded), failure); + assert_eq!(failure.to_status_with_code(Code::Ok).code(), Code::Unknown); + for status in [0, 99, 600] { + assert_eq!(failure.with_upstream_status(status).upstream_status(), None); } - let failure = sample().at_stage(TokenGrantStage::Injection); - assert_eq!(failure.stage(), TokenGrantStage::Injection); - assert_eq!(failure.kind(), sample().kind()); - assert_eq!(failure.upstream_status(), Some(409)); + assert_eq!(failure.to_string(), FAILURE_MESSAGE); + assert_eq!(format!("{failure:?}"), FAILURE_MESSAGE); + assert!(std::error::Error::source(&failure).is_none()); } #[test] fn rejects_unknown_missing_or_noncanonical_metadata() { for (key, value) in [ ("version", "2"), - ("version", "01"), ("code", "SYNTHETIC_SECRET"), ("stage", "SYNTHETIC_SECRET"), ("unknown", "SYNTHETIC_SECRET"), - ("upstream_http_status", "099"), ("upstream_http_status", "600"), ("upstream_http_status", "0409"), - ("upstream_http_status", "+409"), - ("upstream_http_status", "409 "), ("upstream_http_status", "4e2"), - ("upstream_http_status", ""), ] { assert_rejected(&mutate_info(|info| { info.metadata.insert(key.into(), value.into()); @@ -500,77 +434,43 @@ mod tests { } assert_rejected(&mutate_info(|info| info.reason = "SYNTHETIC_SECRET".into())); assert_rejected(&mutate_info(|info| info.domain = "SYNTHETIC_SECRET".into())); - assert_rejected(&mutate_info(|info| info.reason.clear())); - assert_rejected(&mutate_info(|info| info.domain.clear())); } #[test] - fn rejects_inconsistent_envelopes_and_arbitrary_status_diagnostics() { + fn rejects_malformed_inconsistent_or_foreign_envelopes() { let original = sample().to_status(); - for code in [Code::Ok, Code::Internal] { - assert_rejected(&Status::with_details( - code, - FAILURE_MESSAGE, - original.details().to_vec().into(), - )); - } + let details = || original.details().to_vec().into(); + assert_rejected(&Status::unknown("SYNTHETIC_SECRET")); + assert_rejected(&Status::with_details( + Code::Internal, + FAILURE_MESSAGE, + details(), + )); assert_rejected(&Status::with_details( original.code(), "SYNTHETIC_SECRET", - original.details().to_vec().into(), + details(), )); - assert_rejected(&Status::unknown("SYNTHETIC_SECRET")); - for code in [-1, 0, 17, i32::MAX] { - let mut value = envelope(&original); - value.code = code; - assert_rejected(&replace_envelope(&original, &value)); - } - let mut value = envelope(&original); - value.message = "SYNTHETIC_SECRET".into(); - assert_rejected(&replace_envelope(&original, &value)); - for type_url in [ - "google.rpc.ErrorInfo", - "https://untrusted.invalid/google.rpc.ErrorInfo", - "type.googleapis.com/google.rpc.RetryInfo", - "", - ] { - let mut value = envelope(&original); - value.details[0].type_url = type_url.into(); - assert_rejected(&replace_envelope(&original, &value)); - } - let mut value = envelope(&original); - value.details.clear(); - assert_rejected(&replace_envelope(&original, &value)); - let mut value = envelope(&original); - value.details.push(value.details[0].clone()); - assert_rejected(&replace_envelope(&original, &value)); - } - - #[test] - fn rejects_truncation_oversize_invalid_utf8_and_malformed_messages() { - let original = sample().to_status(); - for length in 0..original.details().len() { - assert_rejected(&Status::with_details( - original.code(), - FAILURE_MESSAGE, - original.details()[..length].to_vec().into(), - )); - } - for details in [ + for bytes in [ + original.details()[..original.details().len() - 1].to_vec(), vec![0; MAX_DETAILS_BYTES + 1], - vec![8, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 2], - vec![18, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 1], - vec![0], - vec![11], // Unterminated protobuf group. ] { assert_rejected(&Status::with_details( original.code(), FAILURE_MESSAGE, - details.into(), + bytes.into(), )); } - let mut value = envelope(&original); - value.details[0].value = vec![10, 1, 0xff]; - assert_rejected(&replace_envelope(&original, &value)); + let mutations: [fn(&mut tonic_types::pb::Status); 4] = [ + |value| value.message = "SYNTHETIC_SECRET".into(), + |value| value.details[0].type_url = "type.googleapis.com/google.rpc.RetryInfo".into(), + |value| value.details.clear(), + |value| value.details.push(value.details[0].clone()), + ]; + for mutate in mutations { + let mut value = envelope(&original); + mutate(&mut value); + assert_rejected(&replace_envelope(&original, &value)); + } } } diff --git a/crates/openshell-server/src/compute/mod.rs b/crates/openshell-server/src/compute/mod.rs index 92cc060901..99c0f13a6b 100644 --- a/crates/openshell-server/src/compute/mod.rs +++ b/crates/openshell-server/src/compute/mod.rs @@ -10743,29 +10743,18 @@ mod tests { #[test] fn driver_snapshot_preserves_endpoint_failure_and_ready_phase() { let mut sandbox = sandbox_record("sandbox-id", "sandbox-name", SandboxPhase::Ready); - let failure = openshell_core::endpoint_failure::EndpointFailure::token_grant( - openshell_core::token_grant_failure::TokenGrantFailure::new( - openshell_core::token_grant_failure::TokenGrantFailureKind::Timeout, - openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, - ), - "github", - "api_token", - "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", - ) - .to_proto(); let endpoint = openshell_core::proto::EndpointStatus { endpoint_id: "endpoint:v1:gateway".to_string(), host: "api.example.com".to_string(), ports: vec![443], path: "/mcp".to_string(), - last_result: openshell_core::proto::EndpointResult::CredentialUnavailable as i32, + last_result: openshell_core::proto::EndpointResult::TransportFailed as i32, last_reported_time: Some("2026-09-05T01:01:00.000Z".parse().unwrap()), - last_failure: Some(failure.clone()), + last_failure: None, }; let grant_failure = Box::new(openshell_core::proto::TokenGrantFailureObservation { destination_host: "api.example.com".to_string(), - destination_port: 443, - failure: Some(failure), + ..Default::default() }); sandbox.status = Some(SandboxStatus { phase: SandboxPhase::Ready as i32, diff --git a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs index e2f78ea1a5..3055211dba 100644 --- a/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs +++ b/crates/openshell-server/src/grpc/policy/endpoint_status_tests.rs @@ -736,22 +736,6 @@ fn endpoint_reconciliation_advances_only_observed_endpoint_timestamp() { ); } -const FAILURE_REQUEST_ID: &str = "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908"; - -fn credentialed_expected() -> BTreeMap { - BTreeMap::from([( - "endpoint:v1:tools".to_string(), - ExpectedEndpoint { - provider_credentialed: true, - status: test_initial_endpoint_status( - "endpoint:v1:tools", - "tools.example.com", - "/mcp/**", - ), - }, - )]) -} - fn reauthorize_failure() -> openshell_core::proto::EndpointFailure { openshell_core::endpoint_failure::EndpointFailure::token_grant( openshell_core::token_grant_failure::TokenGrantFailure::new( @@ -760,7 +744,7 @@ fn reauthorize_failure() -> openshell_core::proto::EndpointFailure { ), "github", "access_token", - FAILURE_REQUEST_ID, + "7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908", ) .to_proto() } @@ -831,7 +815,6 @@ async fn rest_grant_status_fixture( ); let sandbox = stored_sandbox(&state, sandbox_id).await; let context = active_endpoint_context(&state, &sandbox).await.unwrap(); - assert!(context.endpoints.is_empty(), "fixture has no MCP endpoints"); ( state, ReportEndpointStatusRequest { @@ -848,147 +831,173 @@ async fn rest_grant_status_fixture( ) } +async fn stored_grant_failure( + state: &Arc, + sandbox_id: &str, +) -> Option> { + public_status(state, sandbox_id) + .await + .last_token_grant_failure +} + #[test] -fn sandbox_grant_status_validates_destination_cause_and_provider() { +fn token_grant_status_rejects_invalid_fields() { use openshell_core::rpc_error::StatusExt; let attached = ["github".to_string()]; - let validate = |failure| { + let validate = |observation: TokenGrantFailureObservation, attached: &[String]| { validate_token_grant_status( Some(&TokenGrantStatus { - last_failure: Some(failure), + last_failure: Some(observation), }), - &attached, + attached, ) }; - for host in ["api.example.com", "127.0.0.1", "::1", "[2001:db8::1]"] { - let mut observation = sandbox_grant_failure(); - observation.destination_host = host.to_string(); - validate(observation).expect("bounded DNS and IP destinations are accepted"); - } - for host in ["API.EXAMPLE.COM", "bad\nSECRET"] { + validate(sandbox_grant_failure(), &attached).expect("valid observation is accepted"); + // A failure raised before credential selection names no provider. + let mut unnamed = sandbox_grant_failure(); + let cause = unnamed.failure.as_mut().unwrap(); + cause.provider.clear(); + cause.credential_key.clear(); + validate(unnamed, &[]).expect("unnamed failure is accepted"); + + type Mutation = fn(&mut TokenGrantFailureObservation); + let rows: [(Mutation, &str); 5] = [ + ( + |o| o.destination_host = "API.EXAMPLE.COM".to_string(), + "token_grant_status.last_failure", + ), + ( + |o| o.destination_host = "bad\nSECRET".to_string(), + "token_grant_status.last_failure", + ), + ( + |o| o.failure = None, + "token_grant_status.last_failure.failure", + ), + ( + |o| o.failure.as_mut().unwrap().domain = 99, + "token_grant_status.last_failure.failure.domain", + ), + ( + |o| o.failure.as_mut().unwrap().code = "bogus".to_string(), + "token_grant_status.last_failure.failure", + ), + ]; + for (mutate, field) in rows { let mut observation = sandbox_grant_failure(); - observation.destination_host = host.to_string(); - let error = validate(observation).unwrap_err(); - assert_eq!(error.code(), Code::InvalidArgument); + mutate(&mut observation); + let error = validate(observation, &attached).unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument, "{field}"); assert!(!error.message().contains("SECRET")); + assert_eq!( + error + .get_error_details() + .bad_request() + .unwrap() + .field_violations[0] + .field, + field + ); } - let mut missing_cause = sandbox_grant_failure(); - missing_cause.failure = None; - assert_eq!( - validate(missing_cause).unwrap_err().code(), - Code::InvalidArgument - ); - let mut unknown_code = sandbox_grant_failure(); - unknown_code.failure.as_mut().unwrap().code = "bogus".to_string(); - let details = validate(unknown_code).unwrap_err().get_error_details(); - assert_eq!( - details.bad_request().unwrap().field_violations[0].field, - "token_grant_status.last_failure.failure" - ); - let mut unknown_domain = sandbox_grant_failure(); - unknown_domain.failure.as_mut().unwrap().domain = 99; + + let mut foreign = sandbox_grant_failure(); + foreign.failure.as_mut().unwrap().provider = "other-team".to_string(); + let error = validate(foreign, &attached).unwrap_err(); + assert_eq!(error.code(), Code::FailedPrecondition); + let info = openshell_core::rpc_error::decode_details(&error) + .and_then(|details| details.error_info().cloned()) + .expect("stable error reason"); + assert_eq!(info.reason, "ENDPOINT_FAILURE_PROVIDER_NOT_ATTACHED"); +} + +#[test] +fn endpoint_snapshot_stores_failure_only_for_a_result_its_domain_explains() { + let expected = BTreeMap::from([( + "endpoint:v1:tools".to_string(), + ExpectedEndpoint { + provider_credentialed: true, + status: test_initial_endpoint_status( + "endpoint:v1:tools", + "tools.example.com", + "/mcp/**", + ), + }, + )]); + let validate = |result: EndpointResult| { + validate_endpoint_snapshot( + &[EndpointObservation { + endpoint_id: "endpoint:v1:tools".to_string(), + result: result as i32, + failure: Some(reauthorize_failure()), + }], + &["endpoint:v1:tools".to_string()], + &expected, + &["github".to_string()], + ) + }; + let (reports, _) = validate(EndpointResult::CredentialUnavailable) + .expect("a classified credential failure is accepted"); assert_eq!( - validate(unknown_domain).unwrap_err().code(), - Code::InvalidArgument + reports["endpoint:v1:tools"].last_failure, + Some(reauthorize_failure()) ); - let mut foreign_provider = sandbox_grant_failure(); - foreign_provider.failure.as_mut().unwrap().provider = "other-team".to_string(); + + let error = validate(EndpointResult::TransportFailed).unwrap_err(); + assert_eq!(error.code(), Code::InvalidArgument); assert_eq!( - validate(foreign_provider).unwrap_err().code(), - Code::FailedPrecondition + error.message(), + "endpoint failure domain does not explain this endpoint result" ); - let mut unnamed = sandbox_grant_failure(); - let cause = unnamed.failure.as_mut().unwrap(); - cause.provider.clear(); - cause.credential_key.clear(); - validate_token_grant_status( - Some(&TokenGrantStatus { - last_failure: Some(unnamed), - }), - &[], - ) - .expect("selection failures may have no credential name"); } -#[tokio::test] -async fn rest_grant_failure_is_retrievable_retained_and_explicitly_cleared() { - let sandbox_id = "rest-grant-status"; - let (state, report) = rest_grant_status_fixture(sandbox_id).await; +/// Send `report` with a new sequence and grant status; return the public cause. +async fn send_grant_report( + state: &Arc, + report: &mut ReportEndpointStatusRequest, + sequence: u64, + grant_status: Option, +) -> Option> { + report.report_sequence = sequence; + report.token_grant_status = grant_status; + let sandbox_id = report.sandbox_id.clone(); handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), + state, + with_sandbox(Request::new(report.clone()), &sandbox_id), ) .await - .expect("REST failure is reported without MCP observations"); - assert_eq!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure, - Some(Box::new(sandbox_grant_failure())) - ); - let mut changed_retry = report.clone(); - changed_retry - .token_grant_status - .as_mut() - .unwrap() - .last_failure - .as_mut() - .unwrap() - .destination_port = 8443; - assert_eq!( - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(changed_retry), sandbox_id) - ) + .expect("report accepted"); + public_status(state, &sandbox_id) .await - .unwrap_err() - .code(), - Code::InvalidArgument - ); - let mut legacy = report.clone(); - legacy.report_sequence = 2; - legacy.token_grant_status = None; - handle_report_endpoint_status(&state, with_sandbox(Request::new(legacy), sandbox_id)) - .await - .expect("missing extension retains current evidence"); + .last_token_grant_failure +} + +#[tokio::test] +async fn rest_grant_failure_is_retained_cleared_and_reset() { + let sandbox_id = "rest-grant-status"; + let (state, mut report) = rest_grant_status_fixture(sandbox_id).await; + let failure = report.token_grant_status.clone(); + let stored = Some(Box::new(sandbox_grant_failure())); + + // A REST-only sandbox reports a failure without MCP observations. A report + // without the extension retains it; an explicit success clears it. assert_eq!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure, - Some(Box::new(sandbox_grant_failure())) + send_grant_report(&state, &mut report, 1, failure.clone()).await, + stored ); - let mut cleared = report.clone(); - cleared.report_sequence = 3; - cleared.token_grant_status = Some(TokenGrantStatus::default()); - handle_report_endpoint_status(&state, with_sandbox(Request::new(cleared), sandbox_id)) - .await - .expect("success explicitly clears the previous failure"); - assert!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure - .is_none() + assert_eq!( + send_grant_report(&state, &mut report, 2, None).await, + stored ); + let success = Some(TokenGrantStatus::default()); assert_eq!( - handle_report_endpoint_status(&state, with_sandbox(Request::new(report), "other-sandbox")) - .await - .unwrap_err() - .code(), - Code::PermissionDenied + send_grant_report(&state, &mut report, 3, success).await, + None ); -} -#[tokio::test] -async fn rest_grant_failure_resets_on_provider_and_policy_changes() { - let sandbox_id = "rest-grant-revisions"; - let (state, mut report) = rest_grant_status_fixture(sandbox_id).await; - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), - ) - .await - .unwrap(); + // A report without the extension from a new provider revision cannot keep + // the earlier cause. + send_grant_report(&state, &mut report, 4, failure.clone()).await; state .store .update_message_cas::( @@ -1005,61 +1014,12 @@ async fn rest_grant_failure_resets_on_provider_and_policy_changes() { .unwrap(); let sandbox = stored_sandbox(&state, sandbox_id).await; let context = active_endpoint_context(&state, &sandbox).await.unwrap(); - assert_ne!(context.provider_env_revision, report.provider_env_revision); - report.report_sequence = 2; report.policy_hash = context.policy_hash; report.provider_env_revision = context.provider_env_revision; - report.token_grant_status = None; - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), - ) - .await - .expect("a new provider revision clears even a legacy report's cause"); - assert!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure - .is_none() - ); - report.report_sequence = 3; - report.token_grant_status = Some(TokenGrantStatus { - last_failure: Some(sandbox_grant_failure()), - }); - handle_report_endpoint_status(&state, with_sandbox(Request::new(report), sandbox_id)) - .await - .unwrap(); - let mut policy = sandbox.spec.unwrap().policy.unwrap(); - policy.network_policies.get_mut("rest").unwrap().endpoints[0].host = - "changed.example.com".to_string(); - state - .store - .put_policy_revision( - "rest-grant-revisions-v2", - sandbox_id, - "default", - 2, - &policy.encode_to_vec(), - &deterministic_policy_hash(&policy), - ) - .await - .unwrap(); - acknowledge_loaded_policy(&state, sandbox_id, 2).await; - let status = public_status(&state, sandbox_id).await; - assert!(status.endpoint_statuses.is_empty()); - assert!(status.last_token_grant_failure.is_none()); -} + assert_eq!(send_grant_report(&state, &mut report, 5, None).await, None); -#[tokio::test] -async fn rest_grant_failure_resets_on_session_loss_replacement_and_startup() { - let sandbox_id = "rest-grant-sessions"; - let (state, mut report) = rest_grant_status_fixture(sandbox_id).await; - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), - ) - .await - .unwrap(); + // Losing the session clears the cause. + send_grant_report(&state, &mut report, 6, failure).await; state .supervisor_sessions .remove_if_current(sandbox_id, "session-a") @@ -1067,120 +1027,22 @@ async fn rest_grant_failure_resets_on_session_loss_replacement_and_startup() { reset_endpoint_status_after_supervisor_disconnect(&state, sandbox_id) .await .unwrap(); - assert!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure - .is_none() - ); - for session in ["session-b", "session-c"] { - register_session(&state, sandbox_id, session); - reset_endpoint_status_for_supervisor_session(&state, sandbox_id, session) - .await - .unwrap(); - assert!( - public_status(&state, sandbox_id) - .await - .last_token_grant_failure - .is_none() - ); - assert!( - state - .supervisor_sessions - .initialize_endpoint_status_authority(sandbox_id, session) - ); - report.supervisor_session_id = session.to_string(); - handle_report_endpoint_status( - &state, - with_sandbox(Request::new(report.clone()), sandbox_id), - ) - .await - .unwrap(); - } + assert!(stored_grant_failure(&state, sandbox_id).await.is_none()); + + // Startup invalidates a sandbox whose only evidence is a grant failure. state - .supervisor_sessions - .remove_if_current(sandbox_id, "session-c") + .store + .update_message_cas::(sandbox_id, 0, |sandbox| { + sandbox.status.as_mut().unwrap().last_token_grant_failure = stored.clone(); + }) + .await .unwrap(); invalidate_endpoint_status_on_startup(&state).await.unwrap(); - let status = public_status(&state, sandbox_id).await; - assert!(status.endpoint_statuses.is_empty()); - assert!(status.last_token_grant_failure.is_none()); -} - -fn failed_observation( - result: EndpointResult, - failure: openshell_core::proto::EndpointFailure, -) -> Vec { - vec![EndpointObservation { - endpoint_id: "endpoint:v1:tools".to_string(), - result: result as i32, - failure: Some(failure), - }] -} - -#[test] -fn endpoint_snapshot_accepts_bounded_failure_for_credential_failure() { - let attached = ["github".to_string()]; - let (reports, _) = validate_endpoint_snapshot( - &failed_observation(EndpointResult::CredentialUnavailable, reauthorize_failure()), - &["endpoint:v1:tools".to_string()], - &credentialed_expected(), - &attached, - ) - .expect("a classified credential failure is accepted"); - let status = &reports["endpoint:v1:tools"]; - assert_eq!( - status.last_result, - EndpointResult::CredentialUnavailable as i32 - ); - assert_eq!(status.last_failure, Some(reauthorize_failure())); - assert_eq!(status.host, "tools.example.com"); - - // A failure that has not selected a credential names no provider. - let mut unnamed = reauthorize_failure(); - unnamed.provider.clear(); - unnamed.credential_key.clear(); - validate_endpoint_snapshot( - &failed_observation(EndpointResult::CredentialUnavailable, unnamed), - &["endpoint:v1:tools".to_string()], - &credentialed_expected(), - &[], - ) - .expect("an unnamed failure is accepted"); + assert!(stored_grant_failure(&state, sandbox_id).await.is_none()); } #[test] -fn endpoint_snapshot_rejects_failures_outside_the_contract() { - let attached = ["github".to_string()]; - let invalid = |result, failure| { - validate_endpoint_snapshot( - &failed_observation(result, failure), - &["endpoint:v1:tools".to_string()], - &credentialed_expected(), - &attached, - ) - .expect_err("invalid failure is rejected") - }; - - let error = invalid(EndpointResult::TransportFailed, reauthorize_failure()); - assert_eq!(error.code(), Code::InvalidArgument); - assert_eq!( - error.message(), - "endpoint failure domain does not explain this endpoint result" - ); - - let mut foreign = reauthorize_failure(); - foreign.provider = "other-team".to_string(); - let error = invalid(EndpointResult::CredentialUnavailable, foreign); - assert_eq!(error.code(), Code::FailedPrecondition); - let info = openshell_core::rpc_error::decode_details(&error) - .and_then(|details| details.error_info().cloned()) - .expect("stable error reason"); - assert_eq!(info.reason, "ENDPOINT_FAILURE_PROVIDER_NOT_ATTACHED"); -} - -#[test] -fn endpoint_failure_change_keeps_report_time_and_session_loss_clears_it() { +fn endpoint_failure_change_without_marker_keeps_report_time() { let mut sandbox = test_sandbox( "sandbox-id", "sandbox-name", @@ -1192,26 +1054,15 @@ fn endpoint_failure_change_keeps_report_time_and_session_loss_clears_it() { last_failure: Some(reauthorize_failure()), ..test_initial_endpoint_status("endpoint", "api.example.com", "/**") }; - // A gateway replica that predates last_failure can rewrite the record - // without it: the stored result remains and its cause is gone. - let rewritten = BTreeMap::from([( - "endpoint".to_string(), - EndpointStatus { - last_failure: None, - ..endpoint.clone() - }, - )]); - let observed = HashSet::from(["endpoint".to_string()]); reconcile_endpoint_statuses( &mut sandbox, - &rewritten, - &observed, + &BTreeMap::from([("endpoint".to_string(), endpoint.clone())]), + &HashSet::from(["endpoint".to_string()]), ×tamp("2026-09-05T01:00:00.000Z"), ); - // The supervisor's next report carries a cause for the same result without - // marking the endpoint observed. Only a new result needs a marker, so the - // cause is stored and keeps the report time. + // Only a new result needs an observation marker; a new cause for the same + // result is stored and keeps the earlier report time. let mut retry = reauthorize_failure(); retry.request_id = "1a2b3c4d-5e6f-4a1b-8c2d-3e4f5a6b7c8d".to_string(); let changed = BTreeMap::from([( @@ -1235,14 +1086,6 @@ fn endpoint_failure_change_keeps_report_time_and_session_loss_clears_it() { status.last_reported_time, Some(timestamp("2026-09-05T01:00:00.000Z")) ); - - invalidate_endpoint_status_without_session(&mut sandbox); - let status = &sandbox.status.as_ref().unwrap().endpoint_statuses[0]; - assert_eq!( - status.last_result, - EndpointResult::NoObservedExchange as i32 - ); - assert_eq!(status.last_failure, None); } #[test] diff --git a/crates/openshell-server/src/grpc/provider.rs b/crates/openshell-server/src/grpc/provider.rs index 31bc77cb4c..ec973439ea 100644 --- a/crates/openshell-server/src/grpc/provider.rs +++ b/crates/openshell-server/src/grpc/provider.rs @@ -5460,6 +5460,7 @@ mod tests { use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; const SECRET: &str = "GRANT_SECRET_CANARY_3319"; + const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; #[derive(Clone, Default)] struct GrantTraceBuffer(Arc>>); @@ -5475,190 +5476,6 @@ mod tests { } } - fn capture_grant_warnings() -> ( - GrantTraceBuffer, - crate::otel_tracing::test_exporter::ScopedTracingTestGuard, - ) { - let output = GrantTraceBuffer::default(); - let writer = output.clone(); - let subscriber = tracing_subscriber::fmt() - .with_max_level(tracing::Level::WARN) - .with_ansi(false) - .without_time() - .with_writer(move || writer.clone()) - .finish(); - let guard = crate::otel_tracing::test_exporter::install_scoped(subscriber); - // A captured sentinel proves the subscriber was active for WARN - // events, so an empty capture cannot hide a broken test fixture. - tracing::warn!("grant capture active"); - (output, guard) - } - - /// Credential values and handles never reach logs. Provider and - /// credential names are not secrets and can appear in diagnostics. - fn assert_no_credential_warnings(output: &GrantTraceBuffer) { - let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); - assert!(logs.contains("grant capture active")); - assert!(!logs.contains(SECRET), "{logs}"); - } - - #[cfg(unix)] - mod expired_driver { - use super::*; - use openshell_core::proto::credentials::v1::{ - DeleteCredentialRequest, DeleteCredentialResponse, - GetCredentialDriverCapabilitiesRequest, GetCredentialDriverCapabilitiesResponse, - ListCredentialsRequest, ListCredentialsResponse, ResolveCredentialsRequest, - ResolveCredentialsResponse, ResolvedCredential, StoreCredentialRequest, - StoreCredentialResponse, - credential_driver_server::{CredentialDriver, CredentialDriverServer}, - }; - - #[derive(Debug)] - struct ExpiredDriver; - - #[tonic::async_trait] - impl CredentialDriver for ExpiredDriver { - async fn get_capabilities( - &self, - _request: Request, - ) -> Result, Status> - { - Ok(Response::new(GetCredentialDriverCapabilitiesResponse { - driver_name: "expired-fixture".to_string(), - supports_expires_at: true, - extension: Some(openshell_core::extension_protocol::extension_metadata( - openshell_core::extension_protocol::ExtensionFamily::Credentials, - "expired-fixture", - "0.0.0", - [], - )), - ..Default::default() - })) - } - - async fn store_credential( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented("fixture only resolves credentials")) - } - - async fn delete_credential( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented("fixture only resolves credentials")) - } - - async fn list_credentials( - &self, - _request: Request, - ) -> Result, Status> { - Err(Status::unimplemented("fixture only resolves credentials")) - } - - async fn resolve_credentials( - &self, - request: Request, - ) -> Result, Status> { - Ok(Response::new(ResolveCredentialsResponse { - credentials: request - .into_inner() - .credentials - .into_iter() - .map(|request| ResolvedCredential { - request_id: request.request_id, - value: SECRET.to_string(), - expiration_time: Some(ts(0)), - }) - .collect(), - })) - } - } - - #[tokio::test] - async fn driver_returned_subject_expiry_is_generic_and_not_logged() { - let directory = tempfile::tempdir().unwrap(); - let socket = directory.path().join("driver.sock"); - let listener = tokio::net::UnixListener::bind(&socket).unwrap(); - let incoming = futures::stream::unfold(listener, |listener| async move { - Some((listener.accept().await.map(|(stream, _)| stream), listener)) - }); - let (shutdown, stop) = tokio::sync::oneshot::channel::<()>(); - let server = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service(CredentialDriverServer::new(ExpiredDriver)) - .serve_with_incoming_shutdown(incoming, async { - let _ = stop.await; - }) - .await - }); - let config = - openshell_core::Config::new(None).with_credential_drivers(["expired-fixture"]); - let socket_path = toml::Value::String(socket.to_string_lossy().into_owned()); - let file: crate::config_file::ConfigFile = toml::from_str(&format!( - "[openshell.credential_drivers.expired-fixture]\ntransport = \"uds\"\nsocket_path = {socket_path}\n" - )).unwrap(); - let credentials = - crate::credentials::CredentialRuntime::from_config_file(&config, Some(&file)) - .await - .unwrap(); - let mut provider = provider_with_values("grant-provider", "custom"); - provider.credentials.clear(); - provider.credential_handles.insert( - "subject".to_string(), - CredentialHandle { - driver: "expired-fixture".to_string(), - handle: SECRET.to_string(), - metadata: HashMap::new(), - }, - ); - assert!(provider.credential_expiration_times.is_empty()); - let (output, capture) = capture_grant_warnings(); - let status = resolve_subject_token_credential(&credentials, &provider, "subject") - .await - .unwrap_err(); - drop(capture); - let failure = assert_generic(&status, Code::FailedPrecondition); - assert_eq!( - failure.kind(), - TokenGrantFailureKind::ProviderAuthorizationRequired - ); - assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); - assert_no_credential_warnings(&output); - drop(credentials); - shutdown.send(()).unwrap(); - server.await.unwrap().unwrap(); - } - } - - fn sandbox_request( - body: ExchangeProviderSubjectTokenRequest, - principal_id: &str, - ) -> Request { - let mut request = Request::new(body); - request - .extensions_mut() - .insert(Principal::Sandbox(SandboxPrincipal { - sandbox_id: principal_id.to_string(), - source: SandboxIdentitySource::BootstrapJwt { - issuer: SECRET.to_string(), - }, - trust_domain: None, - })); - request - } - - fn request_body() -> ExchangeProviderSubjectTokenRequest { - ExchangeProviderSubjectTokenRequest { - sandbox_id: "grant-sandbox".to_string(), - provider: SECRET.to_string(), - credential_key: SECRET.to_string(), - supervisor_jwt_svid: SECRET.to_string(), - } - } - fn assert_generic(status: &Status, code: Code) -> TokenGrantFailure { assert_eq!(status.code(), code); assert_eq!(status.message(), "dynamic token grant failed"); @@ -5675,7 +5492,6 @@ mod tests { raw.metadata_mut() .insert("x-upstream-secret", SECRET.parse().unwrap()); raw.set_source(Arc::new(std::io::Error::other(SECRET))); - assert!(std::error::Error::source(&raw).is_some()); let status = sanitize_token_grant_status(raw); let failure = assert_generic(&status, Code::Internal); assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); @@ -5698,7 +5514,6 @@ mod tests { #[test] fn only_canonical_request_ids_are_echoed() { - const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; let mut metadata = tonic::metadata::MetadataMap::new(); assert_eq!(canonical_request_id(&metadata), None); metadata.insert("x-request-id", REQUEST_ID.parse().unwrap()); @@ -5715,17 +5530,40 @@ mod tests { #[tokio::test] async fn handler_failure_log_names_request_and_classification_without_secrets() { - const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; let state = test_server_state_without_provider_profiles().await; - let mut request = sandbox_request(request_body(), "grant-sandbox"); + let mut request = Request::new(ExchangeProviderSubjectTokenRequest { + sandbox_id: "grant-sandbox".to_string(), + provider: SECRET.to_string(), + credential_key: SECRET.to_string(), + supervisor_jwt_svid: SECRET.to_string(), + }); + request + .extensions_mut() + .insert(Principal::Sandbox(SandboxPrincipal { + sandbox_id: "grant-sandbox".to_string(), + source: SandboxIdentitySource::BootstrapJwt { + issuer: SECRET.to_string(), + }, + trust_domain: None, + })); request .metadata_mut() .insert("x-request-id", REQUEST_ID.parse().unwrap()); - let (output, capture) = capture_grant_warnings(); + + let output = GrantTraceBuffer::default(); + let writer = output.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::WARN) + .with_ansi(false) + .without_time() + .with_writer(move || writer.clone()) + .finish(); + let capture = crate::otel_tracing::test_exporter::install_scoped(subscriber); let error = handle_exchange_provider_subject_token(&state, request) .await .unwrap_err(); drop(capture); + let failure = assert_generic(&error, Code::NotFound); let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); assert!( @@ -5736,104 +5574,27 @@ mod tests { )), "{logs}" ); - // The authenticated supervisor's own sandbox receives the line. assert!(logs.contains("sandbox_id=grant-sandbox"), "{logs}"); assert!(!logs.contains(SECRET), "{logs}"); - - // Another sandbox's supervisor cannot add a line to this sandbox's - // stream; its attempt is attributed to its own sandbox. - let (output, capture) = capture_grant_warnings(); - handle_exchange_provider_subject_token( - &state, - sandbox_request(request_body(), "other"), - ) - .await - .unwrap_err(); - drop(capture); - let logs = String::from_utf8(output.0.lock().unwrap().clone()).unwrap(); - assert!( - logs.contains("dynamic token grant failed request_id=none"), - "{logs}" - ); - assert!(logs.contains("sandbox_id=other"), "{logs}"); - assert!(!logs.contains("sandbox_id=grant-sandbox"), "{logs}"); } #[tokio::test] - async fn handler_auth_and_required_fields_are_generic() { - let state = test_server_state_without_provider_profiles().await; - let missing = - handle_exchange_provider_subject_token(&state, Request::new(request_body())) - .await - .unwrap_err(); - let failure = assert_generic(&missing, Code::Unauthenticated); - assert_eq!( - failure.kind(), - TokenGrantFailureKind::WorkloadIdentityRejected - ); - let user = - handle_exchange_provider_subject_token(&state, authed_request(request_body())) - .await - .unwrap_err(); - assert_generic(&user, Code::PermissionDenied); - let wrong_sandbox = handle_exchange_provider_subject_token( - &state, - sandbox_request(request_body(), SECRET), - ) - .await - .unwrap_err(); - assert_generic(&wrong_sandbox, Code::PermissionDenied); - - for missing_field in 0..3 { - let mut body = request_body(); - match missing_field { - 0 => body.provider.clear(), - 1 => body.credential_key.clear(), - _ => body.supervisor_jwt_svid.clear(), - } - let error = handle_exchange_provider_subject_token( - &state, - sandbox_request(body, "grant-sandbox"), - ) - .await - .unwrap_err(); - let failure = assert_generic(&error, Code::InvalidArgument); - assert_eq!(failure.kind(), TokenGrantFailureKind::Configuration); - } - } - - #[tokio::test] - async fn handler_unattached_provider_is_generic() { - let state = test_server_state_without_provider_profiles().await; - state - .store - .put_message(&Sandbox { - metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta { - id: "grant-sandbox".to_string(), - name: "grant-sandbox".to_string(), - workspace: "default".to_string(), - ..Default::default() - }), - spec: Some(SandboxSpec::default()), - ..Default::default() - }) - .await - .unwrap(); - let unattached = handle_exchange_provider_subject_token( - &state, - sandbox_request(request_body(), "grant-sandbox"), - ) - .await - .unwrap_err(); - assert_generic(&unattached, Code::PermissionDenied); - } - - #[tokio::test] - async fn credential_driver_diagnostics_and_expiration_are_generic() { + async fn missing_expired_or_unresolvable_subject_credential_is_generic() { let config = openshell_core::Config::new(None).with_credential_drivers(["test-static"]); let credentials = crate::credentials::CredentialRuntime::from_config(&config).unwrap(); let mut provider = provider_with_values("grant-provider", "custom"); provider.credentials.clear(); + let missing = resolve_subject_token_credential(&credentials, &provider, "subject") + .await + .unwrap_err(); + let failure = assert_generic(&missing, Code::FailedPrecondition); + assert_eq!( + failure.kind(), + TokenGrantFailureKind::ProviderAuthorizationRequired + ); + assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); + + // Driver diagnostics name the unknown driver; the gateway drops them. provider.credential_handles.insert( "subject".to_string(), CredentialHandle { @@ -5842,97 +5603,59 @@ mod tests { metadata: HashMap::new(), }, ); - let raw = credentials - .resolve_provider_handles(&provider, 0) + let unresolvable = resolve_subject_token_credential(&credentials, &provider, "subject") .await .unwrap_err(); - assert!(raw.message().contains(SECRET)); - let failure = resolve_subject_token_credential(&credentials, &provider, "subject") - .await - .unwrap_err(); - assert_generic(&failure, Code::InvalidArgument); + assert_generic(&unresolvable, Code::InvalidArgument); + // Expiry is checked before the driver is asked. provider .credential_expiration_times .insert("subject".to_string(), ts(0)); let expired = resolve_subject_token_credential(&credentials, &provider, "subject") .await .unwrap_err(); - let failure = assert_generic(&expired, Code::FailedPrecondition); assert_eq!( - failure.kind(), + assert_generic(&expired, Code::FailedPrecondition).kind(), TokenGrantFailureKind::ProviderAuthorizationRequired ); - assert_eq!(failure.stage(), TokenGrantStage::GatewayRpc); - } - - #[tokio::test] - async fn expired_sibling_handle_is_skipped_without_logging_subject() { - let config = openshell_core::Config::new(None).with_credential_drivers(["test-static"]); - let credentials = crate::credentials::CredentialRuntime::from_config(&config).unwrap(); - let mut provider = provider_with_values("grant-provider", "custom"); - provider.credentials.clear(); - provider.credential_handles = credentials - .store_provider_credentials( - provider.object_name(), - provider.object_workspace(), - "grant-fixture-provider", - &HashMap::from([ - ("subject".to_string(), SECRET.to_string()), - ("expired-sibling".to_string(), SECRET.to_string()), - ]), - &HashMap::new(), - ) - .await - .unwrap(); - provider - .credential_expiration_times - .insert("expired-sibling".to_string(), ts(0)); - let (output, capture) = capture_grant_warnings(); - let subject = resolve_subject_token_credential(&credentials, &provider, "subject") - .await - .unwrap(); - drop(capture); - assert_eq!(subject, SECRET); - assert_no_credential_warnings(&output); } #[test] fn invalid_grant_configuration_fails_before_acquisition() { - let mut grant = token_grant_credential("access_token").token_grant.unwrap(); - let mut provider = provider_with_values("grant-provider", "custom"); - grant.cache_ttl = Some(prost_types::Duration { - seconds: -1, - nanos: 0, - }); - let invalid_ttl = - validate_intermediate_grant_configuration(&grant, &provider, "subject") - .unwrap_err(); - assert_eq!( - assert_generic(&invalid_ttl, Code::InvalidArgument).kind(), - TokenGrantFailureKind::Configuration, - ); - grant.cache_ttl = None; - provider.credential_expiration_times.insert( + let valid = token_grant_credential("access_token").token_grant.unwrap(); + let provider = provider_with_values("grant-provider", "custom"); + let mut bad_expiry = provider.clone(); + bad_expiry.credential_expiration_times.insert( "subject".to_string(), prost_types::Timestamp { seconds: 0, nanos: -1, }, ); - let invalid_expiration = - validate_intermediate_grant_configuration(&grant, &provider, "subject") - .unwrap_err(); - assert_generic(&invalid_expiration, Code::FailedPrecondition); - provider.credential_expiration_times.clear(); - grant.token_endpoint = format!("ftp://example.com/{SECRET}"); - let invalid_endpoint = - validate_intermediate_grant_configuration(&grant, &provider, "subject") + let negative_ttl = ProviderCredentialTokenGrant { + cache_ttl: Some(prost_types::Duration { + seconds: -1, + nanos: 0, + }), + ..valid.clone() + }; + let bad_endpoint = ProviderCredentialTokenGrant { + token_endpoint: format!("ftp://example.com/{SECRET}"), + ..valid.clone() + }; + for (grant, provider, code) in [ + (&negative_ttl, &provider, Code::InvalidArgument), + (&valid, &bad_expiry, Code::FailedPrecondition), + (&bad_endpoint, &provider, Code::FailedPrecondition), + ] { + let status = validate_intermediate_grant_configuration(grant, provider, "subject") .unwrap_err(); - assert_eq!( - assert_generic(&invalid_endpoint, Code::FailedPrecondition).kind(), - TokenGrantFailureKind::Configuration, - ); + assert_eq!( + assert_generic(&status, code).kind(), + TokenGrantFailureKind::Configuration, + ); + } } #[test] @@ -5995,25 +5718,11 @@ mod tests { assert_generic(&supervisor, Code::PermissionDenied); } - fn secret_exchange_params() -> TokenExchangeParams<'static> { - TokenExchangeParams { - client_assertion: SECRET, - client_assertion_type: "", - subject_token: SECRET, - subject_token_type: "", - audience: SECRET, - scopes: &[], - requested_token_type: "", - } - } - - /// Serve one token response and return the request headers it received. - async fn token_endpoint_capturing_headers( - body: String, - ) -> (String, tokio::task::JoinHandle) { + #[tokio::test] + async fn intermediate_http_failure_keeps_stage_status_and_request_id_without_body() { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let endpoint = format!("http://{}/token", listener.local_addr().unwrap()); - let task = tokio::spawn(async move { + let server = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.unwrap(); let mut request = Vec::new(); let mut chunk = [0_u8; 1024]; @@ -6022,92 +5731,38 @@ mod tests { assert_ne!(count, 0, "request headers ended early"); request.extend_from_slice(&chunk[..count]); } + let body = format!( + "{{\"error\":\"provider_authorization_required\",\"openshell_error_contract\":\"openshell.token-grant.v1\",\"error_description\":\"{SECRET}\"}}" + ); let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", + "HTTP/1.1 409 Conflict\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", body.len() ); stream.write_all(response.as_bytes()).await.unwrap(); - String::from_utf8_lossy(&request).into_owned() + String::from_utf8_lossy(&request).to_ascii_lowercase() }); - (endpoint, task) - } - - #[tokio::test] - async fn intermediate_exchange_forwards_the_supervisor_request_id() { - const REQUEST_ID: &str = "2f3e4d5c-6b7a-4891-a0b1-c2d3e4f5a6b7"; - let body = format!("{{\"access_token\":\"{SECRET}\",\"token_type\":\"Bearer\"}}"); - let (endpoint, task) = token_endpoint_capturing_headers(body).await; - perform_intermediate_token_exchange( - &endpoint, - &secret_exchange_params(), - Some(REQUEST_ID), - ) - .await - .unwrap(); - let headers = task.await.unwrap().to_ascii_lowercase(); + let params = TokenExchangeParams { + client_assertion: SECRET, + client_assertion_type: "", + subject_token: SECRET, + subject_token_type: "", + audience: SECRET, + scopes: &[], + requested_token_type: "", + }; + let result = + perform_intermediate_token_exchange(&endpoint, ¶ms, Some(REQUEST_ID)).await; + let headers = server.await.unwrap(); assert!( headers.contains(&format!("x-openshell-request-id: {REQUEST_ID}")), "{headers}" ); - } - - async fn token_endpoint_response( - status: u16, - body: String, - ) -> (String, tokio::task::JoinHandle<()>) { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let endpoint = format!("http://{}/token", listener.local_addr().unwrap()); - let task = tokio::spawn(async move { - let (mut stream, _) = listener.accept().await.unwrap(); - // This fixture returns an error as soon as the request starts. - let mut request = [0_u8; 1]; - stream.read_exact(&mut request).await.unwrap(); - let response = format!( - "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{body}", - body.len() - ); - stream.write_all(response.as_bytes()).await.unwrap(); - }); - (endpoint, task) - } - - #[tokio::test] - async fn intermediate_http_failure_keeps_stage_and_status_without_body() { - let body = format!( - "{{\"error\":\"provider_authorization_required\",\"openshell_error_contract\":\"openshell.token-grant.v1\",\"error_description\":\"{SECRET}\"}}" - ); - let (endpoint, task) = token_endpoint_response(409, body).await; - let result = - perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) - .await; - task.await.unwrap(); let status = result.unwrap_err(); let failure = assert_generic(&status, status.code()); assert_eq!(failure.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(failure.stage(), TokenGrantStage::IntermediateExchange); assert_eq!(failure.upstream_status(), Some(409)); } - - #[tokio::test] - async fn intermediate_success_remains_cacheable() { - let body = format!( - "{{\"access_token\":\"{SECRET}\",\"token_type\":\"Bearer\",\"expires_in\":300}}" - ); - let (endpoint, task) = token_endpoint_response(200, body).await; - let token = - perform_intermediate_token_exchange(&endpoint, &secret_exchange_params(), None) - .await - .unwrap(); - task.await.unwrap(); - assert_eq!(token.access_token, SECRET); - let cache = IntermediateTokenCache::new(); - let expires_at_ms = intermediate_token_cache_expires_at_ms(&token, None, 0, 0); - cache.set("cache-key".to_string(), &token, expires_at_ms); - let cached = cache.get("cache-key").unwrap(); - assert_eq!(cached.access_token, SECRET); - assert_eq!(cached.token_type, "Bearer"); - assert!(cached.expires_in > 0 && cached.expires_in <= 270); - } } #[test] diff --git a/crates/openshell-server/src/supervisor_session.rs b/crates/openshell-server/src/supervisor_session.rs index bf3e0db1a3..99f1a13cde 100644 --- a/crates/openshell-server/src/supervisor_session.rs +++ b/crates/openshell-server/src/supervisor_session.rs @@ -3258,18 +3258,7 @@ mod tests { last_result: EndpointResult::CredentialUnavailable as i32, last_reported_time: Some("2026-09-05T01:01:00.000Z".parse().unwrap()), // Evidence without live authority must drop its cause with the result. - last_failure: Some( - openshell_core::endpoint_failure::EndpointFailure::token_grant( - openshell_core::token_grant_failure::TokenGrantFailure::new( - openshell_core::token_grant_failure::TokenGrantFailureKind::Timeout, - openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, - ), - "github", - "access_token", - "3e2d1c0b-9a8f-4e7d-b6c5-a4f3e2d1c0b9", - ) - .to_proto(), - ), + last_failure: Some(openshell_core::proto::EndpointFailure::default()), }; let ready = SandboxCondition { r#type: "Ready".to_string(), @@ -3278,8 +3267,7 @@ mod tests { }; let grant_failure = Box::new(openshell_core::proto::TokenGrantFailureObservation { destination_host: "api.example.com".to_string(), - destination_port: 443, - failure: endpoint.last_failure.clone(), + ..Default::default() }); sandbox.status = Some(SandboxStatus { endpoint_statuses: vec![endpoint.clone()], diff --git a/crates/openshell-supervisor-network/src/l7/relay.rs b/crates/openshell-supervisor-network/src/l7/relay.rs index fdf96ffbe1..9603f65a02 100644 --- a/crates/openshell-supervisor-network/src/l7/relay.rs +++ b/crates/openshell-supervisor-network/src/l7/relay.rs @@ -150,14 +150,7 @@ pub(crate) async fn prepare_inspected_request( observation_context: Option<&openshell_core::endpoint_status::EndpointObservationContext>, ) -> Result<(crate::l7::provider::L7Request, L7EvalContext)> { let mut scoped = ctx.clone(); - let snapshot = ctx - .provider_credentials - .as_ref() - .map(openshell_core::provider_credentials::ProviderCredentialState::snapshot); - if let Some(snapshot) = &snapshot { - scoped.provider_credential_revision = Some(snapshot.revision); - scoped.provider_credential_installation_id = Some(snapshot.installation_id.clone()); - } + let snapshot = crate::l7::token_grant_injection::pin_live_credentials(&mut scoped); // Admission-gated injection reads credentials only from the pinned snapshot, // so a context without one never acquires a grant. let grant_snapshot = snapshot.as_deref().filter(|snapshot| { @@ -3644,14 +3637,21 @@ where req }; - let Ok(req_with_auth) = crate::l7::token_grant_injection::inject_if_needed_captured( - req, - &mut grant_ctx, - &request_id, - observation_context.as_ref(), - ) - .await - else { + let injected = match crate::l7::token_grant_injection::pin_live_credentials(&mut grant_ctx) + { + Some(snapshot) => { + crate::l7::token_grant_injection::inject_for_selectors( + req, + &grant_ctx, + &snapshot, + &request_id, + observation_context.as_ref(), + ) + .await + } + None => Ok(req), + }; + let Ok(req_with_auth) = injected else { // The refusal event names the credential and request ID. // The sandbox-level grant record does not require an MCP endpoint identity. write_bad_gateway_response(client, Some(&request_id)).await?; @@ -3755,42 +3755,22 @@ mod tests { TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::GatewayRpc, ) - .with_upstream_status(409) } - /// A refused grant returns the generic empty 502 plus a request ID, and no - /// part of the response reveals the classification of `failure`. - pub(super) fn assert_grant_refusal_response( - response: &str, - failure: TokenGrantFailure, - ) -> String { + /// Assert the generic refusal and return its request ID header value. + pub(super) fn grant_refusal_request_id(response: &str) -> String { assert!( response.starts_with("HTTP/1.1 502 Bad Gateway\r\n"), "{response}" ); - assert!(response.contains("Content-Length: 0\r\n"), "{response}"); - let request_id = response + response .lines() .find_map(|line| { - line.strip_prefix(openshell_core::token_grant_failure::REQUEST_ID_HEADER) - .and_then(|value| value.strip_prefix(": ")) + line.strip_prefix(openshell_core::token_grant_failure::REQUEST_ID_HEADER)? + .strip_prefix(": ") }) - .expect("refusal carries the request ID"); - assert!(uuid::Uuid::parse_str(request_id).is_ok(), "{request_id}"); - // The random ID can contain any digits; check everything else. - let without_id = response.replace(request_id, ""); - let status = failure.upstream_status().map(|status| status.to_string()); - let recovery = failure - .recovery() - .as_str_name() - .trim_start_matches("PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_"); - for classification in [failure.code(), failure.stage().code(), recovery] - .into_iter() - .chain(status.as_deref()) - { - assert!(!without_id.contains(classification), "{response}"); - } - request_id.to_string() + .expect("refusal carries the request ID") + .to_string() } #[tokio::test] @@ -5365,11 +5345,11 @@ network_policies: .unwrap() .unwrap(); let response = String::from_utf8_lossy(&response[..n]); - if identity_result.is_ok() { - assert!(response.contains("204 No Content")); + assert!(response.contains(if identity_result.is_ok() { + "204 No Content" } else { - assert_grant_refusal_response(&response, relay_grant_failure()); - } + "502 Bad Gateway" + })); assert!(!response.contains("service-token")); drop(app); tokio::time::timeout(std::time::Duration::from_secs(2), relay) @@ -5496,10 +5476,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response( - &String::from_utf8_lossy(&client_response[..n]), - relay_grant_failure(), - ); + grant_refusal_request_id(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -5619,10 +5596,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response( - &String::from_utf8_lossy(&client_response[..n]), - relay_grant_failure(), - ); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -8044,10 +8018,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response( - &String::from_utf8_lossy(&client_response[..n]), - relay_grant_failure(), - ); + grant_refusal_request_id(&String::from_utf8_lossy(&client_response[..n])); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( @@ -8164,10 +8135,7 @@ network_policies: .await .expect("bad gateway response should reach client") .unwrap(); - assert_grant_refusal_response( - &String::from_utf8_lossy(&client_response[..n]), - relay_grant_failure(), - ); + assert!(String::from_utf8_lossy(&client_response[..n]).contains("502 Bad Gateway")); let mut upstream_request = [0u8; 128]; let n = tokio::time::timeout( diff --git a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs index 033c4a5965..eb34e325b9 100644 --- a/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs +++ b/crates/openshell-supervisor-network/src/l7/relay/token_grant_ownership_tests.rs @@ -16,6 +16,9 @@ use openshell_core::proto::{ ProviderCredentialTokenGrant, ProviderCredentialTokenGrantType, ProviderProfileCredential, }; use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_core::token_grant_failure::{ + TokenGrantFailure, TokenGrantFailureKind, TokenGrantStage, +}; use std::collections::HashMap; use std::future::Future; use std::path::PathBuf; @@ -53,17 +56,8 @@ impl TokenGrantResolver for ScriptedResolver { fn obtain<'a>( &'a self, request: TokenGrantRequest<'a>, - ) -> Pin< - Box< - dyn Future< - Output = std::result::Result< - String, - openshell_core::token_grant_failure::TokenGrantFailure, - >, - > + Send - + 'a, - >, - > { + ) -> Pin> + Send + 'a>> + { Box::pin(async move { self.calls .lock() @@ -80,9 +74,9 @@ impl TokenGrantResolver for ScriptedResolver { .expect("test releases the held grant"); } if self.fail_once.swap(false, Ordering::SeqCst) { - return Err(openshell_core::token_grant_failure::TokenGrantFailure::new( - openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, - openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, + return Err(TokenGrantFailure::new( + TokenGrantFailureKind::Unavailable, + TokenGrantStage::TokenEndpoint, )); } Ok(format!("inert-{}", request.audience)) @@ -688,17 +682,10 @@ async fn refused_grant( Vec, ) { use openshell_core::endpoint_status::{ - EndpointConfigVersion, EndpointInventoryEntry, EndpointStatusCommand, - endpoint_status_channel, + EndpointConfigVersion, EndpointInventoryEntry, endpoint_status_channel, }; - let mut fixture = if protocol == "rest" { - let mut endpoint = route("native", "/native"); - endpoint.method = "POST"; - Fixture::new(&[endpoint], &[("/native", "native", "aud-native")], 1) - } else { - native_protocol_fixture(protocol, false) - }; + let mut fixture = native_protocol_fixture(protocol, false); for config in &mut fixture.configs { // The gateway derives these from policy; OPA fixtures carry none. config.endpoint_id = "endpoint:v1:native".into(); @@ -706,6 +693,10 @@ async fn refused_grant( config.provider_credentialed = provider_credentialed; } let (sender, mut receiver) = endpoint_status_channel(); + let inventory = vec![EndpointInventoryEntry { + endpoint_id: "endpoint:v1:native".into(), + uses_provider_credentials: provider_credentialed, + }]; sender .reset( EndpointConfigVersion { @@ -713,30 +704,21 @@ async fn refused_grant( provider_env_revision: 1, }, if protocol == "mcp" { - vec![EndpointInventoryEntry { - endpoint_id: "endpoint:v1:native".into(), - uses_provider_credentials: provider_credentialed, - }] + inventory } else { Vec::new() }, ) .await .expect("install endpoint inventory"); - assert!(matches!( - receiver.recv().await, - Some(EndpointStatusCommand::Reset { .. }) - )); + receiver.recv().await.expect("reset command"); fixture.observation = Some(sender); fixture.resolver.fail_once.store(true, Ordering::SeqCst); let mut connection = fixture.connect(); - let body = if protocol == "rest" { - Vec::new() - } else { - native_protocol_body(protocol, "echo") - }; - let response = connection.exchange_body("/native", &body).await; + let response = connection + .exchange_body("/native", &native_protocol_body(protocol, "echo")) + .await; let capture = connection.finish().await; assert_eq!(capture.bytes_seen, 0, "failed grant forwards nothing"); let mut commands = Vec::new(); @@ -746,101 +728,50 @@ async fn refused_grant( (response, commands) } -fn refusal_request_id(response: &str) -> String { - tests::assert_grant_refusal_response( - response, - openshell_core::token_grant_failure::TokenGrantFailure::new( - openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, - openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, - ), - ) -} - +/// The workload sees only the generic refusal and a request ID. Trusted status +/// receives the classified cause for that request, plus an MCP endpoint +/// observation only for an inventoried endpoint that uses provider credentials. #[tokio::test] -async fn mcp_grant_failure_reports_its_cause_to_endpoint_status_only() { +async fn grant_refusal_reports_cause_to_endpoint_status_only() { use openshell_core::endpoint_status::EndpointStatusCommand; - use openshell_core::proto::ProviderCredentialRefreshRecoveryAction; - - let (response, commands) = refused_grant("mcp", true).await; - // The workload sees the generic refusal and an opaque request ID. - let request_id = refusal_request_id(&response); - - // The trusted status path receives the classified cause for that request. - let [ - EndpointStatusCommand::ObserveTokenGrant { - result: Err(grant_failure), - .. - }, - EndpointStatusCommand::Observe { - endpoint_id, - result, - failure: Some(failure), - .. - }, - ] = commands.as_slice() - else { - panic!("one grant failure observation with a cause: {commands:?}"); - }; - assert_eq!(endpoint_id, "endpoint:v1:native"); - assert_eq!(*result, EndpointResult::CredentialUnavailable); - let failure = failure.to_proto(); - assert_eq!(failure.code, "unavailable"); - assert_eq!(failure.stage, "token_endpoint"); - assert_eq!( - failure.recovery_action, - ProviderCredentialRefreshRecoveryAction::Retry as i32 - ); - assert_eq!(failure.provider, "native"); - assert_eq!(failure.credential_key, "access_token"); - assert_eq!(failure.request_id, request_id); - assert_eq!(grant_failure.to_proto(), failure); - assert_eq!( - openshell_core::endpoint_failure::validate_endpoint_failure( - &failure, - openshell_core::proto::EndpointResult::CredentialUnavailable - ), - Ok(()) - ); -} - -#[tokio::test] -async fn grant_failure_on_uncredentialed_endpoint_reports_only_sandbox_cause() { - // The gateway rejects credential failures for endpoints that do not use - // provider credentials, so the supervisor must not report one. - let (response, commands) = refused_grant("mcp", false).await; - refusal_request_id(&response); - assert!( - matches!( - commands.as_slice(), - [openshell_core::endpoint_status::EndpointStatusCommand::ObserveTokenGrant { .. }] - ), - "{commands:?}" - ); -} -#[tokio::test] -async fn non_mcp_grant_failures_report_without_an_endpoint_inventory() { - for protocol in ["rest", "graphql", "json-rpc"] { - let (response, commands) = refused_grant(protocol, true).await; - let request_id = refusal_request_id(&response); - let [ - openshell_core::endpoint_status::EndpointStatusCommand::ObserveTokenGrant { - destination_host, - destination_port, - result: Err(failure), + for (protocol, credentialed, endpoint_observed) in [ + ("mcp", true, true), + ("mcp", false, false), + ("graphql", true, false), + ] { + let (response, commands) = refused_grant(protocol, credentialed).await; + let request_id = tests::grant_refusal_request_id(&response); + let Some(( + EndpointStatusCommand::ObserveTokenGrant { + result: Err(grant_failure), .. }, - ] = commands.as_slice() + rest, + )) = commands.split_first() else { - panic!("{protocol}: bounded grant cause without MCP inventory: {commands:?}"); + panic!("{protocol}: grant failure observation first: {commands:?}"); }; - assert_eq!(destination_host, HOST); - assert_eq!(*destination_port, PORT); - let failure = failure.to_proto(); - assert_eq!(failure.request_id, request_id); - assert_eq!(failure.provider, "native"); - assert_eq!(failure.credential_key, "access_token"); - assert_eq!(failure.code, "unavailable"); + let grant_failure = grant_failure.to_proto(); + assert_eq!(grant_failure.request_id, request_id); + assert_eq!(grant_failure.provider, "native"); + assert_eq!(grant_failure.credential_key, "access_token"); + assert_eq!(grant_failure.code, "unavailable"); + match rest { + [] => assert!(!endpoint_observed, "{protocol}: {commands:?}"), + [ + EndpointStatusCommand::Observe { + result, + failure: Some(failure), + .. + }, + ] => { + assert!(endpoint_observed, "{protocol}: {commands:?}"); + assert_eq!(*result, EndpointResult::CredentialUnavailable); + assert_eq!(failure.to_proto(), grant_failure); + } + _ => panic!("{protocol}: unexpected status commands: {commands:?}"), + } } } diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 19942d18e9..1319e86100 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -15,8 +15,6 @@ use std::sync::Arc; use openshell_core::dynamic_credential_key::{credential_identity, endpoint_selector}; use openshell_core::endpoint_failure::EndpointFailure; use openshell_core::endpoint_status::EndpointObservationContext; -#[cfg(test)] -use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::proto::{ProviderCredentialTokenGrant, ProviderProfileCredential}; use openshell_core::provider_credentials::{ProviderCredentialSnapshot, ProviderCredentialState}; use openshell_core::token_grant_failure::{ @@ -148,57 +146,45 @@ impl std::error::Error for TokenGrantInjectionError {} impl miette::Diagnostic for TokenGrantInjectionError {} -/// Resolves one endpoint-bound grant per protected header before rewriting a request. +/// Pin `ctx` to the live provider installation and return its snapshot. /// -/// Each header independently uses its most-specific matching binding. Every selected -/// grant must succeed; callers must not forward the request when this returns an error. -/// `request_id` is the supervisor-minted ID of this exchange; it is sent to the -/// token service and gateway and recorded on the refusal event. -#[cfg(test)] -async fn inject_if_needed( - req: L7Request, - ctx: &L7EvalContext, - request_id: &str, -) -> Result { - let observation_context = ctx - .endpoint_observation_tx +/// A connection can sit idle across a refresh before its first request, so a +/// grant always uses the installation current when the request arrives. The +/// pinned revision then scopes the injected keys, the status report and the +/// credential generation guard for the upstream write. +pub fn pin_live_credentials(ctx: &mut L7EvalContext) -> Option> { + let snapshot = ctx + .provider_credentials .as_ref() - .and_then(EndpointObservationSender::capture); - inject_if_needed_captured( - req, - &mut ctx.clone(), - request_id, - observation_context.as_ref(), - ) - .await + .map(ProviderCredentialState::snapshot)?; + ctx.provider_credential_revision = Some(snapshot.revision); + ctx.provider_credential_installation_id = Some(snapshot.installation_id.clone()); + Some(snapshot) } -/// Use authority captured before the proxy selected policy or awaited input. -pub async fn inject_if_needed_captured( +/// Inject the grants whose endpoint selectors match the request. +/// +/// L4-only forwarding uses this selector contract; it has no per-request L7 +/// admission decision. Each header independently uses its most-specific +/// matching binding, and every selected grant must succeed: callers must not +/// forward the request when this returns an error. `ctx` must be pinned to +/// `snapshot` by [`pin_live_credentials`]. `request_id` is the supervisor-minted +/// ID of this exchange; it is sent to the token service and gateway and +/// recorded on the refusal event. +pub async fn inject_for_selectors( req: L7Request, - ctx: &mut L7EvalContext, + ctx: &L7EvalContext, + snapshot: &ProviderCredentialSnapshot, request_id: &str, observation_context: Option<&EndpointObservationContext>, ) -> Result { - // A connection can sit idle across a refresh before its first request. - // Select and pin the live installation. - let Some(snapshot) = ctx - .provider_credentials - .as_ref() - .map(ProviderCredentialState::snapshot) - else { - return Ok(req); - }; - ctx.provider_credential_revision = Some(snapshot.revision); - ctx.provider_credential_installation_id = Some(snapshot.installation_id.clone()); - let selected = select_scoped(&snapshot, ctx, request_path(&req.target), None); + let selected = select_scoped(snapshot, ctx, request_path(&req.target), None); inject_or_report(req, ctx, selected, request_id, observation_context).await } /// Inject only grants owned by endpoints that admitted the inspected request. /// An empty owner set permits no acquisition, including audit-only forwarding. -/// L4-only forwarding uses `inject_if_needed_captured` with its existing -/// endpoint-selector contract; it has no per-request L7 admission decision. +/// L4-only forwarding uses [`inject_for_selectors`] instead. /// /// Credentials come from the pinned `snapshot`, and each selected key is scoped /// to its revision and installation after selection so the one-grant-per-header @@ -731,6 +717,7 @@ fn inject_header( #[cfg(test)] pub mod test_support { use super::*; + use openshell_core::endpoint_status::EndpointObservationSender; use openshell_core::proto::{ ProviderCredentialTokenGrant, ProviderCredentialTokenGrantSubjectToken, ProviderCredentialTokenGrantType, ProviderProfileCredential, @@ -913,14 +900,17 @@ pub mod test_support { /// Installs this fixture's credentials as a pinned provider-credential /// snapshot. Admission-gated injection requests grants by revision-scoped /// key, so each scripted response is also registered under that key. - pub fn provider_credentials(&self) -> ProviderCredentialState { + pub fn provider_credentials( + &self, + ) -> openshell_core::provider_credentials::ProviderCredentialState { let credentials = self.dynamic_credentials.read().unwrap().clone(); - let state = ProviderCredentialState::from_environment( - 1, - HashMap::new(), - HashMap::new(), - credentials.clone(), - ); + let state = + openshell_core::provider_credentials::ProviderCredentialState::from_environment( + 1, + HashMap::new(), + HashMap::new(), + credentials.clone(), + ); let snapshot = state.snapshot(); let mut responses = self.responses.lock().unwrap(); for key in credentials.keys() { @@ -1069,13 +1059,38 @@ pub mod test_support { }) } } + + /// Pin a copy of `ctx` and inject selector-matched grants, as the + /// fallback forwarding path does, capturing the context's own authority. + pub async fn inject_if_needed( + req: L7Request, + ctx: &L7EvalContext, + request_id: &str, + ) -> Result { + let observation_context = ctx + .endpoint_observation_tx + .as_ref() + .and_then(EndpointObservationSender::capture); + let mut pinned = ctx.clone(); + let Some(snapshot) = pin_live_credentials(&mut pinned) else { + return Ok(req); + }; + inject_for_selectors( + req, + &pinned, + &snapshot, + request_id, + observation_context.as_ref(), + ) + .await + } } #[cfg(test)] mod tests { use super::*; use crate::l7::provider::{BodyLength, L7Request}; - use crate::l7::token_grant_injection::test_support::TokenGrantTestFixture; + use crate::l7::token_grant_injection::test_support::{TokenGrantTestFixture, inject_if_needed}; use openshell_core::endpoint_status::{ EndpointConfigVersion, EndpointStatusReceiver, EndpointStatusTracker, endpoint_status_channel, @@ -1302,9 +1317,7 @@ mod tests { ) .expect_err("malformed token must not be injected"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(err.stage(), TokenGrantStage::Injection); - assert_eq!(err.to_string(), "dynamic token grant failed"); } #[tokio::test] @@ -1418,34 +1431,16 @@ mod tests { assert_eq!(err.failure().stage(), TokenGrantStage::Injection); assert_eq!(err.credential(), Some(("provider", "access_token"))); - assert_eq!(err.to_string(), "dynamic token grant failed"); fixture.assert_one_request("api.example.com\t443\t/v1/**\tprovider:access_token"); } - #[tokio::test] - async fn refused_grant_keeps_resolver_classification_and_failed_credential() { - let failure = TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::GatewayRpc, - ); - let fixture = TokenGrantTestFixture::failure_token_exchange( - "api.example.com\t443\t/v1/**\tprovider:access_token", - failure, - ); + const KEY: &str = "api.example.com\t443\t/v1/**\tprovider:access_token"; - let err = inject_if_needed( - grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), - &grant_context(&fixture), - REQUEST_ID, + fn refusal() -> TokenGrantFailure { + TokenGrantFailure::new( + TokenGrantFailureKind::ProviderAuthorizationRequired, + TokenGrantStage::TokenEndpoint, ) - .await - .expect_err("failed grant refuses the request"); - - assert_eq!(err.failure(), failure); - assert_eq!(err.credential(), Some(("provider", "access_token"))); - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{err:?}"), "dynamic token grant failed"); - assert_eq!(fixture.requested_ids(), [REQUEST_ID]); } fn grant_context(fixture: &TokenGrantTestFixture) -> L7EvalContext { @@ -1458,142 +1453,104 @@ mod tests { } } - fn grant_request(raw_header: &[u8]) -> L7Request { + fn grant_request() -> L7Request { L7Request { action: "GET".to_string(), target: "/v1/projects".to_string(), query_params: HashMap::new(), - raw_header: raw_header.to_vec(), + raw_header: b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n".to_vec(), body_length: BodyLength::None, } } - #[tokio::test] - async fn malformed_request_framing_fails_before_acquisition() { - for raw_header in [ - b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com".as_slice(), - b"GET /v1/projects HTTP/1.1\r\nX-Secret: \xff\r\n\r\n".as_slice(), - b"\r\n\r\n".as_slice(), - ] { - let fixture = TokenGrantTestFixture::success( - "api.example.com\t443\t/v1/**\tprovider:access_token", - "grant-token", - ); - let err = inject_if_needed( - grant_request(raw_header), - &grant_context(&fixture), - REQUEST_ID, - ) - .await - .expect_err("invalid request framing refuses the request"); - assert_eq!(err.failure(), injection_configuration_failure()); - fixture.assert_no_requests(); - } + async fn installed_status( + revision: u64, + ) -> ( + EndpointObservationSender, + EndpointStatusReceiver, + EndpointStatusTracker, + EndpointObservationContext, + ) { + let (sender, mut receiver) = endpoint_status_channel(); + let mut tracker = receiver.tracker(); + reset_status(&sender, revision).await; + assert!(tracker.apply(receiver.try_recv().unwrap())); + let authority = sender.capture().unwrap(); + (sender, receiver, tracker, authority) + } + + async fn reset_status(sender: &EndpointObservationSender, revision: u64) { + let version = EndpointConfigVersion { + policy_hash: "policy".into(), + provider_env_revision: revision, + }; + sender.reset(version, Vec::new()).await.unwrap(); + } + + /// Apply the next status command and return the recorded grant failure. + fn next_grant_failure( + tracker: &mut EndpointStatusTracker, + receiver: &mut EndpointStatusReceiver, + ) -> Option { + assert!(tracker.apply(receiver.try_recv().expect("status command sent"))); + tracker.snapshot().unwrap().last_token_grant_failure } #[tokio::test] async fn missing_resolver_refuses_without_naming_a_credential() { - let fixture = TokenGrantTestFixture::success( - "api.example.com\t443\t/v1/**\tprovider:access_token", - "grant-token", - ); + let fixture = TokenGrantTestFixture::success(KEY, "grant-token"); let (sender, mut receiver, mut tracker, _) = installed_status(1).await; let mut ctx = grant_context(&fixture); ctx.token_grant_resolver = None; ctx.endpoint_observation_tx = Some(sender); - let err = inject_if_needed( - grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), - &ctx, - REQUEST_ID, - ) - .await - .expect_err("a grant without a resolver refuses the request"); + let err = inject_if_needed(grant_request(), &ctx, REQUEST_ID) + .await + .expect_err("a grant without a resolver refuses the request"); - assert_eq!(err.failure(), injection_configuration_failure()); assert_eq!(err.credential(), None); fixture.assert_no_requests(); - assert!(tracker.apply(receiver.try_recv().unwrap())); - let failure = tracker - .snapshot() - .unwrap() - .last_token_grant_failure + let failure = next_grant_failure(&mut tracker, &mut receiver) .unwrap() .failure .unwrap(); assert_eq!(failure.code, injection_configuration_failure().code()); assert_eq!(failure.request_id, REQUEST_ID); assert!(failure.provider.is_empty()); - assert!(failure.credential_key.is_empty()); } - async fn installed_status( - revision: u64, - ) -> ( - EndpointObservationSender, - EndpointStatusReceiver, - EndpointStatusTracker, - EndpointObservationContext, - ) { - let (sender, mut receiver) = endpoint_status_channel(); - let mut tracker = receiver.tracker(); - sender - .reset( - EndpointConfigVersion { - policy_hash: "policy".into(), - provider_env_revision: revision, - }, - Vec::new(), - ) - .await - .unwrap(); - assert!(tracker.apply(receiver.try_recv().unwrap())); - let authority = sender.capture().unwrap(); - (sender, receiver, tracker, authority) + async fn inject_pinned( + ctx: &mut L7EvalContext, + authority: &EndpointObservationContext, + ) -> Result { + let snapshot = pin_live_credentials(ctx).expect("live provider credentials"); + inject_for_selectors(grant_request(), ctx, &snapshot, REQUEST_ID, Some(authority)).await } #[tokio::test] async fn fallback_reports_live_credential_refusal() { const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; - let failure = TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::TokenEndpoint, - ); - let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, refusal()); let mut ctx = grant_context(&fixture); let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); let (sender, mut receiver, mut tracker, authority) = installed_status(snapshot.revision).await; ctx.endpoint_observation_tx = Some(sender); - let error = inject_if_needed_captured( - grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), - &mut ctx, - REQUEST_ID, - Some(&authority), - ) - .await - .expect_err("the live credential refuses"); - assert_eq!(error.failure(), failure); + let error = inject_pinned(&mut ctx, &authority) + .await + .expect_err("the live credential refuses"); + assert_eq!( error.credential(), Some(("current-provider", "access_token")) ); assert_eq!(ctx.provider_credential_revision, Some(snapshot.revision)); - assert_eq!( - ctx.provider_credential_installation_id.as_deref(), - Some(snapshot.installation_id.as_str()) - ); fixture.assert_requested_keys(&[CURRENT_KEY]); - assert!(tracker.apply(receiver.try_recv().expect("live refusal is reported"))); - let observation = tracker - .snapshot() - .unwrap() - .last_token_grant_failure + let cause = next_grant_failure(&mut tracker, &mut receiver) + .expect("live refusal is reported") + .failure .unwrap(); - assert_eq!(observation.destination_host, "api.example.com"); - assert_eq!(observation.destination_port, 443); - let cause = observation.failure.unwrap(); assert_eq!(cause.provider, "current-provider"); assert_eq!(cause.credential_key, "access_token"); assert_eq!(cause.request_id, REQUEST_ID); @@ -1602,111 +1559,48 @@ mod tests { #[tokio::test] async fn fallback_live_credential_success_clears_its_failure() { const CURRENT_KEY: &str = "api.example.com\t443\t/v1/projects\tprovider:access_token"; - let failure = TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::TokenEndpoint, - ); - let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, refusal()); let mut ctx = grant_context(&fixture); let snapshot = ctx.provider_credentials.as_ref().unwrap().snapshot(); let (sender, mut receiver, mut tracker, authority) = installed_status(snapshot.revision).await; ctx.endpoint_observation_tx = Some(sender); - let raw_header = b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"; - let error = inject_if_needed_captured( - grant_request(raw_header), - &mut ctx, - REQUEST_ID, - Some(&authority), - ) - .await - .expect_err("current grant first records its failure"); - assert_eq!(error.failure(), failure); - assert!(tracker.apply(receiver.try_recv().unwrap())); - let observation = tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .unwrap(); - assert_eq!(observation.failure.unwrap().provider, "provider"); + inject_pinned(&mut ctx, &authority) + .await + .expect_err("current grant first records its failure"); + assert!(next_grant_failure(&mut tracker, &mut receiver).is_some()); fixture.add_credential( &snapshot.scoped_key(CURRENT_KEY), snapshot.dynamic_credentials[CURRENT_KEY].clone(), Ok("current-token"), ); - let injected = inject_if_needed_captured( - grant_request(raw_header), - &mut ctx, - REQUEST_ID, - Some(&authority), - ) - .await - .expect("the live credential now succeeds"); + let injected = inject_pinned(&mut ctx, &authority) + .await + .expect("the live credential now succeeds"); assert!( String::from_utf8(injected.raw_header) .unwrap() .contains("Authorization: Bearer current-token\r\n") ); - fixture.assert_requested_keys(&[CURRENT_KEY, CURRENT_KEY]); - assert!(tracker.apply(receiver.try_recv().unwrap())); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() - ); + assert!(next_grant_failure(&mut tracker, &mut receiver).is_none()); } #[tokio::test] async fn fallback_rejects_telemetry_authority_captured_before_provider_refresh() { - const CURRENT_KEY: &str = "api.example.com\t443\t/v1/**\tcurrent-provider:access_token"; - let failure = TokenGrantFailure::new( - TokenGrantFailureKind::ProviderAuthorizationRequired, - TokenGrantStage::TokenEndpoint, - ); - let fixture = TokenGrantTestFixture::failure(CURRENT_KEY, failure); + let fixture = TokenGrantTestFixture::failure(KEY, refusal()); let mut ctx = grant_context(&fixture); - let (sender, mut receiver, mut tracker, accepted_authority) = installed_status(0).await; - sender - .reset( - EndpointConfigVersion { - policy_hash: "policy".into(), - provider_env_revision: 1, - }, - Vec::new(), - ) - .await - .unwrap(); - assert!(tracker.apply(receiver.try_recv().unwrap())); + let (sender, mut receiver, _, stale_authority) = installed_status(0).await; + reset_status(&sender, 1).await; + receiver.try_recv().unwrap(); ctx.endpoint_observation_tx = Some(sender); - let error = inject_if_needed_captured( - grant_request(b"GET /v1/projects HTTP/1.1\r\nHost: api.example.com\r\n\r\n"), - &mut ctx, - REQUEST_ID, - Some(&accepted_authority), - ) - .await - .expect_err("the live grant still fails closed under old reporting authority"); - assert_eq!(error.failure(), failure); - assert_eq!( - error.credential(), - Some(("current-provider", "access_token")) - ); - fixture.assert_requested_keys(&[CURRENT_KEY]); - assert!(matches!( - receiver.try_recv(), - Err(tokio::sync::mpsc::error::TryRecvError::Empty) - )); - assert!( - tracker - .snapshot() - .unwrap() - .last_token_grant_failure - .is_none() - ); + inject_pinned(&mut ctx, &stale_authority) + .await + .expect_err("the live grant still fails closed under old reporting authority"); + + fixture.assert_requested_keys(&[KEY]); + assert!(receiver.try_recv().is_err(), "stale observation is dropped"); } } diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs index 0eb8741252..b4c3253672 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection/multiple_grants.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use super::test_support::TokenGrantTestFixture; +use super::test_support::{TokenGrantTestFixture, inject_if_needed}; use super::*; use crate::l7::provider::BodyLength; use std::collections::HashMap; @@ -144,8 +144,6 @@ async fn rejects_tied_credentials_for_same_header_before_acquisition() { .await .unwrap_err(); assert_eq!(error.failure().kind(), TokenGrantFailureKind::Configuration); - assert_eq!(error.failure().stage(), TokenGrantStage::Injection); - assert!(error.credential().is_some()); fixture.assert_no_requests(); } } @@ -167,7 +165,6 @@ async fn rejects_malformed_second_token_after_first_grant_succeeds() { let error = inject_if_needed(request(), &ctx, REQUEST_ID) .await .unwrap_err(); - assert_eq!(error.failure().stage(), TokenGrantStage::Injection); assert_eq!(error.credential(), Some(("provider", "identity"))); fixture.assert_requested_keys(&[SERVICE, IDENTITY]); } @@ -212,8 +209,7 @@ async fn second_grant_failure_redacts_errors_and_emits_no_success() { let failure = TokenGrantFailure::new( TokenGrantFailureKind::ProviderAuthorizationRequired, TokenGrantStage::GatewayRpc, - ) - .with_upstream_status(409); + ); let (fixture, ctx) = fixture(Err(failure)); let logs = Arc::new(Mutex::new(Vec::new())); let events = Arc::new(Mutex::new(Vec::new())); @@ -230,23 +226,16 @@ async fn second_grant_failure_redacts_errors_and_emits_no_success() { .with(openshell_ocsf::OcsfShorthandLayer::new(Capture( shorthand.clone(), ))); - let error = inject_if_needed(request(), &ctx, REQUEST_ID) + inject_if_needed(request(), &ctx, REQUEST_ID) .with_subscriber(subscriber) .await .unwrap_err(); - assert_eq!(error.failure(), failure); - assert_eq!(error.to_string(), "dynamic token grant failed"); let logs = String::from_utf8(logs.lock().unwrap().clone()).unwrap(); let events = String::from_utf8(events.lock().unwrap().clone()).unwrap(); let shorthand = String::from_utf8(shorthand.lock().unwrap().clone()).unwrap(); - // The policy advisor serves the agent exactly these shorthand lines. - let denials = shorthand - .lines() - .filter(|line| line.contains(" OCSF ") && line.contains(" DENIED ")) - .collect::>() - .join("\n"); + // The policy advisor serves the agent these shorthand lines. assert!( - denials.contains("Token grant failed for provider:identity"), + shorthand.contains("Token grant failed for provider:identity"), "{shorthand}" ); // Every sink names the exchange by request ID. Only the operator log line @@ -257,18 +246,11 @@ async fn second_grant_failure_redacts_errors_and_emits_no_success() { assert!(!output.contains("Token grant successful")); } assert!(logs.contains(failure.code()), "{logs}"); - for output in [&events, &shorthand, &denials] { - for classification in [ - failure.code(), - failure.stage().code(), - "upstream_http_status", - "REAUTHORIZE", - ] { - assert!( - !output.contains(classification), - "classification {classification} leaked: {output}" - ); - } + for output in [&events, &shorthand] { + assert!( + !output.contains(failure.code()), + "classification leaked: {output}" + ); } fixture.assert_requested_keys(&[SERVICE, IDENTITY]); } diff --git a/crates/openshell-supervisor-network/src/proxy.rs b/crates/openshell-supervisor-network/src/proxy.rs index bb4f3146c6..7de435869a 100644 --- a/crates/openshell-supervisor-network/src/proxy.rs +++ b/crates/openshell-supervisor-network/src/proxy.rs @@ -5215,9 +5215,13 @@ async fn inject_token_grant_for_forward_request( // L4-only forwarding keeps selector-based credentials; it has no inspected // endpoint owner decision. Inspected requests always use preparation above. - Ok(crate::l7::token_grant_injection::inject_if_needed_captured( + let Some(snapshot) = crate::l7::token_grant_injection::pin_live_credentials(l7_ctx) else { + return Ok(request.raw_header); + }; + Ok(crate::l7::token_grant_injection::inject_for_selectors( request, l7_ctx, + &snapshot, request_id, observation_context, ) @@ -10951,19 +10955,34 @@ network_policies: assert!(String::from_utf8_lossy(&response).contains(&header)); } + type ForwardGrantResponse<'a> = + std::result::Result<&'a str, openshell_core::token_grant_failure::TokenGrantFailure>; + fn forward_grant_failure() -> openshell_core::token_grant_failure::TokenGrantFailure { openshell_core::token_grant_failure::TokenGrantFailure::new( openshell_core::token_grant_failure::TokenGrantFailureKind::Unavailable, openshell_core::token_grant_failure::TokenGrantStage::TokenEndpoint, ) - .with_upstream_status(503) + } + + async fn forward_inject( + ctx: &mut crate::l7::relay::L7EvalContext, + raw: Vec, + ) -> Result> { + inject_token_grant_for_forward_request( + "GET", + "/v1/projects", + raw, + ctx, + None, + FORWARD_REQUEST_ID, + None, + ) + .await } fn forward_token_grant_context( - resolver_response: std::result::Result< - &str, - openshell_core::token_grant_failure::TokenGrantFailure, - >, + resolver_response: ForwardGrantResponse<'_>, ) -> ( crate::l7::relay::L7EvalContext, crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, @@ -10988,10 +11007,7 @@ network_policies: } fn forward_token_exchange_context( - resolver_response: std::result::Result< - &str, - openshell_core::token_grant_failure::TokenGrantFailure, - >, + resolver_response: ForwardGrantResponse<'_>, ) -> ( crate::l7::relay::L7EvalContext, crate::l7::token_grant_injection::test_support::TokenGrantTestFixture, @@ -11007,10 +11023,7 @@ network_policies: fn forward_token_grant_fixture( provider_key: &str, - resolver_response: std::result::Result< - &str, - openshell_core::token_grant_failure::TokenGrantFailure, - >, + resolver_response: ForwardGrantResponse<'_>, token_exchange: bool, ) -> crate::l7::token_grant_injection::test_support::TokenGrantTestFixture { match (resolver_response, token_exchange) { @@ -13196,17 +13209,9 @@ network_policies: let (mut ctx, fixture) = forward_token_grant_context(Ok("grant-token")); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer stale-token\r\nConnection: close\r\n\r\n".to_vec(); - let with_token = inject_token_grant_for_forward_request( - "GET", - "/v1/projects", - raw, - &mut ctx, - None, - FORWARD_REQUEST_ID, - None, - ) - .await - .expect("forward token grant should inject"); + let with_token = forward_inject(&mut ctx, raw) + .await + .expect("forward token grant should inject"); let rewritten = rewrite_forward_request( &with_token, with_token.len(), @@ -13229,17 +13234,9 @@ network_policies: let (mut ctx, fixture) = forward_token_exchange_context(Ok("grant-token")); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nAuthorization: Bearer stale-token\r\nConnection: close\r\n\r\n".to_vec(); - let with_token = inject_token_grant_for_forward_request( - "GET", - "/v1/projects", - raw, - &mut ctx, - None, - FORWARD_REQUEST_ID, - None, - ) - .await - .expect("forward token exchange should inject"); + let with_token = forward_inject(&mut ctx, raw) + .await + .expect("forward token exchange should inject"); let rewritten = rewrite_forward_request( &with_token, with_token.len(), @@ -13264,24 +13261,15 @@ network_policies: let (mut ctx, fixture) = forward_token_grant_context(Err(forward_grant_failure())); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - let err = inject_token_grant_for_forward_request( - "GET", - "/v1/projects", - raw, - &mut ctx, - None, - FORWARD_REQUEST_ID, - None, - ) - .await - .expect_err("forward token grant failure should stop request rewriting"); + let err = forward_inject(&mut ctx, raw) + .await + .expect_err("forward token grant failure should stop request rewriting"); + // The forward proxy answers a typed refusal with the request ID header. let refusal = err .downcast_ref::() .expect("typed grant refusal"); assert_eq!(refusal.failure(), forward_grant_failure()); - assert_eq!(refusal.credential(), Some(("provider", "access_token"))); - assert_eq!(err.to_string(), "dynamic token grant failed"); fixture.assert_one_request("api.example.test\t8080\t/v1/**\tprovider:access_token"); } @@ -13290,17 +13278,9 @@ network_policies: let (mut ctx, fixture) = forward_token_exchange_context(Err(forward_grant_failure())); let raw = b"GET http://api.example.test:8080/v1/projects HTTP/1.1\r\nHost: api.example.test:8080\r\nConnection: close\r\n\r\n".to_vec(); - inject_token_grant_for_forward_request( - "GET", - "/v1/projects", - raw, - &mut ctx, - None, - FORWARD_REQUEST_ID, - None, - ) - .await - .expect_err("forward token exchange failure should stop request rewriting"); + forward_inject(&mut ctx, raw) + .await + .expect_err("forward token exchange failure should stop request rewriting"); fixture.assert_one_token_exchange_request( "api.example.test\t8080\t/v1/**\tprovider:access_token", diff --git a/crates/openshell-supervisor-network/src/token_grant.rs b/crates/openshell-supervisor-network/src/token_grant.rs index a1a2ca6635..e165754105 100644 --- a/crates/openshell-supervisor-network/src/token_grant.rs +++ b/crates/openshell-supervisor-network/src/token_grant.rs @@ -945,92 +945,29 @@ mod tests { .await .expect_err("invalid local configuration must fail closed"); assert_eq!(failure, configuration_failure()); - assert_eq!(failure.to_string(), "dynamic token grant failed"); - assert!(std::error::Error::source(&failure).is_none()); } assert_eq!(grant_calls.load(Ordering::SeqCst), 0); } #[test] - fn workload_api_failures_discard_arbitrary_diagnostics() { + fn workload_api_failures_are_classified() { for (raw, kind) in [ ( WorkloadApiError::PermissionDenied("secret-spiffe-canary".to_string()), TokenGrantFailureKind::WorkloadIdentityRejected, ), ( - WorkloadApiError::NoIdentityIssued, - TokenGrantFailureKind::WorkloadIdentityRejected, - ), - ( - WorkloadApiError::HintNotFound("secret-jwt-canary".to_string()), - TokenGrantFailureKind::Unclassified, - ), - ( - WorkloadApiError::from(tonic::Status::deadline_exceeded("secret-timeout-canary")), + WorkloadApiError::from(tonic::Status::deadline_exceeded("secret-canary")), TokenGrantFailureKind::Timeout, ), ( - WorkloadApiError::from(tonic::Status::unavailable("secret-socket-canary")), + WorkloadApiError::from(tonic::Status::unavailable("secret-canary")), TokenGrantFailureKind::Unavailable, ), - ( - WorkloadApiError::from(tonic::Status::unknown("secret-rpc-canary")), - TokenGrantFailureKind::Unclassified, - ), ] { let failure = workload_api_failure(raw); assert_eq!(failure.kind(), kind); assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); - assert_eq!(failure.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); - assert!(std::error::Error::source(&failure).is_none()); - } - } - - #[tokio::test] - async fn spiffe_connection_failure_does_not_expose_socket_or_audience() { - let failure = fetch_jwt_svid_from_socket( - "/private/tmp/openshell-nonexistent-secret-socket-canary/agent.sock", - "secret-audience-canary", - ) - .await - .expect_err("missing socket must fail closed"); - assert_eq!(failure.kind(), TokenGrantFailureKind::Unavailable); - assert_eq!(failure.stage(), TokenGrantStage::WorkloadIdentity); - assert_eq!(failure.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{failure:?}"), "dynamic token grant failed"); - assert!(std::error::Error::source(&failure).is_none()); - } - - #[tokio::test] - async fn grant_failure_preserves_classification_stage_and_status() { - let cache = TokenCache::new(); - for stage in [ - TokenGrantStage::IntermediateExchange, - TokenGrantStage::TokenEndpoint, - ] { - let expected = - TokenGrantFailure::new(TokenGrantFailureKind::ProviderAuthorizationRequired, stage) - .with_upstream_status(409); - let failure = obtain_provider_token_with_grant( - ObtainProviderTokenInput { - cache: &cache, - provider_name: "provider:access_token", - token_endpoint: "https://auth.example.com/token", - jwt_svid_audience: "", - client_assertion_type: "", - audience: "", - scopes: &[], - cache_ttl_override: None, - grant_type: ProviderCredentialTokenGrantType::ClientCredentials, - requested_token_type: "", - }, - |_| async { Err(expected) }, - ) - .await - .expect_err("grant must fail"); - assert_eq!(failure, expected); } } @@ -1572,8 +1509,6 @@ mod tests { .expect_err("malformed access token should fail closed"); let _request = request.await.expect("token endpoint task should finish"); - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(err.upstream_status(), Some(200)); } @@ -1586,8 +1521,6 @@ mod tests { .expect_err("redirect response should fail closed"); let _request = request.await.expect("token endpoint task should finish"); - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(err.upstream_status(), Some(302)); } @@ -1612,10 +1545,10 @@ mod tests { } #[tokio::test] - async fn perform_token_grant_discards_oauth_error_description() { + async fn perform_token_grant_reports_sanitized_oauth_error_response() { let (endpoint, request) = token_endpoint_once( "401 Unauthorized", - r#"{"error":"invalid_client","error_description":"secret-jwt-assertion-canary"}"#, + r#"{"error":"invalid_client","error_description":"bad jwt assertion"}"#, ) .await; @@ -1635,11 +1568,8 @@ mod tests { request.form.get("audience").map(String::as_str), Some("api://resource") ); - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(format!("{err:?}"), "dynamic token grant failed"); - assert!(std::error::Error::source(&err).is_none()); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(err.upstream_status(), Some(401)); + assert!(!format!("{err:?}").contains("bad jwt assertion")); } #[tokio::test] @@ -1672,8 +1602,6 @@ mod tests { .expect_err("malformed token response should fail"); let _request = request.await.expect("token endpoint task should finish"); - assert_eq!(err.to_string(), "dynamic token grant failed"); - assert_eq!(err.kind(), TokenGrantFailureKind::Unclassified); assert_eq!(err.upstream_status(), Some(200)); } } diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index f7b3ddb40a..e56d38b683 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -687,7 +687,7 @@ The stage names the step that observed the failure: `configuration`, `workload_i - Without a recognized extension code, any other failed HTTP status is `unclassified`, including 401, 403, and 409. OpenShell never infers `provider_authorization_required` from a 401. - For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these causes only over its verified TLS connection to the gateway. -OpenShell keeps only the code, stage, and HTTP status. It discards response descriptions, headers, endpoint diagnostics, and error chains. A successful response without a nonempty header-compatible `access_token` is `unclassified`. +Sandbox status and the workload response keep only the code, stage, and HTTP status. When a token service returns an `OAuth` error body, the log of the component that called it, the gateway or the supervisor, also records its `error` and `error_description`, limited to 256 characters each, with the request ID. The workload cannot read either log. A successful response without a nonempty header-compatible `access_token` is `unclassified`. A token service can also state a cause by adopting the `openshell.token-grant.v1` error extension. For example, a service that requires the user to authorize a provider returns HTTP 409 with: diff --git a/e2e/rust/Cargo.lock b/e2e/rust/Cargo.lock index f68c7dade5..b5403e2ac3 100644 --- a/e2e/rust/Cargo.lock +++ b/e2e/rust/Cargo.lock @@ -1590,7 +1590,6 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", - "tokio-rustls", "tokio-stream", "tower", "tower-layer", diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 47c7387518..01315baef5 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -256,7 +256,7 @@ rustls-pemfile = "2" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yml = { package = "noyalib", version = "0.0.53", default-features = false, features = ["std", "compat-serde-yaml"] } -tonic = { version = "0.14", features = ["transport", "tls-aws-lc"] } +tonic = { version = "0.14", features = ["transport"] } tonic-prost = "0.14" tokio-rustls = { version = "0.26", default-features = false, features = ["logging", "tls12", "aws_lc_rs"] } tower = "0.5" diff --git a/e2e/rust/src/harness/gateway.rs b/e2e/rust/src/harness/gateway.rs index 849807e8b0..edb6f4570a 100644 --- a/e2e/rust/src/harness/gateway.rs +++ b/e2e/rust/src/harness/gateway.rs @@ -22,7 +22,6 @@ pub struct ManagedGateway { log: PathBuf, pid_file: PathBuf, supervisor_image: Option, - native_ca_bundle: Option, } impl ManagedGateway { @@ -39,7 +38,6 @@ impl ManagedGateway { log: log.into(), pid_file: pid_file.into(), supervisor_image: None, - native_ca_bundle: None, } } @@ -58,7 +56,6 @@ impl ManagedGateway { log: env_path("OPENSHELL_E2E_GATEWAY_LOG")?, pid_file: env_path("OPENSHELL_E2E_GATEWAY_PID_FILE")?, supervisor_image: None, - native_ca_bundle: None, })) } @@ -67,11 +64,6 @@ impl ManagedGateway { self.supervisor_image = Some(image.to_owned()); } - /// Set the native TLS roots for this fixture's gateway child process. - pub fn set_native_ca_bundle(&mut self, path: impl Into) { - self.native_ca_bundle = Some(path.into()); - } - /// Start the gateway if it is not already running. pub fn start(&self) -> Result<(), String> { if let Some(pid) = self.current_pid()? { @@ -104,9 +96,6 @@ impl ManagedGateway { if let Some(image) = &self.supervisor_image { command.env("OPENSHELL_SUPERVISOR_IMAGE", image); } - if let Some(bundle) = &self.native_ca_bundle { - command.env("SSL_CERT_FILE", bundle); - } let child = command .spawn() .map_err(|err| format!("start openshell-gateway '{}': {err}", self.bin.display()))?; diff --git a/e2e/rust/tests/provider_token_exchange.rs b/e2e/rust/tests/provider_token_exchange.rs index 017ba10e50..7493a8fe89 100644 --- a/e2e/rust/tests/provider_token_exchange.rs +++ b/e2e/rust/tests/provider_token_exchange.rs @@ -3,9 +3,6 @@ #![cfg(feature = "e2e-podman")] -#[path = "provider_token_exchange/closure.rs"] -mod closure; - use std::collections::HashMap; use std::convert::Infallible; use std::fs; @@ -146,14 +143,10 @@ struct SpiffeWorkloadApi { subject: Arc, jwks: Arc>, encoding_key: Arc, - deny_identity: Arc, } impl SpiffeWorkloadApi { fn jwt_svid(&self, audience: &[String]) -> Result { - if self.deny_identity.load(Ordering::SeqCst) { - return Err(Status::permission_denied("identity-secret-canary")); - } let now = unix_timestamp(); let mut header = Header::new(Algorithm::RS256); header.kid = Some(KEY_ID.to_string()); @@ -321,14 +314,6 @@ fn jwks() -> Vec { } async fn start_spiffe_workload_api(path: &Path, subject: &str) -> FixtureHandle { - start_controlled_spiffe_workload_api(path, subject, Arc::default()).await -} - -async fn start_controlled_spiffe_workload_api( - path: &Path, - subject: &str, - deny_identity: Arc, -) -> FixtureHandle { let api = SpiffeWorkloadApi { subject: Arc::::from(subject), jwks: Arc::new(jwks()), @@ -336,7 +321,6 @@ async fn start_controlled_spiffe_workload_api( EncodingKey::from_rsa_pem(TEST_RSA_PRIVATE_KEY.as_bytes()) .expect("test RSA key should parse"), ), - deny_identity, }; let endpoint = path.to_string_lossy(); if endpoint.starts_with("tcp:") { @@ -1021,8 +1005,8 @@ async fn podman_provider_token_exchange_injects_independent_grants_across_sandbo // Read one whole fixture request without consuming the following keepalive // request. OAuth forms can arrive separately from their headers. -async fn read_route_fixture_request( - stream: &mut S, +async fn read_route_fixture_request( + stream: &mut TcpStream, observed_bytes: Option<&AtomicUsize>, ) -> Result, String> { tokio::time::timeout(Duration::from_secs(30), async { @@ -1251,6 +1235,39 @@ async fn start_route_target( } } +async fn exec_within(sandbox: &SandboxGuard, script: &str) -> Result { + tokio::time::timeout( + Duration::from_secs(60), + sandbox.exec(&["python3", "-c", script]), + ) + .await + .map_err(|_| "route workload timed out".to_owned())? +} + +/// Poll `sandbox get` until the last token-grant failure is present or absent. +async fn wait_for_token_grant_failure( + name: &str, + present: bool, +) -> Result { + for _ in 0..30 { + let output = openshell_cmd() + .args(["sandbox", "get", name, "--output", "json"]) + .output() + .await + .map_err(|_| "could not spawn openshell CLI".to_owned())?; + if output.status.success() + && let Ok(sandbox) = serde_json::from_slice::(&output.stdout) + && sandbox["last_token_grant_failure"].is_null() != present + { + return Ok(sandbox["last_token_grant_failure"].clone()); + } + tokio::time::sleep(Duration::from_secs(1)).await; + } + Err(format!( + "last token-grant failure present={present} not observed" + )) +} + #[tokio::test] async fn podman_provider_routes_recover_and_select_a_b_a_on_one_connection() { let _fixture_guard = SPIFFE_FIXTURE_LOCK.lock().await; @@ -1327,11 +1344,9 @@ async fn podman_provider_routes_recover_and_select_a_b_a_on_one_connection() { grants.fail_next_b.store(true, Ordering::SeqCst); // The supervisor mediates direct workload connections as CONNECT. // HTTPConnection keeps one client socket for the successful A/B/A run. - let script = format!( + let failure_script = format!( r#"import http.client, uuid -host = 'host.openshell.internal' -port = {target_port} -first = http.client.HTTPConnection(host, port, timeout=10) +first = http.client.HTTPConnection('host.openshell.internal', {target_port}, timeout=10) first.request('GET', '/b/failure', headers={{'Authorization': 'Bearer stale-value'}}) failed = first.getresponse() failed.read() @@ -1341,6 +1356,12 @@ assert str(uuid.UUID(request_id)) == request_id, 'the refusal must carry a canon print('failed-request-id=' + request_id) assert failed.will_close, 'the failed tunnel must close' first.close() +"# + ); + let recovery_script = format!( + r#"import http.client +host = 'host.openshell.internal' +port = {target_port} recovered = http.client.HTTPConnection(host, port, timeout=10) connection = None for path in ['/a/first', '/b/second', '/a/third']: @@ -1356,14 +1377,19 @@ recovered.close() print('route-recovery-a-b-a-ok') "# ); - let output = tokio::time::timeout( - Duration::from_secs(60), - sandbox.exec(&["python3", "-c", &script]), - ) - .await - .map_err(|_| "route workload timed out".to_owned()); + let output = async { + let mut output = exec_within(&sandbox, &failure_script).await?; + // An authorized reader sees the cause of the refused grant. + let failure = wait_for_token_grant_failure(&sandbox.name, true).await?; + output.push_str(&format!("reported-failure={failure}\n")); + output.push_str(&exec_within(&sandbox, &recovery_script).await?); + // The matching grant's later success clears it. + wait_for_token_grant_failure(&sandbox.name, false).await?; + Ok::<_, String>(output) + } + .await; sandbox.cleanup().await; - output? + output } .await; for name in &names { @@ -1386,6 +1412,22 @@ print('route-recovery-a-b-a-ok') Some(response_request_id), "the token service and refused response must identify the same exchange" ); + let reported: serde_json::Value = serde_json::from_str( + output + .lines() + .find_map(|line| line.strip_prefix("reported-failure=")) + .expect("the refused grant reached sandbox status"), + ) + .expect("reported failure is JSON"); + assert_eq!(reported["destination_port"], u64::from(target_port)); + let failure = &reported["failure"]; + assert_eq!(failure["request_id"], response_request_id); + assert_eq!(failure["code"], "unavailable"); + assert_eq!(failure["recovery_action"], "retry"); + assert_eq!(failure["stage"], "token_endpoint"); + assert_eq!(failure["upstream_http_status"], 503); + assert_eq!(failure["provider"], names[1].as_str()); + assert_eq!(failure["credential_key"], "access_token"); let requests = observed.lock().expect("route target observations lock"); assert_eq!(requests.len(), 3, "failed grant must not reach the target"); assert_eq!( diff --git a/e2e/rust/tests/provider_token_exchange/closure.rs b/e2e/rust/tests/provider_token_exchange/closure.rs deleted file mode 100644 index 191aeca7d1..0000000000 --- a/e2e/rust/tests/provider_token_exchange/closure.rs +++ /dev/null @@ -1,794 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Exercise trusted grant failure retrieval through a real REST workload. - -use super::*; -use openshell_e2e::harness::cli::wait_for_healthy; -use openshell_e2e::harness::container::ContainerEngine; -use openshell_e2e::harness::gateway::ManagedGateway; -use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; -use serde_json::Value; -use tempfile::TempDir; -use tokio::process::Command; -use tonic::transport::{Certificate, ClientTlsConfig, Endpoint as GrpcEndpoint}; - -const ERROR_CANARY: &str = "grant-response-secret-canary"; -const INTERMEDIATE: &str = "intermediate-closure"; - -async fn checked(command: &mut Command, label: &str) -> Result { - command.kill_on_drop(true); - let output = tokio::time::timeout(Duration::from_secs(300), command.output()) - .await - .map_err(|_| format!("{label} timed out"))? - .map_err(|_| format!("{label} could not start"))?; - if !output.status.success() { - return Err(format!("{label} failed")); - } - Ok(String::from_utf8_lossy(&output.stdout).into_owned()) -} - -async fn cli(args: &[&str]) -> Result { - checked(openshell_cmd().args(args), "closure CLI operation").await -} - -fn no_secrets(output: &str) -> Result<(), String> { - if [ - ERROR_CANARY, - "identity-secret-canary", - "stored-closure", - INTERMEDIATE, - ] - .iter() - .any(|secret| output.contains(secret)) - { - return Err("grant diagnostics exposed fixture credential material".into()); - } - Ok(()) -} - -// Both OAuth clients use native roots. Install the public fixture CA in the -// supervisor image and in the gateway child's own SSL_CERT_FILE. Never change -// verification flags or the test runner's process-wide environment. -struct GatewayTrust { - directory: TempDir, - engine: ContainerEngine, - image: String, - changed: bool, -} - -impl GatewayTrust { - async fn new() -> Result { - if std::env::var_os("OPENSHELL_GATEWAY_ENDPOINT").is_some() - || std::env::var("OPENSHELL_E2E_DRIVER").as_deref() != Ok("podman") - || std::env::var("OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER") - .is_ok_and(|value| value != "0") - { - return Err("grant closure requires a wrapper-owned bundled Podman gateway".into()); - } - let args_path = std::env::var_os("OPENSHELL_E2E_GATEWAY_ARGS_FILE") - .ok_or("managed gateway arguments are missing")?; - let bytes = fs::read(args_path).map_err(|_| "could not read gateway arguments")?; - let args = bytes - .split(|byte| *byte == 0) - .filter(|arg| !arg.is_empty()) - .map(std::str::from_utf8) - .collect::, _>>() - .map_err(|_| "gateway arguments are not UTF-8")?; - let argument = |name| { - args.windows(2) - .find(|pair| pair[0] == name) - .map(|pair| pair[1]) - .ok_or("gateway argument is missing") - }; - let config = fs::read_to_string(argument("--config")?) - .map_err(|_| "could not read fixture gateway configuration")?; - let base_image = std::env::var("OPENSHELL_SUPERVISOR_IMAGE") - .ok() - .or_else(|| { - config.lines().find_map(|line| { - let value = line.trim().strip_prefix("supervisor_image = ")?; - serde_json::from_str::(value).ok() - }) - }) - .ok_or("fixture supervisor image is missing")?; - let directory = TempDir::new().map_err(|_| "could not create private TLS fixture")?; - generate_tls(directory.path()).await?; - let roots = std::env::var_os("SSL_CERT_FILE").map_or_else( - || PathBuf::from("/etc/ssl/certs/ca-certificates.crt"), - PathBuf::from, - ); - let mut bundle = fs::read(roots).map_err(|_| "native host CA bundle is required")?; - bundle.push(b'\n'); - bundle.extend( - fs::read(directory.path().join("ca.crt")).map_err(|_| "fixture CA is missing")?, - ); - fs::write(directory.path().join("host-roots.crt"), bundle) - .map_err(|_| "could not stage gateway roots")?; - let context = directory.path().join("image"); - fs::create_dir(&context).map_err(|_| "could not create public image context")?; - fs::copy( - directory.path().join("ca.crt"), - context.join("fixture-ca.crt"), - ) - .map_err(|_| "could not stage public fixture CA")?; - fs::write(context.join("Dockerfile"), format!( - "FROM {base_image} AS original\nFROM {E2E_WORKLOAD_IMAGE} AS bundle\nUSER 0\nCOPY --from=original /etc/ssl/certs/ca-certificates.crt /tmp/roots.crt\nCOPY fixture-ca.crt /tmp/fixture-ca.crt\nRUN [\"/usr/bin/python3\", \"-c\", \"from pathlib import Path; p=Path('/tmp/roots.crt'); p.write_bytes(p.read_bytes()+Path('/tmp/fixture-ca.crt').read_bytes())\"]\nFROM {base_image}\nCOPY --from=bundle /tmp/roots.crt /etc/ssl/certs/ca-certificates.crt\n" - )).map_err(|_| "could not write supervisor fixture Dockerfile")?; - let engine = ContainerEngine::from_env()?; - let image = format!( - "localhost/openshell-grant-closure-{:016x}:fixture", - rand::random::() - ); - let mut build = Command::from(engine.command()); - build.args(["build", "--tag", &image]).arg(&context); - build.kill_on_drop(true); - let status = tokio::time::timeout(Duration::from_secs(300), build.status()) - .await - .map_err(|_| "build trusted supervisor fixture timed out")? - .map_err(|_| "build trusted supervisor fixture could not start")?; - if !status.success() { - return Err("build trusted supervisor fixture failed".into()); - } - Ok(Self { - directory, - engine, - image, - changed: false, - }) - } - - async fn install(&mut self) -> Result<(), String> { - self.changed = true; - let mut gateway = ManagedGateway::from_env()?.ok_or("managed gateway is missing")?; - gateway.set_supervisor_image(&self.image); - gateway.set_native_ca_bundle(self.directory.path().join("host-roots.crt")); - gateway.stop()?; - gateway.start()?; - wait_for_healthy(Duration::from_secs(120)).await - } - - async fn restore(&mut self) -> Result<(), String> { - if self.changed { - let gateway = ManagedGateway::from_env()?.ok_or("managed gateway is missing")?; - gateway.stop()?; - gateway.start()?; - wait_for_healthy(Duration::from_secs(120)).await?; - self.changed = false; - } - let mut remove = Command::from(self.engine.command()); - checked( - remove.args(["image", "rm", "--force", &self.image]), - "remove supervisor fixture image", - ) - .await?; - Ok(()) - } -} - -async fn generate_tls(directory: &Path) -> Result<(), String> { - checked( - Command::new("openssl") - .args([ - "req", - "-x509", - "-newkey", - "rsa:2048", - "-nodes", - "-days", - "1", - "-subj", - "/CN=OpenShell grant fixture CA", - "-addext", - "basicConstraints=critical,CA:TRUE", - "-keyout", - ]) - .arg(directory.join("ca.key")) - .arg("-out") - .arg(directory.join("ca.crt")), - "generate fixture CA", - ) - .await?; - checked( - Command::new("openssl") - .args([ - "req", - "-newkey", - "rsa:2048", - "-nodes", - "-subj", - "/CN=localhost", - "-keyout", - ]) - .arg(directory.join("server.key")) - .arg("-out") - .arg(directory.join("server.csr")), - "generate token service CSR", - ) - .await?; - fs::write(directory.join("extensions"), "subjectAltName=IP:127.0.0.1,DNS:localhost\nbasicConstraints=critical,CA:FALSE\nkeyUsage=critical,digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\n") - .map_err(|_| "could not write token service certificate extensions")?; - checked( - Command::new("openssl") - .args(["x509", "-req", "-days", "1", "-in"]) - .arg(directory.join("server.csr")) - .arg("-CA") - .arg(directory.join("ca.crt")) - .arg("-CAkey") - .arg(directory.join("ca.key")) - .arg("-CAcreateserial") - .arg("-extfile") - .arg(directory.join("extensions")) - .arg("-out") - .arg(directory.join("server.crt")), - "sign token service certificate", - ) - .await?; - Ok(()) -} - -#[derive(Clone, Copy)] -enum Mode { - IntermediateAuthorization, - FinalAuthorization, - Bare401, - Success, -} - -struct TokenService { - mode: Mutex, - requests: Mutex>, -} - -async fn token_server( - directory: &Path, - port: u16, - state: Arc, -) -> Result { - let certificate = - fs::read(directory.join("server.crt")).map_err(|_| "token certificate is missing")?; - let key = fs::read(directory.join("server.key")).map_err(|_| "token key is missing")?; - let certificates = rustls_pemfile::certs(&mut certificate.as_slice()) - .collect::, _>>() - .map_err(|_| "invalid token certificate")?; - let private_key = rustls_pemfile::private_key(&mut key.as_slice()) - .map_err(|_| "invalid token key")? - .ok_or("empty token key")?; - let config = rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(certificates, private_key) - .map_err(|_| "invalid token TLS configuration")?; - let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config)); - let listener = TcpListener::bind((Ipv4Addr::LOCALHOST, port)) - .await - .map_err(|_| "could not bind HTTPS token fixture")?; - let endpoint = format!("https://127.0.0.1:{port}/token"); - let task = tokio::spawn(async move { - let mut requests = tokio::task::JoinSet::new(); - loop { - tokio::select! { - accepted = listener.accept() => { - let Ok((stream, _)) = accepted else { break; }; - let (acceptor, state, endpoint) = (acceptor.clone(), state.clone(), endpoint.clone()); - requests.spawn(async move { - let Ok(mut stream) = acceptor.accept(stream).await else { return; }; - let Ok(Some(request)) = read_route_fixture_request(&mut stream, None).await else { return; }; - let fields = token_form(&request); - let assertion = fields.get("client_assertion").map_or("", String::as_str); - let mut validation = Validation::new(Algorithm::RS256); - validation.set_audience(&[&endpoint]); - validation.set_issuer(&[ISSUER]); - let jwks: Value = serde_json::from_slice(&jwks()).unwrap(); - let key = DecodingKey::from_rsa_components(jwks["keys"][0]["n"].as_str().unwrap(), jwks["keys"][0]["e"].as_str().unwrap()).unwrap(); - if jsonwebtoken::decode::(assertion, &key, &validation).is_err() { return; } - let subject = fields.get("subject_token").map_or("", String::as_str); - let final_hop = subject == INTERMEDIATE; - if !final_hop && !subject.starts_with("stored-closure") { return; } - let request_id = request.split_once("\r\n\r\n").unwrap().0.lines() - .filter_map(|line| line.split_once(':')) - .find(|(name, _)| name.eq_ignore_ascii_case("x-openshell-request-id")) - .map_or("", |(_, value)| value.trim()).to_owned(); - state.requests.lock().unwrap().push((final_hop, request_id)); - let mode = *state.mode.lock().unwrap(); - let authorization = matches!((mode, final_hop), (Mode::IntermediateAuthorization, false) | (Mode::FinalAuthorization, true)); - let (status, body) = if authorization { - ("409 Conflict", json!({"error":"provider_authorization_required", "openshell_error_contract":"openshell.token-grant.v1", "error_description":ERROR_CANARY, "authorization_url":format!("https://{ERROR_CANARY}.invalid/")})) - } else if matches!((mode, final_hop), (Mode::Bare401, true)) { - ("401 Unauthorized", json!({"error":"invalid_client", "error_description":ERROR_CANARY})) - } else { - ("200 OK", json!({"access_token":if final_hop {"final-b"} else {INTERMEDIATE}, "token_type":"Bearer", "expires_in":0})) - }; - let body = body.to_string(); - let response = format!("HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()); - let _ = stream.write_all(response.as_bytes()).await; - }); - } - _ = requests.join_next(), if !requests.is_empty() => {} - } - } - }); - Ok(FixtureHandle { task }) -} - -async fn probe(sandbox: &SandboxGuard, target_port: u16, denied: bool) -> Result { - let script = format!( - r"import urllib.request, urllib.error, uuid -try: - response = urllib.request.urlopen('http://host.openshell.internal:{target_port}/closure/probe', timeout=10) - response.read() - assert not {denied}, 'grant unexpectedly forwarded' - print('success') -except urllib.error.HTTPError as error: - body = error.read().decode('utf-8', errors='replace') - text = str(error.headers) + body - assert {denied} and error.code == 502, 'unexpected grant refusal' - for value in ['{ERROR_CANARY}', 'identity-secret-canary', 'stored-closure', '{INTERMEDIATE}', 'final-b', 'provider_authorization_required', 'workload_identity_rejected', 'authorization_url']: - assert value not in text, 'grant refusal exposed private diagnostics' - request_id = error.headers.get('x-openshell-request-id', '') - assert str(uuid.UUID(request_id)) == request_id, 'invalid refusal request ID' - print(request_id) -", - denied = if denied { "True" } else { "False" } - ); - let output = sandbox.exec(&["python3", "-c", &script]).await?; - no_secrets(&output)?; - Ok(output.trim().to_owned()) -} - -async fn snapshot(name: &str) -> Result { - let output = cli(&["sandbox", "get", name, "--output", "json"]).await?; - no_secrets(&output)?; - serde_json::from_str(&output).map_err(|_| "sandbox snapshot was invalid JSON".into()) -} - -async fn wait_cause(name: &str, request_id: Option<&str>) -> Result { - let deadline = tokio::time::Instant::now() + Duration::from_secs(60); - loop { - let value = snapshot(name).await?; - let cause = value - .get("last_token_grant_failure") - .ok_or("sandbox snapshot omitted grant failures")?; - if request_id.map_or_else( - || cause.is_null(), - |id| cause["failure"]["request_id"] == id, - ) { - return Ok(value); - } - if tokio::time::Instant::now() >= deadline { - return Err("grant observation did not settle".into()); - } - tokio::time::sleep(Duration::from_millis(200)).await; - } -} - -fn check_cause( - value: &Value, - port: u16, - code: &str, - stage: &str, - recovery: &str, -) -> Result<(), String> { - let cause = &value["last_token_grant_failure"]; - let failure = &cause["failure"]; - if cause["destination_host"] != "host.openshell.internal" - || cause["destination_port"].as_u64() != Some(u64::from(port)) - || failure["code"] != code - || failure["stage"] != stage - || failure["recovery_action"] != recovery - { - return Err("trusted grant cause did not match the refused exchange".into()); - } - if !value["endpoint_statuses"] - .as_array() - .is_some_and(Vec::is_empty) - { - return Err("REST fixture unexpectedly depended on MCP endpoint status".into()); - } - Ok(()) -} - -async fn check_logs(sandbox: &SandboxGuard, request_id: &str) -> Result<(), String> { - let deadline = tokio::time::Instant::now() + Duration::from_secs(10); - loop { - let logs = cli(&[ - "logs", - &sandbox.name, - "--source", - "sandbox", - "--since", - "5m", - ]) - .await?; - no_secrets(&logs)?; - if logs - .lines() - .any(|line| line.contains(request_id) && line.contains("Token grant failed")) - { - break; - } - if tokio::time::Instant::now() >= deadline { - return Err("supervisor log omitted the refused request ID".into()); - } - tokio::time::sleep(Duration::from_millis(200)).await; - } - let gateway_log = - std::env::var_os("OPENSHELL_E2E_GATEWAY_LOG").ok_or("gateway log path is missing")?; - let gateway = - fs::read_to_string(gateway_log).map_err(|_| "could not read gateway fixture log")?; - no_secrets(&gateway)?; - if !gateway - .lines() - .any(|line| line.contains(request_id) && line.contains("provider_authorization_required")) - { - return Err("gateway log omitted the classified exchange".into()); - } - let denials = sandbox.exec(&["python3", "-c", "import urllib.request; print(urllib.request.urlopen('http://policy.local/v1/denials?last=20').read().decode())"]).await?; - no_secrets(&denials)?; - if !denials.contains(request_id) || denials.contains("provider_authorization_required") { - return Err("agent denial stream lost correlation or exposed classification".into()); - } - Ok(()) -} - -#[derive(Clone, PartialEq, prost::Message)] -struct GetSandbox { - #[prost(string, tag = "1")] - name: String, - #[prost(message, optional, tag = "2")] - workspace_scope: Option, -} -#[derive(Clone, PartialEq, prost::Message)] -struct Workspace { - #[prost(string, tag = "1")] - workspace: String, -} -#[derive(Clone, PartialEq, prost::Message)] -struct Empty {} - -async fn assert_untrusted_reader(name: &str, bearer: Option<&str>) -> Result<(), String> { - let gateway = std::env::var("OPENSHELL_GATEWAY").map_err(|_| "gateway name is missing")?; - let home = std::env::var_os("XDG_CONFIG_HOME").ok_or("gateway config directory is missing")?; - let directory = PathBuf::from(home).join("openshell/gateways").join(gateway); - let metadata: Value = serde_json::from_slice( - &fs::read(directory.join("metadata.json")).map_err(|_| "gateway metadata is missing")?, - ) - .map_err(|_| "gateway metadata is invalid")?; - let endpoint = metadata["gateway_endpoint"] - .as_str() - .ok_or("gateway endpoint is missing")?; - let ca = fs::read(directory.join("mtls/ca.crt")).map_err(|_| "gateway CA is missing")?; - let channel = GrpcEndpoint::from_shared(endpoint.to_owned()) - .map_err(|_| "invalid gateway endpoint")? - .tls_config(ClientTlsConfig::new().ca_certificate(Certificate::from_pem(ca))) - .map_err(|_| "invalid gateway TLS configuration")? - .connect() - .await - .map_err(|_| "could not connect untrusted reader over verified TLS")?; - let mut client = tonic::client::Grpc::new(channel); - client - .ready() - .await - .map_err(|_| "untrusted reader transport is not ready")?; - let mut request = Request::new(GetSandbox { - name: name.to_owned(), - workspace_scope: Some(Workspace { - workspace: "default".into(), - }), - }); - if let Some(token) = bearer { - request.metadata_mut().insert( - "authorization", - format!("Bearer {token}") - .parse() - .map_err(|_| "invalid fixture bearer")?, - ); - } - let result: Result, Status> = client - .unary( - request, - http::uri::PathAndQuery::from_static("/openshell.v1.OpenShell/GetSandbox"), - tonic_prost::ProstCodec::default(), - ) - .await; - match result { - Err(status) - if matches!( - status.code(), - tonic::Code::Unauthenticated | tonic::Code::PermissionDenied - ) => - { - Ok(()) - } - _ => Err("untrusted reader was not denied access to grant details".into()), - } -} - -async fn supervisor_token(engine: &ContainerEngine, name: &str) -> Result { - let mut list = Command::from(engine.command()); - let output = checked( - list.args([ - "ps", - "--quiet", - "--filter", - &format!("label=openshell.ai/sandbox-name={name}"), - "--filter", - "label=openshell.ai/isolation-role=supervisor", - ]), - "locate fixture supervisor", - ) - .await?; - let ids = output.lines().collect::>(); - let [id] = ids.as_slice() else { - return Err("fixture supervisor is ambiguous".into()); - }; - let directory = - TempDir::new().map_err(|_| "could not create private authentication fixture")?; - let path = directory.path().join("auth.json"); - let mut copy = Command::from(engine.command()); - checked( - copy.arg("cp") - .arg(format!("{id}:/.openshell/supervisor/auth.json")) - .arg(&path), - "copy private fixture supervisor authentication", - ) - .await?; - let value: Value = - serde_json::from_slice(&fs::read(&path).map_err(|_| "fixture authentication is missing")?) - .map_err(|_| "fixture authentication is invalid")?; - value["gateway_token"] - .as_str() - .map(str::to_owned) - .ok_or("fixture supervisor token is missing".into()) -} - -async fn failure_cases( - sandbox: &SandboxGuard, - port: u16, - service: &TokenService, - denied_identity: &AtomicBool, -) -> Result<(), String> { - for (mode, stage) in [ - (Mode::IntermediateAuthorization, "intermediate_exchange"), - (Mode::FinalAuthorization, "token_endpoint"), - ] { - *service.mode.lock().unwrap() = mode; - let request_id = probe(sandbox, port, true).await?; - let value = wait_cause(&sandbox.name, Some(&request_id)).await?; - check_cause( - &value, - port, - "provider_authorization_required", - stage, - "reauthorize", - )?; - if value["last_token_grant_failure"]["failure"]["upstream_http_status"] != 409 { - return Err("trusted cause lost the token service HTTP status".into()); - } - let hops = { - let requests = service.requests.lock().unwrap(); - requests - .iter() - .filter(|(_, id)| id == &request_id) - .map(|(final_hop, _)| *final_hop) - .collect::>() - }; - let expected = if stage == "intermediate_exchange" { - vec![false] - } else { - vec![false, true] - }; - if hops != expected { - return Err( - "actual token-service hops did not preserve the workload request ID".into(), - ); - } - if stage == "intermediate_exchange" { - check_logs(sandbox, &request_id).await?; - } - } - *service.mode.lock().unwrap() = Mode::Bare401; - let request_id = probe(sandbox, port, true).await?; - let value = wait_cause(&sandbox.name, Some(&request_id)).await?; - check_cause( - &value, - port, - "unclassified", - "token_endpoint", - "investigate", - )?; - let requests = service.requests.lock().unwrap().len(); - denied_identity.store(true, Ordering::SeqCst); - let request_id = probe(sandbox, port, true).await?; - let value = wait_cause(&sandbox.name, Some(&request_id)).await?; - denied_identity.store(false, Ordering::SeqCst); - check_cause( - &value, - port, - "workload_identity_rejected", - "workload_identity", - "fix_configuration", - )?; - if service.requests.lock().unwrap().len() != requests { - return Err("rejected workload identity reached the token service".into()); - } - Ok(()) -} - -async fn create_rest_fixture( - name: &str, - token_port: u16, - port: u16, -) -> Result { - let profile = write_route_profile(name, token_port, port, "closure"); - let contents = fs::read_to_string(profile.path()) - .map_err(|_| "fixture profile is missing")? - .replace("http://127.0.0.1:", "https://127.0.0.1:") - .replace("cache_ttl_seconds: 300", "cache_ttl: \"0s\""); - fs::write(profile.path(), contents).map_err(|_| "could not set verified token endpoint")?; - run_cli(&[ - "profile", - "import", - "-f", - profile.path().to_str().ok_or("profile path is invalid")?, - ]) - .await?; - run_cli(&[ - "provider", - "create", - "--name", - name, - "--type", - name, - "--credential", - "subject_token=stored-closure", - ]) - .await?; - let sandbox = SandboxGuard::create_keep_with_args( - &["--name", name, "--provider", name], - &["sh", "-lc", &sandbox_script()], - "token-server-ready", - ) - .await?; - cli(&[ - "settings", - "set", - name, - "--key", - "agent_policy_proposals_enabled", - "--value", - "true", - ]) - .await?; - sandbox - .exec(&[ - "python3", - "-c", - r"import time, urllib.request, urllib.error -deadline = time.monotonic() + 60 -while True: - try: - with urllib.request.urlopen('http://policy.local/v1/denials?last=1', timeout=2): - break - except urllib.error.HTTPError as error: - if error.code != 404 or time.monotonic() >= deadline: - raise - time.sleep(0.2) -", - ]) - .await?; - Ok(sandbox) -} - -async fn recovery_cases( - sandbox: &SandboxGuard, - port: u16, - service: &TokenService, -) -> Result<(), String> { - *service.mode.lock().unwrap() = Mode::Success; - if probe(sandbox, port, false).await? != "success" { - return Err("reauthorized request did not reach its destination".into()); - } - wait_cause(&sandbox.name, None).await?; - *service.mode.lock().unwrap() = Mode::FinalAuthorization; - let request_id = probe(sandbox, port, true).await?; - wait_cause(&sandbox.name, Some(&request_id)).await?; - run_cli(&[ - "provider", - "update", - &sandbox.name, - "--credential", - "subject_token=stored-closure-replacement", - ]) - .await?; - wait_cause(&sandbox.name, None).await?; - Ok(()) -} - -#[tokio::test] -async fn verified_grant_failures_reach_trusted_rest_status_and_recover() -> Result<(), String> { - let _fixture_guard = SPIFFE_FIXTURE_LOCK.lock().await; - let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); - let gateway_socket = PathBuf::from( - std::env::var("OPENSHELL_E2E_GATEWAY_SPIFFE_SOCKET") - .map_err(|_| "gateway Workload API fixture is missing")?, - ); - let provider_socket = PathBuf::from( - std::env::var("OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET") - .map_err(|_| "provider Workload API fixture is missing")?, - ); - let token_port = std::env::var("OPENSHELL_E2E_TOKEN_PORT").map_or_else( - |_| Ok(find_free_port()), - |value| { - value - .parse::() - .map_err(|_| "invalid fixed token fixture port") - }, - )?; - let mut trust = GatewayTrust::new().await?; - let gateway_spiffe = start_spiffe_workload_api( - &gateway_socket, - &format!("spiffe://{TRUST_DOMAIN}/openshell/gateway"), - ) - .await; - let deny_identity = Arc::new(AtomicBool::new(false)); - let provider_spiffe = start_controlled_spiffe_workload_api( - &provider_socket, - &format!("spiffe://{TRUST_DOMAIN}/openshell/sandbox/e2e"), - deny_identity.clone(), - ) - .await; - let port = find_free_port(); - let service = Arc::new(TokenService { - mode: Mutex::new(Mode::IntermediateAuthorization), - requests: Mutex::new(Vec::new()), - }); - let tokens = match token_server(trust.directory.path(), token_port, service.clone()).await { - Ok(tokens) => tokens, - Err(error) => { - trust.restore().await?; - return Err(error); - } - }; - let observed = Arc::new(Mutex::new(Vec::new())); - let observed_bytes = Arc::new(AtomicUsize::new(0)); - let target = start_route_target(port, observed.clone(), observed_bytes.clone()).await; - let name = format!("closure-{:08x}", rand::random::()); - let mut sandbox = None; - let result = async { - trust.install().await?; - sandbox = Some(create_rest_fixture(&name, token_port, port).await?); - let active = sandbox.as_ref().unwrap(); - failure_cases(active, port, &service, &deny_identity).await?; - if !observed.lock().unwrap().is_empty() || observed_bytes.load(Ordering::Relaxed) != 0 { - return Err("a refused grant forwarded protected request bytes".into()); - } - assert_untrusted_reader(&name, None).await?; - let bearer = supervisor_token(&trust.engine, &name).await?; - assert_untrusted_reader(&name, Some(&bearer)).await?; - recovery_cases(active, port, &service).await?; - Ok::<(), String>(()) - } - .await; - if let Some(mut sandbox) = sandbox { - sandbox.cleanup().await; - } - run_cli_ignore_error(&["sandbox", "delete", &name]).await; - run_cli_ignore_error(&["provider", "delete", &name, "--yes"]).await; - run_cli_ignore_error(&["profile", "delete", &name, "--yes"]).await; - let drained = target.drain().await; - tokens.shutdown().await; - provider_spiffe.shutdown().await; - gateway_spiffe.shutdown().await; - let restored = trust.restore().await; - result?; - drained?; - restored?; - let requests = observed.lock().unwrap(); - if requests.len() != 1 - || observed_bytes.load(Ordering::Relaxed) - != requests.iter().map(|request| request.bytes).sum::() - { - return Err("grant recovery reached an unexpected number of protected requests".into()); - } - Ok(()) -} diff --git a/sdk/go/openshell/v1/fake/fake_test.go b/sdk/go/openshell/v1/fake/fake_test.go index d196ba0ed3..75f9ba0440 100644 --- a/sdk/go/openshell/v1/fake/fake_test.go +++ b/sdk/go/openshell/v1/fake/fake_test.go @@ -236,69 +236,29 @@ func TestFakeClient_SandboxGrantFailures_DeepCopy(t *testing.T) { fc := NewClient() ctx := context.Background() upstreamStatus := uint32(401) - sb := &types.Sandbox{ - Name: "grant-sandbox", - Status: types.SandboxStatus{EndpointStatuses: []types.EndpointStatus{{ - EndpointID: "endpoint-grant", - Host: "tools.example.test", - Ports: []uint32{443}, - Path: "/mcp", - LastResult: types.EndpointCredentialUnavailable, - LastFailure: &types.EndpointFailure{ - Domain: types.EndpointFailureDomainTokenGrant, - Code: "provider_authorization_required", - RecoveryAction: types.RefreshRecoveryActionReauthorize, - Provider: "github", - CredentialKey: "token", - Stage: "gateway_rpc", - UpstreamHTTPStatus: &upstreamStatus, - RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", - }, - }}}, - } - sb.Status.LastTokenGrantFailure = &types.TokenGrantFailureObservation{ - DestinationHost: "api.example.test", - DestinationPort: 8443, - Failure: sb.Status.EndpointStatuses[0].LastFailure, - } + failure := &types.EndpointFailure{Code: "provider_authorization_required", UpstreamHTTPStatus: &upstreamStatus} + sb := &types.Sandbox{Name: "grant-sandbox", Status: types.SandboxStatus{ + EndpointStatuses: []types.EndpointStatus{{EndpointID: "endpoint-grant", LastFailure: failure}}, + LastTokenGrantFailure: &types.TokenGrantFailureObservation{DestinationHost: "api.example.test", Failure: failure}, + }} fc.AddSandbox("default", sb) // Neither the seed object nor returned snapshots may mutate the stored failure. - sb.Status.EndpointStatuses[0].LastFailure.Code = "seed_changed" + failure.Code = "seed_changed" upstreamStatus = 500 got, err := fc.Sandboxes().Get(ctx, "default", "grant-sandbox") require.NoError(t, err) - failure := got.Status.EndpointStatuses[0].LastFailure - require.NotNil(t, failure) - assert.Equal(t, "provider_authorization_required", failure.Code) - require.NotNil(t, failure.UpstreamHTTPStatus) - assert.Equal(t, uint32(401), *failure.UpstreamHTTPStatus) - failure.Stage = "changed" - *failure.UpstreamHTTPStatus = 503 - observation := got.Status.LastTokenGrantFailure - require.NotNil(t, observation) - assert.Equal(t, "api.example.test", observation.DestinationHost) - require.NotNil(t, observation.Failure) - assert.Equal(t, "provider_authorization_required", observation.Failure.Code) - observation.DestinationHost = "changed.example.test" - observation.Failure.Code = "changed" - *observation.Failure.UpstreamHTTPStatus = 503 + got.Status.EndpointStatuses[0].LastFailure.Code = "changed" + got.Status.LastTokenGrantFailure.DestinationHost = "changed.example.test" + *got.Status.LastTokenGrantFailure.Failure.UpstreamHTTPStatus = 503 - listed, err := fc.Sandboxes().ListAll(ctx, "default") + stored, err := fc.Sandboxes().Get(ctx, "default", "grant-sandbox") require.NoError(t, err) - require.Len(t, listed, 1) - stored := listed[0].Status.EndpointStatuses[0].LastFailure - require.NotNil(t, stored) - assert.Equal(t, "gateway_rpc", stored.Stage) - require.NotNil(t, stored.UpstreamHTTPStatus) - assert.Equal(t, uint32(401), *stored.UpstreamHTTPStatus) - storedObservation := listed[0].Status.LastTokenGrantFailure - require.NotNil(t, storedObservation) - assert.Equal(t, "api.example.test", storedObservation.DestinationHost) - assert.Equal(t, uint32(8443), storedObservation.DestinationPort) - require.NotNil(t, storedObservation.Failure) - assert.Equal(t, "provider_authorization_required", storedObservation.Failure.Code) - assert.Equal(t, uint32(401), *storedObservation.Failure.UpstreamHTTPStatus) + assert.Equal(t, "provider_authorization_required", stored.Status.EndpointStatuses[0].LastFailure.Code) + assert.Equal(t, uint32(401), *stored.Status.EndpointStatuses[0].LastFailure.UpstreamHTTPStatus) + assert.Equal(t, "api.example.test", stored.Status.LastTokenGrantFailure.DestinationHost) + assert.Equal(t, "provider_authorization_required", stored.Status.LastTokenGrantFailure.Failure.Code) + assert.Equal(t, uint32(401), *stored.Status.LastTokenGrantFailure.Failure.UpstreamHTTPStatus) } func TestFakeClient_AddProvider(t *testing.T) { diff --git a/sdk/go/openshell/v1/internal/converter/sandbox_test.go b/sdk/go/openshell/v1/internal/converter/sandbox_test.go index dce84bc302..e3886a4b9e 100644 --- a/sdk/go/openshell/v1/internal/converter/sandbox_test.go +++ b/sdk/go/openshell/v1/internal/converter/sandbox_test.go @@ -256,126 +256,61 @@ func TestSandboxFromProto_EndpointStatuses(t *testing.T) { assert.Equal(t, uint32(443), input.Status.EndpointStatuses[1].Ports[0]) } -func TestSandboxFromProto_EndpointLastFailure(t *testing.T) { - upstreamStatus := uint32(401) - reportedAt := timestamppb.New(time.Date(2026, 9, 11, 10, 0, 0, 0, time.UTC)) - input := &pb.Sandbox{Status: &pb.SandboxStatus{EndpointStatuses: []*pb.EndpointStatus{ - { - EndpointId: "endpoint-grant", Host: "tools.example.test", Ports: []uint32{443}, Path: "/mcp", - LastResult: pb.EndpointResult_ENDPOINT_RESULT_CREDENTIAL_UNAVAILABLE, - LastReportedTime: reportedAt, - LastFailure: &pb.EndpointFailure{ - Domain: pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, - Code: "provider_authorization_required", - RecoveryAction: pb.ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE, - Provider: "github", - CredentialKey: "token", - Stage: "gateway_rpc", - UpstreamHttpStatus: &upstreamStatus, - RequestId: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", - }, - }, - { - EndpointId: "endpoint-plain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/other", - LastResult: pb.EndpointResult_ENDPOINT_RESULT_TRANSPORT_FAILED, - LastReportedTime: reportedAt, - }, - }}} - - s := SandboxFromProto(input) - require.NotNil(t, s) - wantStatus := uint32(401) - require.Equal(t, []v1.EndpointStatus{ - { - EndpointID: "endpoint-grant", Host: "tools.example.test", Ports: []uint32{443}, Path: "/mcp", - LastResult: v1.EndpointCredentialUnavailable, LastReportedAt: "2026-09-11T10:00:00Z", - LastFailure: &v1.EndpointFailure{ - Domain: v1.EndpointFailureDomainTokenGrant, - Code: "provider_authorization_required", - RecoveryAction: v1.RefreshRecoveryActionReauthorize, - Provider: "github", - CredentialKey: "token", - Stage: "gateway_rpc", - UpstreamHTTPStatus: &wantStatus, - RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", - }, - }, - { - EndpointID: "endpoint-plain", Host: "tools.example.test", Ports: []uint32{443}, Path: "/other", - LastResult: v1.EndpointTransportFailed, LastReportedAt: "2026-09-11T10:00:00Z", - }, - }, s.Status.EndpointStatuses) - - // The SDK failure must not share the wire message or its optional status. - input.Status.EndpointStatuses[0].LastFailure.Code = "changed" - upstreamStatus = 500 - assert.Equal(t, "provider_authorization_required", s.Status.EndpointStatuses[0].LastFailure.Code) - assert.Equal(t, uint32(401), *s.Status.EndpointStatuses[0].LastFailure.UpstreamHTTPStatus) - *s.Status.EndpointStatuses[0].LastFailure.UpstreamHTTPStatus = 503 - assert.Equal(t, uint32(500), *input.Status.EndpointStatuses[0].LastFailure.UpstreamHttpStatus) -} - -func TestSandboxFromProto_TokenGrantFailureOutsideEndpointInventory(t *testing.T) { +func TestSandboxFromProto_TokenGrantFailures(t *testing.T) { upstreamStatus := uint32(409) + failure := &pb.EndpointFailure{ + Domain: pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, + Code: "provider_authorization_required", + RecoveryAction: pb.ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE, + Provider: "github", + CredentialKey: "token", + Stage: "token_endpoint", + UpstreamHttpStatus: &upstreamStatus, + RequestId: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + } input := &pb.Sandbox{Status: &pb.SandboxStatus{ + EndpointStatuses: []*pb.EndpointStatus{{EndpointId: "with-failure", LastFailure: failure}, {EndpointId: "without-failure"}}, LastTokenGrantFailure: &pb.TokenGrantFailureObservation{ DestinationHost: "api.example.test", DestinationPort: 8443, - Failure: &pb.EndpointFailure{ - Domain: pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, - Code: "provider_authorization_required", - RecoveryAction: pb.ProviderCredentialRefreshRecoveryAction_PROVIDER_CREDENTIAL_REFRESH_RECOVERY_ACTION_REAUTHORIZE, - Provider: "github", - CredentialKey: "token", - Stage: "token_endpoint", - UpstreamHttpStatus: &upstreamStatus, - RequestId: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", - }, + Failure: failure, }, }} - converted := SandboxFromProto(input) - require.Empty(t, converted.Status.EndpointStatuses) + + status := SandboxFromProto(input).Status wantStatus := uint32(409) - require.Equal(t, &v1.TokenGrantFailureObservation{ + want := &v1.EndpointFailure{ + Domain: v1.EndpointFailureDomainTokenGrant, + Code: "provider_authorization_required", + RecoveryAction: v1.RefreshRecoveryActionReauthorize, + Provider: "github", + CredentialKey: "token", + Stage: "token_endpoint", + UpstreamHTTPStatus: &wantStatus, + RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", + } + assert.Equal(t, want, status.EndpointStatuses[0].LastFailure) + assert.Nil(t, status.EndpointStatuses[1].LastFailure) + assert.Equal(t, &v1.TokenGrantFailureObservation{ DestinationHost: "api.example.test", DestinationPort: 8443, - Failure: &v1.EndpointFailure{ - Domain: v1.EndpointFailureDomainTokenGrant, - Code: "provider_authorization_required", - RecoveryAction: v1.RefreshRecoveryActionReauthorize, - Provider: "github", - CredentialKey: "token", - Stage: "token_endpoint", - UpstreamHTTPStatus: &wantStatus, - RequestID: "0b6f1f6e-7c55-4f1c-9a43-2d8f5b1f0e11", - }, - }, converted.Status.LastTokenGrantFailure) - input.Status.LastTokenGrantFailure.DestinationHost = "changed.example.test" - input.Status.LastTokenGrantFailure.Failure.Code = "changed" + Failure: want, + }, status.LastTokenGrantFailure) + + // The SDK values must not share the wire message's optional status. upstreamStatus = 500 - assert.Equal(t, "api.example.test", converted.Status.LastTokenGrantFailure.DestinationHost) - assert.Equal(t, "provider_authorization_required", converted.Status.LastTokenGrantFailure.Failure.Code) - assert.Equal(t, uint32(409), *converted.Status.LastTokenGrantFailure.Failure.UpstreamHTTPStatus) + assert.Equal(t, uint32(409), *status.LastTokenGrantFailure.Failure.UpstreamHTTPStatus) + + // Unknown wire domains must not select a closed code set. + failure.Domain = pb.EndpointFailureDomain(99) + input.Status.LastTokenGrantFailure.Failure = nil + status = SandboxFromProto(input).Status + assert.Equal(t, v1.EndpointFailureDomainUnspecified, status.EndpointStatuses[0].LastFailure.Domain) + assert.Nil(t, status.LastTokenGrantFailure.Failure) input.Status.LastTokenGrantFailure = nil assert.Nil(t, SandboxFromProto(input).Status.LastTokenGrantFailure) } -func TestEndpointFailureDomainFromProto(t *testing.T) { - for _, tc := range []struct { - input pb.EndpointFailureDomain - want v1.EndpointFailureDomain - }{ - {pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_UNSPECIFIED, v1.EndpointFailureDomainUnspecified}, - {pb.EndpointFailureDomain_ENDPOINT_FAILURE_DOMAIN_TOKEN_GRANT, v1.EndpointFailureDomainTokenGrant}, - {pb.EndpointFailureDomain(99), v1.EndpointFailureDomainUnspecified}, - } { - t.Run(tc.input.String(), func(t *testing.T) { - assert.Equal(t, tc.want, endpointFailureDomainFromProto(tc.input)) - }) - } - assert.Nil(t, endpointFailureFromProto(nil)) -} - func TestEndpointResultFromProto(t *testing.T) { for _, tc := range []struct { input pb.EndpointResult From 9e86dfc97c550a1e21de89465e42def88dea47a6 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 19:08:54 +0530 Subject: [PATCH 07/13] fix(providers): restore test import and features lockfile The injection tests named EndpointObservationSender through the parent module, which no longer imports it. The features suite lockfile kept the tonic TLS dependency that the removed E2E fixture added. Signed-off-by: Shiju --- .../src/l7/token_grant_injection.rs | 21 ++++++++----------- tests/suites/features/Cargo.lock | 1 - 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs index 1319e86100..49338c9917 100644 --- a/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs +++ b/crates/openshell-supervisor-network/src/l7/token_grant_injection.rs @@ -900,17 +900,14 @@ pub mod test_support { /// Installs this fixture's credentials as a pinned provider-credential /// snapshot. Admission-gated injection requests grants by revision-scoped /// key, so each scripted response is also registered under that key. - pub fn provider_credentials( - &self, - ) -> openshell_core::provider_credentials::ProviderCredentialState { + pub fn provider_credentials(&self) -> ProviderCredentialState { let credentials = self.dynamic_credentials.read().unwrap().clone(); - let state = - openshell_core::provider_credentials::ProviderCredentialState::from_environment( - 1, - HashMap::new(), - HashMap::new(), - credentials.clone(), - ); + let state = ProviderCredentialState::from_environment( + 1, + HashMap::new(), + HashMap::new(), + credentials.clone(), + ); let snapshot = state.snapshot(); let mut responses = self.responses.lock().unwrap(); for key in credentials.keys() { @@ -1092,8 +1089,8 @@ mod tests { use crate::l7::provider::{BodyLength, L7Request}; use crate::l7::token_grant_injection::test_support::{TokenGrantTestFixture, inject_if_needed}; use openshell_core::endpoint_status::{ - EndpointConfigVersion, EndpointStatusReceiver, EndpointStatusTracker, - endpoint_status_channel, + EndpointConfigVersion, EndpointObservationSender, EndpointStatusReceiver, + EndpointStatusTracker, endpoint_status_channel, }; const REQUEST_ID: &str = "6a1f0e2d-3c4b-4a59-8e7f-0123456789ab"; diff --git a/tests/suites/features/Cargo.lock b/tests/suites/features/Cargo.lock index b096620b70..b92ccd05f5 100644 --- a/tests/suites/features/Cargo.lock +++ b/tests/suites/features/Cargo.lock @@ -1586,7 +1586,6 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", - "tokio-rustls", "tokio-stream", "tower", "tower-layer", From e61e60aa2dd29cc2af9f20530a802b07f7a62396 Mon Sep 17 00:00:00 2001 From: Shiju Date: Tue, 13 Oct 2026 19:21:25 +0530 Subject: [PATCH 08/13] test(providers): fix body-bound fixture width and table type Pass the response-size width as a named argument; an implicitly captured constant panics at runtime. Name the mutation type in the validation table for Clippy. Signed-off-by: Shiju --- crates/openshell-core/src/endpoint_failure.rs | 3 ++- crates/openshell-core/src/oauth.rs | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/openshell-core/src/endpoint_failure.rs b/crates/openshell-core/src/endpoint_failure.rs index 8826ce67d5..77bf6fcda6 100644 --- a/crates/openshell-core/src/endpoint_failure.rs +++ b/crates/openshell-core/src/endpoint_failure.rs @@ -242,11 +242,12 @@ mod tests { #[test] fn rejects_each_invalid_field() { use EndpointFailureValidationError as E; + type Mutation = fn(&mut proto::EndpointFailure); assert_eq!( validate_endpoint_failure(&reported(), proto::EndpointResult::PolicyDenied), Err(E::Result) ); - let cases: [(fn(&mut proto::EndpointFailure), E); 9] = [ + let cases: [(Mutation, E); 9] = [ (|f| f.domain = 99, E::Domain), (|f| f.code = "reauthorization_required".into(), E::Code), (|f| f.stage = "token service said no".into(), E::Stage), diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index 7fa2119321..511776f0a2 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -770,7 +770,7 @@ mod tests { } let token = r#"{"access_token":"access-token"}"#; - let body = format!("{token: Date: Tue, 13 Oct 2026 19:32:50 +0530 Subject: [PATCH 09/13] test(providers): pad exact-bound body without a format width MAX_TOKEN_RESPONSE_BYTES exceeds u16::MAX, the largest format width, so the padded fixture panicked. Pad with repeat_n as before. Signed-off-by: Shiju --- crates/openshell-core/src/oauth.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index 511776f0a2..da39509c26 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -769,8 +769,12 @@ mod tests { assert_eq!(failure.upstream_status(), Some(503)); } - let token = r#"{"access_token":"access-token"}"#; - let body = format!("{token: Date: Tue, 13 Oct 2026 23:44:20 +0530 Subject: [PATCH 10/13] test(server): update public RPC schema hash after rebase Main added the ListProvidersRequest label selector. The merged public schema hash comes from the Branch Checks storage_proto test output. Signed-off-by: Shiju --- crates/openshell-server/src/storage_proto.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openshell-server/src/storage_proto.rs b/crates/openshell-server/src/storage_proto.rs index fdfd57fe2a..3ffff8945e 100644 --- a/crates/openshell-server/src/storage_proto.rs +++ b/crates/openshell-server/src/storage_proto.rs @@ -155,7 +155,7 @@ mod tests { // closures and TokenGrantStatus to public reports. Older rows have no grant // record; absent report extensions retain compatibility without a rewrite. const PUBLIC_RPC_SCHEMA_SHA256: &str = - "10fce7e572f1513551d6b7abec7dcc8fc2cb40c92fb394f8321232f5b9921fd4"; + "cc06b38c9ecb4c6518e0d4963f40c4f16f992483ce111c7b23cbcb729db946ea"; const DURABLE_SCHEMA_SHA256: &str = "d4dc7abdb91b919db1f7be0d4652e30cbfda86f3b2ed9b59d5de1f76a2bf691d"; const PUBLIC_DURABLE_OVERLAP_SHA256: &str = From c47fa3a3ac30af6ad35f1d660ee145826464a005 Mon Sep 17 00:00:00 2001 From: Shiju Date: Thu, 15 Oct 2026 09:14:35 +0530 Subject: [PATCH 11/13] fix(providers): keep token-service error text out of grant logs A token service can echo a submitted credential in its OAuth error or error_description. Log only the classified failure code, HTTP status and request ID, and cover the canary echo in a dedicated test binary. Signed-off-by: Shiju --- Cargo.lock | 1 + crates/openshell-core/Cargo.toml | 1 + crates/openshell-core/src/oauth.rs | 39 ++---- .../tests/oauth_error_logging.rs | 120 ++++++++++++++++++ docs/how-it-works/providers/profiles.mdx | 2 +- 5 files changed, 133 insertions(+), 30 deletions(-) create mode 100644 crates/openshell-core/tests/oauth_error_logging.rs diff --git a/Cargo.lock b/Cargo.lock index 0c9e17bca4..cd0cfdf0fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4343,6 +4343,7 @@ dependencies = [ "tonic-prost-build", "tonic-types", "tracing", + "tracing-subscriber", "url", "uuid", "zeroize", diff --git a/crates/openshell-core/Cargo.toml b/crates/openshell-core/Cargo.toml index f0c1d85847..93c4c4b0b2 100644 --- a/crates/openshell-core/Cargo.toml +++ b/crates/openshell-core/Cargo.toml @@ -66,6 +66,7 @@ protoc-bin-vendored = { workspace = true } [dev-dependencies] tempfile = "3" +tracing-subscriber = { workspace = true } rcgen = { workspace = true } tokio-rustls = { workspace = true } diff --git a/crates/openshell-core/src/oauth.rs b/crates/openshell-core/src/oauth.rs index da39509c26..51dc40a9f3 100644 --- a/crates/openshell-core/src/oauth.rs +++ b/crates/openshell-core/src/oauth.rs @@ -24,7 +24,6 @@ pub const ACCESS_TOKEN_TYPE: &str = "urn:ietf:params:oauth:token-type:access_tok const TOKEN_EXCHANGE_GRANT_TYPE: &str = "urn:ietf:params:oauth:grant-type:token-exchange"; const ERROR_CONTRACT: &str = "openshell.token-grant.v1"; const MAX_TOKEN_RESPONSE_BYTES: usize = 64 * 1024; -const MAX_LOGGED_ERROR_CHARS: usize = 256; /// HTTP client that authenticates HTTPS endpoints and never redirects assertions. /// @@ -96,7 +95,6 @@ struct RawTokenResponse { #[derive(Deserialize)] struct OAuthErrorResponse { error: Option, - error_description: Option, openshell_error_contract: Option, } @@ -140,8 +138,16 @@ async fn post_oauth_token_request( let status = response.status(); let body = read_bounded_body(response).await?; if !status.is_success() { - log_token_service_error(status, &body, request_id); - return Err(classify_http_failure(status, &body, authenticated)); + let failure = classify_http_failure(status, &body, authenticated); + // A token service can echo submitted credentials in `error` or + // `error_description`, so the log keeps only the closed classification. + tracing::warn!( + request_id = request_id.unwrap_or("none"), + upstream_http_status = status.as_u16(), + failure_code = failure.code(), + "token service rejected the grant" + ); + return Err(failure); } // Raw parse failures can quote response bytes. Retain only a closed failure @@ -378,31 +384,6 @@ async fn read_bounded_body(mut response: reqwest::Response) -> Result, T Ok(body) } -/// Record the token service's `OAuth` error text for operators. -/// -/// Only the gateway and supervisor logs receive it; the workload cannot read -/// either, and status reports and responses keep only the bounded failure. -fn log_token_service_error(status: reqwest::StatusCode, body: &[u8], request_id: Option<&str>) { - let Ok(response) = serde_json::from_slice::(body) else { - return; - }; - let printable = |value: Option| -> String { - value - .unwrap_or_default() - .chars() - .filter(|character| !character.is_control()) - .take(MAX_LOGGED_ERROR_CHARS) - .collect() - }; - tracing::warn!( - request_id = request_id.unwrap_or("none"), - upstream_http_status = status.as_u16(), - error = %printable(response.error), - error_description = %printable(response.error_description), - "token service rejected the grant" - ); -} - fn classify_http_failure( status: reqwest::StatusCode, body: &[u8], diff --git a/crates/openshell-core/tests/oauth_error_logging.rs b/crates/openshell-core/tests/oauth_error_logging.rs new file mode 100644 index 0000000000..a8f5cd4951 --- /dev/null +++ b/crates/openshell-core/tests/oauth_error_logging.rs @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Token-service error text must never reach logs. +//! +//! This test captures a scoped subscriber. It lives in its own binary because a +//! parallel test can register the shared `warn!` callsite as disabled first. + +#![cfg(feature = "oauth")] + +use std::io::Write; +use std::sync::{Arc, Mutex}; + +use openshell_core::oauth::{TokenExchangeParams, TokenGrantClient, post_oauth_token_exchange}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; + +const CANARY: &str = "SUBJECT_SECRET_CANARY"; + +#[derive(Clone, Default)] +struct CapturedLogs(Arc>>); + +impl Write for CapturedLogs { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.lock().expect("log buffer").extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Serve one request with HTTP 400 and `body`, after reading the whole request. +async fn rejecting_endpoint_once(body: String) -> (String, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local addr"); + let task = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.expect("accept"); + let mut request = Vec::new(); + let mut chunk = [0u8; 1024]; + loop { + let read = stream.read(&mut chunk).await.expect("read"); + assert!(read > 0, "request ended early"); + request.extend_from_slice(&chunk[..read]); + let text = String::from_utf8_lossy(&request); + let Some((headers, form)) = text.split_once("\r\n\r\n") else { + continue; + }; + let length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + if form.len() >= length { + break; + } + } + let response = format!( + "HTTP/1.1 400 Bad Request\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len(), + ); + stream.write_all(response.as_bytes()).await.expect("write"); + stream.shutdown().await.expect("shutdown"); + }); + (format!("http://{addr}/token"), task) +} + +#[tokio::test] +async fn rejected_grant_log_omits_token_service_error_text() { + let logs = CapturedLogs::default(); + let subscriber = tracing_subscriber::fmt() + .with_writer({ + let logs = logs.clone(); + move || logs.clone() + }) + .with_ansi(false) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + let client = TokenGrantClient::new().expect("client"); + let echoes = [ + format!(r#"{{"error":"invalid_grant","error_description":"subject_token={CANARY}"}}"#), + format!(r#"{{"error":"subject_token={CANARY}"}}"#), + ]; + for body in echoes { + let (endpoint, task) = rejecting_endpoint_once(body).await; + let failure = post_oauth_token_exchange( + &client, + &endpoint, + &TokenExchangeParams { + client_assertion: "jwt-svid-token", + client_assertion_type: "", + subject_token: CANARY, + subject_token_type: "", + audience: "", + scopes: &[], + requested_token_type: "", + }, + Some("request-1"), + ) + .await + .expect_err("token service rejected the exchange"); + task.await.expect("endpoint task"); + assert_eq!(failure.code(), "unclassified"); + } + + let output = String::from_utf8(logs.0.lock().expect("log buffer").clone()).expect("UTF-8"); + assert_eq!( + output.matches("token service rejected the grant").count(), + 2, + "{output}" + ); + assert!(output.contains("upstream_http_status=400"), "{output}"); + assert!(output.contains("failure_code=\"unclassified\""), "{output}"); + assert!(!output.contains(CANARY), "{output}"); +} diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index e56d38b683..0d1090a438 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -687,7 +687,7 @@ The stage names the step that observed the failure: `configuration`, `workload_i - Without a recognized extension code, any other failed HTTP status is `unclassified`, including 401, 403, and 409. OpenShell never infers `provider_authorization_required` from a 401. - For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these causes only over its verified TLS connection to the gateway. -Sandbox status and the workload response keep only the code, stage, and HTTP status. When a token service returns an `OAuth` error body, the log of the component that called it, the gateway or the supervisor, also records its `error` and `error_description`, limited to 256 characters each, with the request ID. The workload cannot read either log. A successful response without a nonempty header-compatible `access_token` is `unclassified`. +Sandbox status and the workload response keep only the code, stage, and HTTP status. The log of the component that called the token service, the gateway or the supervisor, records the same code with the HTTP status and request ID. OpenShell never logs the token service's `error` or `error_description` text, because a token service can echo a submitted credential there. A successful response without a nonempty header-compatible `access_token` is `unclassified`. A token service can also state a cause by adopting the `openshell.token-grant.v1` error extension. For example, a service that requires the user to authorize a provider returns HTTP 409 with: From b5e5457d6a4deac71b602ef382aae2f998443743 Mon Sep 17 00:00:00 2001 From: Shiju Date: Thu, 15 Oct 2026 22:45:28 +0530 Subject: [PATCH 12/13] docs(providers): tighten token grant failure documentation Signed-off-by: Shiju --- docs/how-it-works/providers/profiles.mdx | 35 ++++++++++------------ docs/how-it-works/sandboxes/overview.mdx | 38 +++++++++++------------- docs/observability/logging.mdx | 2 +- 3 files changed, 34 insertions(+), 41 deletions(-) diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 0d1090a438..657343006e 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -660,36 +660,31 @@ The gateway only brokers an intermediate token for a sandbox principal, and only ### Token Grant Failures -When a token grant fails, OpenShell refuses the protected request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header and the body described in [Proxy Error Responses](/observability/logging#proxy-error-responses). The response, the supervisor's OCSF events, and `policy.local` denials never say why the grant failed, because the sandboxed agent can read them. Users authorized to read the sandbox can retrieve the cause from `Sandbox.status.last_token_grant_failure` and `openshell logs`. +When a token grant fails, OpenShell refuses the request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header and the body described in [Proxy Error Responses](/observability/logging#proxy-error-responses). The agent can read that response, the OCSF events, and `policy.local` denials, so none of them state the cause. Users who can read the sandbox get it from `Sandbox.status.last_token_grant_failure` and `openshell logs`. -The request ID is a random UUID that the supervisor mints for each HTTP exchange. It names the same exchange everywhere: +The supervisor mints a random UUID request ID for each HTTP exchange. It sends the ID to the token service as `x-openshell-request-id` and, for `token_exchange` grants, to the gateway as gRPC `x-request-id` metadata, which the gateway forwards to the intermediate token endpoint. The supervisor's `Token grant failed` OCSF event carries it as `request_id`. Log lines from the supervisor and gateway add the code, stage, and upstream HTTP status. -- The supervisor sends it to the token service in the `x-openshell-request-id` request header. -- For `token_exchange` grants, the supervisor sends it to the gateway as gRPC `x-request-id` metadata, and the gateway forwards it to the intermediate token endpoint. -- The supervisor's `Token grant failed` OCSF event carries it as `request_id`. In `openshell logs`, the supervisor's log line with the same message adds the failure code, stage, and upstream HTTP status. -- When the gateway hop fails, the gateway's `dynamic token grant failed` log line carries it with the failure code, stage, and upstream HTTP status. - -OpenShell classifies each failure into one of these codes. The recovery action uses the same values as provider refresh status: +Each failure gets one code. Recovery actions match provider refresh status: | Code | Meaning | Recovery action | |---|---|---| | `provider_authorization_required` | The user must authorize the provider again. | `reauthorize` | -| `workload_identity_rejected` | The SPIFFE Workload API, the gateway, or the token service rejected the sandbox's workload identity. | `fix_configuration` | +| `workload_identity_rejected` | The SPIFFE Workload API, gateway, or token service rejected the workload identity. | `fix_configuration` | | `configuration_error` | The grant, provider, or token-service configuration is invalid. | `fix_configuration` | | `timeout` | A grant step exceeded its time limit. | `retry` | | `unavailable` | A required service or connection was unavailable. | `retry` | -| `unclassified` | The available evidence does not establish a more specific cause. | `investigate` | +| `unclassified` | The evidence does not establish a more specific cause. | `investigate` | -The stage names the step that observed the failure: `configuration`, `workload_identity`, `gateway_rpc`, `intermediate_exchange`, `token_endpoint`, or `injection`. OpenShell maps token-service responses as follows: +The stage is the step that observed the failure: `configuration`, `workload_identity`, `gateway_rpc`, `intermediate_exchange`, `token_endpoint`, or `injection`. OpenShell maps token-service responses as follows: -- A response body over 64 KiB is `unclassified`, whatever its HTTP status. -- HTTP 429 and 5xx responses are `unavailable` unless they carry a recognized extension code, described below. Failures to connect, including TLS certificate or hostname verification failures, are also `unavailable`, but a connection that fails after it is established is `unclassified`. A timeout, including a slow response body, is `timeout`. -- Without a recognized extension code, any other failed HTTP status is `unclassified`, including 401, 403, and 409. OpenShell never infers `provider_authorization_required` from a 401. -- For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these causes only over its verified TLS connection to the gateway. +- A response body over 64 KiB is `unclassified`. +- HTTP 429, 5xx, and failures to connect, including TLS certificate or hostname verification, are `unavailable`. A timeout, including a slow body, is `timeout`. +- Everything else is `unclassified`: other failed statuses including 401, 403, and 409, a connection lost after it is established, and a success without a nonempty, header-compatible `access_token`. OpenShell never infers `provider_authorization_required` from a 401. +- For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these only over its verified TLS connection to the gateway. -Sandbox status and the workload response keep only the code, stage, and HTTP status. The log of the component that called the token service, the gateway or the supervisor, records the same code with the HTTP status and request ID. OpenShell never logs the token service's `error` or `error_description` text, because a token service can echo a submitted credential there. A successful response without a nonempty header-compatible `access_token` is `unclassified`. +Sandbox status and the workload response keep only the code, stage, and HTTP status. Logs add the request ID. OpenShell never logs the token service's `error` or `error_description`, which can echo a submitted credential. -A token service can also state a cause by adopting the `openshell.token-grant.v1` error extension. For example, a service that requires the user to authorize a provider returns HTTP 409 with: +A token service can state a cause with the `openshell.token-grant.v1` error extension. For example, a service that needs the user to authorize a provider returns HTTP 409 with: ```json { @@ -698,15 +693,15 @@ A token service can also state a cause by adopting the `openshell.token-grant.v1 } ``` -OpenShell accepts this cause only from the configured endpoint over verified HTTPS. It verifies the server certificate and hostname and does not follow redirects. Loopback and Kubernetes-service HTTP endpoints cannot authenticate these signals. The extension recognizes these status and code pairs: +OpenShell accepts the extension only from the configured endpoint over verified HTTPS, without following redirects. Plain HTTP, including loopback and Kubernetes-service endpoints, cannot authenticate it. Each response needs the exact `openshell_error_contract` marker and one of these pairs: - HTTP 409 with `provider_authorization_required`. - HTTP 401 or 403 with `workload_identity_rejected`. - HTTP 400 with `configuration_error`. -Each response also requires the exact `openshell_error_contract` marker. OpenShell classifies a response with an unknown contract version or an unrecognized code by its HTTP status alone. A recognized code with a different HTTP status, or one received over plain HTTP, is `unclassified`. OpenShell does not infer identity rejection from generic OAuth errors such as `invalid_client` or `invalid_grant`, which can have more than one cause. +An unknown contract version or code falls back to HTTP status. A recognized code with another status, or over plain HTTP, is `unclassified`. Generic OAuth errors such as `invalid_client` and `invalid_grant` are never read as identity rejection, because they have more than one cause. -Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) for how to read `last_token_grant_failure` and when it clears. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. The no-bytes guarantee excludes connection setup: an inspected CONNECT path can already have completed an upstream TLS handshake before a grant fails. +Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) to read `last_token_grant_failure` and see when it clears. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. The no-bytes guarantee does not cover connection setup: on an inspected CONNECT path, the upstream TLS handshake can finish before a grant fails. ## Provider Instances diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index 58325c7556..e28814fee7 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -726,9 +726,9 @@ openshell sandbox get my-sandbox --policy-only ### Diagnose Failed Calls to Tool Servers -A sandbox can be `Ready` while an agent's call to an external tool server fails with `fetch failed`. Run `openshell sandbox get my-sandbox` and look under `Tool server connections` for the server's address, `Last result`, and `Reported at`. This information is available for configured endpoints that use MCP over HTTP. It helps you find where the call failed without changing sandbox readiness. +A sandbox can be `Ready` while an agent's call to an external tool server fails with `fetch failed`. Run `openshell sandbox get my-sandbox` and look under `Tool server connections` for the server's address, `Last result`, and `Reported at`. This is available for configured MCP-over-HTTP endpoints and does not change sandbox readiness. -JSON and YAML output include an `endpoint_statuses` list with `last_reported_at`. The protobuf API exposes the corresponding timestamp as `Sandbox.status.endpoint_statuses[].last_reported_time`; SDKs render it using their native or curated time representation. Select the endpoint by its address to read the result directly. Treat `endpoint_id` as opaque. +JSON and YAML output include an `endpoint_statuses` list with `last_reported_at`. The protobuf API exposes the same timestamp as `Sandbox.status.endpoint_statuses[].last_reported_time`, which SDKs render in their own time type. Select an endpoint by its address; `endpoint_id` is opaque. For example, use `jq` to select the tool server at `tools.example.com`, path `/mcp`, and port `443`: @@ -742,16 +742,16 @@ openshell sandbox get my-sandbox --output json | jq -e \ Read `last_result` to choose the next check: -- `NoObservedExchange`: no result has been reported for the current configuration and supervisor session. Try the operation and inspect its logs if no result appears. -- `HttpResponseReceived`: the server returned a final HTTP status below 400, including a protocol upgrade. Informational responses alone do not establish success. Check the client's response for protocol or tool errors; an HTTP 200 response can still contain an error. +- `NoObservedExchange`: nothing has been reported for the current configuration and supervisor session. Try the operation, then check the logs if no result appears. +- `HttpResponseReceived`: the server returned a final HTTP status below 400, including a protocol upgrade. Check the client's response for protocol or tool errors, because an HTTP 200 can still carry one. - `PolicyDenied`: OpenShell denied the request, including MCP protocol-version, request-header, request-body, and HTTP-method checks. Check the sandbox policy and denial logs. -- `CredentialUnavailable`: required credentials were unavailable. Check the endpoint's attached provider and `last_failure`. +- `CredentialUnavailable`: a required credential was unavailable. Check the endpoint's attached provider and `last_failure`. - `TlsFailed`: TLS setup or the handshake failed. Check certificates and TLS configuration. - `TransportFailed`: the network exchange failed. Check name resolution, connectivity, and the server process. -- `UpstreamRejected`: the server returned an HTTP status of 400 or higher. Check the server's response and logs. -- `Unspecified`: the result is unrecognized or absent. Do not infer success from it. +- `UpstreamRejected`: the server returned HTTP 400 or higher. Check the server's response and logs. +- `Unspecified`: the result is unrecognized or absent. Do not infer success. -For a provider-credentialed MCP-over-HTTP endpoint, a failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) also carries `last_failure`, which explains the `CredentialUnavailable` result. It describes why OpenShell could not complete the exchange; the tool server can still be healthy. `openshell sandbox get` prints it under the result: +For a provider-credentialed MCP-over-HTTP endpoint, a failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) sets `last_failure`, which explains `CredentialUnavailable`. The tool server can still be healthy. `openshell sandbox get` prints it under the result: ```text Last result: Required OpenShell-managed credential was unavailable. @@ -761,34 +761,32 @@ Recovery: Reauthorize the provider credential. Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908 ``` -In JSON and YAML output, `last_failure` has `domain`, `code`, `recovery_action`, `provider`, `credential_key`, `stage`, `upstream_http_status`, and `request_id`; it is `null` when no cause was reported. `recovery_action` says who acts next: +`last_failure` has `domain`, `code`, `recovery_action`, `provider`, `credential_key`, `stage`, `upstream_http_status`, and `request_id` in JSON and YAML, and is `null` when no cause was reported. `recovery_action` is one of: - `reauthorize`: the user authorizes the provider again. - `fix_configuration`: an operator repairs the provider, grant, or workload identity configuration. -- `retry`: the failure can be temporary; retry later. -- `investigate`: an operator inspects the supervisor and gateway logs for the request ID. +- `retry`: the failure can be temporary. +- `investigate`: an operator checks the supervisor and gateway logs for the request ID. -`request_id` matches the `x-openshell-request-id` header of the refused response. `provider` and `credential_key` are names, never credential values; they are empty when the grant failed before a credential was selected. Like `last_result`, `last_failure` describes only the last observation: a later exchange replaces it, and a change to the attached provider, such as storing a new credential, clears it. +`request_id` matches the `x-openshell-request-id` header of the refused response. `provider` and `credential_key` are names, never values, and are empty if the grant failed before credential selection. Like `last_result`, `last_failure` is the last observation only: a later exchange replaces it, and changing the attached provider clears it. -For example, a stopped server can produce `TransportFailed` while the sandbox stays `Ready`. After the server recovers, an HTTP response below 400 updates the result to `HttpResponseReceived`. +For example, a stopped server can produce `TransportFailed` while the sandbox stays `Ready`. After it recovers, an HTTP response below 400 updates the result to `HttpResponseReceived`. -Results come from actual traffic and do not expire. A result can remain unchanged after the server stops until another observed exchange or configuration/session reset. Reports combine recent observations and can drop them when full, so this list is not a request history. The address remains available when a result resets to `NoObservedExchange`. To verify current tool availability, run the actual operation. +Results come from actual traffic and do not expire, so one can outlast the server until another exchange is observed or the configuration or session resets. Reports can drop observations when full, so this is not a request history. The address remains after a reset to `NoObservedExchange`. To verify current availability, run the operation. -`last_reported_at` is the RFC 3339 rendering of the protobuf `last_reported_time` when the gateway accepted the result. It is empty until a result is reported. New accepted observations advance it, including repeated results; identical report retries do not. If a configuration reset supersedes a report whose acknowledgement was lost, the gateway can accept still-valid pending evidence again and advance this timestamp without another exchange. +`last_reported_at` is the RFC 3339 form of `last_reported_time`, the time the gateway accepted the result. It is empty until a result is reported. Each newly accepted observation advances it, including a repeated result; a retry of an identical report does not. -Addresses use lowercase hosts, canonical paths, and sorted distinct effective ports. Equivalent addresses share one record that combines observations from all callers and ports; it does not establish that every caller or port works. A failure before the HTTP path is known, such as a TLS failure during CONNECT, updates status only when the host and port identify one distinct endpoint. If several configured paths share that host and port, inspect the logs for the failure. +Addresses use lowercase hosts, canonical paths, and sorted distinct effective ports. Equivalent addresses share one record across all callers and ports, so it does not show that every one works. A failure before the HTTP path is known, such as a TLS failure during CONNECT, updates status only when the host and port identify one endpoint. Otherwise, check the logs. ### Read Token Grant Failures -`openshell sandbox get` shows `last_token_grant_failure` for the latest classified grant refusal, including REST requests and failures before an HTTP path is known. The protobuf field is `Sandbox.status.last_token_grant_failure`. It carries `destination_host`, `destination_port`, and a `failure` with the bounded fields described above. It describes the attempted call; the remote server can still be healthy. - -Read it in JSON or YAML, or select it directly: +`openshell sandbox get` shows `last_token_grant_failure` for the latest classified grant refusal, including REST requests and failures before an HTTP path is known. The protobuf field is `Sandbox.status.last_token_grant_failure`. It has `destination_host`, `destination_port`, and a `failure` with the fields above. It describes the attempted call, not server health. ```shell openshell sandbox get my-sandbox --output json | jq '.last_token_grant_failure' ``` -Match `failure.request_id` with the `x-openshell-request-id` from the refused response before starting recovery. A newer grant failure can replace this record. A successful grant for the same destination, provider, and credential clears it; when a failure happened before credential selection, the next successful grant to the same destination clears it. Policy, provider, and supervisor-session resets also clear it. JSON/YAML show `null` when no cause is retained. This is one latest observation, not a request history. +Match `failure.request_id` with the refused response's `x-openshell-request-id` before recovering, because a newer failure replaces this record. A successful grant for the same destination, provider, and credential clears it, or for the same destination if the failure came before credential selection. A policy, provider, or supervisor-session reset also clears it. The value is `null` when nothing is retained. ### Inspect Logs and Activity diff --git a/docs/observability/logging.mdx b/docs/observability/logging.mdx index 0879fcd855..2f6983b129 100644 --- a/docs/observability/logging.mdx +++ b/docs/observability/logging.mdx @@ -232,7 +232,7 @@ The `error` field is a short machine-readable code (`policy_denied`, `middleware For L7 REST policy denials, the body also includes structured policy fields such as `method`, `path`, `rule_missing`, and `next_steps`. When the policy advisor is enabled, the body also includes `agent_guidance`, a short plain-language instruction telling the agent to read `/etc/openshell/skills/policy_advisor.md`, propose the narrowest rule through `http://policy.local/v1/proposals`, wait for `policy_reloaded: true`, and retry. A middleware denial instead identifies the policy-local config in `middleware` and can include a validated `reason_code`. A fail-closed runtime failure uses `middleware_failed` with platform-owned text. Both middleware responses omit `rule_missing`, `next_steps`, and `agent_guidance` because no policy rule is missing. -A failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) returns `502 Bad Gateway` with an `x-openshell-request-id` header. The forward proxy returns `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`; L7 relays return an empty body. The response does not say why the grant failed. Give the request ID to someone who can read the sandbox. +A failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) returns `502 Bad Gateway` with an `x-openshell-request-id` header. The forward proxy body is `{"error":"token_grant_failed","detail":"dynamic token grant failed"}`; L7 relays send an empty body. Neither states the cause, so give the request ID to someone who can read the sandbox. ## Filesystem Sandbox Logs From ea9ff195e77162f47d4d7127b34a455489b2a47a Mon Sep 17 00:00:00 2001 From: John Myers <9696606+johntmyers@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:07:13 -0700 Subject: [PATCH 13/13] docs(providers): condense token grant failure guidance Signed-off-by: John Myers <9696606+johntmyers@users.noreply.github.com> --- docs/how-it-works/providers/profiles.mdx | 43 ++++++++++-------------- docs/how-it-works/sandboxes/overview.mdx | 38 +++++++++------------ 2 files changed, 33 insertions(+), 48 deletions(-) diff --git a/docs/how-it-works/providers/profiles.mdx b/docs/how-it-works/providers/profiles.mdx index 657343006e..1617e00afe 100644 --- a/docs/how-it-works/providers/profiles.mdx +++ b/docs/how-it-works/providers/profiles.mdx @@ -660,31 +660,22 @@ The gateway only brokers an intermediate token for a sandbox principal, and only ### Token Grant Failures -When a token grant fails, OpenShell refuses the request before sending any of its bytes upstream. The workload receives `502 Bad Gateway` with an `x-openshell-request-id` header and the body described in [Proxy Error Responses](/observability/logging#proxy-error-responses). The agent can read that response, the OCSF events, and `policy.local` denials, so none of them state the cause. Users who can read the sandbox get it from `Sandbox.status.last_token_grant_failure` and `openshell logs`. +A failed grant returns HTTP 502 without forwarding the protected request. Connection setup, including an upstream TLS handshake, may already have completed. The workload response hides the cause; authorized sandbox readers can [diagnose the failure](/how-it-works/sandboxes/overview#read-token-grant-failures). The request ID correlates the response, sandbox status, and supervisor/gateway logs. OpenShell never logs the token service's `error` or `error_description`, which can echo credentials. -The supervisor mints a random UUID request ID for each HTTP exchange. It sends the ID to the token service as `x-openshell-request-id` and, for `token_exchange` grants, to the gateway as gRPC `x-request-id` metadata, which the gateway forwards to the intermediate token endpoint. The supervisor's `Token grant failed` OCSF event carries it as `request_id`. Log lines from the supervisor and gateway add the code, stage, and upstream HTTP status. - -Each failure gets one code. Recovery actions match provider refresh status: +Each failure has a code, a stage identifying the failed step, and a recovery action: | Code | Meaning | Recovery action | |---|---|---| -| `provider_authorization_required` | The user must authorize the provider again. | `reauthorize` | -| `workload_identity_rejected` | The SPIFFE Workload API, gateway, or token service rejected the workload identity. | `fix_configuration` | -| `configuration_error` | The grant, provider, or token-service configuration is invalid. | `fix_configuration` | -| `timeout` | A grant step exceeded its time limit. | `retry` | -| `unavailable` | A required service or connection was unavailable. | `retry` | -| `unclassified` | The evidence does not establish a more specific cause. | `investigate` | - -The stage is the step that observed the failure: `configuration`, `workload_identity`, `gateway_rpc`, `intermediate_exchange`, `token_endpoint`, or `injection`. OpenShell maps token-service responses as follows: - -- A response body over 64 KiB is `unclassified`. -- HTTP 429, 5xx, and failures to connect, including TLS certificate or hostname verification, are `unavailable`. A timeout, including a slow body, is `timeout`. -- Everything else is `unclassified`: other failed statuses including 401, 403, and 409, a connection lost after it is established, and a success without a nonempty, header-compatible `access_token`. OpenShell never infers `provider_authorization_required` from a 401. -- For `token_exchange` grants, the gateway reports `provider_authorization_required` when the user's stored subject credential is missing or expired, and `workload_identity_rejected` when it rejects the supervisor's JWT-SVID. The supervisor accepts these only over its verified TLS connection to the gateway. +| `provider_authorization_required` | Provider authorization is required, including a missing or expired stored subject credential. | `reauthorize` | +| `workload_identity_rejected` | The workload identity was rejected. | `fix_configuration` | +| `configuration_error` | Provider, grant, or token-service configuration is invalid. | `fix_configuration` | +| `timeout` | A grant step timed out. | `retry` | +| `unavailable` | A service or connection was unavailable. | `retry` | +| `unclassified` | No more specific cause is established. | `investigate` | -Sandbox status and the workload response keep only the code, stage, and HTTP status. Logs add the request ID. OpenShell never logs the token service's `error` or `error_description`, which can echo a submitted credential. +Without a recognized error extension, HTTP 429, 5xx, and connection failures, including TLS verification failures, are `unavailable`; timeouts are `timeout`. Other failures are `unclassified`, including response bodies over 64 KiB and successful responses without a valid `access_token`. A generic 401 never implies reauthorization. -A token service can state a cause with the `openshell.token-grant.v1` error extension. For example, a service that needs the user to authorize a provider returns HTTP 409 with: +A token service can report a specific cause using the `openshell.token-grant.v1` extension. For example, return HTTP 409 with: ```json { @@ -693,15 +684,15 @@ A token service can state a cause with the `openshell.token-grant.v1` error exte } ``` -OpenShell accepts the extension only from the configured endpoint over verified HTTPS, without following redirects. Plain HTTP, including loopback and Kubernetes-service endpoints, cannot authenticate it. Each response needs the exact `openshell_error_contract` marker and one of these pairs: +OpenShell requires the exact contract marker and one of these pairs: -- HTTP 409 with `provider_authorization_required`. -- HTTP 401 or 403 with `workload_identity_rejected`. -- HTTP 400 with `configuration_error`. - -An unknown contract version or code falls back to HTTP status. A recognized code with another status, or over plain HTTP, is `unclassified`. Generic OAuth errors such as `invalid_client` and `invalid_grant` are never read as identity rejection, because they have more than one cause. +| HTTP status | Error code | +|---|---| +| 409 | `provider_authorization_required` | +| 401 or 403 | `workload_identity_rejected` | +| 400 | `configuration_error` | -Refer to [Read Token Grant Failures](/how-it-works/sandboxes/overview#read-token-grant-failures) to read `last_token_grant_failure` and see when it clears. MCP-over-HTTP endpoint status also carries the cause as `last_failure`. The no-bytes guarantee does not cover connection setup: on an inspected CONNECT path, the upstream TLS handshake can finish before a grant fails. +The extension requires the configured endpoint over verified HTTPS; redirects are not followed. Recognized codes over plain HTTP or with a mismatched status are `unclassified`. Unknown versions or codes fall back to HTTP-status classification. Generic OAuth errors such as `invalid_client` and `invalid_grant` do not establish identity rejection. ## Provider Instances diff --git a/docs/how-it-works/sandboxes/overview.mdx b/docs/how-it-works/sandboxes/overview.mdx index e28814fee7..45938d22ed 100644 --- a/docs/how-it-works/sandboxes/overview.mdx +++ b/docs/how-it-works/sandboxes/overview.mdx @@ -745,31 +745,12 @@ Read `last_result` to choose the next check: - `NoObservedExchange`: nothing has been reported for the current configuration and supervisor session. Try the operation, then check the logs if no result appears. - `HttpResponseReceived`: the server returned a final HTTP status below 400, including a protocol upgrade. Check the client's response for protocol or tool errors, because an HTTP 200 can still carry one. - `PolicyDenied`: OpenShell denied the request, including MCP protocol-version, request-header, request-body, and HTTP-method checks. Check the sandbox policy and denial logs. -- `CredentialUnavailable`: a required credential was unavailable. Check the endpoint's attached provider and `last_failure`. +- `CredentialUnavailable`: a required credential was unavailable. Check the attached provider and [diagnose the token grant failure](#read-token-grant-failures). - `TlsFailed`: TLS setup or the handshake failed. Check certificates and TLS configuration. - `TransportFailed`: the network exchange failed. Check name resolution, connectivity, and the server process. - `UpstreamRejected`: the server returned HTTP 400 or higher. Check the server's response and logs. - `Unspecified`: the result is unrecognized or absent. Do not infer success. -For a provider-credentialed MCP-over-HTTP endpoint, a failed [dynamic token grant](/how-it-works/providers/profiles#token-grant-failures) sets `last_failure`, which explains `CredentialUnavailable`. The tool server can still be healthy. `openshell sandbox get` prints it under the result: - -```text -Last result: Required OpenShell-managed credential was unavailable. -Cause: provider_authorization_required (token grant, stage gateway_rpc) -Credential: github/access_token -Recovery: Reauthorize the provider credential. -Request ID: 7e6d5c4b-3a29-4180-9f6e-5d4c3b2a1908 -``` - -`last_failure` has `domain`, `code`, `recovery_action`, `provider`, `credential_key`, `stage`, `upstream_http_status`, and `request_id` in JSON and YAML, and is `null` when no cause was reported. `recovery_action` is one of: - -- `reauthorize`: the user authorizes the provider again. -- `fix_configuration`: an operator repairs the provider, grant, or workload identity configuration. -- `retry`: the failure can be temporary. -- `investigate`: an operator checks the supervisor and gateway logs for the request ID. - -`request_id` matches the `x-openshell-request-id` header of the refused response. `provider` and `credential_key` are names, never values, and are empty if the grant failed before credential selection. Like `last_result`, `last_failure` is the last observation only: a later exchange replaces it, and changing the attached provider clears it. - For example, a stopped server can produce `TransportFailed` while the sandbox stays `Ready`. After it recovers, an HTTP response below 400 updates the result to `HttpResponseReceived`. Results come from actual traffic and do not expire, so one can outlast the server until another exchange is observed or the configuration or session resets. Reports can drop observations when full, so this is not a request history. The address remains after a reset to `NoObservedExchange`. To verify current availability, run the operation. @@ -780,13 +761,26 @@ Addresses use lowercase hosts, canonical paths, and sorted distinct effective po ### Read Token Grant Failures -`openshell sandbox get` shows `last_token_grant_failure` for the latest classified grant refusal, including REST requests and failures before an HTTP path is known. The protobuf field is `Sandbox.status.last_token_grant_failure`. It has `destination_host`, `destination_port`, and a `failure` with the fields above. It describes the attempted call, not server health. +When OpenShell cannot obtain a required token, the workload receives HTTP 502 with an `x-openshell-request-id` header. The response hides the cause. If you can read the sandbox, inspect its latest failure: ```shell openshell sandbox get my-sandbox --output json | jq '.last_token_grant_failure' ``` -Match `failure.request_id` with the refused response's `x-openshell-request-id` before recovering, because a newer failure replaces this record. A successful grant for the same destination, provider, and credential clears it, or for the same destination if the failure came before credential selection. A policy, provider, or supervisor-session reset also clears it. The value is `null` when nothing is retained. +Match `failure.request_id` to the response header; a newer failure may have replaced the record. It identifies the destination host and port, failure code, stage, and recovery action. Provider and credential names appear when known; credential values never appear. + +| Recovery action | Next step | +|---|---| +| `reauthorize` | Authorize the provider again. | +| `fix_configuration` | Repair the provider, grant, or workload identity configuration. | +| `retry` | Retry later. | +| `investigate` | Check supervisor and gateway logs using the request ID. | + +For MCP-over-HTTP endpoints, `Tool server connections` also shows the cause as `last_failure` under `CredentialUnavailable`. A credential failure does not establish that the remote server is unhealthy. + +The sandbox retains one latest failure, not a request history. A successful grant for the same destination, provider, and credential clears it; failures before credential selection clear on the next successful grant to that destination. Policy, provider, and supervisor-session resets also clear it. JSON/YAML shows `null` when no failure remains. + +See [Token Grant Failures](/how-it-works/providers/profiles#token-grant-failures) for failure-code definitions and token-service behavior. ### Inspect Logs and Activity