From 325922aaf025148eb23afe6eb48dd112c3172db9 Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Sat, 10 Oct 2026 21:19:53 +0100 Subject: [PATCH 1/2] fix(otel): enable gzip and https OTLP export Signed-off-by: Eric Curtin --- Cargo.lock | 3 + Cargo.toml | 2 +- crates/openshell-otel-test-support/Cargo.toml | 6 +- crates/openshell-otel-test-support/src/lib.rs | 36 +++++++++++- .../tests/tls_gzip_export.rs | 56 +++++++++++++++++++ crates/openshell-otel/src/lib.rs | 15 +++-- docs/how-it-works/gateways/configuration.mdx | 5 +- 7 files changed, 111 insertions(+), 12 deletions(-) create mode 100644 crates/openshell-otel-test-support/tests/tls_gzip_export.rs diff --git a/Cargo.lock b/Cargo.lock index 407213cf88..b027c616db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4719,6 +4719,8 @@ version = "0.0.0" dependencies = [ "openshell-otel", "opentelemetry-proto", + "rcgen", + "tempfile", "tokio", "tokio-stream", "tonic", @@ -8216,6 +8218,7 @@ dependencies = [ "axum", "base64 0.22.1", "bytes", + "flate2", "h2", "http 1.4.0", "http-body 1.0.1", diff --git a/Cargo.toml b/Cargo.toml index c6136bdc26..705bc5f74a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,7 @@ tracing-appender = "0.2" # tonic 0.14 / prost 0.14 via opentelemetry-proto's `grpc-tonic` feature. opentelemetry = "0.32" opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] } -opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] } +opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "gzip-tonic", "tls-aws-lc", "tls-roots", "trace"] } tracing-opentelemetry = { version = "0.33", default-features = false, features = ["tracing-log"] } # Metrics diff --git a/crates/openshell-otel-test-support/Cargo.toml b/crates/openshell-otel-test-support/Cargo.toml index 48be77e1ce..e288531b6c 100644 --- a/crates/openshell-otel-test-support/Cargo.toml +++ b/crates/openshell-otel-test-support/Cargo.toml @@ -15,9 +15,13 @@ openshell-otel = { path = "../openshell-otel" } opentelemetry-proto = { version = "0.32", default-features = false, features = ["gen-tonic", "trace"] } tokio = { workspace = true } tokio-stream = { workspace = true } -tonic = { workspace = true } +tonic = { workspace = true, features = ["gzip", "tls-aws-lc"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } +[dev-dependencies] +rcgen = { workspace = true } +tempfile = "3" + [lints] workspace = true diff --git a/crates/openshell-otel-test-support/src/lib.rs b/crates/openshell-otel-test-support/src/lib.rs index 5d6db481a6..b2634f0136 100644 --- a/crates/openshell-otel-test-support/src/lib.rs +++ b/crates/openshell-otel-test-support/src/lib.rs @@ -71,6 +71,7 @@ pub struct ReceivedTraces { pub service_names: Vec, pub gateway_names: Vec, pub compute_drivers: Vec, + pub encodings: Vec, } #[derive(Clone)] @@ -90,6 +91,13 @@ impl TraceService for Collector { .received .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some(encoding) = request + .metadata() + .get("grpc-encoding") + .and_then(|value| value.to_str().ok()) + { + received.encodings.push(encoding.to_string()); + } for resource_span in request.into_inner().resource_spans { if let Some(resource) = resource_span.resource { for attribute in resource.attributes { @@ -133,20 +141,42 @@ pub struct OtlpTestServer { impl OtlpTestServer { pub async fn start() -> Self { + Self::start_with(None).await + } + + /// Serve over TLS at `https://localhost:` using `identity`. + pub async fn start_tls(identity: tonic::transport::Identity) -> Self { + Self::start_with(Some(identity)).await + } + + async fn start_with(identity: Option) -> Self { let received = Arc::new(Mutex::new(ReceivedTraces::default())); let exported = Arc::new(tokio::sync::Notify::new()); let listener = tokio::net::TcpListener::bind("127.0.0.1:0") .await .expect("OTLP test collector should bind a loopback listener"); - let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let port = listener.local_addr().unwrap().port(); + let endpoint = if identity.is_some() { + format!("https://localhost:{port}") + } else { + format!("http://127.0.0.1:{port}") + }; let collector = Collector { received: Arc::clone(&received), exported: Arc::clone(&exported), }; + let mut server = tonic::transport::Server::builder(); + if let Some(identity) = identity { + server = server + .tls_config(tonic::transport::ServerTlsConfig::new().identity(identity)) + .expect("OTLP test collector should accept its TLS identity"); + } + let service = TraceServiceServer::new(collector) + .accept_compressed(tonic::codec::CompressionEncoding::Gzip); let (shutdown, shutdown_rx) = tokio::sync::oneshot::channel(); let task = tokio::spawn(async move { - tonic::transport::Server::builder() - .add_service(TraceServiceServer::new(collector)) + server + .add_service(service) .serve_with_incoming_shutdown( tokio_stream::wrappers::TcpListenerStream::new(listener), async { diff --git a/crates/openshell-otel-test-support/tests/tls_gzip_export.rs b/crates/openshell-otel-test-support/tests/tls_gzip_export.rs new file mode 100644 index 0000000000..e892b26c01 --- /dev/null +++ b/crates/openshell-otel-test-support/tests/tls_gzip_export.rs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The shared exporter delivers gzip-compressed spans to a collector over TLS. + +use openshell_otel::{OtlpTraceConfig, ServiceName, provider_for}; +use openshell_otel_test_support::OtlpTestServer; +use rcgen::{BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair}; +use tonic::transport::Identity; +use tracing_subscriber::layer::SubscriberExt as _; + +#[tokio::test(flavor = "multi_thread")] +#[allow(unsafe_code)] +async fn gzip_spans_reach_a_collector_trusted_through_platform_roots() { + let ca_key = KeyPair::generate().unwrap(); + let mut ca_params = CertificateParams::new(Vec::::new()).unwrap(); + ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let ca = ca_params.self_signed(&ca_key).unwrap(); + + let server_key = KeyPair::generate().unwrap(); + let mut server_params = CertificateParams::new(vec!["localhost".to_string()]).unwrap(); + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let server_cert = server_params.signed_by(&server_key, &ca, &ca_key).unwrap(); + + let roots = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(roots.path(), ca.pem()).unwrap(); + // This binary has one test, so nothing else reads the environment. + unsafe { + std::env::set_var("SSL_CERT_FILE", roots.path()); + std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", "gzip"); + } + + let collector = OtlpTestServer::start_tls(Identity::from_pem( + server_cert.pem(), + server_key.serialize_pem(), + )) + .await; + let (provider, error) = provider_for(Some(OtlpTraceConfig { + endpoint: collector.endpoint(), + service_name: ServiceName::Fixed("tls-test"), + service_version: None, + resource_attributes: Vec::new(), + })); + assert!(error.is_none(), "{error:?}"); + let provider = provider.unwrap(); + let subscriber = + tracing_subscriber::registry().with(openshell_otel::layer(&provider, "tls-test")); + tracing::subscriber::with_default(subscriber, || drop(tracing::info_span!("tls.span"))); + provider.force_flush().unwrap(); + collector.wait_for_export().await; + provider.shutdown().unwrap(); + let received = collector.shutdown().await; + + assert!(received.spans.iter().any(|span| span.name == "tls.span")); + assert_eq!(received.encodings, ["gzip"]); +} diff --git a/crates/openshell-otel/src/lib.rs b/crates/openshell-otel/src/lib.rs index 182a1e0837..8fcbecd761 100644 --- a/crates/openshell-otel/src/lib.rs +++ b/crates/openshell-otel/src/lib.rs @@ -25,7 +25,8 @@ pub use propagation::{ use opentelemetry::KeyValue; use opentelemetry::trace::TracerProvider as _; -use opentelemetry_otlp::{SpanExporter, WithExportConfig}; +use opentelemetry_otlp::tonic_types::transport::ClientTlsConfig; +use opentelemetry_otlp::{SpanExporter, WithExportConfig, WithTonicConfig}; use opentelemetry_sdk::Resource; use opentelemetry_sdk::trace::SdkTracer; pub use opentelemetry_sdk::trace::SdkTracerProvider; @@ -189,17 +190,19 @@ pub fn build_provider(config: &OtlpTraceConfig<'_>) -> Result() .map_err(|source| SetupError::InvalidEndpoint { endpoint: endpoint.to_string(), source, })?; - let exporter = SpanExporter::builder() - .with_tonic() - .with_endpoint(endpoint) - .build()?; + let mut builder = SpanExporter::builder().with_tonic().with_endpoint(endpoint); + if uri.scheme() == Some(&http::uri::Scheme::HTTPS) { + // Tonic trusts nothing by default, so enable the platform roots. + builder = builder.with_tls_config(ClientTlsConfig::new().with_enabled_roots()); + } + let exporter = builder.build()?; Ok(SdkTracerProvider::builder() .with_batch_exporter(exporter) diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 21eda0a59d..d812484bba 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -381,6 +381,8 @@ service_name = "openshell-gateway" The transport is **OTLP over gRPC only**. HTTP/protobuf and HTTP/JSON are not supported, and `OTEL_EXPORTER_OTLP_PROTOCOL` has no effect. Point `endpoint` at a collector's gRPC receiver, conventionally port `4317`, not the HTTP receiver on `4318`. A URI alone cannot distinguish the two, so an HTTP endpoint is accepted at startup and then fails on export. +Use an `https://` endpoint for TLS. The gateway verifies the collector certificate against the system trust roots. + The OpenTelemetry SDK logs export failures after startup. Spans in a failed batch are dropped rather than retried. `service_name` sets the gateway's `service.name` resource attribute and defaults to `openshell-gateway`. The gateway also reports `service.version`, `openshell.gateway.name` from the gateway's configured `name`, and `openshell.gateway.compute_driver`. @@ -426,7 +428,8 @@ This table decides whether and where to export. How the SDK exports is controlle | `OTEL_BSP_SCHEDULE_DELAY`, `OTEL_BSP_MAX_QUEUE_SIZE`, `OTEL_BSP_MAX_EXPORT_BATCH_SIZE`, `OTEL_BSP_EXPORT_TIMEOUT` | Batch span processor tuning. | | `OTEL_RESOURCE_ATTRIBUTES` | Additional resource attributes, such as `deployment.environment=prod`. | | `OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT`, `OTEL_SPAN_EVENT_COUNT_LIMIT`, `OTEL_SPAN_LINK_COUNT_LIMIT` | Per-span limits. | -| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_COMPRESSION`, `OTEL_EXPORTER_OTLP_TIMEOUT` | Exporter transport tuning. | +| `OTEL_EXPORTER_OTLP_HEADERS`, `OTEL_EXPORTER_OTLP_TIMEOUT` | Exporter transport tuning. | +| `OTEL_EXPORTER_OTLP_COMPRESSION` | Set to `gzip` to compress exports. `zstd` is not supported. | | `OTEL_EXPORTER_OTLP_PROTOCOL` | No effect. The gateway is built with the gRPC exporter only. | To sample 10% of traces: From aecc4d248d1476af4a8ef9aaece839205578438d Mon Sep 17 00:00:00 2001 From: Eric Curtin Date: Mon, 12 Oct 2026 21:09:11 +0100 Subject: [PATCH 2/2] test(otel): reject untrusted OTLP collector certificates Signed-off-by: Eric Curtin --- .../tests/common/mod.rs | 75 +++++++++++++++++++ .../tests/tls_gzip_export.rs | 47 +----------- .../tests/tls_untrusted_export.rs | 14 ++++ 3 files changed, 92 insertions(+), 44 deletions(-) create mode 100644 crates/openshell-otel-test-support/tests/common/mod.rs create mode 100644 crates/openshell-otel-test-support/tests/tls_untrusted_export.rs diff --git a/crates/openshell-otel-test-support/tests/common/mod.rs b/crates/openshell-otel-test-support/tests/common/mod.rs new file mode 100644 index 0000000000..6fa3f39409 --- /dev/null +++ b/crates/openshell-otel-test-support/tests/common/mod.rs @@ -0,0 +1,75 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Shared TLS export scenario for the one-test-per-binary TLS tests. + +use openshell_otel::{OtlpTraceConfig, ServiceName, provider_for}; +use openshell_otel_test_support::{OtlpTestServer, ReceivedTraces}; +use rcgen::{ + BasicConstraints, Certificate, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair, +}; +use tonic::transport::Identity; +use tracing_subscriber::layer::SubscriberExt as _; + +fn new_ca() -> (Certificate, KeyPair) { + let key = KeyPair::generate().unwrap(); + let mut params = CertificateParams::new(Vec::::new()).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + (params.self_signed(&key).unwrap(), key) +} + +/// Export one gzip span to a TLS collector and return the flush outcome and +/// what the collector received. Only one CA is in the platform roots. The +/// collector's certificate is signed by that CA when `server_trusted`, and by +/// an unrelated CA otherwise. +/// +/// Callers must be the only test in their binary, since this sets the +/// process environment. +#[allow(unsafe_code)] +pub async fn export_span_over_tls(server_trusted: bool) -> (bool, ReceivedTraces) { + let (trusted_ca, trusted_key) = new_ca(); + let (other_ca, other_key) = new_ca(); + let (signer, signer_key) = if server_trusted { + (&trusted_ca, &trusted_key) + } else { + (&other_ca, &other_key) + }; + + let server_key = KeyPair::generate().unwrap(); + let mut server_params = CertificateParams::new(vec!["localhost".to_string()]).unwrap(); + server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; + let server_cert = server_params + .signed_by(&server_key, signer, signer_key) + .unwrap(); + + let roots = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(roots.path(), trusted_ca.pem()).unwrap(); + // The caller's binary has one test, so nothing else reads the environment. + unsafe { + std::env::set_var("SSL_CERT_FILE", roots.path()); + std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", "gzip"); + } + + let collector = OtlpTestServer::start_tls(Identity::from_pem( + server_cert.pem(), + server_key.serialize_pem(), + )) + .await; + let (provider, error) = provider_for(Some(OtlpTraceConfig { + endpoint: collector.endpoint(), + service_name: ServiceName::Fixed("tls-test"), + service_version: None, + resource_attributes: Vec::new(), + })); + assert!(error.is_none(), "{error:?}"); + let provider = provider.unwrap(); + let subscriber = + tracing_subscriber::registry().with(openshell_otel::layer(&provider, "tls-test")); + tracing::subscriber::with_default(subscriber, || drop(tracing::info_span!("tls.span"))); + let flushed = provider.force_flush().is_ok(); + if flushed { + collector.wait_for_export().await; + } + let _ = provider.shutdown(); + (flushed, collector.shutdown().await) +} diff --git a/crates/openshell-otel-test-support/tests/tls_gzip_export.rs b/crates/openshell-otel-test-support/tests/tls_gzip_export.rs index e892b26c01..97d2881bb1 100644 --- a/crates/openshell-otel-test-support/tests/tls_gzip_export.rs +++ b/crates/openshell-otel-test-support/tests/tls_gzip_export.rs @@ -3,54 +3,13 @@ //! The shared exporter delivers gzip-compressed spans to a collector over TLS. -use openshell_otel::{OtlpTraceConfig, ServiceName, provider_for}; -use openshell_otel_test_support::OtlpTestServer; -use rcgen::{BasicConstraints, CertificateParams, ExtendedKeyUsagePurpose, IsCa, KeyPair}; -use tonic::transport::Identity; -use tracing_subscriber::layer::SubscriberExt as _; +mod common; #[tokio::test(flavor = "multi_thread")] -#[allow(unsafe_code)] async fn gzip_spans_reach_a_collector_trusted_through_platform_roots() { - let ca_key = KeyPair::generate().unwrap(); - let mut ca_params = CertificateParams::new(Vec::::new()).unwrap(); - ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); - let ca = ca_params.self_signed(&ca_key).unwrap(); - - let server_key = KeyPair::generate().unwrap(); - let mut server_params = CertificateParams::new(vec!["localhost".to_string()]).unwrap(); - server_params.extended_key_usages = vec![ExtendedKeyUsagePurpose::ServerAuth]; - let server_cert = server_params.signed_by(&server_key, &ca, &ca_key).unwrap(); - - let roots = tempfile::NamedTempFile::new().unwrap(); - std::fs::write(roots.path(), ca.pem()).unwrap(); - // This binary has one test, so nothing else reads the environment. - unsafe { - std::env::set_var("SSL_CERT_FILE", roots.path()); - std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", "gzip"); - } - - let collector = OtlpTestServer::start_tls(Identity::from_pem( - server_cert.pem(), - server_key.serialize_pem(), - )) - .await; - let (provider, error) = provider_for(Some(OtlpTraceConfig { - endpoint: collector.endpoint(), - service_name: ServiceName::Fixed("tls-test"), - service_version: None, - resource_attributes: Vec::new(), - })); - assert!(error.is_none(), "{error:?}"); - let provider = provider.unwrap(); - let subscriber = - tracing_subscriber::registry().with(openshell_otel::layer(&provider, "tls-test")); - tracing::subscriber::with_default(subscriber, || drop(tracing::info_span!("tls.span"))); - provider.force_flush().unwrap(); - collector.wait_for_export().await; - provider.shutdown().unwrap(); - let received = collector.shutdown().await; + let (flushed, received) = common::export_span_over_tls(true).await; + assert!(flushed); assert!(received.spans.iter().any(|span| span.name == "tls.span")); assert_eq!(received.encodings, ["gzip"]); } diff --git a/crates/openshell-otel-test-support/tests/tls_untrusted_export.rs b/crates/openshell-otel-test-support/tests/tls_untrusted_export.rs new file mode 100644 index 0000000000..a62fb9c047 --- /dev/null +++ b/crates/openshell-otel-test-support/tests/tls_untrusted_export.rs @@ -0,0 +1,14 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The shared exporter refuses a collector whose certificate is not trusted. + +mod common; + +#[tokio::test(flavor = "multi_thread")] +async fn spans_do_not_reach_a_collector_with_an_untrusted_certificate() { + let (flushed, received) = common::export_span_over_tls(false).await; + + assert!(!flushed, "export to an untrusted collector must fail"); + assert!(received.spans.is_empty()); +}