From 596bc12fe8e45fa29d8c862f7693dec2444d5d23 Mon Sep 17 00:00:00 2001 From: Balaji Ganesan Date: Wed, 14 Oct 2026 18:37:46 -0700 Subject: [PATCH] fix(vm): boot a digest-pinned image from the cache without a registry call A registry-sourced sandbox boot authenticated with the registry, and for a tag resolved the manifest, before it looked at the root disk cache. It did so on every boot, even when the image reference carries a digest and the root disk for that digest is already cached. Each boot then waits on the registry, and a registry outage stops every boot of an image the host already has. A digest names the image's content, and since #4342 a cached root disk is published only after its manifest was verified against that digest. On a cache hit for a digest-pinned reference the registry cannot add anything, so return the hit before authenticating. Authentication and the pull stay on the miss path, and a tag still resolves through the registry. The hit emits the same Pulling, CacheHit and Pulled events as before. Behaviour change: a host whose registry credentials were revoked still boots digest-pinned images it has already cached, as a kubelet does with IfNotPresent and a digest-pinned image. Test: bootstrap_registry_cache_hit_for_a_digest_pin_makes_no_registry_request covers a miss (authenticates and fetches), a cached digest pin (no request at all) and a tag for the same content (still resolved). It fails without this change with ["GET /v2/"]. Closes #4581 Signed-off-by: Balaji Ganesan --- crates/openshell-driver-vm/src/driver.rs | 90 ++++++++++++++----- .../src/registry_cache_tests.rs | 60 +++++++++++++ 2 files changed, 126 insertions(+), 24 deletions(-) diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index d62e9812b3..821bb6820e 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -3223,6 +3223,43 @@ impl VmDriver { ) } + fn publish_pulling_event(&self, sandbox_id: &str, image_ref: &str) { + self.publish_platform_event( + sandbox_id.to_string(), + platform_event( + "vm", + "Normal", + "Pulling", + format!("Pulling image \"{image_ref}\""), + ), + ); + } + + /// A registry-sourced root disk found in the cache: `Pulling`, the + /// `CacheHit` progress and `Pulled`, so the image step still advances. + async fn publish_registry_cache_hit( + &self, + sandbox_id: &str, + image_ref: &str, + image_identity: &str, + image_path: &Path, + ) { + self.publish_pulling_event(sandbox_id, image_ref); + self.publish_vm_progress( + sandbox_id, + "CacheHit", + format!("Using cached VM root disk for image \"{image_ref}\""), + HashMap::from([ + ("image_ref".to_string(), image_ref.to_string()), + ("image_source".to_string(), "registry".to_string()), + ("cache_hit".to_string(), "true".to_string()), + ("image_identity".to_string(), image_identity.to_string()), + ]), + ); + self.publish_pulled_event(sandbox_id, image_ref, image_path) + .await; + } + async fn ensure_cached_registry_rootfs_image( &self, sandbox_id: &str, @@ -3233,6 +3270,31 @@ impl VmDriver { let span_status = openshell_otel::ErrorStatusGuard::current(); info!(image_ref = %image_ref, "vm driver: ensuring cached root disk image (registry)"); let reference = parse_registry_reference(image_ref)?; + // A digest pin already names the image's content, and a cached root + // disk is only published after its manifest was verified against + // that digest. On a hit the registry cannot add anything, so neither + // authenticate nor resolve: the sandbox boots from the cache without a + // network round trip, and keeps booting while the registry is down. + // Tags still resolve through the registry below. + if let Some(digest) = reference.digest() { + let image_identity = bootstrap_image_cache_identity(digest); + let image_path = image_cache_rootfs_image(&self.config.state_dir, &image_identity); + if tokio::fs::metadata(&image_path).await.is_ok() { + info!( + image_identity = %image_identity, + image_path = %image_path.display(), + "vm driver: root disk image cache hit (digest-pinned, no registry call)" + ); + self.publish_registry_cache_hit( + sandbox_id, + image_ref, + &image_identity, + &image_path, + ) + .await; + return span_status.finish(Ok(image_identity)); + } + } info!(image_ref = %image_ref, "vm driver: authenticating with registry"); self.publish_vm_progress( sandbox_id, @@ -3277,40 +3339,20 @@ impl VmDriver { let image_identity = bootstrap_image_cache_identity(&source_image_identity); let image_path = image_cache_rootfs_image(&self.config.state_dir, &image_identity); - // Emit a driver progress hint for cache hits too and immediately - // follow with `Pulled` so the image step still advances cleanly. - self.publish_platform_event( - sandbox_id.to_string(), - platform_event( - "vm", - "Normal", - "Pulling", - format!("Pulling image \"{image_ref}\""), - ), - ); - if tokio::fs::metadata(&image_path).await.is_ok() { info!( image_identity = %image_identity, image_path = %image_path.display(), "vm driver: root disk image cache hit (no build needed)" ); - self.publish_vm_progress( - sandbox_id, - "CacheHit", - format!("Using cached VM root disk for image \"{image_ref}\""), - HashMap::from([ - ("image_ref".to_string(), image_ref.to_string()), - ("image_source".to_string(), "registry".to_string()), - ("cache_hit".to_string(), "true".to_string()), - ("image_identity".to_string(), image_identity.clone()), - ]), - ); - self.publish_pulled_event(sandbox_id, image_ref, &image_path) + self.publish_registry_cache_hit(sandbox_id, image_ref, &image_identity, &image_path) .await; return span_status.finish(Ok(image_identity)); } + // A driver progress hint before the build, so the image step advances. + self.publish_pulling_event(sandbox_id, image_ref); + info!( image_identity = %image_identity, "vm driver: root disk image cache miss, acquiring build lock" diff --git a/crates/openshell-driver-vm/src/registry_cache_tests.rs b/crates/openshell-driver-vm/src/registry_cache_tests.rs index 21c843e319..85a5f2084c 100644 --- a/crates/openshell-driver-vm/src/registry_cache_tests.rs +++ b/crates/openshell-driver-vm/src/registry_cache_tests.rs @@ -478,3 +478,63 @@ async fn registry_cache_preserves_caller_digest_when_head_uses_another_algorithm .all(|request| request == "GET /v2/") ); } + +#[tokio::test] +async fn bootstrap_registry_cache_hit_for_a_digest_pin_makes_no_registry_request() { + let body = manifest("pinned"); + let digest = sha256(&body); + let registry = Registry::start( + digest.clone(), + HashMap::from([("latest".to_string(), body.clone()), (digest.clone(), body)]), + None, + false, + ) + .await; + let root = tempfile::tempdir().unwrap(); + let driver = driver(root.path()); + let client = client(); + + // Not cached yet: a digest pin still authenticates and fetches (and here + // fails at the blobs the fixture does not serve). + ensure_image(&driver, ®istry.reference(Some(&digest)), &client, true) + .await + .unwrap_err(); + assert!( + registry + .requests() + .iter() + .any(|request| request.contains(&format!("/manifests/{digest}"))), + "{:?}", + registry.requests() + ); + + // Cached: the same pin boots from the cache with no request at all. + let identity = bootstrap_image_cache_identity(&digest); + let path = image_cache_rootfs_image(root.path(), &identity); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(&path, b"cached disk").unwrap(); + let before = registry.requests().len(); + let resolved = ensure_image(&driver, ®istry.reference(Some(&digest)), &client, true) + .await + .unwrap(); + assert_eq!(resolved, identity); + assert_eq!( + registry.requests().len(), + before, + "a digest-pinned cache hit reached the registry: {:?}", + ®istry.requests()[before..] + ); + + // A tag naming the same content still resolves through the registry. + let resolved = ensure_image(&driver, ®istry.reference(None), &client, true) + .await + .unwrap(); + assert_eq!(resolved, identity); + assert!( + registry.requests()[before..] + .iter() + .any(|request| request == "HEAD /v2/test/image/manifests/latest"), + "{:?}", + registry.requests() + ); +}