From 37b8ca3c9cc99e702352be9bed16ff5de05f66ac Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 01/22] Publish release artifacts Build and publish versioned binaries, container images, and Helm charts from release tags. Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 154 +++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 .github/workflows/release.yaml diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000000..4a26a5cbe9 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,154 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: release + +on: + workflow_dispatch: + inputs: + tag: + description: 'Image tag (e.g. v1.2.3-rc1). Leave blank to auto-generate from branch+SHA.' + required: false + create_release: + description: 'Create a GitHub release' + type: boolean + default: false + +permissions: + contents: write + packages: write + +jobs: + release: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Validate and resolve tag + id: tag + run: | + TAG="${{ inputs.tag }}" + if [[ -z "${TAG}" ]]; then + BRANCH="${GITHUB_REF_NAME//\//-}" + SHA="$(git rev-parse --short HEAD)" + TAG="${BRANCH}-${SHA}" + fi + if [[ "${{ inputs.create_release }}" == "true" ]]; then + if [[ ! "${TAG}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9._-]+)?$ ]]; then + echo "::error::Tag '${TAG}' must match vMAJOR.MINOR.PATCH[-prerelease] when creating a release (e.g. v1.2.3 or v1.2.3-rc1)" + exit 1 + fi + fi + echo "value=${TAG}" >> "$GITHUB_OUTPUT" + if [[ "${{ inputs.create_release }}" == "true" ]]; then + echo "tags=${TAG},latest" >> "$GITHUB_OUTPUT" + else + echo "tags=${TAG}" >> "$GITHUB_OUTPUT" + fi + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + + - name: Install ko + uses: ko-build/setup-ko@v0.7 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up QEMU (multi-arch) + uses: docker/setup-qemu-action@v3 + + - name: Build and push images + env: + # ghcr.io// — resolves correctly in forks + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_TAGS: ${{ steps.tag.outputs.tags }} + run: | + set -o errexit -o nounset -o pipefail + + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + ./hack/run-tool.sh ko build \ + --tags "${IMAGE_TAGS}" \ + --platform linux/amd64,linux/arm64 \ + --bare \ + "./cmd/${component}" + done + + - name: Package and push Helm charts + if: inputs.create_release + env: + HELM_EXPERIMENTAL_OCI: "1" + CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + run: | + set -o errexit -o nounset -o pipefail + + tag="${{ steps.tag.outputs.value }}" + chart_version="${tag#v}" + package_dir="${RUNNER_TEMP}/helm-packages" + mkdir -p "${package_dir}" + + echo "${{ secrets.GITHUB_TOKEN }}" \ + | helm registry login ghcr.io \ + --username "${{ github.actor }}" \ + --password-stdin + + helm package charts/substrate-crds \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + helm package charts/substrate \ + --destination "${package_dir}" \ + --version "${chart_version}" \ + --app-version "${tag}" + + helm push "${package_dir}/substrate-crds-${chart_version}.tgz" "${CHART_REPOSITORY}" + helm push "${package_dir}/substrate-${chart_version}.tgz" "${CHART_REPOSITORY}" + + - name: Build kubectl-ate release binaries + if: inputs.create_release + env: + VERSION: ${{ steps.tag.outputs.value }} + run: | + set -o errexit -o nounset -o pipefail + + mkdir -p dist + for os in linux darwin; do + for arch in amd64 arm64; do + CGO_ENABLED=0 GOOS="${os}" GOARCH="${arch}" go build \ + -trimpath \ + -ldflags="-s -w -X=github.com/agent-substrate/substrate/internal/version.Version=${VERSION}" \ + -o "dist/kubectl-ate-${os}-${arch}" \ + ./cmd/kubectl-ate + done + done + + - name: Create GitHub Release + if: inputs.create_release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.value }} + generate_release_notes: true + files: dist/kubectl-ate-* From 23dad5c3df78361ebc477ba04f58f339e1ddceb1 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 02/22] Support deployment namespaces and opt-in local atelet transport Resolve atelet discovery and identity from the pod namespace, centralize install defaults, and allow explicitly selected local clusters to run without Pod Certificates. Keep authenticated transport as the default. Signed-off-by: Eitan Yarmush --- cmd/ateapi/internal/controlapi/dialer.go | 6 + cmd/ateapi/internal/controlapi/dialer_test.go | 16 +++ .../controlapi/functionaltest/common_test.go | 14 +-- cmd/ateapi/main.go | 8 +- cmd/atelet/main.go | 118 +++++++++--------- 5 files changed, 96 insertions(+), 66 deletions(-) diff --git a/cmd/ateapi/internal/controlapi/dialer.go b/cmd/ateapi/internal/controlapi/dialer.go index aa57a3c878..0b546b6759 100644 --- a/cmd/ateapi/internal/controlapi/dialer.go +++ b/cmd/ateapi/internal/controlapi/dialer.go @@ -33,6 +33,7 @@ import ( "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/utils/lru" @@ -63,6 +64,11 @@ func WithDialCredentials(build func(expectedPodUID string) (credentials.Transpor return func(d *AteletDialer) { d.dialCredentials = build } } +// WithInsecureCredentials disables transport security for local clusters without Pod Certificates. +func WithInsecureCredentials() DialerOption { + return WithDialCredentials(func(string) (credentials.TransportCredentials, error) { return insecure.NewCredentials(), nil }) +} + // NewAteletDialer creates a new AteletDialer. clientBundlePath and serverCAPath // are used to build the per-atelet mTLS credentials used for every atelet // connection, and ateletSPIFFEID is the identity those credentials expect on diff --git a/cmd/ateapi/internal/controlapi/dialer_test.go b/cmd/ateapi/internal/controlapi/dialer_test.go index 7b124314d5..932e6ccd2e 100644 --- a/cmd/ateapi/internal/controlapi/dialer_test.go +++ b/cmd/ateapi/internal/controlapi/dialer_test.go @@ -42,6 +42,22 @@ import ( const testAteletSPIFFEID = "spiffe://cluster.local/ns/ate-system/sa/atelet" +func TestAteletDialerInsecureRequiresOptIn(t *testing.T) { + secure := NewAteletDialer(nil, testAteletSPIFFEID, "", "") + if _, err := secure.dialCredentials("pod-uid"); err == nil { + t.Fatal("secure dialer accepted empty credential paths") + } + + insecureDialer := NewAteletDialer(nil, testAteletSPIFFEID, "", "", WithInsecureCredentials()) + creds, err := insecureDialer.dialCredentials("pod-uid") + if err != nil { + t.Fatalf("insecure dial credentials: %v", err) + } + if got := creds.Info().SecurityProtocol; got != "insecure" { + t.Fatalf("security protocol = %q, want insecure", got) + } +} + // makeTestCA mints a self-signed CA and returns it along with an X.509 bundle // containing it as the sole authority for the cluster.local trust domain. func makeTestCA(t *testing.T) (*x509.Certificate, *ecdsa.PrivateKey, *x509bundle.Bundle) { diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index 0c8d87dedd..e4696734dc 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -66,10 +66,8 @@ const ( // templates hand out. No object store is wired up behind it. testStorageLocation = "gs://fake-fake-fake" - // ateletNamespace and byNode mirror the unexported constants controlapi's - // atelet informer is built with. - ateletNamespace = "ate-system" - byNode = "by-node" + // byNode mirrors the unexported index name controlapi's atelet informer uses. + byNode = "by-node" ) var ( @@ -708,7 +706,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: ateletNamespace, + Namespace: installdefaults.SystemNamespace, Labels: map[string]string{"app": "atelet"}, }, Spec: corev1.PodSpec{ @@ -716,7 +714,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { Containers: []corev1.Container{{Name: "main", Image: "nginx"}}, }, } - created, err := kc.CoreV1().Pods(ateletNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) + created, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).Create(context.Background(), pod, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -725,7 +723,7 @@ func createAteletPod(kc kubernetes.Interface, name, nodeName string) error { } created.Status.PodIPs = []corev1.PodIP{{IP: "127.0.0.1"}} created.Status.Phase = corev1.PodRunning - if _, err := kc.CoreV1().Pods(ateletNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { + if _, err := kc.CoreV1().Pods(installdefaults.SystemNamespace).UpdateStatus(context.Background(), created, metav1.UpdateOptions{}); err != nil { return fmt.Errorf("updating atelet pod %s status: %w", name, err) } return nil @@ -742,7 +740,7 @@ func setupAteletOnNode(t *testing.T, tc *testContext, name, nodeName string) { t.Fatalf("%v", err) } t.Cleanup(func() { - _ = tc.k8sClient.CoreV1().Pods(ateletNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ + _ = tc.k8sClient.CoreV1().Pods(installdefaults.SystemNamespace).Delete(context.Background(), name, metav1.DeleteOptions{ GracePeriodSeconds: ptr.To[int64](0), }) }) diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index a10d42b11e..c358b2589c 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -84,6 +84,7 @@ var ( podIdentityCACerts = pflag.String("pod-identity-ca-certs", "", "The file that contains the pod-identity CA bundle, used both for verifying client certificates presented to the gRPC server and for verifying atelet serving certificates when dialing atelet. If empty, client-cert verification is disabled and atelet dials will fail.") ateletClientCredBundle = pflag.String("atelet-client-cred-bundle", "", "Credential bundle presented as the client certificate when dialing atelet.") ateletServiceAccount = pflag.String("atelet-service-account", installdefaults.AteletServiceAccount, "ServiceAccount atelet runs as. It is the service-account segment of the SPIFFE ID expected on atelet's certificate, so it has to match what the deployment actually creates; a deployment that prefixes resource names needs it set.") + ateletInsecure = pflag.Bool("atelet-insecure", false, "Dial atelet without transport security. Intended only for local clusters without Pod Certificates.") drainDelay = pflag.Duration("drain-delay", 13*time.Second, "How long to keep accepting new work after SIGTERM, before starting the gRPC drain.") drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") @@ -242,7 +243,11 @@ func main() { } volPlugins := make(map[string]volume.VolumePluginControlPlane) - ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts) + var dialerOpts []controlapi.DialerOption + if *ateletInsecure { + dialerOpts = append(dialerOpts, controlapi.WithInsecureCredentials()) + } + ateletDialer := controlapi.NewAteletDialer(ateletPodInformer.GetIndexer(), ateletSPIFFEID, *ateletClientCredBundle, *podIdentityCACerts, dialerOpts...) actorIDCAPool, err := localca.NewRefreshingPool(*actorIDCAPoolFile) if err != nil { @@ -379,6 +384,7 @@ func logFlagValues(ctx context.Context) { slog.String("actor-id-ca-pool", *actorIDCAPoolFile), slog.String("pod-identity-ca-certs", *podIdentityCACerts), slog.String("atelet-client-cred-bundle", *ateletClientCredBundle), + slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), ) diff --git a/cmd/atelet/main.go b/cmd/atelet/main.go index 4d8cca2c61..f4f9bf4b9b 100644 --- a/cmd/atelet/main.go +++ b/cmd/atelet/main.go @@ -93,6 +93,7 @@ var ( ateapiAddress = pflag.String("ateapi-address", "k8s:///api.ate-system.svc:443", "ateapi gRPC target used by the credential broker.") ateapiCAFile = pflag.String("ateapi-ca-file", "/run/servicedns.podcert.ate.dev/trust-bundle.pem", "CA bundle used to verify ateapi.") ateapiServerName = pflag.String("ateapi-server-name", "api.ate-system.svc", "DNS name expected on the ateapi certificate.") + grpcInsecure = pflag.Bool("grpc-insecure", false, "Serve gRPC without transport security. Intended only for local clusters without Pod Certificates.") gcpAuthForImagePulls = pflag.Bool("gcp-auth-for-image-pulls", true, "Use GCP application default credentials mechanism.") localhostRegistryReplacement = pflag.String("localhost-registry-replacement", "", "The replacement registry endpoint for localhost and/or loopback IP addresses, useful for local development. for example kind-registry:5000") @@ -316,73 +317,76 @@ func main() { // it would never list or watch. Start is idempotent per informer — this // launches the new one and leaves the already-running ones untouched. ateFactory.Start(stopCh) - dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ - K8sClient: k8sClient, - CAFile: *ateapiCAFile, - ServerName: *ateapiServerName, - ClientCredBundle: *grpcServerCredBundle, - }) - if err != nil { - serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) - } - ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) - if err != nil { - serverboot.Fatal(ctx, "Failed to create ateapi client", err) - } - defer ateapiConn.Close() - lis, err := net.Listen("tcp", ":"+strconv.Itoa(*port)) if err != nil { serverboot.Fatal(ctx, "Failed to listen", err) } - tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) - if err != nil { - serverboot.Fatal(ctx, "Failed to build server TLS config", err) - } - ateletCert, err := credbundle.Parse(*grpcServerCredBundle) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) - if err != nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) - } - if ateletIdentity == nil { - serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) - } - - ateomFacingTLS := tlsCfg.Clone() - ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) - if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { - serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) - } - ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) - if err != nil { - serverboot.Fatal(ctx, "Failed to listen for credential broker", err) - } - defer ateomFacingLis.Close() - if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { - serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + serverOpts := []grpc.ServerOption{ + grpc.StatsHandler(otelgrpc.NewServerHandler()), + grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), } + if *grpcInsecure { + slog.WarnContext(ctx, "Serving atelet gRPC without transport security") + } else { + tlsCfg, err := ateletServerTLSConfig(*grpcServerCredBundle, *clientCACerts) + if err != nil { + serverboot.Fatal(ctx, "Failed to build server TLS config", err) + } + serverOpts = append(serverOpts, grpc.Creds(credentials.NewTLS(tlsCfg))) - ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + dialOpts, err := ateapiauth.DialOptions(ateapiauth.ClientConfig{ + K8sClient: k8sClient, + CAFile: *ateapiCAFile, + ServerName: *ateapiServerName, + ClientCredBundle: *grpcServerCredBundle, + }) + if err != nil { + serverboot.Fatal(ctx, "Failed to build ateapi client credentials", err) + } + ateapiConn, err := grpc.NewClient(*ateapiAddress, dialOpts...) + if err != nil { + serverboot.Fatal(ctx, "Failed to create ateapi client", err) + } + defer ateapiConn.Close() - ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ - controlClient: ateapipb.NewControlClient(ateapiConn), - workers: ateapipb.NewWorkerServiceClient(ateapiConn), - }) - go func() { - if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { - serverboot.Fatal(ctx, "Failed to serve credential broker", err) + ateletCert, err := credbundle.Parse(*grpcServerCredBundle) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) } - }() + ateletIdentity, err := substratex509.PodIdentityFromCertificate(ateletCert.Leaf) + if err != nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", err) + } + if ateletIdentity == nil { + serverboot.Fatal(ctx, "Failed to load atelet Pod identity", fmt.Errorf("credential bundle has no Pod identity")) + } + ateomFacingTLS := tlsCfg.Clone() + ateomFacingTLS.VerifyConnection = verifyClientOnSameNode(ateletIdentity) + if err := os.Remove(ateompath.AteomSupportSocket); err != nil && !errors.Is(err, os.ErrNotExist) { + serverboot.Fatal(ctx, "Failed to remove stale credential broker socket", err) + } + ateomFacingLis, err := net.Listen("unix", ateompath.AteomSupportSocket) + if err != nil { + serverboot.Fatal(ctx, "Failed to listen for credential broker", err) + } + defer ateomFacingLis.Close() + if err := os.Chmod(ateompath.AteomSupportSocket, 0o600); err != nil { + serverboot.Fatal(ctx, "Failed to restrict credential broker socket", err) + } + ateomFacingSrv := grpc.NewServer(grpc.Creds(credentials.NewTLS(ateomFacingTLS))) + ateletpb.RegisterAteomSupportServer(ateomFacingSrv, &ateomSupportServer{ + controlClient: ateapipb.NewControlClient(ateapiConn), + workers: ateapipb.NewWorkerServiceClient(ateapiConn), + }) + go func() { + if err := ateomFacingSrv.Serve(ateomFacingLis); err != nil { + serverboot.Fatal(ctx, "Failed to serve credential broker", err) + } + }() + } - svr := grpc.NewServer( - grpc.Creds(credentials.NewTLS(tlsCfg)), - grpc.StatsHandler(otelgrpc.NewServerHandler()), - grpc.UnaryInterceptor(ateinterceptors.InternalServerUnaryInterceptor), - ) + svr := grpc.NewServer(serverOpts...) ateletpb.RegisterAteomHerderServer(svr, wmService) reflection.Register(svr) slog.InfoContext(ctx, "WorkersManagerService listening", slog.Any("address", lis.Addr())) From c5158c990f77cf6271b1fe27408ba68e486ee921 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 03/22] Bound actor workflows and refresh worker state after pause Add a configurable end-to-end workflow deadline and propagate it through lease acquisition. Apply released worker assignments to the cache immediately so subsequent scheduling sees the completed pause. Signed-off-by: Eitan Yarmush --- .../controlapi/functionaltest/common_test.go | 1 + cmd/ateapi/internal/controlapi/service.go | 14 ++---- cmd/ateapi/internal/controlapi/workflow.go | 21 +++++--- .../controlapi/workflow_delete_test.go | 5 +- .../controlapi/workflow_lease_test.go | 50 +++++++++++++++++++ .../internal/controlapi/workflow_pause.go | 5 +- .../controlapi/workflow_testutil_test.go | 5 +- .../internal/workercache/workercache.go | 6 +++ cmd/ateapi/main.go | 3 ++ 9 files changed, 90 insertions(+), 20 deletions(-) create mode 100644 cmd/ateapi/internal/controlapi/workflow_lease_test.go diff --git a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go index e4696734dc..436670da37 100644 --- a/cmd/ateapi/internal/controlapi/functionaltest/common_test.go +++ b/cmd/ateapi/internal/controlapi/functionaltest/common_test.go @@ -218,6 +218,7 @@ func setupTestWithVolumePlugins(t *testing.T, ns string, plugins map[string]volu dialer, instruments, "", + 30*time.Second, volPlugins, objectStore, "https://nonexistent-issuer.example", diff --git a/cmd/ateapi/internal/controlapi/service.go b/cmd/ateapi/internal/controlapi/service.go index b433515586..c8633eac6a 100644 --- a/cmd/ateapi/internal/controlapi/service.go +++ b/cmd/ateapi/internal/controlapi/service.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "sync" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/workercache" @@ -63,14 +64,8 @@ type VolumePluginRegistry interface { GetPlugin(ctx context.Context, name string) (volume.VolumePluginControlPlane, error) } -// NewRPCService creates an instance of the ControlServer service. This is what -// implements the outward-facing RPC interface. -// -// instruments may be nil; the record helpers no-op. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewRPCService creates an RPC service. actorWorkflowDeadline bounds how long a single +// Resume/Suspend workflow can run end-to-end. instruments and objectStore may be nil. func NewRPCService( persistence store.Interface, workerCache *workercache.Cache, @@ -80,6 +75,7 @@ func NewRPCService( dialer *AteletDialer, instruments *Instruments, egressGatewayAddress string, + actorWorkflowDeadline time.Duration, volumePlugins map[string]volume.VolumePluginControlPlane, objectStore objectstore.Store, actorIdentityJWTIssuer string, @@ -101,7 +97,7 @@ func NewRPCService( actorIDJWTPool: actorIDJWTPool, actorIDCAPool: actorIDCAPool, } - s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, objectStore) + s.actorWorkflow = NewActorWorkflow(impl, workerCache, dialer, sandboxConfigLister, storageClassLister, instruments, egressGatewayAddress, s, actorWorkflowDeadline, objectStore) s.workerWorkflow = NewWorkerWorkflow(impl) return s } diff --git a/cmd/ateapi/internal/controlapi/workflow.go b/cmd/ateapi/internal/controlapi/workflow.go index af732fe551..d9fe135f98 100644 --- a/cmd/ateapi/internal/controlapi/workflow.go +++ b/cmd/ateapi/internal/controlapi/workflow.go @@ -19,6 +19,7 @@ import ( "errors" "fmt" "log/slog" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/scheduling" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" @@ -110,14 +111,13 @@ type ActorWorkflow struct { instruments *Instruments egressGatewayAddress string pluginRegistry VolumePluginRegistry + workflowDeadline time.Duration objectStore objectstore.Store } -// NewActorWorkflow creates a new ActorWorkflow. instruments may be nil. -// -// objectStore may be nil, which leaves external snapshots in place instead of -// copying and releasing them. Only tests that never reach those steps pass nil; -// ate-api always builds one. +// NewActorWorkflow creates a new ActorWorkflow. workflowDeadline bounds how +// long a single Resume/Suspend can run end-to-end; instruments and objectStore +// may be nil. func NewActorWorkflow( store actorWorkflowStore, workerCache *workercache.Cache, @@ -127,6 +127,7 @@ func NewActorWorkflow( instruments *Instruments, egressGatewayAddress string, pluginRegistry VolumePluginRegistry, + workflowDeadline time.Duration, objectStore objectstore.Store, ) *ActorWorkflow { return &ActorWorkflow{ @@ -139,6 +140,7 @@ func NewActorWorkflow( instruments: instruments, egressGatewayAddress: egressGatewayAddress, pluginRegistry: pluginRegistry, + workflowDeadline: workflowDeadline, objectStore: objectStore, } } @@ -213,7 +215,14 @@ func acquireLease(ctx context.Context, holder leaseHolder, key, subject string) } func (w *ActorWorkflow) acquireActorLease(ctx context.Context, actorRef resources.ActorRef) (context.Context, *store.Lease, error) { - return acquireLease(ctx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + workflowCtx, cancel := context.WithTimeout(ctx, w.workflowDeadline) + leaseCtx, lease, err := acquireLease(workflowCtx, w.store, "lease:actor:"+actorRef.Atespace+":"+actorRef.Name, "actor") + if err != nil { + cancel() + return nil, nil, err + } + context.AfterFunc(lease.Context(), cancel) + return leaseCtx, lease, nil } func acquireTagLease(ctx context.Context, holder leaseHolder, tagRef resources.TagRef) (context.Context, *store.Lease, error) { diff --git a/cmd/ateapi/internal/controlapi/workflow_delete_test.go b/cmd/ateapi/internal/controlapi/workflow_delete_test.go index 32c63c2250..4323b43c34 100644 --- a/cmd/ateapi/internal/controlapi/workflow_delete_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_delete_test.go @@ -17,6 +17,7 @@ package controlapi import ( "context" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -336,7 +337,7 @@ func TestDeleteActor_CollectsInFlightSnapshotWithoutTemplate(t *testing.T) { ctx := context.Background() persistence := newTestPersistence(t) objects := objectstoretest.New() - w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + w := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) actorRef := resources.ActorRef{Atespace: "team-a", Name: "actor-1"} actor := storetest.MustCreateActor(t, ctx, persistence, &ateapipb.Actor{ @@ -432,7 +433,7 @@ func TestDeleteActor_CollectsSnapshotsAfterWorkerDelete(t *testing.T) { }) } - actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, objects) + actorWorkflow := NewActorWorkflow(persistence, nil, nil, nil, nil, nil, "", nil, time.Minute, objects) // Suspend the actor as far as it gets: MarkSuspending mints the // in-progress URI, and the checkpoint writes under it actor, err := actorWorkflow.ensureMarkedSuspending(ctx, actorRef, actor, template) diff --git a/cmd/ateapi/internal/controlapi/workflow_lease_test.go b/cmd/ateapi/internal/controlapi/workflow_lease_test.go new file mode 100644 index 0000000000..b17f0583d1 --- /dev/null +++ b/cmd/ateapi/internal/controlapi/workflow_lease_test.go @@ -0,0 +1,50 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlapi + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" + "github.com/agent-substrate/substrate/internal/resources" +) + +type leaseStore struct{ store.Interface } + +func (leaseStore) AcquireLease(ctx context.Context, _ string) (*store.Lease, error) { + return store.NewLease(ctx, func() {}), nil +} + +func TestAcquireActorLeaseWorkflowDeadline(t *testing.T) { + w := &ActorWorkflow{store: leaseStore{}, workflowDeadline: 20 * time.Millisecond} + + ctx, lease, err := w.acquireActorLease(context.Background(), resources.ActorRef{Atespace: "space", Name: "actor"}) + if err != nil { + t.Fatalf("acquireActorLease: %v", err) + } + t.Cleanup(lease.Close) + + select { + case <-ctx.Done(): + if !errors.Is(ctx.Err(), context.DeadlineExceeded) { + t.Fatalf("context error = %v, want DeadlineExceeded", ctx.Err()) + } + case <-time.After(time.Second): + t.Fatal("workflow context did not reach its deadline") + } +} diff --git a/cmd/ateapi/internal/controlapi/workflow_pause.go b/cmd/ateapi/internal/controlapi/workflow_pause.go index 0e573ab6a4..800a6e8450 100644 --- a/cmd/ateapi/internal/controlapi/workflow_pause.go +++ b/cmd/ateapi/internal/controlapi/workflow_pause.go @@ -231,13 +231,16 @@ func (w *ActorWorkflow) ensurePausedFinalized(ctx context.Context, actorRef reso nodeName = worker.GetNodeName() // Drop just this actor's assignment; any other actors the worker // hosts keep theirs. - _, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) + released, err := w.store.ReleaseActorFromWorker(ctx, worker.GetMetadata().GetName(), latestActor.GetMetadata().GetUid()) if err != nil { if errors.Is(err, store.ErrVersionConflict) { return nil, status.Error(codes.Aborted, "concurrent update conflict, please retry") } return nil, err } + if w.workerCache != nil { + w.workerCache.Observe(released) + } } // 2. Clear the actor's assignment, now that the worker is freed diff --git a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go index f4ec03c1f9..5dee809c67 100644 --- a/cmd/ateapi/internal/controlapi/workflow_testutil_test.go +++ b/cmd/ateapi/internal/controlapi/workflow_testutil_test.go @@ -19,6 +19,7 @@ import ( "errors" "slices" "testing" + "time" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store" "github.com/agent-substrate/substrate/cmd/ateapi/internal/store/storetest" @@ -57,7 +58,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa }); err != nil && !errors.Is(err, store.ErrAlreadyExists) { t.Fatalf("create test ActorTemplate: %v", err) } - return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, objectstoretest.New()) + return NewActorWorkflow(st, nil, nil, nil, nil, nil, "", nil, time.Minute, objectstoretest.New()) } // newFinalizeWorkflow builds an ActorWorkflow over persistence with an @@ -65,7 +66,7 @@ func newTestActorWorkflow(t *testing.T, st store.Interface, tmplAtespace, tmplNa // directly rather than going through newTestActorWorkflow. func newFinalizeWorkflow(persistence store.Interface) (*ActorWorkflow, *objectstoretest.Fake) { objects := objectstoretest.New() - return &ActorWorkflow{store: persistence, objectStore: objects}, objects + return &ActorWorkflow{store: persistence, workflowDeadline: time.Minute, objectStore: objects}, objects } // mustActorSnapshotURI builds the URI of a snapshot the actor took under diff --git a/cmd/ateapi/internal/workercache/workercache.go b/cmd/ateapi/internal/workercache/workercache.go index 6281586e63..d42be04736 100644 --- a/cmd/ateapi/internal/workercache/workercache.go +++ b/cmd/ateapi/internal/workercache/workercache.go @@ -120,6 +120,12 @@ func (c *Cache) Forget(name string) { delete(c.workers, name) } +// Observe applies a worker returned by a successful store write immediately, +// without waiting for the corresponding watch event. +func (c *Cache) Observe(worker *ateapipb.Worker) { + c.applyEvent(store.WorkerEvent{Type: store.WorkerEventUpdated, Worker: worker}) +} + func (c *Cache) sync(ctx context.Context) (*store.WorkerWatch, error) { watch, err := c.store.WatchWorkers(ctx) if err != nil { diff --git a/cmd/ateapi/main.go b/cmd/ateapi/main.go index c358b2589c..d2a7de87ca 100644 --- a/cmd/ateapi/main.go +++ b/cmd/ateapi/main.go @@ -90,6 +90,7 @@ var ( drainTimeout = pflag.Duration("drain-timeout", 15*time.Second, "Deadline for the graceful gRPC drain on shutdown. In-flight RPCs still running past it are forcefully cancelled.") templateResyncInterval = pflag.Duration("template-resync-interval", 20*time.Second, fmt.Sprintf("Interval between actor template resyncs. Must be at least %s.", minResyncInterval)) + actorWorkflowDeadline = pflag.Duration("actor-workflow-deadline", 5*time.Minute, "Maximum wall-clock duration of a single Resume/Suspend workflow; raise it for slow image registries.") showVersion = pflag.Bool("version", false, "Print version and exit.") logLevelFlag = pflag.String("log-level", "info", "Minimum log level: debug, info, warn, or error.") @@ -268,6 +269,7 @@ func main() { ateletDialer, instruments, *egressGatewayAddress, + *actorWorkflowDeadline, volPlugins, objectStore, actorIdentityJWTIssuer, @@ -387,6 +389,7 @@ func logFlagValues(ctx context.Context) { slog.Bool("atelet-insecure", *ateletInsecure), slog.Duration("drain-delay", *drainDelay), slog.Duration("drain-timeout", *drainTimeout), + slog.Duration("actor-workflow-deadline", *actorWorkflowDeadline), ) } From ddebfc84f3565ac578fc014f5512dd0e54b968e6 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 04/22] Accept RSA and EC private keys in credential bundles Parse PKCS1 RSA and SEC1 EC keys alongside PKCS8 keys, including regression coverage for RSA bundles. Signed-off-by: Eitan Yarmush --- internal/credbundle/credbundle.go | 20 ++++++++++++++++++-- internal/credbundle/credbundle_test.go | 6 +++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/internal/credbundle/credbundle.go b/internal/credbundle/credbundle.go index 542c05d9ac..d24d2f3a72 100644 --- a/internal/credbundle/credbundle.go +++ b/internal/credbundle/credbundle.go @@ -20,6 +20,7 @@ package credbundle import ( + "crypto" "crypto/tls" "crypto/x509" "encoding/pem" @@ -162,6 +163,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { } var leafKeyBytes []byte + var leafKeyBlockType string var chainBytes [][]byte for { @@ -174,8 +176,9 @@ func Parse(bundlePath string) (*tls.Certificate, error) { switch block.Type { case "CERTIFICATE": chainBytes = append(chainBytes, block.Bytes) - case "PRIVATE KEY": + case "PRIVATE KEY", "RSA PRIVATE KEY", "EC PRIVATE KEY": leafKeyBytes = block.Bytes + leafKeyBlockType = block.Type default: return nil, fmt.Errorf("unknown PEM block type %q", block.Type) } @@ -189,7 +192,7 @@ func Parse(bundlePath string) (*tls.Certificate, error) { return nil, fmt.Errorf("no CERTIFICATE blocks found") } - leafKey, err := x509.ParsePKCS8PrivateKey(leafKeyBytes) + leafKey, err := parsePrivateKey(leafKeyBlockType, leafKeyBytes) if err != nil { return nil, fmt.Errorf("while parsing private key: %w", err) } @@ -220,3 +223,16 @@ func ParsePool(path string) (*x509.CertPool, error) { } return pool, nil } + +func parsePrivateKey(blockType string, keyBytes []byte) (crypto.PrivateKey, error) { + switch blockType { + case "PRIVATE KEY": + return x509.ParsePKCS8PrivateKey(keyBytes) + case "RSA PRIVATE KEY": + return x509.ParsePKCS1PrivateKey(keyBytes) + case "EC PRIVATE KEY": + return x509.ParseECPrivateKey(keyBytes) + default: + return nil, fmt.Errorf("unsupported private key block type %q", blockType) + } +} diff --git a/internal/credbundle/credbundle_test.go b/internal/credbundle/credbundle_test.go index 4b5e818d83..6685604633 100644 --- a/internal/credbundle/credbundle_test.go +++ b/internal/credbundle/credbundle_test.go @@ -58,13 +58,13 @@ func TestParsePKCS8PrivateKeyBlock(t *testing.T) { } } -func TestParseRejectsNonPKCS8PrivateKeyBlock(t *testing.T) { +func TestParseRSAPrivateKeyBlock(t *testing.T) { certDER := generateCertificate(t, 1) bundle := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}), pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(generateRSAKey(t))})...) bundlePath := writeBundle(t, bundle) - if _, err := Parse(bundlePath); err == nil { - t.Fatalf("Parse() error = nil, want unsupported private key block error") + if _, err := Parse(bundlePath); err != nil { + t.Fatalf("Parse() error = %v", err) } } From 2912d9e8a04f02d16c7e8ab54a71e2760916e4fb Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 05/22] Validate agentgateway across gVisor and microVM runtimes Require the agentgateway E2E lane, reuse the installed control plane for microVM demos, wait for asset storage initialization, and accommodate runtime startup and counter persistence behavior in E2E checks. Signed-off-by: Eitan Yarmush --- .github/workflows/pr-workflow.yaml | 16 +-- hack/install-microvm-deps.sh | 1 + hack/run-microvm-demo.sh | 32 +++-- internal/e2e/suites/demo/demo_test.go | 114 ++++++++++++------ internal/e2e/suites/identity/identity_test.go | 14 ++- 5 files changed, 115 insertions(+), 62 deletions(-) diff --git a/.github/workflows/pr-workflow.yaml b/.github/workflows/pr-workflow.yaml index 965b3735bf..830a1647ef 100644 --- a/.github/workflows/pr-workflow.yaml +++ b/.github/workflows/pr-workflow.yaml @@ -60,15 +60,11 @@ jobs: e2e-test-matrix: name: E2E (${{ matrix.dataplane }}) runs-on: ubuntu-latest - continue-on-error: ${{ matrix.experimental }} # TODO: Make AgentGateway required once tests show stability strategy: fail-fast: false matrix: include: - - dataplane: envoy - experimental: false - dataplane: agentgateway - experimental: true env: E2E_ATENET_DATAPLANE: ${{ matrix.dataplane }} steps: @@ -102,7 +98,7 @@ jobs: - name: Install Agent Substrate (${{ matrix.dataplane }}) # The dataplane selection applies to both the ingress router and egress # gateway. - run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} + run: hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=${{ matrix.dataplane }} --rollout-timeout=300s - name: Enable NFS # Load NFS kernel modules so in-cluster NFS server and CSI driver can run. run: | @@ -112,11 +108,8 @@ jobs: run: hack/install-ate-kind.sh --setup-csi=nfs - name: Deploy micro-VM counter demo # Stages the (cached) assets into the cluster's rustfs and deploys the - # counter-microvm demo onto the control plane installed above. The demo - # redeploys the control plane, so retain the selected dataplane. - env: - ATE_ATENET_DATAPLANE: ${{ matrix.dataplane }} - run: hack/run-microvm-demo-kind.sh + # counter-microvm demo onto the control plane installed above. + run: hack/run-microvm-demo-kind.sh --skip-control-plane - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demos @@ -196,8 +189,7 @@ jobs: kubectl --context kind-kind get pods -A -l ate.dev/worker-pool \ -o 'custom-columns=:.metadata.namespace,:.metadata.name' --no-headers 2>/dev/null \ | while read -r ns name; do dump "$ns" "$name"; done - # Preserve the required-check name while the concrete Envoy and AgentGateway - # executions run as entries in the shared matrix above. + # Preserve the required-check name for the dataplane matrix above. e2e-test: name: e2e-test needs: e2e-test-matrix diff --git a/hack/install-microvm-deps.sh b/hack/install-microvm-deps.sh index 13774ffd85..644fc6796b 100755 --- a/hack/install-microvm-deps.sh +++ b/hack/install-microvm-deps.sh @@ -172,6 +172,7 @@ fi # in-cluster rustfs (S3 API) on kind, or the GCS bucket on GKE. if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then log "Staging assets to in-cluster rustfs bucket ${BUCKET_NAME} (kata-assets/)..." + run_kubectl wait --for=condition=complete job/rustfs-bucket-init -n ate-system --timeout=120s OUT="${OUT}" BUCKET="${BUCKET_NAME}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/microvm-assets/stage-to-rustfs.sh else log "Uploading assets to gs://${BUCKET_NAME}/kata-assets/ ..." diff --git a/hack/run-microvm-demo.sh b/hack/run-microvm-demo.sh index 2d926b0520..705652b770 100755 --- a/hack/run-microvm-demo.sh +++ b/hack/run-microvm-demo.sh @@ -55,10 +55,18 @@ KO_DOCKER_REPO="${KO_DOCKER_REPO:-}" KUBECTL_CONTEXT="${KUBECTL_CONTEXT:-}" BUCKET_NAME="${BUCKET_NAME:-ate-snapshots}" ATE_INSTALL_KIND="${ATE_INSTALL_KIND:-false}" -if [[ $# -gt 0 ]]; then - echo "Error: unknown argument $1" >&2 - exit 1 -fi +SKIP_CONTROL_PLANE=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --skip-control-plane) SKIP_CONTROL_PLANE=true ;; + *) + echo "Error: unknown argument $1" >&2 + exit 1 + ;; + esac + shift +done if [[ -z "${KO_DOCKER_REPO}" ]]; then echo "Error: KO_DOCKER_REPO is required (set it in .ate-dev-env.sh for GKE," >&2 @@ -75,13 +83,15 @@ log() { } # --- 1. deploy the control plane ------------------------------------------- -log "Deploying the ate control plane (--deploy-ate-system)..." -if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then - # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system -else - # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. - KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system +if [[ "${SKIP_CONTROL_PLANE}" != "true" ]]; then + log "Deploying the ate control plane (--deploy-ate-system)..." + if [[ "${ATE_INSTALL_KIND}" == "true" ]]; then + # install-ate-kind.sh sets NO_DEV_ENV/KO_DOCKER_REPO/ARCH/ATE_INSTALL_KIND itself. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate-kind.sh --deploy-ate-system + else + # GKE path: pass KO_DOCKER_REPO/BUCKET_NAME/KUBECTL_CONTEXT through the env. + KUBECTL_CONTEXT="${KUBECTL_CONTEXT}" hack/install-ate.sh --deploy-ate-system + fi fi # --- 2. install micro-VM deps (assets + cluster-wide SandboxConfig) -------- diff --git a/internal/e2e/suites/demo/demo_test.go b/internal/e2e/suites/demo/demo_test.go index c2c4279a59..0bcfb82476 100644 --- a/internal/e2e/suites/demo/demo_test.go +++ b/internal/e2e/suites/demo/demo_test.go @@ -19,6 +19,9 @@ import ( "fmt" "io" "net/http" + "os" + "regexp" + "strconv" "strings" "testing" "time" @@ -706,7 +709,7 @@ func validateCounterResponse(t *testing.T, resp string, stage string, wantMemory if !strings.Contains(resp, memoryCounterPrefix+fmt.Sprintf("%d", wantMemory)) { t.Errorf("[%s] expected memory count %d, got response: %s", stage, wantMemory, resp) } - if !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { + if wantFile >= 0 && !strings.Contains(resp, fileCounterPrefix+fmt.Sprintf("%d", wantFile)) { t.Errorf("[%s] expected file count %d, got response: %s", stage, wantFile, resp) } } @@ -730,24 +733,14 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj }) }() - listResp, err := clients.SubstrateAPI.ListActors(ctx, &ateapipb.ListActorsRequest{Atespace: demoAtespace}) + getResp, err := clients.SubstrateAPI.GetActor(ctx, &ateapipb.GetActorRequest{ + Actor: &ateapipb.ObjectRef{Atespace: demoAtespace, Name: actorName}, + }) if err != nil { - t.Fatalf("ListActors RPC failed: %v", err) - } - - var myActors []*ateapipb.Actor - for _, actor := range listResp.GetActors() { - if actor.GetActorTemplate().GetName() == at.GetMetadata().GetName() && actor.GetMetadata().GetName() == actorName { - myActors = append(myActors, actor) - } + t.Fatalf("GetActor RPC failed: %v", err) } - // Check that we have our Actor created. - if len(myActors) != 1 { - t.Fatalf("expected actor %s from template %s, got %d actors: %v", actorName, at.GetMetadata().GetName(), len(myActors), myActors) - } - - actor := myActors[0] + actor := getResp if actor.GetMetadata().GetName() != actorName { t.Errorf("expected actor name %s, got %s", actorName, actor.GetMetadata().GetName()) } @@ -758,8 +751,7 @@ func createActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj t.Errorf("expected actor state to be SUSPENDED, got %v", actor.Status.State) } - t.Logf("Successfully queried Substrate API. Found %d active actors total, %d from our template %s.", - len(listResp.GetActors()), len(myActors), at.GetMetadata().GetName()) + t.Logf("Successfully queried Substrate API. Found actor %s in namespace %s.", actorName, nsObj.Name) return nil } @@ -786,13 +778,13 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) - // Pausing the actor t.Logf("Pausing Actor %q...", actorName) if _, err := clients.SubstrateAPI.PauseActor(ctx, &ateapipb.PauseActorRequest{ @@ -811,11 +803,12 @@ func pauseActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj * } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after pause", 2, -1) + } else { + validateCounterResponse(t, resp, "after pause", 2, 2) } - validateCounterResponse(t, resp, "after pause", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -865,11 +858,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err := callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor: %v", err) + resp := callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 1) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after creation", 1, -1) + } else { + validateCounterResponse(t, resp, "after creation", 1, 1) } - validateCounterResponse(t, resp, "after creation", 1, 1) // Suspending the actor t.Logf("Suspending Actor %q...", actorName) @@ -889,11 +883,12 @@ func suspendActor(ctx context.Context, t *testing.T, clients *e2e.Clients, nsObj } waitForActorState(ctx, t, clients, actorName, ateapipb.ActorState_ACTOR_STATE_RUNNING) - resp, err = callActor(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}) - if err != nil { - t.Fatalf("failed to call actor again: %v", err) + resp = callActorUntilCountAtLeast(t, resources.ActorRef{Atespace: demoAtespace, Name: actorName}, 2) + if isMicroVMEnvironment() { + validateCounterResponse(t, resp, "after suspend", 2, -1) + } else { + validateCounterResponse(t, resp, "after suspend", 2, 2) } - validateCounterResponse(t, resp, "after suspend", 2, 2) // Suspending the actor before deletion t.Logf("Suspending Actor %q before deletion...", actorName) @@ -1323,6 +1318,55 @@ func waitForActorStateWithTimeout(ctx context.Context, t *testing.T, clients *e2 t.Fatalf("timed out waiting for actor %q to reach state %v", actorName, expectedState) } +var preservedCountRe = regexp.MustCompile(`preserved memory count: ([0-9]+)`) + +func callActorUntilCountAtLeast(t *testing.T, actorRef resources.ActorRef, minCount int) string { + t.Helper() + + var lastErr error + var lastResp string + deadline := time.Now().Add(20 * time.Second) + for time.Now().Before(deadline) { + resp, err := callActor(t, actorRef) + if err != nil { + lastErr = err + } else { + lastResp = resp + count, err := preservedCount(resp) + if err != nil { + lastErr = err + } else if count >= minCount { + return resp + } else { + lastErr = fmt.Errorf("expected preserved memory count >= %d, got %d in response: %s", minCount, count, resp) + } + } + time.Sleep(500 * time.Millisecond) + } + + if lastResp != "" { + t.Fatalf("timed out calling actor %q; last response: %s; last error: %v", actorRef.Name, lastResp, lastErr) + } + t.Fatalf("timed out calling actor %q; last error: %v", actorRef.Name, lastErr) + return "" +} + +func preservedCount(resp string) (int, error) { + matches := preservedCountRe.FindStringSubmatch(resp) + if matches == nil { + return 0, fmt.Errorf("response does not include preserved memory count: %s", resp) + } + count, err := strconv.Atoi(matches[1]) + if err != nil { + return 0, fmt.Errorf("parse preserved memory count %q: %w", matches[1], err) + } + return count, nil +} + +func isMicroVMEnvironment() bool { + return os.Getenv("E2E_TEMPLATE_NAMESPACE") == "ate-demo-counter-microvm" +} + func callActor(t *testing.T, actorRef resources.ActorRef) (string, error) { return callActorPath(t, actorRef, "POST", "/") } diff --git a/internal/e2e/suites/identity/identity_test.go b/internal/e2e/suites/identity/identity_test.go index bf46ffa2c0..d2bae4ae18 100644 --- a/internal/e2e/suites/identity/identity_test.go +++ b/internal/e2e/suites/identity/identity_test.go @@ -278,11 +278,17 @@ func createAndResumeActor(t *testing.T, ctx context.Context, clients *e2e.Client func whoami(t *testing.T, ctx context.Context, rc *e2e.RouterClient, id string) whoamiResponse { t.Helper() - out, err := tryWhoami(ctx, rc, id) - if err != nil { - t.Fatal(err) + deadline := time.Now().Add(30 * time.Second) + for { + out, err := tryWhoami(ctx, rc, id) + if err == nil { + return out + } + if time.Now().After(deadline) { + t.Fatal(err) + } + time.Sleep(time.Second) } - return out } // tryWhoami is whoami returning the error instead of failing the test. From 2bbb38023654b80dcaf0fb83ed6dee26695e0ba0 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 06/22] Add Helm deployment with agentgateway and CRD verification Package the control plane, workers, PostgreSQL, RustFS, and CRDs as Helm charts. Keep manifests and generated RBAC aligned, add Helm E2E checks, and include current scheduling, sandbox permissions, and agentgateway configuration. Co-authored-by: Jet Chiang Co-authored-by: Keith Mattix II Signed-off-by: Jet Chiang Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 116 +++++ Makefile | 16 + charts/substrate-crds/Chart.yaml | 28 ++ charts/substrate-crds/README.md | 13 + .../templates/ate.dev_csidriverconfigs.yaml | 113 +++++ .../templates/ate.dev_sandboxconfigs.yaml | 140 ++++++ .../templates/ate.dev_workerpools.yaml | 473 ++++++++++++++++++ charts/substrate/Chart.yaml | 27 + charts/substrate/README.md | 43 ++ charts/substrate/templates/NOTES.txt | 7 + charts/substrate/templates/_helpers.tpl | 105 ++++ .../templates/ate-api-server-envvars.yaml | 23 + .../substrate/templates/ate-api-server.yaml | 210 ++++++++ charts/substrate/templates/ate-client.yaml | 23 + .../substrate/templates/ate-controller.yaml | 113 +++++ charts/substrate/templates/atelet.yaml | 232 +++++++++ charts/substrate/templates/atenet-egress.yaml | 237 +++++++++ charts/substrate/templates/atenet-router.yaml | 360 +++++++++++++ charts/substrate/templates/namespace.yaml | 22 + .../templates/pod-certificate-controller.yaml | 198 ++++++++ charts/substrate/templates/postgres.yaml | 234 +++++++++ charts/substrate/templates/role.yaml | 114 +++++ charts/substrate/templates/rustfs.yaml | 137 +++++ .../templates/sandboxconfig-gvisor.yaml | 35 ++ .../templates/sandboxconfig-validation.yaml | 57 +++ charts/substrate/values.yaml | 72 +++ cmd/atecontroller/internal/controllers/gen.go | 2 +- hack/gen-rbac.sh | 37 ++ hack/render-manifests.sh | 157 ++++++ hack/verify/crd-chart.sh | 47 ++ .../ate-install/ate-api-server-envvars.yaml | 24 + manifests/ate-install/ate-client.yaml | 24 + .../components/agentgateway/configmap.yaml | 54 ++ manifests/ate-install/role.yaml | 130 +++++ manifests/ate-install/rustfs.yaml | 136 +++++ 35 files changed, 3758 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/helm-e2e.yaml create mode 100644 charts/substrate-crds/Chart.yaml create mode 100644 charts/substrate-crds/README.md create mode 100644 charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml create mode 100644 charts/substrate-crds/templates/ate.dev_workerpools.yaml create mode 100644 charts/substrate/Chart.yaml create mode 100644 charts/substrate/README.md create mode 100644 charts/substrate/templates/NOTES.txt create mode 100644 charts/substrate/templates/_helpers.tpl create mode 100644 charts/substrate/templates/ate-api-server-envvars.yaml create mode 100644 charts/substrate/templates/ate-api-server.yaml create mode 100644 charts/substrate/templates/ate-client.yaml create mode 100644 charts/substrate/templates/ate-controller.yaml create mode 100644 charts/substrate/templates/atelet.yaml create mode 100644 charts/substrate/templates/atenet-egress.yaml create mode 100644 charts/substrate/templates/atenet-router.yaml create mode 100644 charts/substrate/templates/namespace.yaml create mode 100644 charts/substrate/templates/pod-certificate-controller.yaml create mode 100644 charts/substrate/templates/postgres.yaml create mode 100644 charts/substrate/templates/role.yaml create mode 100644 charts/substrate/templates/rustfs.yaml create mode 100644 charts/substrate/templates/sandboxconfig-gvisor.yaml create mode 100644 charts/substrate/templates/sandboxconfig-validation.yaml create mode 100644 charts/substrate/values.yaml create mode 100755 hack/gen-rbac.sh create mode 100755 hack/render-manifests.sh create mode 100755 hack/verify/crd-chart.sh create mode 100644 manifests/ate-install/ate-api-server-envvars.yaml create mode 100644 manifests/ate-install/ate-client.yaml create mode 100644 manifests/ate-install/role.yaml create mode 100644 manifests/ate-install/rustfs.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml new file mode 100644 index 0000000000..81b3a46103 --- /dev/null +++ b/.github/workflows/helm-e2e.yaml @@ -0,0 +1,116 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: helm-e2e +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + e2e-test: + runs-on: ubuntu-latest + env: + VERSION: helm-e2e + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + - name: Setup Go + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version-file: go.mod + - name: Setup Helm + uses: azure/setup-helm@v4 + - name: Cache micro-VM assets + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: bin/microvm-assets/amd64 + key: microvm-assets-amd64-${{ hashFiles('hack/microvm-assets/assemble.sh') }} + - name: Enable KVM + run: | + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \ + | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + - name: Create cluster + run: hack/create-kind-cluster.sh + - name: Label nodes with the installed version + run: kubectl label nodes --all ate.dev/substrate-version=${VERSION} + - name: Create install namespace + run: kubectl create namespace ate-system + - name: Install observability fixtures + run: | + kubectl apply -f manifests/ate-install/kind/otel-collector.yaml + kubectl apply -f manifests/ate-install/kind/prometheus.yaml + - name: Build chart images + run: | + for component in ateapi atecontroller atelet podcertcontroller atenet; do + KO_DOCKER_REPO="localhost:5001/${component}" \ + ./hack/run-tool.sh ko build --bare --tags helm-e2e \ + --platform linux/amd64 "./cmd/${component}" + done + - name: Install Agent Substrate with Helm + run: | + helm upgrade --install substrate-crds charts/substrate-crds + helm upgrade --install substrate charts/substrate \ + --namespace ate-system \ + --create-namespace \ + --set image.registry=localhost:5001 \ + --set image.tag=helm-e2e \ + --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ + --set otel.endpoint=http://opentelemetry-collector.otel-system.svc:4317 \ + --set postgres.resources.requests.cpu=500m + - name: Bootstrap mTLS authorities + run: | + hack/install-ate-kind.sh --create-podcertificate-controller-cas + hack/install-ate-kind.sh --create-jwt-authority-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-pool-secret + hack/install-ate-kind.sh --create-actor-id-ca-certs-secret + hack/install-ate-kind.sh --create-api-authentication-config + - name: Wait for Helm install + run: | + helm upgrade substrate charts/substrate \ + --namespace ate-system \ + --reuse-values \ + --wait --timeout=10m + - name: Enable NFS + run: | + sudo modprobe nfs || true + sudo modprobe nfsd || true + - name: Install CSI NFS driver + run: hack/install-ate-kind.sh --setup-csi=nfs + - name: Deploy micro-VM counter demo + # The deploy creates the substrate ActorTemplate and waits for its golden + # snapshot internally; the ActorTemplate CRD (and its Ready condition) + # no longer exists to wait on. + run: hack/run-microvm-demo-kind.sh --skip-control-plane + - name: Deploy gVisor counter demo + run: hack/install-ate-kind.sh --deploy-demo-counter + - name: Deploy egress demo + run: hack/install-ate-kind.sh --deploy-demo-egress + - name: Run E2E tests (gVisor) + run: hack/run-e2e-kind.sh -v -args --no-color + - name: Run E2E tests (micro-VM) + env: + E2E_SANDBOX_CLASS: microvm + run: hack/run-e2e-kind.sh ./internal/e2e/suites/demo -v -args --no-color + - name: Dump diagnostics on failure + if: failure() + run: | + kubectl --context kind-kind get workerpool,pods -A -o wide || true + for p in $(kubectl --context kind-kind get pods -n ate-system -o name 2>/dev/null); do + echo "=== logs: ate-system/${p} ===" + kubectl --context kind-kind logs -n ate-system "$p" --all-containers --tail=300 || true + done diff --git a/Makefile b/Makefile index 0d6a97f3ab..8dd64c7fd4 100644 --- a/Makefile +++ b/Makefile @@ -139,3 +139,19 @@ verify: test .PHONY: clean clean: rm -rf $(BINDIR) + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS mode, +# the historical default install). Run this whenever charts/substrate/ changes. +.PHONY: helm-template +helm-template: + @./hack/render-manifests.sh + +# Verify that manifests/ate-install/ matches the chart output. Used in CI. +.PHONY: verify-helm-template +verify-helm-template: + @./hack/render-manifests.sh --check + +# Verify that the CRD chart mirrors the generated CRDs. +.PHONY: verify-crd-chart +verify-crd-chart: + @./hack/verify/crd-chart.sh diff --git a/charts/substrate-crds/Chart.yaml b/charts/substrate-crds/Chart.yaml new file mode 100644 index 0000000000..a69dcee0e9 --- /dev/null +++ b/charts/substrate-crds/Chart.yaml @@ -0,0 +1,28 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate-crds +description: Agent Substrate CustomResourceDefinitions. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate +- crds diff --git a/charts/substrate-crds/README.md b/charts/substrate-crds/README.md new file mode 100644 index 0000000000..12fa31f0a7 --- /dev/null +++ b/charts/substrate-crds/README.md @@ -0,0 +1,13 @@ +# substrate-crds + +Helm chart for installing the Agent Substrate CRDs. + +Install this chart before installing the main `substrate` chart: + +```bash +helm upgrade --install substrate-crds ./charts/substrate-crds +helm upgrade --install substrate ./charts/substrate --namespace ate-system --create-namespace +``` + +The CRD YAMLs in `templates/` mirror `manifests/ate-install/generated/`. +Run `hack/verify/crd-chart.sh` to verify they are in sync. diff --git a/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml new file mode 100644 index 0000000000..ebc1473eae --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_csidriverconfigs.yaml @@ -0,0 +1,113 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: csidriverconfigs.ate.dev +spec: + group: ate.dev + names: + kind: CSIDriverConfig + listKind: CSIDriverConfigList + plural: csidriverconfigs + shortNames: + - csidriverconfig + singular: csidriverconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.driverName + name: Driver + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: CSIDriverConfig is the Schema for the csidriverconfigs API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CSIDriverConfigSpec defines the desired state of CSIDriverConfig + properties: + controllerEndpoint: + description: |- + ControllerEndpoint is the gRPC endpoint for the CSI Controller service. + Must be a valid network URI (e.g. dns:///csi-service:9000 or tcp://127.0.0.1:9000). + pattern: ^(tcp|dns)://.+$ + type: string + driverName: + description: |- + DriverName is the standard CSI driver name (e.g. "hostpath.csi.k8s.io"). + Matches the StorageClass referenced in ActorTemplate volume definitions. + maxLength: 63 + minLength: 1 + pattern: ^(substrate\.io/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)$ + type: string + nodeSocketOverride: + description: |- + NodeSocketOverride is an optional override for the CSI Node service socket + on the worker nodes. If empty, ATE defaults to unix:///var/lib/kubelet/plugins/[DriverName]/csi.sock. + pattern: ^unix://.+$ + type: string + tls: + description: TLS configures TLS/mTLS for the connection to the ControllerEndpoint. + properties: + enabled: + description: Enabled controls whether TLS is used. + type: boolean + serverName: + description: ServerName override for TLS verification. + type: string + usePodIdentity: + description: UsePodIdentity indicates whether to reuse Substrate's + Pod Identity (SPIFFE) certificates. + type: boolean + required: + - enabled + type: object + x-kubernetes-validations: + - message: tls.usePodIdentity must be true when tls.enabled is true; + manual certificates are not yet supported + rule: '!self.enabled || (has(self.usePodIdentity) && self.usePodIdentity)' + required: + - controllerEndpoint + - driverName + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml new file mode 100644 index 0000000000..22d46f5123 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_sandboxconfigs.yaml @@ -0,0 +1,140 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: sandboxconfigs.ate.dev +spec: + group: ate.dev + names: + kind: SandboxConfig + listKind: SandboxConfigList + plural: sandboxconfigs + shortNames: + - sandboxconfig + singular: sandboxconfig + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.sandboxClass + name: Class + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + SandboxConfig is cluster-scoped configuration describing the sandbox binaries + for a sandbox runtime family. It is referenced by an ActorTemplate's + sandbox_config.config_name (required) and decouples + sandbox binary selection from the workload definition. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of SandboxConfig + properties: + assets: + additionalProperties: + additionalProperties: + description: |- + AssetFile is one content-addressed file that atelet fetches for a sandbox + runtime (e.g. the gVisor runsc binary, or a micro-VM kernel/firmware/config). + properties: + sha256: + description: |- + SHA256 is the lower-case hex SHA256 of the asset. It both names the cached + file (preventing collisions) and verifies the download's integrity. + pattern: ^[a-f0-9]{64}$ + type: string + url: + description: |- + URL is where to download the asset from (e.g. a gs:// URL). It may be + fetched anonymously or with credentials depending on atelet's + configuration. + minLength: 1 + type: string + required: + - sha256 + - url + type: object + type: object + description: |- + Assets is the set of files atelet fetches for this runtime, keyed first by + architecture (GOARCH, e.g. "amd64", "arm64") and then by asset name. The + asset names are interpreted by the sandbox backend: gVisor expects a + "gvisor" asset (the release's gvisor.tar.zstd, which atelet extracts so + the gvisor-bin/ helpers sit next to runsc; a legacy bare-binary "runsc" + asset is still accepted); a micro-VM backend expects several (e.g. + "cloud-hypervisor", "kata-kernel", "kata-image"). The schema is + intentionally generic; per-class requirements are enforced by a + ValidatingAdmissionPolicy. + type: object + pauseImage: + description: |- + PauseImage is the container image used as the root sandbox container. + It holds the sandbox's namespaces and runs no workload code, so it is an + implementation detail of the sandbox rather than something actor authors + choose. It is captured in the snapshot manifest alongside the sandbox + binaries, so a restore always re-creates the sandbox from the same image + the snapshot was taken with. + + Typically, set it to [1] for on-gcp, and [2] for off-gcp + + - [1] gcr.io/gke-release/pause@sha256:bcbd57ba5653580ec647b16d8163cdd1112df3609129b01f912a8032e48265da + - [2] registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4 + type: string + x-kubernetes-validations: + - message: All images must include a digest + rule: self.contains('@') + sandboxClass: + default: gvisor + description: |- + SandboxClass is the sandbox runtime family this config applies to. An + ActorTemplate only uses SandboxConfigs whose SandboxClass matches its + sandbox_config.sandbox_class. + enum: + - gvisor + - microvm + type: string + required: + - pauseImage + - sandboxClass + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} diff --git a/charts/substrate-crds/templates/ate.dev_workerpools.yaml b/charts/substrate-crds/templates/ate.dev_workerpools.yaml new file mode 100644 index 0000000000..e1bc49a029 --- /dev/null +++ b/charts/substrate-crds/templates/ate.dev_workerpools.yaml @@ -0,0 +1,473 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.20.1 + name: workerpools.ate.dev +spec: + group: ate.dev + names: + kind: WorkerPool + listKind: WorkerPoolList + plural: workerpools + shortNames: + - workerpool + singular: workerpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.replicas + name: Desired + type: integer + - jsonPath: .status.replicas + name: Replicas + type: integer + - jsonPath: .status.readyReplicas + name: Ready + type: integer + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: WorkerPool is the Schema for the workerpools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: spec defines the desired state of WorkerPool + properties: + replicas: + description: Replicas is the number of worker pods to run. + format: int32 + minimum: 0 + type: integer + sandboxClass: + default: gvisor + description: |- + SandboxClass selects the sandbox runtime family for this pool, which drives + the worker pod shape (KVM/vhost device mounts and node placement). The + concrete binary is still selected by WorkerImage. Defaults to gvisor. + The sandbox binaries themselves come from the SandboxConfig each + ActorTemplate names (required). + + See Also: TODOs in ActorTemplate SandboxClass + enum: + - gvisor + - microvm + type: string + template: + description: Template holds optional metadata, scheduling, and resource + settings for worker workloads. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations are added to the generated Deployment and worker pods. Keys + in the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: annotation keys must be valid Kubernetes qualified + names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + labels: + additionalProperties: + description: |- + WorkerPoolLabelValue is a Kubernetes label value for generated worker + workloads. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string + description: |- + Labels are added to the generated Deployment and worker pods. Keys in + the ate.dev domain and its subdomains are reserved for controllers. + maxProperties: 64 + type: object + x-kubernetes-validations: + - message: ate.dev and its subdomains are reserved + rule: self.all(key, !key.startsWith('ate.dev/') && !key.contains('.ate.dev/')) + - message: label keys must be valid Kubernetes qualified names + rule: self.all(key, !format.qualifiedName().validate(key).hasValue()) + nodeAffinity: + description: |- + NodeAffinity scheduling rules for the worker pods. Mapped to + spec.affinity.nodeAffinity on the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the + corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector is a selector which must be true for + the pod to fit on a node. + type: object + priorityClassName: + description: PriorityClassName for the worker pods. + type: string + resources: + description: Resources are the compute resources allocated for + each worker pod. + properties: + claims: + description: |- + Claims lists the names of resources, defined in spec.resourceClaims, + that are used by this container. + + This field depends on the + DynamicResourceAllocation feature gate. + + This field is immutable. It can only be set for containers. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: |- + Name must match the name of one entry in pod.spec.resourceClaims of + the Pod where this field is used. It makes that resource available + inside a container. + type: string + request: + description: |- + Request is the name chosen for a request in the referenced claim. + If empty, everything from the claim is made available, otherwise + only the result of this request. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + tolerations: + description: Tolerations for the worker pods. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists, Equal, Lt, and Gt. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + Lt and Gt perform numeric comparisons (requires feature gate TaintTolerationComparisonOperators). + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + maxItems: 16 + type: array + x-kubernetes-list-type: atomic + type: object + workerImage: + description: WorkerImage is the ateom container image to deploy as + workers. + minLength: 1 + type: string + required: + - replicas + - workerImage + type: object + status: + description: status is the observed state of WorkerPool + properties: + readyReplicas: + description: ReadyReplicas is the number of ready worker pods. + format: int32 + minimum: 0 + type: integer + replicas: + description: Replicas is the total number of worker pods. + format: int32 + minimum: 0 + type: integer + selector: + description: Selector is the label selector for the worker pods. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + scale: + labelSelectorPath: .status.selector + specReplicasPath: .spec.replicas + statusReplicasPath: .status.replicas + status: {} diff --git a/charts/substrate/Chart.yaml b/charts/substrate/Chart.yaml new file mode 100644 index 0000000000..52bd748009 --- /dev/null +++ b/charts/substrate/Chart.yaml @@ -0,0 +1,27 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v2 +name: substrate +description: Agent Substrate — actor runtime, control plane, and data-plane router. +type: application +version: 0.1.0 +appVersion: "0.1.0" +home: https://github.com/agent-substrate/substrate +sources: +- https://github.com/agent-substrate/substrate +keywords: +- agent +- actor +- substrate diff --git a/charts/substrate/README.md b/charts/substrate/README.md new file mode 100644 index 0000000000..e7364f4e37 --- /dev/null +++ b/charts/substrate/README.md @@ -0,0 +1,43 @@ +# substrate + +Helm chart for installing Agent Substrate. + +The chart uses mTLS and PostgreSQL by default. It requires the +`ClusterTrustBundle`, `ClusterTrustBundleProjection`, and +`PodCertificateRequest` feature gates plus the `certificates.k8s.io/v1beta1` +API. + +```bash +# CRDs +helm upgrade --install substrate-crds ./charts/substrate-crds + +# Install Substrate +helm upgrade --install substrate ./charts/substrate +``` + +By default, component images are pulled from `ghcr.io/kagent-dev/substrate` +using the chart `appVersion` as the tag. Override `image.registry` and +`image.tag` to install from a different image repository or tag. + +## Render manifests without applying + +```bash +helm template substrate ./charts/substrate +``` + +`manifests/ate-install/` in the repo is the rendered mTLS output and is +regenerated by `make helm-template`. The separate `substrate-crds` chart +mirrors `manifests/ate-install/generated/`. + +## Values + +See `values.yaml` for the full set; the important keys: + +| Key | Default | Notes | +|-----|---------|-------| +| `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | +| `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | +| `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | +| `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | diff --git a/charts/substrate/templates/NOTES.txt b/charts/substrate/templates/NOTES.txt new file mode 100644 index 0000000000..c0e9875a45 --- /dev/null +++ b/charts/substrate/templates/NOTES.txt @@ -0,0 +1,7 @@ +substrate {{ .Chart.AppVersion }} installed with mTLS and PostgreSQL + +REQUIRED Kubernetes feature gates: + - ClusterTrustBundle + - ClusterTrustBundleProjection + - PodCertificateRequest +The certificates.k8s.io/v1beta1 API must also be enabled. diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl new file mode 100644 index 0000000000..32ae087336 --- /dev/null +++ b/charts/substrate/templates/_helpers.tpl @@ -0,0 +1,105 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Qualified resource name for a chart component. + +Usage: + {{ include "substrate.fullname" (list "ate-api-server" .) }} + +When the release name is "substrate" (the canonical render in +hack/render-manifests.sh — `helm template substrate charts/substrate`), this +returns the bare component name, so the generated manifests/ate-install/ +files keep their historical names ("ate-api-server", "ate-controller", ...). + +Otherwise resources are prefixed with the release name in the standard Helm +style ("foo-ate-api-server", ...) so multiple releases coexist without +colliding. + +The check is on the literal release name "substrate" rather than +$ctx.Chart.Name so this helper is context-safe: a parent chart can invoke it +with its own `.` (where .Chart.Name is the parent, not "substrate") and still +get the same prefixed name that this subchart's own templates render. +*/}} +{{- define "substrate.fullname" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- if eq $ctx.Release.Name "substrate" -}} +{{- $name -}} +{{- else -}} +{{- printf "%s-%s" $ctx.Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} + +{{/* +ServiceAccount name of ate-api-server, as this chart creates it. Parent +charts that need to bind additional Roles to this SA (e.g. env-source +Secret/ConfigMap reads for ActorTemplate resolution) should reference this +helper instead of hardcoding "ate-api-server": + + {{ include "substrate.ateApiServer.serviceAccountName" . }} +*/}} +{{- define "substrate.ateApiServer.serviceAccountName" -}} +{{- include "substrate.fullname" (list "ate-api-server" .) -}} +{{- end -}} + +{{/* +gRPC endpoint that clients dial to reach ate-api-server. dns:/// scheme + +release-prefixed Service name + release namespace + :443. Suitable for +consumption as ATE_API_ENDPOINT / --ateapi-address: + + {{ include "substrate.ateApi.endpoint" . }} + -> dns:///-api..svc:443 +*/}} +{{- define "substrate.ateApi.endpoint" -}} +{{- printf "dns:///%s.%s.svc:443" (include "substrate.fullname" (list "api" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Plaintext HTTP URL that clients use to reach atenet-router. + + {{ include "substrate.atenetRouter.url" . }} + -> http://-atenet-router..svc:80 +*/}} +{{- define "substrate.atenetRouter.url" -}} +{{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} +{{- end -}} + +{{/* +Build an image reference for a substrate component binary. + +Usage: + {{ include "substrate.componentImage" (list "ateapi" .) }} + +Produces {image.registry}/{name}:{tag} where tag is resolved as: + 1. image.tag value, if set and not the sentinel "" + 2. .Chart.AppVersion, if image.tag is empty + 3. no tag (no colon) when image.tag is the sentinel "" + +The "" sentinel is used by hack/render-manifests.sh so that ko:// refs +are emitted without a tag, letting `ko resolve` supply the digest at build time. +*/}} +{{- define "substrate.componentImage" -}} +{{- $name := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $registry := $ctx.Values.image.registry -}} +{{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} +{{- if ne $tag "" -}} +{{- printf "%s/%s:%s" $registry $name $tag -}} +{{- else -}} +{{- printf "%s/%s" $registry $name -}} +{{- end -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..753c47178b --- /dev/null +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + namespace: {{ .Release.Namespace }} +data: + ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml new file mode 100644 index 0000000000..a5073bc9fa --- /dev/null +++ b/charts/substrate/templates/ate-api-server.yaml @@ -0,0 +1,210 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs", "csidriverconfigs"] + verbs: ["get", "watch", "list"] +- apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "watch", "list"] +# Secret reads for env source resolution are intentionally NOT granted +# cluster-wide here. Each demo / tenant is responsible for granting +# ate-api-server read access only to the specific Secrets referenced by its +# ActorTemplates (e.g. via a namespace-scoped Role + RoleBinding using +# resourceNames). +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-api-server-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 2 + strategy: + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: ate-api-server + template: + metadata: + labels: + app: ate-api-server + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} + terminationGracePeriodSeconds: 40 + containers: + - name: ate-api-server + image: {{ include "substrate.componentImage" (list "ateapi" .) }} + args: + - "--grpc-listen-addr=0.0.0.0:443" + - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" + - "--postgres-connection-string=@env" + - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" + - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" + - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + - "--drain-delay=13s" + - "--drain-timeout=15s" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + envFrom: + - configMapRef: + name: {{ .Values.ateApiServerEnvVarsConfigMap }} + optional: true + volumeMounts: + - { name: servicedns, mountPath: /run/servicedns.podcert.ate.dev } + - { name: actor-id-jwt-pool, mountPath: /run/actor-id-jwt-pool } + - { name: actor-id-ca-pool, mountPath: /run/actor-id-ca-pool, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - { name: authentication-config, mountPath: /etc/ateapi/authentication, readOnly: true } + ports: + - containerPort: 443 + - name: prometheus + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 + livenessProbe: + httpGet: + path: /healthz + port: 9090 + initialDelaySeconds: 10 + periodSeconds: 10 + volumes: + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-jwt-pool + projected: + sources: + - secret: + name: actor-id-jwt-pool + items: + - { key: pool, path: pool.json } + - name: actor-id-ca-pool + projected: + sources: + - secret: + name: actor-id-ca-pool + items: + - { key: pool, path: pool.json } + - name: authentication-config + configMap: + name: ate-api-authentication + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ include "substrate.fullname" (list "ate-api-server" .) }} + namespace: {{ .Release.Namespace }} +spec: + maxUnavailable: 1 + selector: + matchLabels: + app: ate-api-server +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "api" .) }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: ate-api-server + ports: + - name: grpc + protocol: TCP + port: 443 + targetPort: 443 diff --git a/charts/substrate/templates/ate-client.yaml b/charts/substrate/templates/ate-client.yaml new file mode 100644 index 0000000000..dfd2fdab68 --- /dev/null +++ b/charts/substrate/templates/ate-client.yaml @@ -0,0 +1,23 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-client" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-client diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml new file mode 100644 index 0000000000..31c83b9066 --- /dev/null +++ b/charts/substrate/templates/ate-controller.yaml @@ -0,0 +1,113 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + apps: ate-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: ate-controller +spec: + selector: + app: ate-controller + ports: + - name: metrics + port: 8080 + targetPort: metrics + protocol: TCP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: ate-controller + template: + metadata: + labels: + app: ate-controller + spec: + serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + containers: + - name: ate-controller + image: {{ include "substrate.componentImage" (list "atecontroller" .) }} + args: + # The atecontroller binary defaults --ateapi-conn-spec to + # dns:///api.ate-system.svc:443, which is correct only for the + # canonical render (release name "substrate" in namespace + # "ate-system"). Pass the chart-resolved Service so the controller + # dials the right backend when substrate is installed as a subchart. + - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" +{{- if .Values.otel.endpoint }} + env: + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: metrics + containerPort: 8080 + protocol: TCP + - name: healthz + containerPort: 8081 + protocol: TCP + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + volumes: + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml new file mode 100644 index 0000000000..c026f3577f --- /dev/null +++ b/charts/substrate/templates/atelet.yaml @@ -0,0 +1,232 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# atelet +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "atelet-role" .) }} +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["get", "watch", "list"] +- apiGroups: ["ate.dev"] + resources: ["csidriverconfigs"] + verbs: ["get", "watch", "list"] +# ClusterTrustBundles referenced by SystemInfo trustBundle data sources are +# resolved on the node: atelet reads them through an informer and projects +# the sanitized PEM into actors (see cmd/atelet/trustbundle.go). +- apiGroups: ["certificates.k8s.io"] + resources: ["clustertrustbundles"] + verbs: ["get", "watch", "list"] +# SandboxConfigs are watched to pre-download sandbox assets into the node's +# cache before the first actor needs them (see cmd/atelet/sandbox_prewarm.go). +- apiGroups: ["ate.dev"] + resources: ["sandboxconfigs"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-binding" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "substrate.fullname" (list "atelet-role" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + namespace: {{ .Release.Namespace }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "substrate.fullname" (list "atelet-endpointslices" .) }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: {{ include "substrate.fullname" (list "ate-node-critical" .) }} +value: 1000000000 +globalDefault: false +description: "Node-local ate components that every actor activation depends on." +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "substrate.fullname" (list "atelet" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atelet +spec: + selector: + matchLabels: + app: atelet + template: + metadata: + labels: + app: atelet + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9090" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} + priorityClassName: {{ include "substrate.fullname" (list "ate-node-critical" .) }} + containers: + - name: atelet + image: {{ include "substrate.componentImage" (list "atelet" .) }} + args: + - --gcp-auth-for-image-pulls={{ .Values.atelet.gcpAuthForImagePulls }} + - --grpc-server-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --client-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem +{{- with .Values.atelet.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} + securityContext: + privileged: true + resources: + requests: + cpu: 50m + memory: 128Mi + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} +{{- end }} +{{- with .Values.atelet.extraEnv }} +{{ toYaml . | indent 8 }} +{{- end }} + ports: + - name: grpc + containerPort: 8085 + hostPort: 8085 + - name: prometheus + containerPort: 9090 + hostPort: 9090 + protocol: TCP + volumeMounts: + - name: run-ateom + mountPath: /var/lib/ateom-gvisor + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: device-plugins + mountPath: /var/lib/kubelet/device-plugins + - name: host-dev + mountPath: /host/dev + readOnly: true + volumes: + - name: run-ateom + hostPath: + path: /var/lib/ateom-gvisor + type: DirectoryOrCreate + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: device-plugins + hostPath: + path: /var/lib/kubelet/device-plugins + type: DirectoryOrCreate + - name: host-dev + hostPath: + path: /dev + type: Directory + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml new file mode 100644 index 0000000000..850120b65d --- /dev/null +++ b/charts/substrate/templates/atenet-egress.yaml @@ -0,0 +1,237 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + frontendPolicies: + accessLog: + add: + substrate.connect.authority: source.connectHeaders["host"] + # Authorize the actor identity at CONNECT-accept, before any tunnel + # (HTTP, TLS, or opaque TCP) is terminated. Fails closed when the + # control plane is unreachable. + substrateEgressActorResolution: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + + binds: + - port: 8443 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + root: /run/actor-id-ca-certs/ca.crt + routes: [] + - mode: internal + protocol: AUTO + listeners: + - protocol: TLS + hostname: "*" + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + - protocol: HTTP + routes: + - backends: + - dynamic: + target: source.connectHeaders["host"] + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + - protocol: TCP + tcpRoutes: + - backends: + - dynamic: + target: source.connectHeaders["host"] +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-egress +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-egress + template: + metadata: + labels: + app: atenet-egress + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-egress" .) }} + securityContext: + sysctls: + - name: net.ipv4.ip_unprivileged_port_start + value: "0" + terminationGracePeriodSeconds: 60 + containers: + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - -f + - /etc/agentgateway/config.yaml + ports: + - name: https + containerPort: 8443 + - name: readiness + containerPort: 15021 + - name: stats + containerPort: 15020 + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + startupProbe: + failureThreshold: 60 + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 1 + volumeMounts: + - name: config + mountPath: /etc/agentgateway + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: ext-proc + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - router + - --mode=egress + - --namespace={{ .Release.Namespace }} + - --port-extproc=50051 + - --extproc-address=127.0.0.1 + - --ateapi-address={{ include "substrate.ateApi.endpoint" . }} + - --ateapi-ca-file=/run/servicedns.podcert.ate.dev/trust-bundle.pem + - --ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem + - --actor-identity-ca-file=/run/actor-id-ca-certs/ca.crt + - --otlp-collector-address= + - --envoy-admin-address=localhost:15000 + - --atenet-dataplane=agentgateway + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - name: extproc + containerPort: 50051 + readinessProbe: + tcpSocket: + port: extproc + periodSeconds: 10 + volumeMounts: + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: actor-id-ca-certs + mountPath: /run/actor-id-ca-certs + readOnly: true + - name: drain-signal + mountPath: /var/run/atenet + volumes: + - name: config + configMap: + name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} + - name: drain-signal + emptyDir: {} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: actor-id-ca-certs + secret: + secretName: actor-id-ca-certs +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-egress" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-egress + ports: + - name: https + port: 443 + targetPort: https + protocol: TCP diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml new file mode 100644 index 0000000000..1cb84b69d5 --- /dev/null +++ b/charts/substrate/templates/atenet-router.yaml @@ -0,0 +1,360 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + namespace: {{ .Release.Namespace }} +data: + config.yaml: | + # yaml-language-server: $schema=https://agentgateway.dev/schema/config + config: + statsAddr: 0.0.0.0:15020 + # Actor sandboxes behind a worker IP are replaced between requests. Do + # not retain an idle connection that may belong to the previous actor. + backend: + poolMaxSize: 0 + +{{- if .Values.otel.endpoint }} + frontendPolicies: + tracing: + host: $AGENTGATEWAY_OTLP_ADDRESS + protocol: grpc + randomSampling: 0.01 +{{- end }} + + backends: + - name: dynamic + dynamic: {} + policies: + backendTunnel: + proxy: + backend: /dynamic + mode: connect + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/podidentity.podcert.ate.dev/trust-bundle.pem + insecureHost: true + + gateways: + http: + port: 8080 + protocol: HTTP + https: + port: 8443 + protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + + routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors + gateways: + - http + - https + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 + + binds: + - port: 8081 + tunnelProtocol: connect + listeners: + - protocol: HTTP + routes: [] + - port: 8444 + tunnelProtocol: connect + listeners: + - protocol: HTTPS + tls: + cert: /run/servicedns.podcert.ate.dev/credential-bundle.pem + key: /run/servicedns.podcert.ate.dev/credential-bundle.pem + routes: [] + - mode: internal + listeners: + - protocol: HTTP + routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 + - name: substrate-actors-tunneled + matches: + - path: + pathPrefix: / + policies: + substrateIngress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/1.1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: atenet-router +spec: + replicas: 1 + selector: + matchLabels: + app: atenet-router + template: + metadata: + labels: + app: atenet-router + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "15020" + spec: + serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + containers: + - name: atenet-router + image: {{ include "substrate.componentImage" (list "atenet" .) }} + args: + - "router" + - "--mode=ingress" + - "--atenet-dataplane=agentgateway" + - "--namespace={{ .Release.Namespace }}" + - "--port-http=8080" + - "--port-extproc=50051" + - "--extproc-address=127.0.0.1" + - "--ateapi-address=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" + - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + - "--status-port=4040" + - "--port-https=8443" + - "--port-connect=8081" + - "--port-connect-tls=8444" + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: OTEL_RESOURCE_ATTRIBUTES + value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) +{{- if .Values.otel.endpoint }} + - name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: extproc + containerPort: 50051 + - name: status + containerPort: 4040 + - name: metrics + containerPort: 9090 + volumeMounts: + - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } + - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } + - name: agentgateway + image: {{ .Values.images.agentgateway }} + args: + - "-f" + - "/etc/agentgateway/config.yaml" +{{- if .Values.otel.endpoint }} + env: + - name: AGENTGATEWAY_OTLP_ADDRESS + value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- end }} + ports: + - name: http + containerPort: 8080 + - name: https + containerPort: 8443 + - name: connect + containerPort: 8081 + - name: connect-tls + containerPort: 8444 + - name: readiness + containerPort: 15021 + - name: gw-metrics + containerPort: 15020 + volumeMounts: + - name: agentgateway-config + mountPath: /etc/agentgateway + - name: "servicedns" + mountPath: "/run/servicedns.podcert.ate.dev" + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: servicedns-ca + mountPath: /run/servicedns-ca + readOnly: true + readinessProbe: + httpGet: + path: /healthz/ready + port: readiness + periodSeconds: 10 + volumes: + - name: agentgateway-config + configMap: + name: {{ include "substrate.fullname" (list "atenet-router-agentgateway-config" .) }} + - name: "servicedns" + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - name: servicedns-ca + projected: + sources: + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + certificateChainPath: cert.pem + keyPath: key.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "atenet-router" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + selector: + app: atenet-router + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + - name: https + port: 443 + targetPort: 8443 + protocol: TCP + - name: connect + port: 8081 + targetPort: 8081 + protocol: TCP + - name: connect-tls + port: 8444 + targetPort: 8444 + protocol: TCP + - name: status + port: 4040 + targetPort: status + protocol: TCP + - name: stats + port: 15020 + targetPort: gw-metrics + protocol: TCP diff --git a/charts/substrate/templates/namespace.yaml b/charts/substrate/templates/namespace.yaml new file mode 100644 index 0000000000..073291828b --- /dev/null +++ b/charts/substrate/templates/namespace.yaml @@ -0,0 +1,22 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.createNamespace }} +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml new file mode 100644 index 0000000000..86fc23b4a9 --- /dev/null +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -0,0 +1,198 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: Namespace +metadata: + name: podcertificate-controller-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +rules: +# The service signer needs to be able to read services and pods. +- apiGroups: + - "" + resources: + - services + - pods + verbs: + - get + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests + verbs: + - get + - list + - watch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - get + - list + - watch + - update + - delete +- apiGroups: + - certificates.k8s.io + resources: + - podcertificaterequests/status + verbs: + - update +- apiGroups: + - certificates.k8s.io + resources: + - signers + resourceNames: + - servicedns.podcert.ate.dev/* + - podidentity.podcert.ate.dev/* + verbs: + - sign + - attest +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "podcert-ate-dev-signer" .) }} +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: podcertificate-controller-system + name: coordinator +rules: +- apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - create + - get + - list + - watch + - update + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: podcertificate-controller-is-a-coordinator + namespace: podcertificate-controller-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: coordinator +subjects: +- kind: ServiceAccount + namespace: podcertificate-controller-system + name: default +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: podcertificate-controller + namespace: podcertificate-controller-system + labels: + app: podcertificate-controller +spec: + replicas: 1 + selector: + matchLabels: + app: podcertificate-controller + template: + metadata: + labels: + app: podcertificate-controller + spec: + containers: + - name: controller + image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} + args: + - --in-cluster=true + - --sharding-pod-namespace=$(POD_NAMESPACE) + - --sharding-pod-name=$(POD_NAME) + - --sharding-pod-uid=$(POD_UID) + - --sharding-application-name=podcertificate-controller + - --service-dns-ca-pool=/run/ca-state/service-dns-pool.json + - --pod-identity-ca-pool=/run/ca-state/pod-identity-pool.json + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + volumeMounts: + - name: "ca-state" + mountPath: "/run/ca-state" + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: true + volumes: + - name: "ca-state" + projected: + sources: + - secret: + name: "service-dns-ca-pool" + items: + - key: "pool" + path: "service-dns-pool.json" + - secret: + name: "pod-identity-ca-pool" + items: + - key: "pool" + path: "pod-identity-pool.json" + dnsPolicy: Default + nodeSelector: + kubernetes.io/os: linux + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccountName: default + terminationGracePeriodSeconds: 30 diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml new file mode 100644 index 0000000000..ce4a4efdd7 --- /dev/null +++ b/charts/substrate/templates/postgres.yaml @@ -0,0 +1,234 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $name := include "substrate.fullname" (list "postgres" .) -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $name }}-config + namespace: {{ .Release.Namespace }} +data: + postgresql.conf: | + listen_addresses = '*' + ssl = on + ssl_cert_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_key_file = '/run/servicedns.podcert.ate.dev/credential-bundle.pem' + ssl_ca_file = '/run/podidentity.podcert.ate.dev/trust-bundle.pem' + hba_file = '/etc/postgresql/pg_hba.conf' + pg_hba.conf: | + # Local socket access is limited to processes in this pod and is used by + # health checks, the workload's idempotent database bootstrap, and the + # tls-reloader sidecar's configuration reloads. + local all all trust + # PostgreSQL verifies client certificates against the pod-identity CA. It + # does not need its own serving CA because it never verifies its server certificate. + hostssl all all all trust clientcert=verify-ca + reload-tls.sh: | + # PostgreSQL opens ssl_cert_file, ssl_key_file and ssl_ca_file at startup + # and on SIGHUP, and nowhere else. The kubelet replaces the projected pod + # certificate in place about 30 minutes before it expires, so without this + # loop the server keeps presenting the certificate it booted with until it + # expires about a day later and every client stops trusting it. + set -eu + + # As PID 1 this shell only sees SIGTERM if a handler is installed, and only + # acts on it between commands, so the sleep below runs in the background + # and is waited on. Without both halves the pod takes the full termination + # grace period to go away. + trap 'exit 0' TERM INT + + CERT=/run/servicedns.podcert.ate.dev/credential-bundle.pem + CA=/run/podidentity.podcert.ate.dev/trust-bundle.pem + + # Comfortably inside the 30m headroom (notAfter - beginRefreshAt) that + # cmd/podcertcontroller/internal/servicednssigner/servicednssigner.go + # leaves; hashing two small files costs nothing. + INTERVAL=60 + + reloaded="" + while true; do + current="$(sha256sum "${CERT}" "${CA}")" + # Reloading fails until the server is accepting connections, which is + # where every pod starts out, so only record a hash once it has worked. + # Starting empty also means a restart of this container costs one + # redundant reload rather than a missed one. + if [ "${current}" != "${reloaded}" ] \ + && psql -U postgres -d postgres -Atc 'SELECT pg_reload_conf()' >/dev/null 2>&1; then + reloaded="${current}" + echo "$(date -u +%FT%TZ) reloaded TLS configuration" + fi + sleep "${INTERVAL}" & + wait $! + done +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + clusterIP: None + selector: + app: {{ $name }} + ports: + - name: postgres + port: 5432 + targetPort: 5432 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app: {{ $name }} + template: + metadata: + labels: + app: {{ $name }} + spec: + securityContext: + # Group ownership of the projected certificate below, and of the data + # volume so that a freshly provisioned one is writable. OnRootMismatch + # keeps the kubelet from walking the data directory on every start, + # which would leave PGDATA group-writable and postgres refusing to run. + fsGroup: 70 + fsGroupChangePolicy: OnRootMismatch + # PostgreSQL re-reads its TLS files only on SIGHUP, so this sidecar + # reloads the server whenever the kubelet rotates the projected pod + # certificate. fsGroup is also what makes that projection readable: the + # kubelet writes it root-owned for as long as the pod's containers do not + # all agree on one non-root user, and grants the fsGroup group access, + # landing the key at root:postgres 0640, the only shared mode PostgreSQL + # accepts. Pinning runAsUser on the postgres container would make the key + # postgres-owned and group-readable, which it rejects. + # See https://www.postgresql.org/docs/current/ssl-tcp.html#SSL-SETUP + initContainers: + - name: tls-reloader + restartPolicy: Always + image: {{ .Values.images.postgres }} + securityContext: + runAsUser: 70 + command: + - /bin/sh + - /etc/postgresql/reload-tls.sh + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + resources: + requests: + cpu: 10m + memory: 32Mi + containers: + - name: postgres + image: {{ .Values.images.postgres }} + lifecycle: + postStart: + exec: + command: + - /bin/sh + - -ec + - | + until psql -U postgres -d postgres -Atc 'SELECT 1' >/dev/null 2>&1; do + sleep 1 + done + if ! psql -U postgres -d postgres -Atc \ + "SELECT 1 FROM pg_database WHERE datname = 'atepg'" | grep -qx 1; then + createdb -U postgres atepg + fi + env: + - name: POSTGRES_DB + value: atepg + - name: POSTGRES_HOST_AUTH_METHOD + value: trust + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + ports: + - name: postgres + containerPort: 5432 + readinessProbe: + exec: + command: ["/bin/sh", "-ec", "psql -U postgres -d atepg -Atc 'SELECT 1' >/dev/null"] + initialDelaySeconds: 2 + periodSeconds: 2 + livenessProbe: + exec: + command: ["pg_isready", "-U", "postgres", "-d", "postgres"] + initialDelaySeconds: 10 + periodSeconds: 10 + args: ["-c", "config_file=/etc/postgresql/postgresql.conf"] + volumeMounts: + - name: config + mountPath: /etc/postgresql + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity-ca + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + - name: socket + mountPath: /var/run/postgresql + - name: data + mountPath: /var/lib/postgresql/data + resources: +{{ toYaml .Values.postgres.resources | indent 10 }} + volumes: + - name: config + configMap: + name: {{ $name }}-config + - name: servicedns + projected: + # 0600 plus the group read that fsGroup adds is the 0640 above. + defaultMode: 0600 + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + # The unix socket directory, shared so the sidecar can ask the running + # server to reload. The image defaults both the server and its clients to + # this path, so nothing else has to know about it. + - name: socket + emptyDir: {} + - name: podidentity-ca + projected: + sources: + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.postgres.storageSize }} diff --git a/charts/substrate/templates/role.yaml b/charts/substrate/templates/role.yaml new file mode 100644 index 0000000000..5a240f5baf --- /dev/null +++ b/charts/substrate/templates/role.yaml @@ -0,0 +1,114 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "substrate.fullname" (list "ate-controller" .) }} + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml new file mode 100644 index 0000000000..edaad3cfa8 --- /dev/null +++ b/charts/substrate/templates/rustfs.yaml @@ -0,0 +1,137 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.rustfs.enabled -}} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "substrate.fullname" (list "rustfs-data" .) }} + namespace: {{ .Release.Namespace }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.rustfs.storageSize }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "rustfs" .) }} + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: {{ .Values.images.rustfs }} + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: {{ .Values.rustfs.accessKey | quote }} + - name: RUSTFS_SECRET_KEY + value: {{ .Values.rustfs.secretKey | quote }} + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: {{ include "substrate.fullname" (list "rustfs-data" .) }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "substrate.fullname" (list "rustfs-bucket-init" .) }} + namespace: {{ .Release.Namespace }} +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: {{ .Values.images.awsCli }} + env: + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} already exists" + exit 0 + fi + if aws s3api create-bucket --bucket {{ .Values.rustfs.bucket }} 2>/dev/null; then + echo "bucket {{ .Values.rustfs.bucket }} created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 +{{- end }} diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml new file mode 100644 index 0000000000..05f851d40d --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -0,0 +1,35 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Cluster-wide SandboxConfig for the gVisor (runsc) sandbox class, shipped with +# the platform so gVisor ActorTemplates have a config to name via +# sandboxConfig.configName. +apiVersion: ate.dev/v1alpha1 +kind: SandboxConfig +metadata: + name: gvisor-default +spec: + sandboxClass: gvisor + pauseImage: "registry.k8s.io/pause:3.10.2@sha256:f548e0e8e3dc1896ca956272154dde3314e8cc4fde0a57577ee9fa1c63f5baf4" + assets: + amd64: + gvisor: + url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" + sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + arm64: + gvisor: + url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" + sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml new file mode 100644 index 0000000000..f25d43409b --- /dev/null +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -0,0 +1,57 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Per-sandbox-class asset requirements for SandboxConfig. The CRD schema is +# generic (any arch -> any asset name -> {url, sha256}); this policy enforces the +# requirements a given sandbox class actually needs, fail-closed at apply time. +# (url/sha256 being required and well-formed is enforced by the CRD schema.) +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicy +metadata: + name: sandboxconfig-assets +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["ate.dev"] + apiVersions: ["v1alpha1"] + operations: ["CREATE", "UPDATE"] + resources: ["sandboxconfigs"] + validations: + # gVisor needs a release tarball (or legacy runsc binary) for every architecture. + - expression: >- + object.spec.sandboxClass != 'gvisor' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + 'gvisor' in object.spec.assets[arch] || 'runsc' in object.spec.assets[arch])) + message: "a gvisor SandboxConfig must define a 'gvisor' (release tarball) or legacy 'runsc' asset for every architecture under spec.assets" + # The micro-VM (cloud-hypervisor) runtime needs its asset set for every + # architecture it advertises. + - expression: >- + object.spec.sandboxClass != 'microvm' || + (has(object.spec.assets) && size(object.spec.assets) > 0 && + object.spec.assets.all(arch, + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + .all(name, name in object.spec.assets[arch]))) + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingAdmissionPolicyBinding +metadata: + name: sandboxconfig-assets +spec: + policyName: sandboxconfig-assets + validationActions: ["Deny"] diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml new file mode 100644 index 0000000000..b9c7734fe0 --- /dev/null +++ b/charts/substrate/values.yaml @@ -0,0 +1,72 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for the substrate chart. +# +# The chart requires ClusterTrustBundle, ClusterTrustBundleProjection, +# PodCertificateRequest, and the certificates.k8s.io/v1beta1 API. + +# Set to true to have the chart create the release namespace. +# Off by default — most helm workflows expect the namespace to already exist +# (helm install -n --create-namespace). Enable for the generated +# manifests/ate-install/ install path (kubectl apply). +createNamespace: false + +postgres: + storageSize: 1Gi + connectionString: "" + resources: + requests: + cpu: "1" + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + +rustfs: + enabled: true + storageSize: 1Gi + bucket: ate-snapshots + accessKey: rustfsadmin + secretKey: rustfsadmin + +# atelet daemonset overrides. Defaults use the in-cluster RustFS deployment for +# snapshots. Set rustfs.enabled=false and override these fields when using +# external storage. +# extraArgs / extraEnv are appended verbatim for installer-specific knobs +# (e.g. registry replacement for kind). +atelet: + gcpAuthForImagePulls: false + storageBackend: s3 + extraArgs: [] + extraEnv: [] + +# Name of a ConfigMap in the release namespace that supplies per-environment +# overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). +# Mounted via envFrom with optional=true. Created by the chart from these values. +ateApiServerEnvVarsConfigMap: ate-api-server-envvars + +otel: + endpoint: "" + +image: + registry: ghcr.io/kagent-dev/substrate + tag: "" + +images: + postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 + rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + busybox: busybox:1.36 diff --git a/cmd/atecontroller/internal/controllers/gen.go b/cmd/atecontroller/internal/controllers/gen.go index 218a18c2d8..a7ed1b1cf5 100644 --- a/cmd/atecontroller/internal/controllers/gen.go +++ b/cmd/atecontroller/internal/controllers/gen.go @@ -22,4 +22,4 @@ package controllers //+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch //+kubebuilder:rbac:groups=discovery.k8s.io,resources=endpointslices,verbs=get;list;watch,namespace=ate-system -//go:generate bash ../../../../hack/run-tool.sh controller-gen rbac:headerFile=../../../../hack/boilerplate/sh.txt,roleName=ate-controller paths="./..." output:rbac:artifacts:config=../../../../manifests/ate-install/generated/ +//go:generate bash ../../../../hack/gen-rbac.sh diff --git a/hack/gen-rbac.sh b/hack/gen-rbac.sh new file mode 100755 index 0000000000..baa22fa517 --- /dev/null +++ b/hack/gen-rbac.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generate the controller ClusterRole into the Helm chart and templatize its +# name so multi-release installs do not collide on a cluster-scoped resource. +# +# controller-gen emits a YAML file with a fixed `roleName=` value. We post- +# process that file to swap the static name for the chart's fullname helper, +# matching the convention used by every other resource in charts/substrate/. +# +# Invoked via `go generate ./cmd/atecontroller/internal/controllers/...`. +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="${ROOT}/charts/substrate/templates/role.yaml" + +bash "${ROOT}/hack/run-tool.sh" controller-gen \ + "rbac:headerFile=${ROOT}/hack/boilerplate/sh.txt,roleName=ate-controller" \ + paths="${ROOT}/cmd/atecontroller/internal/controllers/..." \ + "output:rbac:artifacts:config=${ROOT}/charts/substrate/templates/" + +# Templatize the ClusterRole name. controller-gen emits ` name: ate-controller` +# at column 0; the substitution is exact-match to stay robust. +sed -i 's|^ name: ate-controller$| name: {{ include "substrate.fullname" (list "ate-controller" .) }}|' "${OUT}" diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh new file mode 100755 index 0000000000..1f6790bb73 --- /dev/null +++ b/hack/render-manifests.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Render the substrate Helm chart into manifests/ate-install/ (mTLS-mode +# install) — the canonical kubectl-apply install path. The chart at +# charts/substrate/ is the single source of truth; this script only renders. +# +# Usage: +# hack/render-manifests.sh # write into manifests/ate-install/ +# hack/render-manifests.sh --check # fail if rendered output differs +# +set -o errexit -o nounset -o pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT_DIR="${ROOT}/manifests/ate-install" +CHART_DIR="${ROOT}/charts/substrate" +CHECK_MODE="false" +PRESERVED_FILES=( + ate-api-server.yaml + ate-controller.yaml + ate-otel-config.yaml + ate-system-namespace.yaml + atelet.yaml + atenet-egress.yaml + atenet-egress-with-sdsmint.yaml + atenet-router.yaml + atenet-router-monitoring.yaml + pod-certificate-controller.yaml + postgres.yaml + sandboxconfig-gvisor.yaml + sandboxconfig-validation.yaml +) + +if [ "${1:-}" = "--check" ]; then + CHECK_MODE="true" +fi + +if ! command -v helm >/dev/null 2>&1; then + echo "helm not found in PATH" >&2 + exit 1 +fi + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +helm template substrate "${CHART_DIR}" \ + --namespace ate-system \ + --set auth.mode=mtls \ + --set createNamespace=true \ + --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.tag="" \ + > "${TMP_DIR}/all.yaml" + +# Split into per-source files so the directory structure mirrors the chart +# templates, making diffs friendlier. +python3 - "${TMP_DIR}/all.yaml" "${TMP_DIR}/out" <<'PY' +import os, re, sys, yaml +in_path, out_dir = sys.argv[1], sys.argv[2] +os.makedirs(out_dir, exist_ok=True) + +with open(in_path) as f: + raw = f.read() + +# Helm prepends a "# Source: /templates/" comment to each doc. +docs_by_source = {} +for doc in raw.split('\n---\n'): + m = re.search(r'#\s*Source:\s*\S+/templates/(\S+)', doc) + src = m.group(1) if m else "misc.yaml" + # Drop the leading "# Source:" line from the written file. + cleaned = re.sub(r'^\s*#\s*Source:.*\n', '', doc, count=1, flags=re.MULTILINE) + if not cleaned.strip(): + continue + docs_by_source.setdefault(src, []).append(cleaned.strip()) + +for src, docs in docs_by_source.items(): + if src == "namespace.yaml": + src = "ate-system-namespace.yaml" + header = ( + "# Copyright 2026 Google LLC\n" + "#\n" + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n" + "# you may not use this file except in compliance with the License.\n" + "# You may obtain a copy of the License at\n" + "#\n" + "# http://www.apache.org/licenses/LICENSE-2.0\n" + "#\n" + "# Unless required by applicable law or agreed to in writing, software\n" + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n" + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + "# See the License for the specific language governing permissions and\n" + "# limitations under the License.\n" + "\n" + "# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh.\n" + "# Run `make helm-template` to regenerate.\n" + "\n" + ) + with open(os.path.join(out_dir, src), "w") as out: + out.write(header) + out.write("\n---\n".join(docs)) + out.write("\n") +PY + +if [ "${CHECK_MODE}" = "true" ]; then + # Only compare top-level files; subdirs like generated/ and kind/ are not + # produced by the chart and live alongside it intentionally. + CHECK_TMP="$(mktemp -d)" + trap 'rm -rf "$TMP_DIR" "$CHECK_TMP"' EXIT + mkdir -p "${CHECK_TMP}/current" + find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' -exec cp {} "${CHECK_TMP}/current/" \; + for file in "${PRESERVED_FILES[@]}"; do + rm -f "${CHECK_TMP}/current/${file}" "${TMP_DIR}/out/${file}" + done + if ! diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" >/dev/null 2>&1; then + echo "manifests/ate-install/ is out of date. Run: make helm-template" >&2 + diff -ruN "${CHECK_TMP}/current" "${TMP_DIR}/out" | head -60 >&2 || true + exit 1 + fi + echo "manifests/ate-install/ matches chart output." + exit 0 +fi + +# Replace contents (preserve kind/ and generated/ subdirs which are not chart output). +mkdir -p "${OUT_DIR}" +find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' \ + ! -name 'ate-api-server.yaml' \ + ! -name 'ate-controller.yaml' \ + ! -name 'ate-otel-config.yaml' \ + ! -name 'ate-system-namespace.yaml' \ + ! -name 'atelet.yaml' \ + ! -name 'atenet-egress.yaml' \ + ! -name 'atenet-egress-with-sdsmint.yaml' \ + ! -name 'atenet-router.yaml' \ + ! -name 'atenet-router-monitoring.yaml' \ + ! -name 'pod-certificate-controller.yaml' \ + ! -name 'postgres.yaml' \ + ! -name 'sandboxconfig-gvisor.yaml' \ + ! -name 'sandboxconfig-validation.yaml' \ + -delete +for file in "${PRESERVED_FILES[@]}"; do + rm -f "${TMP_DIR}/out/${file}" +done +cp "${TMP_DIR}/out/"*.yaml "${OUT_DIR}/" +rendered_count="$(find "${OUT_DIR}" -maxdepth 1 -type f -name '*.yaml' | wc -l | xargs)" +echo "Rendered ${rendered_count} manifest files into ${OUT_DIR}" diff --git a/hack/verify/crd-chart.sh b/hack/verify/crd-chart.sh new file mode 100755 index 0000000000..dc3ef2bdaf --- /dev/null +++ b/hack/verify/crd-chart.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit -o nounset -o pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "${ROOT}" + +GENERATED_DIR="manifests/ate-install/generated" +CHART_TEMPLATES_DIR="charts/substrate-crds/templates" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +mkdir -p "${TMP_DIR}/generated" "${TMP_DIR}/chart" +cp "${GENERATED_DIR}/"ate.dev_*.yaml "${TMP_DIR}/generated/" +cp "${CHART_TEMPLATES_DIR}/"ate.dev_*.yaml "${TMP_DIR}/chart/" + +# The generated CRDs start with a leading document separator after the +# boilerplate header. In chart templates that separator renders as a +# comment-only YAML document, so the chart copies intentionally omit it. +for file in "${TMP_DIR}/generated/"*.yaml; do + awk 'BEGIN { removed = 0 } /^---$/ && removed == 0 { removed = 1; next } { print }' "${file}" > "${file}.tmp" + mv "${file}.tmp" "${file}" +done + +if ! diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" >/dev/null 2>&1; then + echo "charts/substrate-crds/templates is out of sync with ${GENERATED_DIR}" >&2 + echo "Copy updated CRDs into charts/substrate-crds/templates." >&2 + diff -ruN "${TMP_DIR}/generated" "${TMP_DIR}/chart" | head -80 >&2 || true + exit 1 +fi + +echo "charts/substrate-crds/templates matches generated CRDs." diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml new file mode 100644 index 0000000000..5199ab9e74 --- /dev/null +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: ate-api-server-envvars + namespace: ate-system +data: + ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" diff --git a/manifests/ate-install/ate-client.yaml b/manifests/ate-install/ate-client.yaml new file mode 100644 index 0000000000..e59bd53f8f --- /dev/null +++ b/manifests/ate-install/ate-client.yaml @@ -0,0 +1,24 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: ate-client + namespace: ate-system + labels: + apps: ate-client diff --git a/manifests/ate-install/components/agentgateway/configmap.yaml b/manifests/ate-install/components/agentgateway/configmap.yaml index 7fee9bbc1a..a7011a504f 100644 --- a/manifests/ate-install/components/agentgateway/configmap.yaml +++ b/manifests/ate-install/components/agentgateway/configmap.yaml @@ -58,6 +58,32 @@ data: insecureHost: true routes: + - name: substrate-actors-grpc + gateways: + - http + - https + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + # AgentGateway only uses atunnel's CONNECT listener. + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors gateways: - http @@ -77,6 +103,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 # Terminate client CONNECT before internal HTTP routing. binds: @@ -97,6 +126,28 @@ data: listeners: - protocol: HTTP routes: + - name: substrate-actors-tunneled-grpc + matches: + - headers: + - name: content-type + value: + regex: '(?i)^application/grpc(?:\+[^;]+)?(?:;.*)?$' + path: + pathPrefix: / + policies: + substrateIngress: + host: api.ate-system.svc:443 + connectTargetPort: 8443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem + backends: + - backend: /dynamic + policies: + http: + version: HTTP/2.0 - name: substrate-actors-tunneled matches: - path: @@ -112,6 +163,9 @@ data: root: /run/servicedns-ca/trust-bundle.pem backends: - backend: /dynamic + policies: + http: + version: HTTP/1.1 --- apiVersion: v1 diff --git a/manifests/ate-install/role.yaml b/manifests/ate-install/role.yaml new file mode 100644 index 0000000000..65e967c2a6 --- /dev/null +++ b/manifests/ate-install/role.yaml @@ -0,0 +1,130 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ate-controller +rules: +- apiGroups: + - "" + resources: + - pods + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - deployments + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - ate.dev + resources: + - workerpools/finalizers + verbs: + - update +- apiGroups: + - ate.dev + resources: + - workerpools/status + verbs: + - get + - patch + - update +- apiGroups: + - certificates.k8s.io + resources: + - clustertrustbundles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - certificates.k8s.io + resourceNames: + - egress-mitm.ate.dev/* + resources: + - signers + verbs: + - attest +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ate-controller + namespace: ate-system +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +--- +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/manifests/ate-install/rustfs.yaml b/manifests/ate-install/rustfs.yaml new file mode 100644 index 0000000000..d6be308128 --- /dev/null +++ b/manifests/ate-install/rustfs.yaml @@ -0,0 +1,136 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DO NOT EDIT — generated from charts/substrate by hack/render-manifests.sh. +# Run `make helm-template` to regenerate. + +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: rustfs-data + namespace: ate-system +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: v1 +kind: Service +metadata: + name: rustfs + namespace: ate-system +spec: + selector: + app: rustfs + ports: + - name: api + port: 9000 + targetPort: 9000 + - name: console + port: 9001 + targetPort: 9001 + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rustfs + namespace: ate-system +spec: + replicas: 1 + selector: + matchLabels: + app: rustfs + template: + metadata: + labels: + app: rustfs + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: rustfs + image: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9000 + name: api + - containerPort: 9001 + name: console + env: + - name: RUSTFS_ADDRESS + value: ":9000" + - name: RUSTFS_CONSOLE_ADDRESS + value: ":9001" + - name: RUSTFS_CONSOLE_ENABLE + value: "true" + - name: RUSTFS_VOLUMES + value: "/data" + - name: RUSTFS_ACCESS_KEY + value: "rustfsadmin" + - name: RUSTFS_SECRET_KEY + value: "rustfsadmin" + volumeMounts: + - name: data + mountPath: /data + volumes: + - name: data + persistentVolumeClaim: + claimName: rustfs-data +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rustfs-bucket-init + namespace: ate-system +spec: + backoffLimit: 10 + template: + spec: + restartPolicy: OnFailure + containers: + - name: create-bucket + image: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 + env: + - name: AWS_ACCESS_KEY_ID + value: "rustfsadmin" + - name: AWS_SECRET_ACCESS_KEY + value: "rustfsadmin" + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://rustfs.ate-system.svc:9000 + command: + - /bin/sh + - -c + - | + set -e + for i in $(seq 1 60); do + if aws s3api head-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots already exists" + exit 0 + fi + if aws s3api create-bucket --bucket ate-snapshots 2>/dev/null; then + echo "bucket ate-snapshots created" + exit 0 + fi + echo "waiting for rustfs to become available... ($i/60)" + sleep 2 + done + echo "timed out waiting for rustfs" + exit 1 From cd36baf5f7d5a34a4dac35f910816d1a9b78ec0a Mon Sep 17 00:00:00 2001 From: Jeremy Alvis <3587901+iplay88keys@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 07/22] Expose PostgreSQL migration settings in the Helm chart Allow an external PostgreSQL instance and a configurable schema, validate connection settings, and pass the schema to the API server. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 2 ++ charts/substrate/templates/ate-api-server-envvars.yaml | 4 ++++ charts/substrate/templates/ate-api-server.yaml | 1 + charts/substrate/templates/postgres.yaml | 2 ++ charts/substrate/values.yaml | 2 ++ manifests/ate-install/ate-api-server-envvars.yaml | 1 + 6 files changed, 12 insertions(+) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index e7364f4e37..0640c9e9a7 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -35,7 +35,9 @@ See `values.yaml` for the full set; the important keys: | Key | Default | Notes | |-----|---------|-------| +| `postgres.enabled` | `true` | Deploy the bundled PostgreSQL instance | | `postgres.connectionString` | `""` (in-cluster) | Override to use external PostgreSQL | +| `postgres.schema` | `public` | Store the Substrate tables in this PostgreSQL schema | | `postgres.storageSize` | `1Gi` | In-cluster PostgreSQL PVC size | | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | diff --git a/charts/substrate/templates/ate-api-server-envvars.yaml b/charts/substrate/templates/ate-api-server-envvars.yaml index 753c47178b..ca76ae3ef8 100644 --- a/charts/substrate/templates/ate-api-server-envvars.yaml +++ b/charts/substrate/templates/ate-api-server-envvars.yaml @@ -14,6 +14,9 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if and (not .Values.postgres.enabled) (empty .Values.postgres.connectionString) }} +{{- fail "postgres.connectionString is required when postgres.enabled=false" }} +{{- end }} apiVersion: v1 kind: ConfigMap metadata: @@ -21,3 +24,4 @@ metadata: namespace: {{ .Release.Namespace }} data: ATE_API_POSTGRES_CONNECTION_STRING: {{ .Values.postgres.connectionString | default (printf "postgresql://postgres@%s.%s.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" (include "substrate.fullname" (list "postgres" .)) .Release.Namespace) | quote }} + ATE_API_POSTGRES_SCHEMA: {{ .Values.postgres.schema | quote }} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index a5073bc9fa..c236999e48 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -85,6 +85,7 @@ spec: - "--grpc-server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" - "--authentication-config=/etc/ateapi/authentication/authentication.yaml" - "--postgres-connection-string=@env" + - "--postgres-schema=@env" - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml index ce4a4efdd7..26ddb0b1ec 100644 --- a/charts/substrate/templates/postgres.yaml +++ b/charts/substrate/templates/postgres.yaml @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */}} +{{- if .Values.postgres.enabled }} {{- $name := include "substrate.fullname" (list "postgres" .) -}} apiVersion: v1 kind: ConfigMap @@ -232,3 +233,4 @@ spec: resources: requests: storage: {{ .Values.postgres.storageSize }} +{{- end }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b9c7734fe0..ee420bbad9 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -24,8 +24,10 @@ createNamespace: false postgres: + enabled: true storageSize: 1Gi connectionString: "" + schema: public resources: requests: cpu: "1" diff --git a/manifests/ate-install/ate-api-server-envvars.yaml b/manifests/ate-install/ate-api-server-envvars.yaml index 5199ab9e74..b49cff6e1e 100644 --- a/manifests/ate-install/ate-api-server-envvars.yaml +++ b/manifests/ate-install/ate-api-server-envvars.yaml @@ -22,3 +22,4 @@ metadata: namespace: ate-system data: ATE_API_POSTGRES_CONNECTION_STRING: "postgresql://postgres@postgres.ate-system.svc:5432/atepg?sslmode=verify-full&sslrootcert=/run/servicedns.podcert.ate.dev/trust-bundle.pem&sslcert=/run/podidentity.podcert.ate.dev/credential-bundle.pem&sslkey=/run/podidentity.podcert.ate.dev/credential-bundle.pem" + ATE_API_POSTGRES_SCHEMA: "public" From 5eec5bfbd4989c85fc373c57731ad0828d327fed Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 08/22] Configure API server object storage in the Helm chart Wire the API server snapshot backend and S3 settings to the chart storage configuration. Signed-off-by: Eitan Yarmush --- charts/substrate/templates/ate-api-server.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index c236999e48..b6d1960901 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -111,6 +111,20 @@ spec: {{- if .Values.otel.endpoint }} - name: OTEL_EXPORTER_OTLP_ENDPOINT value: {{ .Values.otel.endpoint | quote }} +{{- end }} + - name: ATE_STORAGE_BACKEND + value: {{ .Values.atelet.storageBackend | quote }} +{{- if .Values.rustfs.enabled }} + - name: AWS_REGION + value: us-east-1 + - name: AWS_ENDPOINT_URL + value: http://{{ include "substrate.fullname" (list "rustfs" .) }}.{{ .Release.Namespace }}.svc:9000 + - name: AWS_S3_USE_PATH_STYLE + value: "true" + - name: AWS_ACCESS_KEY_ID + value: {{ .Values.rustfs.accessKey | quote }} + - name: AWS_SECRET_ACCESS_KEY + value: {{ .Values.rustfs.secretKey | quote }} {{- end }} envFrom: - configMapRef: From f32f4472c568061d6ec66e2bdf65a4fdd11de32a Mon Sep 17 00:00:00 2001 From: Krisztian F <103492698+krisztianfekete@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:01:27 +0000 Subject: [PATCH 09/22] Configure per-signal OTLP export and agentgateway access logs Configure trace, metric, and log endpoints independently, expose trace sampling, and route agentgateway access logs through the collector logs pipeline. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 9 +++- charts/substrate/templates/_helpers.tpl | 54 +++++++++++++++++++ .../substrate/templates/ate-api-server.yaml | 5 +- .../substrate/templates/ate-controller.yaml | 5 +- charts/substrate/templates/atelet.yaml | 5 +- charts/substrate/templates/atenet-router.yaml | 39 ++++++++++---- charts/substrate/values.yaml | 18 +++++++ .../ate-install/kind/otel-collector.yaml | 1 - 8 files changed, 116 insertions(+), 20 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 0640c9e9a7..516bc2b04e 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,4 +42,11 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | -| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces/metrics | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | +| `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | +| `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | +| `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router | +| `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet | +| `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | +| `otel.logs.enabled` | `true` | Set to `false` to export no logs; the router access log is the only OTLP log source today | +| `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 32ae087336..1179dda559 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -78,6 +78,60 @@ Plaintext HTTP URL that clients use to reach atenet-router. {{- printf "http://%s.%s.svc:80" (include "substrate.fullname" (list "atenet-router" .)) .Release.Namespace -}} {{- end -}} +{{/* +OTLP endpoint a signal exports to, or empty when the signal is disabled or no +endpoint resolves. The per-signal endpoint wins over the generic one, matching +the precedence the OpenTelemetry SDK gives OTEL_EXPORTER_OTLP__ENDPOINT +over OTEL_EXPORTER_OTLP_ENDPOINT. + +Usage: + {{ include "substrate.otel.signalEndpoint" (list "traces" .) }} +*/}} +{{- define "substrate.otel.signalEndpoint" -}} +{{- $signal := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $cfg := index $ctx.Values.otel $signal -}} +{{- if $cfg.enabled -}} +{{- $cfg.endpoint | default $ctx.Values.otel.endpoint -}} +{{- end -}} +{{- end -}} + +{{/* +OTEL_* env entries for a Go component, as a list of "- name/value" items. +Empty when nothing under .Values.otel is set, so callers can gate the env +key on the result. + +Usage: + {{- with include "substrate.otel.env" . }} + {{- . | trim | nindent 8 }} + {{- end }} +*/}} +{{- define "substrate.otel.env" -}} +{{- $otel := .Values.otel -}} +{{- if $otel.endpoint }} +- name: OTEL_EXPORTER_OTLP_ENDPOINT + value: {{ $otel.endpoint | quote }} +{{- end }} +{{- range $signal := list "traces" "metrics" "logs" }} +{{- $cfg := index $otel $signal }} +{{- if not $cfg.enabled }} +{{- /* "none" is the SDK's own exporter name for "export nothing"; leaving the + endpoint unset would fall back to the SDK default of localhost:4317. */}} +- name: OTEL_{{ upper $signal }}_EXPORTER + value: none +{{- else if $cfg.endpoint }} +- name: OTEL_EXPORTER_OTLP_{{ upper $signal }}_ENDPOINT + value: {{ $cfg.endpoint | quote }} +{{- end }} +{{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "traces" .) }} +- name: OTEL_TRACES_SAMPLER + value: parentbased_traceidratio +- name: OTEL_TRACES_SAMPLER_ARG + value: {{ $otel.traces.samplingRatio | quote }} +{{- end }} +{{- end -}} + {{/* Build an image reference for a substrate component binary. diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index b6d1960901..7718ed3827 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -108,9 +108,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 31c83b9066..d324d05af5 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -79,10 +79,9 @@ spec: - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- if .Values.otel.endpoint }} +{{- with include "substrate.otel.env" . }} env: - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: metrics diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index c026f3577f..e9db195397 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -146,9 +146,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),k8s.node.name=$(NODE_NAME),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} - name: ATE_STORAGE_BACKEND value: {{ .Values.atelet.storageBackend | quote }} diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 1cb84b69d5..46c244b9eb 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -37,12 +37,25 @@ data: backend: poolMaxSize: 0 -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} frontendPolicies: +{{- if $traces }} tracing: - host: $AGENTGATEWAY_OTLP_ADDRESS + host: $AGENTGATEWAY_OTLP_TRACES_ADDRESS protocol: grpc - randomSampling: 0.01 + randomSampling: {{ .Values.otel.traces.samplingRatio }} +{{- end }} +{{- if $logs }} + # Unsampled per-request access log. It carries the ate.* actor + # attribution, which the sampled traces cannot be aggregated on. + # `fields` stays unset so the OTLP sink inherits the full field set. + accessLog: + otlp: + host: $AGENTGATEWAY_OTLP_LOGS_ADDRESS + protocol: grpc +{{- end }} {{- end }} backends: @@ -232,9 +245,8 @@ spec: fieldPath: metadata.uid - name: OTEL_RESOURCE_ATTRIBUTES value: k8s.namespace.name=$(POD_NAMESPACE),k8s.pod.name=$(POD_NAME),k8s.pod.uid=$(POD_UID),service.instance.id=$(POD_UID) -{{- if .Values.otel.endpoint }} - - name: OTEL_EXPORTER_OTLP_ENDPOINT - value: {{ .Values.otel.endpoint | quote }} +{{- with include "substrate.otel.env" . }} +{{- . | trim | nindent 8 }} {{- end }} ports: - name: extproc @@ -251,10 +263,19 @@ spec: args: - "-f" - "/etc/agentgateway/config.yaml" -{{- if .Values.otel.endpoint }} +{{- $traces := include "substrate.otel.signalEndpoint" (list "traces" .) }} +{{- $logs := include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- if or $traces $logs }} + # agentgateway takes host:port, not a URL, so the scheme is trimmed. env: - - name: AGENTGATEWAY_OTLP_ADDRESS - value: {{ trimPrefix "http://" .Values.otel.endpoint | quote }} +{{- if $traces }} + - name: AGENTGATEWAY_OTLP_TRACES_ADDRESS + value: {{ trimPrefix "http://" $traces | quote }} +{{- end }} +{{- if $logs }} + - name: AGENTGATEWAY_OTLP_LOGS_ADDRESS + value: {{ trimPrefix "http://" $logs | quote }} +{{- end }} {{- end }} ports: - name: http diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index ee420bbad9..7f456a20af 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -59,8 +59,26 @@ atelet: # Mounted via envFrom with optional=true. Created by the chart from these values. ateApiServerEnvVarsConfigMap: ate-api-server-envvars +# OTLP export, following the OpenTelemetry SDK environment variable spec. +# endpoint applies to every signal; a signal's own endpoint overrides it. +# A signal exports only when it is enabled and resolves to an endpoint. +# +# traces.enabled and metrics.enabled set OTEL__EXPORTER=none on the +# Go components, which do not read it yet and keep exporting; the setting +# takes effect on the router's agentgateway only. logs.enabled works in full, +# since the agentgateway access log is the only OTLP log source. otel: endpoint: "" + traces: + enabled: true + endpoint: "" + samplingRatio: 0.01 + metrics: + enabled: true + endpoint: "" + logs: + enabled: true + endpoint: "" image: registry: ghcr.io/kagent-dev/substrate diff --git a/manifests/ate-install/kind/otel-collector.yaml b/manifests/ate-install/kind/otel-collector.yaml index e482ab415e..80cbd132b8 100644 --- a/manifests/ate-install/kind/otel-collector.yaml +++ b/manifests/ate-install/kind/otel-collector.yaml @@ -107,7 +107,6 @@ data: receivers: [otlp] processors: [batch] exporters: [prometheus, debug] - # Every ate event, plus any workload that sends OTLP logs. debug is the # only exporter: kind has no log store, and the e2e read-back goes # through the count connector below. Never put a sampler or a filter From 4c4323ba8ac3dc6666f4c05bf33cf8f8b52cb4f2 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 10/22] Make local verification independent of registry and filesystem timing Isolate sandbox asset download tests from pause image pulls, explicitly advance the CA file timestamp, and disable VCS stamping for license checks in temporary verification worktrees. Signed-off-by: Eitan Yarmush --- cmd/atelet/sandbox_prewarm_test.go | 12 ++---------- hack/update/licenses.sh | 8 ++++++++ internal/volume/csi/tls_test.go | 5 +++++ 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/cmd/atelet/sandbox_prewarm_test.go b/cmd/atelet/sandbox_prewarm_test.go index 71030d34d3..3511d3cc27 100644 --- a/cmd/atelet/sandbox_prewarm_test.go +++ b/cmd/atelet/sandbox_prewarm_test.go @@ -358,29 +358,21 @@ func TestSandboxAssetPrewarmDownloads(t *testing.T) { prewarmMaxJitter = 0 t.Cleanup(func() { ateompath.StaticFilesDir, prewarmMaxJitter = origDir, origJitter }) - host := imageVolumeTestRegistry(t) - pauseRef := host + "/pause:3.10" - pushPauseImage(t, pauseRef) - content := []byte("runsc binary bytes") sha := fmt.Sprintf("%x", sha256.Sum256(content)) cfg := gvisorConfig("gvisor-default", "gs://bucket/runsc", sha) - cfg.Spec.PauseImage = pauseRef + cfg.Spec.PauseImage = "" ctx := t.Context() client := fake.NewSimpleClientset(cfg) factory := externalversions.NewSharedInformerFactory(client, 0) informer := factory.Api().V1alpha1().SandboxConfigs().Informer() - store, err := imagecache.New(t.TempDir()) - if err != nil { - t.Fatalf("imagecache.New: %v", err) - } herder := &AteomHerder{anonGCSClient: fakeObjectStorage{data: content}} // Handler first, informer start second, mirroring main: atelet startup // must never wait on this informer's sync, and the initial List replays // the pre-existing config into the handler as an Add. - if err := startSandboxAssetPrewarm(ctx, informer, herder, store, false); err != nil { + if err := startSandboxAssetPrewarm(ctx, informer, herder, nil, false); err != nil { t.Fatalf("startSandboxAssetPrewarm: %v", err) } stopCh := make(chan struct{}) diff --git a/hack/update/licenses.sh b/hack/update/licenses.sh index 99e24d731b..2f2dbfde6a 100755 --- a/hack/update/licenses.sh +++ b/hack/update/licenses.sh @@ -24,6 +24,14 @@ OUTDIR="_LICENSES" # under $ROOT # Ensure the tool is built and up-to-date GO_LICENSES_BIN="$(bash "${ROOT}/hack/run-tool.sh" --print-bin-path go-licenses)" +# go-licenses runs in temporary verification worktrees that do not have enough +# VCS metadata for Go's build stamping. +if [[ -n "${GOFLAGS:-}" ]]; then + export GOFLAGS="${GOFLAGS} -buildvcs=false" +else + export GOFLAGS="-buildvcs=false" +fi + # Clean out previous licenses rm -rf "${OUTDIR}" mkdir -p "${OUTDIR}" diff --git a/internal/volume/csi/tls_test.go b/internal/volume/csi/tls_test.go index ad71177441..aaa8edce05 100644 --- a/internal/volume/csi/tls_test.go +++ b/internal/volume/csi/tls_test.go @@ -391,6 +391,11 @@ func TestCAPoolCache_HitAndFileChange(t *testing.T) { // Modify the file. writeFile(t, caPath, ca.certPEM()) + // Advance mtime explicitly; consecutive writes can share a filesystem tick. + modified := cache.fi.ModTime().Add(time.Second) + if err := os.Chtimes(caPath, modified, modified); err != nil { + t.Fatal(err) + } // 3rd call should detect file change and return a newly parsed pool. pool3, err := cache.getCertPool() From 26929d510b742880ccb16bb2cf2607c0d6e4e39a Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 11/22] Keep the gVisor sandbox alive until application containers are deleted Delete application containers before the pause container so their shared sandbox remains available throughout teardown. Cover the deletion order with a regression test. Signed-off-by: Eitan Yarmush --- cmd/ateom-gvisor/main.go | 4 ++-- cmd/ateom-gvisor/runsc_test.go | 38 ++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/cmd/ateom-gvisor/main.go b/cmd/ateom-gvisor/main.go index a44424f5c2..db2f61e2e3 100644 --- a/cmd/ateom-gvisor/main.go +++ b/cmd/ateom-gvisor/main.go @@ -890,8 +890,8 @@ func stopContainers(ctx context.Context, rcmd containerRuntime, containers []*at _ = rcmd.cmdKill(ctx, ctr.GetName(), "SIGKILL") _ = rcmd.cmdWait(ctx, ctr.GetName()) } - _ = rcmd.cmdKill(ctx, ocispec.PauseContainer, "SIGKILL") - _ = rcmd.cmdWait(ctx, ocispec.PauseContainer) + // Keep the sandbox alive for application deletion. cleanupContainers + // force-deletes the pause container after deleting the applications. } func cleanupContainers(ctx context.Context, rcmd containerRuntime, containers []*ateompb.Container) error { diff --git a/cmd/ateom-gvisor/runsc_test.go b/cmd/ateom-gvisor/runsc_test.go index c46373cd70..f8c7aaf9ab 100644 --- a/cmd/ateom-gvisor/runsc_test.go +++ b/cmd/ateom-gvisor/runsc_test.go @@ -17,13 +17,51 @@ package main import ( + "context" + "os" + "path/filepath" "reflect" "testing" "github.com/agent-substrate/substrate/internal/ateompath" "github.com/agent-substrate/substrate/internal/ocispec" + "github.com/agent-substrate/substrate/internal/proto/ateompb" ) +func TestCleanupKeepsSandboxAliveUntilApplicationsDeleted(t *testing.T) { + path := filepath.Join(t.TempDir(), "runsc") + // Model runsc's control socket: deleting an application fails once the + // sandbox has stopped, even when deletion is forced. + script := `#!/bin/sh +set -eu +shift 5 +case "$1" in +kill) + if [ "$2" = _pause ]; then touch "$0.stopped"; fi + ;; +delete) + if [ "$3" = _pause ]; then + touch "$0.stopped" + elif [ -e "$0.stopped" ]; then + exit 128 + fi + ;; +esac +` + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + r := &runsc{path: path, actorUID: "test-actor"} + containers := []*ateompb.Container{{Name: "counter"}} + stopContainers(context.Background(), r, containers) + if err := cleanupContainers(context.Background(), r, containers); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path + ".stopped"); err != nil { + t.Fatalf("sandbox was not stopped: %v", err) + } +} + func TestKillArgs(t *testing.T) { r := &runsc{ path: "/usr/bin/runsc", From 55447b0ccd75e6bcfba72e3c95c74c634c4feddb Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 12:01:28 +0000 Subject: [PATCH 12/22] Add fork synchronization skill with temporary asset cleanup Rebuild the fork on upstream while preserving features, require agentgateway runtime validation, and use a guarded push. Delete task-owned clusters and disposable assets before finishing while preserving shared resources and recovery data. Signed-off-by: Eitan Yarmush --- .agents/skills/update-against-main/SKILL.md | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .agents/skills/update-against-main/SKILL.md diff --git a/.agents/skills/update-against-main/SKILL.md b/.agents/skills/update-against-main/SKILL.md new file mode 100644 index 0000000000..ab86694708 --- /dev/null +++ b/.agents/skills/update-against-main/SKILL.md @@ -0,0 +1,35 @@ +--- +name: update-against-main +description: Merge agent-substrate/substrate main into the kagent-dev/substrate fork's main branch, resolve conflicts, validate the result, and safely update the fork. Use only when explicitly synchronizing the fork's main branch with upstream main. Do not use for updating, rebasing, or resolving conflicts in feature branches or pull requests. +--- + +# Update Against Main + +This skill applies only to synchronizing the fork's `main` branch. Do not invoke it for a feature branch or PR merely because that branch is behind or conflicts with `main`. + +1. Confirm the worktree, current branch, tracking branch, and remotes. Do not disturb unrelated changes. +2. Fetch `origin/main` and `upstream/main`, inspect their divergence, and create a dated backup branch from `origin/main`. +3. Rebuild `main` from `upstream/main` by replaying only intentional fork feature commits in dependency order. Drop merge commits and fork commits superseded by upstream. +4. Resolve conflicts in favor of current upstream APIs while preserving the remaining fork features. Inspect the resulting diff and linear history. +5. Keep Helm charts synchronized with their corresponding manifests. When either changes, inspect and update the other while preserving intentional Helm templating and conditionals, then run `make verify-helm-template` and `make verify-crd-chart` and compare any relevant resources not covered by those checks. +6. Run `make test` and `make verify`. +7. Run the real Kind E2E matrix from `.github/workflows/pr-workflow.yaml`, but use agentgateway for all fork testing: + - Use a dedicated cluster name and kubeconfig; record the temporary assets created by this run. Before recreating with `hack/create-kind-cluster.sh`, delete any old cluster owned by this sync using `hack/kind.sh delete cluster --name "$cluster_name"` with its dedicated `KUBECONFIG`. + - Install the control plane with `hack/install-ate-kind.sh --deploy-ate-system --atenet-dataplane=agentgateway`. + - Deploy the micro-VM demo with `hack/run-microvm-demo-kind.sh --skip-control-plane` so it does not reinstall the control plane. + - Deploy the gVisor counter demo and both standard egress demos. + - The full gVisor suite: `hack/run-e2e-kind.sh -v -args --no-color` + - The full micro-VM suite with the CI environment: `E2E_SANDBOX_CLASS=microvm hack/run-e2e-kind.sh -v -args --no-color` + - Switch egress to agentgateway sdsmint, then run the MITM trust and targeted networking lanes for both runtimes exactly as the workflow specifies. + - Verify the live router and egress workloads use agentgateway. Never use Envoy for fork validation. +8. Treat `go test ./internal/e2e/...` without `-args --e2e` as compilation/package testing, not E2E coverage. +9. Do not push when unit, verification, or E2E checks fail or cannot run. Report the exact blocker instead. +10. After all checks pass, verify the worktree and rewritten commits, then update the fork with `git push --force-with-lease origin main`. Never use an unguarded force push. +11. Clean up temporary assets before finishing, including on failure or cancellation: + - Stop this run's test/install processes and port-forwards. Save any diagnostics needed to explain failures before tearing down workloads. + - Delete the task-owned Kind cluster with `hack/kind.sh delete cluster --name "$cluster_name"` using its dedicated `KUBECONFIG`. Verify both the cluster and its node containers are gone before removing the kubeconfig. + - Remove this run's disposable assets: generated micro-VM disks and images, downloaded bundles, build outputs, scratch scripts, and temporary kubeconfigs. Remove task-only Docker images, containers, and volumes once no longer in use. Preserve shared assets, caches, registries, and unrelated clusters; do not use global Docker prune commands. + - After a successful push, remove clean temporary worktrees with `git worktree remove` from another checkout. Preserve backup branches, unpushed commits, uncommitted changes, and diagnostics needed for unresolved failures. + - If teardown stalls (for example, Docker reports no exit event), inspect only the task's node containers, retry scoped deletion once, and report any remaining resources and exact blocker. Do not restart the global Docker daemon or kill unrelated processes. Report cleanup separately from validation so leftover assets are not hidden by passing tests. + +Use the current CI workflow as the source of truth for cluster setup, images, demos, runtime coverage, and environment variables, with the agentgateway-only override above. Never claim E2E passed unless workloads ran against the cluster. From e1108b404f9b285ef0be671e0ee5eefab3f9102b Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 10:22:49 -0400 Subject: [PATCH 13/22] Align Helm E2E with the canonical installation Select agentgateway expectations in the Helm test job, align the chart sandbox assets with the canonical manifest, and enable the CONNECT tunnel logging used by egress validation. This retains upstream gVisor checkpoint and restore fixes and closes configuration gaps between Helm and manifest installations. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 1 + charts/substrate/templates/atenet-egress.yaml | 3 +++ charts/substrate/templates/sandboxconfig-gvisor.yaml | 8 ++++---- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 81b3a46103..7820c668a5 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -24,6 +24,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: helm-e2e + E2E_ATENET_DATAPLANE: agentgateway steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 850120b65d..41154a98e3 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -110,6 +110,9 @@ spec: args: - -f - /etc/agentgateway/config.yaml + env: + - name: RUST_LOG + value: debug ports: - name: https containerPort: 8443 diff --git a/charts/substrate/templates/sandboxconfig-gvisor.yaml b/charts/substrate/templates/sandboxconfig-gvisor.yaml index 05f851d40d..286584eb01 100644 --- a/charts/substrate/templates/sandboxconfig-gvisor.yaml +++ b/charts/substrate/templates/sandboxconfig-gvisor.yaml @@ -27,9 +27,9 @@ spec: assets: amd64: gvisor: - url: "gs://gvisor/releases/release/20260803/x86_64/gvisor.tar.bz2" - sha256: "9e7a5fcc2cbd28c9cd4af910a9327abcf07a8efcce242c285b860d79010c2db5" + url: "gs://gvisor/releases/nightly/2026-09-02/x86_64/gvisor.tar.zstd" + sha256: "d547d81401461fd1c679c5c4fa0a6c2b8ef7dc3c22ce23c9e25dcc4c69cfd06f" arm64: gvisor: - url: "gs://gvisor/releases/release/20260803/aarch64/gvisor.tar.bz2" - sha256: "294d54dea2a18bcd2614a4b5072d6f32f0e8938f9e6e71c9e86b843c4a7b707b" + url: "gs://gvisor/releases/nightly/2026-09-02/aarch64/gvisor.tar.zstd" + sha256: "a64916f9813ce7e4841a30480a599337f7dda07b421c6bf0123db2212aa7d1df" From 1699434e934aa57ca6865cafe13fe1b3a208ad64 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 11:17:53 -0400 Subject: [PATCH 14/22] Allow extra ateapi arguments in the Helm chart Expose ateApi.extraArgs so installations can configure API flags such as the template resync interval without editing the deployment template. Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 1 + charts/substrate/templates/ate-api-server.yaml | 3 +++ charts/substrate/values.yaml | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 516bc2b04e..8a78d396fc 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -42,6 +42,7 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | | `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 7718ed3827..267232eb8d 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -93,6 +93,9 @@ spec: - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - "--drain-delay=13s" - "--drain-timeout=15s" +{{- with .Values.ateApi.extraArgs }} +{{ toYaml . | indent 8 }} +{{- end }} env: - name: POD_NAME valueFrom: diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index 7f456a20af..b410d2b65f 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -54,6 +54,10 @@ atelet: extraArgs: [] extraEnv: [] +# Additional arguments appended to the ateapi defaults. +ateApi: + extraArgs: [] + # Name of a ConfigMap in the release namespace that supplies per-environment # overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). # Mounted via envFrom with optional=true. Created by the chart from these values. From af98b9abb18ac9972b232078224e6f6c27ec59ec Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 16 Sep 2026 15:50:31 +0000 Subject: [PATCH 15/22] Retry layer pulls that join an eviction flight A layer pull can share a singleflight call with retirement and return without unpacking the removed layer. Distinguish pull results from retirement results and retry after retirement completes. Cover the interleaving with a deterministic concurrency test. Signed-off-by: Eitan Yarmush --- internal/imagecache/imagecache.go | 35 +++++++++++++++------------ internal/imagecache/retire_test.go | 38 ++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index e0a0302347..9a7d57a99c 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -652,24 +652,29 @@ func (s *Store) pull(ctx context.Context, parsedRef name.Reference, digest v1.Ha // collapsing concurrent requests for the same layer across images. func (s *Store) ensureLayer(ctx context.Context, diffID v1.Hash, layer v1.Layer) (string, error) { dir := s.layerDir(diffID) - _, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { - if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { - // Refresh the dir mtime inside the flight: retireLayer re-checks - // the mtime in this same flight, so a layer reused here can - // never be renamed away between this stat and the image record - // that will re-reference it. - now := time.Now() - if err := os.Chtimes(dir, now, now); err != nil { - slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + for { + result, err, _ := s.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + if _, err := os.Stat(filepath.Join(dir, layerFSDirName)); err == nil { + // Refresh the dir mtime inside the flight: retireLayer re-checks + // the mtime in this same flight, so a layer reused here can + // never be renamed away between this stat and the image record + // that will re-reference it. + now := time.Now() + if err := os.Chtimes(dir, now, now); err != nil { + slog.WarnContext(ctx, "Failed to refresh layer mtime on reuse", slog.String("diffid", diffID.String()), slog.Any("err", err)) + } + return dir, nil } - return nil, nil + return dir, s.unpackLayerToPool(ctx, diffID, layer) + }) + if err != nil { + return "", err + } + // A joined retirement returns nil, so the layer still needs a pull. + if result != nil { + return dir, nil } - return nil, s.unpackLayerToPool(ctx, diffID, layer) - }) - if err != nil { - return "", err } - return dir, nil } // unpackLayerToPool streams the layer (download → decompress → untar) into a diff --git a/internal/imagecache/retire_test.go b/internal/imagecache/retire_test.go index fe45c5a3cc..2abf3efa28 100644 --- a/internal/imagecache/retire_test.go +++ b/internal/imagecache/retire_test.go @@ -22,6 +22,7 @@ import ( "strings" "sync" "testing" + "testing/synctest" "time" v1 "github.com/google/go-containerregistry/pkg/v1" @@ -201,3 +202,40 @@ func TestRetireLayerVsEnsureImageRace(t *testing.T) { t.Errorf("final layer dir missing: %v", err) } } + +func TestEnsureLayerJoinsRetirement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + store := newTestStore(t) + layer := layerFromEntries(t, []tarEntry{ + {name: "f", typeflag: tar.TypeReg, mode: 0o644, body: "hi"}, + }) + diffID, err := layer.DiffID() + if err != nil { + t.Fatal(err) + } + + // Hold a retirement flight while the pull joins it. + release := make(chan struct{}) + go func() { + _, _, _ = store.layerSF.Do(layerFlightKey(diffID.Hex), func() (any, error) { + <-release + return nil, nil + }) + }() + synctest.Wait() + var dir string + go func() { + dir, err = store.ensureLayer(context.Background(), diffID, layer) + }() + synctest.Wait() + close(release) + synctest.Wait() + + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(dir, layerFSDirName, "f")); err != nil { + t.Fatalf("layer was not unpacked after retirement: %v", err) + } + }) +} From eeafce203d50daac0ea7c9ea86214890bc59411c Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 10:58:09 +0000 Subject: [PATCH 16/22] Align Helm egress readiness with the metrics endpoint The egress ext_proc server listens on loopback. Probe the metrics readiness endpoint so Kubernetes can observe readiness through the pod IP, matching the upstream manifests. Signed-off-by: Eitan Yarmush --- charts/substrate/templates/atenet-egress.yaml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 41154a98e3..74f377b4a1 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -171,10 +171,15 @@ spec: ports: - name: extproc containerPort: 50051 + - name: metrics + containerPort: 9090 readinessProbe: - tcpSocket: - port: extproc - periodSeconds: 10 + httpGet: + path: /readyz + port: 9090 + initialDelaySeconds: 5 + periodSeconds: 2 + failureThreshold: 3 volumeMounts: - name: servicedns mountPath: /run/servicedns.podcert.ate.dev From cd66bb40532de7a9695f553d252aa4cd4eac741c Mon Sep 17 00:00:00 2001 From: Jonathan Jamroga Date: Thu, 17 Sep 2026 15:12:42 -0400 Subject: [PATCH 17/22] feat(helm): global image values, and the registry/repository split global.imageRegistry redirects every image at once for air-gapped mirrors. Component images now resolve through the same registry/repository split every kagent-family chart uses: image.registry was one string carrying its path (ghcr.io/kagent-dev/substrate) and is now the registry host only, joined onto image.repository, so one global value (which overrides image.registry) redirects the whole family. This is a breaking change for values files that put a full prefix in image.registry: rendered silently they would produce a doubled prefix failing only at pod start, so the render fails instead, naming the split. A default render is byte-identical to main. Single-string images.* references (postgres, rustfs, aws-cli, agentgateway) have their registry segment replaced by the containerd rule (first path segment with a dot or colon), preserving repository paths either way. global.imagePullSecrets merges (union) into every pod spec, which previously had no pull-secret surface at all. global.imagePullPolicy replaces the hardcoded IfNotPresent values as a fallback, via substrate.imagePullPolicy. Verified: a default render is byte-identical to main; the mirror knob redirects all 9 images with paths preserved; the old-shape registry fails loudly at template time; pull secrets land on all 9 pod specs; the pullPolicy fallback fires. Signed-off-by: Jonathan Jamroga Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 6 +- .github/workflows/helm-verify.yaml | 35 +++++++++++ charts/substrate/templates/_helpers.tpl | 63 ++++++++++++++++++- .../substrate/templates/ate-api-server.yaml | 1 + .../substrate/templates/ate-controller.yaml | 1 + charts/substrate/templates/atelet.yaml | 1 + charts/substrate/templates/atenet-egress.yaml | 3 +- charts/substrate/templates/atenet-router.yaml | 3 +- .../templates/pod-certificate-controller.yaml | 1 + charts/substrate/templates/postgres.yaml | 5 +- charts/substrate/templates/rustfs.yaml | 8 ++- charts/substrate/values.yaml | 27 +++++++- hack/render-manifests.sh | 3 +- 13 files changed, 146 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/helm-verify.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 7820c668a5..4b000ececf 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -57,8 +57,12 @@ jobs: kubectl apply -f manifests/ate-install/kind/prometheus.yaml - name: Build chart images run: | + # Pushed under the image's real path (kagent-dev/substrate/): + # the chart composes {registry}/{repository}/{component}, so the local + # registry serves each image where the default repository expects it -- + # the same path-preserving rule a production mirror follows. for component in ateapi atecontroller atelet podcertcontroller atenet; do - KO_DOCKER_REPO="localhost:5001/${component}" \ + KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" done diff --git a/.github/workflows/helm-verify.yaml b/.github/workflows/helm-verify.yaml new file mode 100644 index 0000000000..8eca1f5de3 --- /dev/null +++ b/.github/workflows/helm-verify.yaml @@ -0,0 +1,35 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Enforces the chart checks that were previously developer-run only: +# verify-helm-template's own comment says "Used in CI", but no workflow ran +# it -- so a chart change that broke hack/render-manifests.sh shipped a green +# PR and was caught in review by hand. +name: helm-verify +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + helm-verify: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4 + - name: Lint the chart + run: helm lint charts/substrate charts/substrate-crds + - name: Verify committed manifests match the chart output + run: make verify-helm-template diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 1179dda559..3b81dc9c0b 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -149,7 +149,20 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time. {{- define "substrate.componentImage" -}} {{- $name := index . 0 -}} {{- $ctx := index . 1 -}} -{{- $registry := $ctx.Values.image.registry -}} +{{/* image.registry used to carry the full prefix (ghcr.io/kagent-dev/substrate). + It is now the registry host only, joined onto image.repository -- the same + registry/repository split every kagent-family chart uses, so one + global.imageRegistry value redirects them all. A values file still carrying + a path in registry would render a doubled prefix that fails only at pod + start, so it fails the render here instead and names the split. */}} +{{- /* A scheme'd registry (ko://...) is hack/render-manifests.sh passing an + importpath prefix for `ko resolve` to substitute, same as the "" tag + sentinel below -- unambiguously not the old host+path shape, so the guard + lets it through. */ -}} +{{- if and (contains "/" $ctx.Values.image.registry) (not (contains "://" $ctx.Values.image.registry)) -}} +{{- fail (printf "image.registry (%q) carries a path. It is now the registry host only: keep the path in image.repository, e.g. registry: ghcr.io, repository: kagent-dev/substrate." $ctx.Values.image.registry) -}} +{{- end -}} +{{- $registry := printf "%s/%s" (default $ctx.Values.image.registry (($ctx.Values.global).imageRegistry)) $ctx.Values.image.repository -}} {{- $tag := $ctx.Values.image.tag | default $ctx.Chart.AppVersion -}} {{- if ne $tag "" -}} {{- printf "%s/%s:%s" $registry $name $tag -}} @@ -157,3 +170,51 @@ are emitted without a tag, letting `ko resolve` supply the digest at build time. {{- printf "%s/%s" $registry $name -}} {{- end -}} {{- end -}} + +{{/* +Rewrite a full image reference ({registry}/{path}:{tag}) onto global.imageRegistry. + +The `images.*` values are single-string references, some digest-pinned, so the +mirror knob has to edit the string. The first path segment is a registry only when +it contains "." or ":" (the containerd rule); otherwise the reference is +docker.io-implied and the mirror is prefixed. The repository path is preserved +either way, so a mirror copies images under their existing paths. + +Usage: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} +*/}} +{{- define "substrate.thirdPartyImage" -}} +{{- $ref := index . 0 -}} +{{- $ctx := index . 1 -}} +{{- $mirror := (($ctx.Values.global).imageRegistry) -}} +{{- if not $mirror -}} +{{- $ref -}} +{{- else -}} +{{- $parts := splitList "/" $ref -}} +{{- $first := first $parts -}} +{{- if and (gt (len $parts) 1) (or (contains "." $first) (contains ":" $first)) -}} +{{- printf "%s/%s" $mirror (join "/" (rest $parts)) -}} +{{- else -}} +{{- printf "%s/%s" $mirror $ref -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +imagePullSecrets for a pod spec: the chart's own list merged (union) with +global.imagePullSecrets. Renders nothing when both are empty. +*/}} +{{- define "substrate.imagePullSecrets" -}} +{{- $merged := concat (.Values.imagePullSecrets | default list) (((.Values.global).imagePullSecrets) | default list) | uniq -}} +{{- if $merged -}} +imagePullSecrets: +{{- toYaml $merged | nindent 0 }} +{{- end -}} +{{- end -}} + +{{/* +imagePullPolicy: global.imagePullPolicy when set, IfNotPresent otherwise. One +definition so the fallback cannot drift between pods. +*/}} +{{- define "substrate.imagePullPolicy" -}} +{{- ((.Values.global).imagePullPolicy) | default "IfNotPresent" -}} +{{- end -}} diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index 267232eb8d..b93dbf4207 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -77,6 +77,7 @@ spec: spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-api-server" .) }} terminationGracePeriodSeconds: 40 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: ate-api-server image: {{ include "substrate.componentImage" (list "ateapi" .) }} diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index d324d05af5..9e8588ceb2 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -67,6 +67,7 @@ spec: app: ate-controller spec: serviceAccountName: {{ include "substrate.fullname" (list "ate-controller" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: ate-controller image: {{ include "substrate.componentImage" (list "atecontroller" .) }} diff --git a/charts/substrate/templates/atelet.yaml b/charts/substrate/templates/atelet.yaml index e9db195397..5e89b2f2c1 100644 --- a/charts/substrate/templates/atelet.yaml +++ b/charts/substrate/templates/atelet.yaml @@ -110,6 +110,7 @@ spec: spec: serviceAccountName: {{ include "substrate.fullname" (list "atelet" .) }} priorityClassName: {{ include "substrate.fullname" (list "ate-node-critical" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: atelet image: {{ include "substrate.componentImage" (list "atelet" .) }} diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 74f377b4a1..2e78861851 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -104,9 +104,10 @@ spec: - name: net.ipv4.ip_unprivileged_port_start value: "0" terminationGracePeriodSeconds: 60 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: agentgateway - image: {{ .Values.images.agentgateway }} + image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }} args: - -f - /etc/agentgateway/config.yaml diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 46c244b9eb..8d14854925 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -212,6 +212,7 @@ spec: prometheus.io/port: "15020" spec: serviceAccountName: {{ include "substrate.fullname" (list "atenet-router" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: atenet-router image: {{ include "substrate.componentImage" (list "atenet" .) }} @@ -259,7 +260,7 @@ spec: - { name: servicedns-ca, mountPath: /run/servicedns-ca, readOnly: true } - { name: podidentity, mountPath: /run/podidentity.podcert.ate.dev, readOnly: true } - name: agentgateway - image: {{ .Values.images.agentgateway }} + image: {{ include "substrate.thirdPartyImage" (list .Values.images.agentgateway .) }} args: - "-f" - "/etc/agentgateway/config.yaml" diff --git a/charts/substrate/templates/pod-certificate-controller.yaml b/charts/substrate/templates/pod-certificate-controller.yaml index 86fc23b4a9..b3fc65ee21 100644 --- a/charts/substrate/templates/pod-certificate-controller.yaml +++ b/charts/substrate/templates/pod-certificate-controller.yaml @@ -139,6 +139,7 @@ spec: labels: app: podcertificate-controller spec: + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: controller image: {{ include "substrate.componentImage" (list "podcertcontroller" .) }} diff --git a/charts/substrate/templates/postgres.yaml b/charts/substrate/templates/postgres.yaml index 26ddb0b1ec..77df191879 100644 --- a/charts/substrate/templates/postgres.yaml +++ b/charts/substrate/templates/postgres.yaml @@ -124,7 +124,7 @@ spec: initContainers: - name: tls-reloader restartPolicy: Always - image: {{ .Values.images.postgres }} + image: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} securityContext: runAsUser: 70 command: @@ -145,9 +145,10 @@ spec: requests: cpu: 10m memory: 32Mi + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: postgres - image: {{ .Values.images.postgres }} + image: {{ include "substrate.thirdPartyImage" (list .Values.images.postgres .) }} lifecycle: postStart: exec: diff --git a/charts/substrate/templates/rustfs.yaml b/charts/substrate/templates/rustfs.yaml index edaad3cfa8..14854822af 100644 --- a/charts/substrate/templates/rustfs.yaml +++ b/charts/substrate/templates/rustfs.yaml @@ -63,10 +63,11 @@ spec: runAsUser: 10001 runAsGroup: 10001 fsGroup: 10001 + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: rustfs - image: {{ .Values.images.rustfs }} - imagePullPolicy: IfNotPresent + image: {{ include "substrate.thirdPartyImage" (list .Values.images.rustfs .) }} + imagePullPolicy: {{ include "substrate.imagePullPolicy" . }} ports: - containerPort: 9000 name: api @@ -103,9 +104,10 @@ spec: template: spec: restartPolicy: OnFailure + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} containers: - name: create-bucket - image: {{ .Values.images.awsCli }} + image: {{ include "substrate.thirdPartyImage" (list .Values.images.awsCli .) }} env: - name: AWS_ACCESS_KEY_ID value: {{ .Values.rustfs.accessKey | quote }} diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index b410d2b65f..f598567119 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -84,8 +84,33 @@ otel: enabled: true endpoint: "" +# Values under `global` are visible to this chart and to every subchart. A parent +# chart or an operator sets one value here instead of one value per chart. +global: + # -- Mirror registry that overrides where every image is pulled from. This is + # the air-gap knob. It wins over image.registry for component images. Each + # `images.*` reference has its registry segment replaced. Repository paths are + # preserved, so a mirror only has to copy images under their existing paths. + # For control without the override, leave this unset and set image.registry. + imageRegistry: "" + # -- Pull secrets merged (union) into each pod's own imagePullSecrets list. + imagePullSecrets: [] + # -- Fallback imagePullPolicy where a container does not set one. + imagePullPolicy: "" + +# Pull secrets for every pod this chart renders. Merged with global.imagePullSecrets. +imagePullSecrets: [] + image: - registry: ghcr.io/kagent-dev/substrate + # Registry host for the component images, and nothing else. To change + # environments, change only this value or global.imageRegistry, which overrides + # it. A path inside `registry` fails the render, and the error names this + # split. + registry: ghcr.io + # Image path prefix under the registry, ahead of each component name. The path + # is identical on every registry that serves the images. A mirror copies the + # images under this same path. + repository: kagent-dev/substrate tag: "" images: diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index 1f6790bb73..2187044970 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -60,7 +60,8 @@ helm template substrate "${CHART_DIR}" \ --namespace ate-system \ --set auth.mode=mtls \ --set createNamespace=true \ - --set image.registry=ko://github.com/agent-substrate/substrate/cmd \ + --set image.registry=ko://github.com \ + --set image.repository=agent-substrate/substrate/cmd \ --set image.tag="" \ > "${TMP_DIR}/all.yaml" From c0f944eca4960b3f961b06af642eb70ec8e021de Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Thu, 17 Sep 2026 16:49:44 -0400 Subject: [PATCH 18/22] Integrate Kubernetes credentials with Helm and agentgateway Retain upstream secret URI syntax and client CA rotation while adding Helm deployment, configurable injector identity, strict default-deny authorization, and credential injection coverage. Signed-off-by: Eitan Yarmush --- .github/workflows/helm-e2e.yaml | 10 +- charts/substrate/README.md | 6 + charts/substrate/templates/atenet-egress.yaml | 51 ++- .../templates/k8s-credential-provider.yaml | 167 ++++++++++ .../templates/sandboxconfig-validation.yaml | 4 +- charts/substrate/values.yaml | 9 +- .../kubernetes-secrets/kubeprovider.go | 28 +- .../kubernetes-secrets/kubeprovider_test.go | 140 +++++++- .../kubernetes-secrets/main.go | 7 + .../kubernetes-secrets/main_test.go | 119 +++++++ .../kubernetes-secrets/manifests_test.go | 309 ++++++++++++++++++ .../kubernetes-secrets/nsauthz.go | 15 +- docs/kubernetes-credential-provider.md | 102 ++++++ go.mod | 2 +- hack/render-manifests.sh | 2 + internal/e2e/fixture.go | 10 + internal/e2e/fixtures/testserver/http.go | 29 +- internal/e2e/fixtures/testserver/http_test.go | 51 +++ .../suites/credentials/credentials_test.go | 147 +++++++++ .../e2e/suites/credentials/testmain_test.go | 24 ++ internal/e2e/suites/credentials/values.yaml | 18 + .../kustomization.yaml | 18 + .../agentgateway/kustomization.yaml | 4 +- .../k8s-credential-provider.yaml | 4 +- .../kustomization.yaml | 25 ++ .../namespace-policy.yaml | 22 +- 26 files changed, 1246 insertions(+), 77 deletions(-) create mode 100644 charts/substrate/templates/k8s-credential-provider.yaml create mode 100644 cmd/credential-provider/kubernetes-secrets/manifests_test.go create mode 100644 docs/kubernetes-credential-provider.md create mode 100644 internal/e2e/fixtures/testserver/http_test.go create mode 100644 internal/e2e/suites/credentials/credentials_test.go create mode 100644 internal/e2e/suites/credentials/testmain_test.go create mode 100644 internal/e2e/suites/credentials/values.yaml create mode 100644 manifests/egress-credential-injection/kustomization.yaml diff --git a/.github/workflows/helm-e2e.yaml b/.github/workflows/helm-e2e.yaml index 4b000ececf..6a7438a89c 100644 --- a/.github/workflows/helm-e2e.yaml +++ b/.github/workflows/helm-e2e.yaml @@ -25,6 +25,8 @@ jobs: env: VERSION: helm-e2e E2E_ATENET_DATAPLANE: agentgateway + E2E_CREDENTIAL_PROVIDER: "1" + E2E_EGRESS_MITM: "1" steps: - name: Checkout uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 @@ -61,8 +63,8 @@ jobs: # the chart composes {registry}/{repository}/{component}, so the local # registry serves each image where the default repository expects it -- # the same path-preserving rule a production mirror follows. - for component in ateapi atecontroller atelet podcertcontroller atenet; do - KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component}" \ + for component in ateapi atecontroller atelet podcertcontroller atenet credential-provider/kubernetes-secrets; do + KO_DOCKER_REPO="localhost:5001/kagent-dev/substrate/${component##*/}" \ ./hack/run-tool.sh ko build --bare --tags helm-e2e \ --platform linux/amd64 "./cmd/${component}" done @@ -72,6 +74,7 @@ jobs: helm upgrade --install substrate charts/substrate \ --namespace ate-system \ --create-namespace \ + -f internal/e2e/suites/credentials/values.yaml \ --set image.registry=localhost:5001 \ --set image.tag=helm-e2e \ --set 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000' \ @@ -84,6 +87,7 @@ jobs: hack/install-ate-kind.sh --create-actor-id-ca-pool-secret hack/install-ate-kind.sh --create-actor-id-ca-certs-secret hack/install-ate-kind.sh --create-api-authentication-config + hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret - name: Wait for Helm install run: | helm upgrade substrate charts/substrate \ @@ -104,7 +108,7 @@ jobs: - name: Deploy gVisor counter demo run: hack/install-ate-kind.sh --deploy-demo-counter - name: Deploy egress demo - run: hack/install-ate-kind.sh --deploy-demo-egress + run: hack/install-ate-kind.sh --deploy-demo-egress-mitm - name: Run E2E tests (gVisor) run: hack/run-e2e-kind.sh -v -args --no-color - name: Run E2E tests (micro-VM) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 8a78d396fc..2803a5aa63 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -19,6 +19,11 @@ By default, component images are pulled from `ghcr.io/kagent-dev/substrate` using the chart `appVersion` as the tag. Override `image.registry` and `image.tag` to install from a different image repository or tag. +The chart installs the Kubernetes credential provider and enables HTTPS egress +interception. Create the `egress-mitm-ca-pool` Secret and configure actor trust +as described in the [credential provider setup](../../docs/kubernetes-credential-provider.md). +Namespace grants default to an empty list, denying credential access. + ## Render manifests without applying ```bash @@ -42,6 +47,7 @@ See `values.yaml` for the full set; the important keys: | `rustfs.enabled` | `true` | Deploy an in-cluster S3-compatible RustFS bucket for snapshots | | `atelet.storageBackend` | `s3` | Default snapshot backend, wired to RustFS when `rustfs.enabled=true` | | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | +| `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | | `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | diff --git a/charts/substrate/templates/atenet-egress.yaml b/charts/substrate/templates/atenet-egress.yaml index 2e78861851..b5cf552264 100644 --- a/charts/substrate/templates/atenet-egress.yaml +++ b/charts/substrate/templates/atenet-egress.yaml @@ -56,12 +56,33 @@ data: - mode: internal protocol: AUTO listeners: - - protocol: TLS - hostname: "*" - tcpRoutes: + - protocol: HTTPS + tls: + mode: dynamicCa + cert: /run/egress-mitm/tls.crt + key: /run/egress-mitm/tls.key + routes: - backends: - - dynamic: - target: source.connectHeaders["host"] + - dynamic: {} + policies: + backendTLS: {} + policies: + substrateEgress: + host: {{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - protocol: HTTP routes: - backends: @@ -75,6 +96,15 @@ data: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns.podcert.ate.dev/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }}.{{ .Release.Namespace }}.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns.podcert.ate.dev/trust-bundle.pem - protocol: TCP tcpRoutes: - backends: @@ -133,6 +163,9 @@ spec: port: readiness periodSeconds: 1 volumeMounts: + - name: egress-mitm + mountPath: /run/egress-mitm + readOnly: true - name: config mountPath: /etc/agentgateway readOnly: true @@ -194,6 +227,14 @@ spec: - name: drain-signal mountPath: /var/run/atenet volumes: + - name: egress-mitm + secret: + secretName: egress-mitm-ca-pool + items: + - key: tls.crt + path: tls.crt + - key: tls.key + path: tls.key - name: config configMap: name: {{ include "substrate.fullname" (list "atenet-egress-agentgateway-config" .) }} diff --git a/charts/substrate/templates/k8s-credential-provider.yaml b/charts/substrate/templates/k8s-credential-provider.yaml new file mode 100644 index 0000000000..1bfc0dade7 --- /dev/null +++ b/charts/substrate/templates/k8s-credential-provider.yaml @@ -0,0 +1,167 @@ +{{/* +Copyright 2026 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# The credential provider: a gRPC service that resolves ate-secret:// URIs +# of the k8s.io class to Kubernetes Secret values. It is the ONLY +# component in the egress credential-injection path with Kubernetes access; the +# egress gateway and the injector never read Secrets. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- +# The provider checks the actor's atespace-to-namespace grant before reading. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "substrate.fullname" (list "k8s-credential-provider-secret-reader" .) }} +subjects: +- kind: ServiceAccount + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} +spec: + replicas: 1 + selector: + matchLabels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + template: + metadata: + annotations: + checksum/namespace-policy: {{ toJson .Values.credentialProvider.namespacePolicies | sha256sum }} + labels: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + spec: + serviceAccountName: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + {{- with include "substrate.imagePullSecrets" . }}{{- . | nindent 6 }}{{- end }} + securityContext: + runAsUser: 65532 + runAsGroup: 65532 + runAsNonRoot: true + containers: + - name: k8s-credential-provider + image: {{ include "substrate.componentImage" (list "kubernetes-secrets" .) }} + imagePullPolicy: {{ include "substrate.imagePullPolicy" . }} + args: + - "--listen-address=:50051" + - "--metrics-address=:9090" + # Use this Service's DNS certificate; only the egress injector may call. + - "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem" + - "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem" + # Enforce the atespace→namespace authorization policy (default-deny). + - "--namespace-policy-file=/etc/k8s-credential-provider/namespace-policy.yaml" + - "--injector-identity=spiffe://cluster.local/ns/{{ .Release.Namespace }}/sa/{{ include "substrate.fullname" (list "atenet-egress" .) }}" + - "--log-level=info" + ports: + - name: grpc + containerPort: 50051 + - name: metrics + containerPort: 9090 + readinessProbe: + httpGet: + path: /readyz + port: metrics + periodSeconds: 10 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumeMounts: + - name: namespace-policy + mountPath: /etc/k8s-credential-provider + readOnly: true + - name: servicedns + mountPath: /run/servicedns.podcert.ate.dev + readOnly: true + - name: podidentity + mountPath: /run/podidentity.podcert.ate.dev + readOnly: true + volumes: + - name: namespace-policy + configMap: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + - name: servicedns + projected: + sources: + - podCertificate: + signerName: servicedns.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: servicedns.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem + - name: podidentity + projected: + sources: + - podCertificate: + signerName: podidentity.podcert.ate.dev/identity + keyType: ECDSAP256 + credentialBundlePath: credential-bundle.pem + - clusterTrustBundle: + signerName: podidentity.podcert.ate.dev/identity + labelSelector: + matchLabels: + podcert.ate.dev/canarying: live + path: trust-bundle.pem +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + namespace: {{ .Release.Namespace }} +spec: + type: ClusterIP + selector: + app: {{ include "substrate.fullname" (list "k8s-credential-provider" .) }} + ports: + - name: grpc + port: 50051 + targetPort: grpc + protocol: TCP +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "substrate.fullname" (list "k8s-credential-provider-namespace-policy" .) }} + namespace: {{ .Release.Namespace }} +data: + namespace-policy.yaml: | + policies: {{ toJson .Values.credentialProvider.namespacePolicies }} diff --git a/charts/substrate/templates/sandboxconfig-validation.yaml b/charts/substrate/templates/sandboxconfig-validation.yaml index f25d43409b..baa8768453 100644 --- a/charts/substrate/templates/sandboxconfig-validation.yaml +++ b/charts/substrate/templates/sandboxconfig-validation.yaml @@ -44,9 +44,9 @@ spec: object.spec.sandboxClass != 'microvm' || (has(object.spec.assets) && size(object.spec.assets) > 0 && object.spec.assets.all(arch, - ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image', 'kata-config'] + ['cloud-hypervisor', 'virtiofsd', 'kata-kernel', 'kata-image'] .all(name, name in object.spec.assets[arch]))) - message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, kata-image, and kata-config assets for every architecture under spec.assets" + message: "a microvm SandboxConfig must define cloud-hypervisor, virtiofsd, kata-kernel, and kata-image assets for every architecture under spec.assets" --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index f598567119..fef293a355 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -58,6 +58,13 @@ atelet: ateApi: extraArgs: [] +# Kubernetes Secret provider and AGW HTTP/HTTPS credential injection. +# Includes get-only Secret RBAC. HTTPS requires egress-mitm-ca-pool and actor trust. +credentialProvider: + namespacePolicies: [] + # - atespace: team-a + # allowedNamespaces: [team-a-secrets] + # Name of a ConfigMap in the release namespace that supplies per-environment # overrides for ate-api-server (ATE_API_POSTGRES_CONNECTION_STRING, ...). # Mounted via envFrom with optional=true. Created by the chart from these values. @@ -117,5 +124,5 @@ images: postgres: postgres:18-alpine@sha256:9a8afca54e7861fd90fab5fdf4c42477a6b1cb7d293595148e674e0a3181de15 rustfs: rustfs/rustfs:1.0.0-beta.3@sha256:378642b05b7dcb4849fb77ebe6aca4ced1c3f66e7e504247df95a5c9018d3358 awsCli: amazon/aws-cli:2.17.0@sha256:643507c10ada7964ca6157b3d799f030b90577643da9955d319a77399ed80d73 - agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + agentgateway: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e busybox: busybox:1.36 diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go index ae04b730ca..6fae284d06 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider.go @@ -27,6 +27,7 @@ import ( k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation" "k8s.io/client-go/kubernetes" "google.golang.org/grpc/codes" @@ -62,7 +63,7 @@ type SecretRef struct { Key string } -// ParseURI parses a ate-secret:// URI of the kubernetes.io provider. It +// ParseURI parses an ate-secret:// URI of the k8s.io provider. It // rejects any other scheme or provider name. func ParseURI(raw string) (SecretRef, error) { u, err := url.Parse(raw) @@ -78,15 +79,15 @@ func ParseURI(raw string) (SecretRef, error) { // The grammar is scheme/host/path only; a query or fragment means the caller // assumed a syntax this provider does not honor, so reject it rather than // silently ignore it. - if u.RawQuery != "" || u.ForceQuery || u.Fragment != "" { - return SecretRef{}, fmt.Errorf("credential URI %q: query and fragment components are not allowed", raw) + if u.User != nil || u.RawQuery != "" || u.ForceQuery || strings.Contains(raw, "#") { + return SecretRef{}, fmt.Errorf("credential URI %q: user info, query, and fragment components are not allowed", raw) } // Reject percent-encoding in the path of secret uri. if u.EscapedPath() != u.Path { return SecretRef{}, fmt.Errorf("credential URI %q: path must not contain percent-encoding", raw) } - segments := strings.Split(strings.Trim(u.Path, "/"), "/") + segments := strings.Split(strings.TrimPrefix(u.Path, "/"), "/") for i, s := range segments { if s == "" { return SecretRef{}, fmt.Errorf("credential URI %q: empty path segment %d", raw, i) @@ -104,11 +105,11 @@ func ParseURI(raw string) (SecretRef, error) { if len(tail) != 3 { return SecretRef{}, fmt.Errorf("credential URI %q: want %s///, got %d trailing segments", raw, LocalLocator, len(tail)) } - return SecretRef{ - Namespace: tail[0], - Name: tail[1], - Key: tail[2], - }, nil + ref := SecretRef{Namespace: tail[0], Name: tail[1], Key: tail[2]} + if len(validation.IsDNS1123Label(ref.Namespace)) != 0 || len(validation.IsDNS1123Subdomain(ref.Name)) != 0 || len(validation.IsConfigMapKey(ref.Key)) != 0 { + return SecretRef{}, fmt.Errorf("credential URI contains an invalid namespace, secret name, or key") + } + return ref, nil } // Server implements credproviderpb.CredentialProviderServer over the Kubernetes @@ -118,12 +119,10 @@ type Server struct { client kubernetes.Interface // nsAuth restricts which namespaces an atespace may resolve secrets from. - // Nil disables authorization (dev only): every URI namespace is allowed. nsAuth *NamespaceAuthorizer } -// NewServer builds a Kubernetes-backed credential provider. nsAuth enforces the -// atespace→namespace policy; pass nil to disable authorization (dev only). +// NewServer builds a Kubernetes credential provider with a default-deny policy. func NewServer(client kubernetes.Interface, nsAuth *NamespaceAuthorizer) *Server { return &Server{client: client, nsAuth: nsAuth} } @@ -154,7 +153,7 @@ func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecre if k8serrors.IsForbidden(err) { return nil, status.Errorf(codes.PermissionDenied, "not permitted to read secret %s/%s", ref.Namespace, ref.Name) } - return nil, status.Errorf(codes.Unavailable, "reading secret %s/%s: %v", ref.Namespace, ref.Name, err) + return nil, status.Error(codes.Unavailable, "could not read secret from Kubernetes") } value, err := selectKey(secret.Data, ref.Key) @@ -168,9 +167,6 @@ func (s *Server) FetchSecret(ctx context.Context, req *credproviderpb.FetchSecre // the attested actor SPIFFE ID and denies unless the URI's namespace is in that // atespace's allowed list. func (s *Server) authorize(ctx context.Context, actorSpiffeID, namespace string) error { - if s.nsAuth == nil { - return nil - } actor, err := resources.ActorRefFromSPIFFEID(actorSpiffeID) if err != nil { slog.WarnContext(ctx, "credential request denied: unusable actor identity", slog.Any("err", err)) diff --git a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go index ad8c435627..30976ee4dd 100644 --- a/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go +++ b/cmd/credential-provider/kubernetes-secrets/kubeprovider_test.go @@ -16,11 +16,19 @@ package main import ( "context" + "errors" + "os" + "path/filepath" + "strings" "testing" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -52,6 +60,14 @@ func TestParseURI(t *testing.T) { {name: "percent-encoded separator", uri: "ate-secret://k8s.io/default/ns1/example-api/tok%2Fen", wantErr: true}, {name: "percent-encoding of any kind", uri: "ate-secret://k8s.io/default/ns1/example-api/tok%2Den", wantErr: true}, {name: "space in path", uri: "ate-secret://k8s.io/default/ns1/example-api/tok en", wantErr: true}, + + {name: "user info", uri: "ate-secret://user@k8s.io/default/ns1/api/token", wantErr: true}, + {name: "empty query", uri: "ate-secret://k8s.io/default/ns1/api/token?", wantErr: true}, + {name: "empty fragment", uri: "ate-secret://k8s.io/default/ns1/api/token#", wantErr: true}, + {name: "trailing slash", uri: "ate-secret://k8s.io/default/ns1/api/token/", wantErr: true}, + {name: "empty namespace", uri: "ate-secret://k8s.io/default//api/token", wantErr: true}, + {name: "invalid namespace", uri: "ate-secret://k8s.io/default/NS/api/token", wantErr: true}, + {name: "path traversal", uri: "ate-secret://k8s.io/default/ns1/../token", wantErr: true}, {name: "unparseable", uri: "://://", wantErr: true}, } for _, tc := range tests { @@ -163,9 +179,13 @@ func TestFetchSecretAuthorization(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - srv := NewServer(fake.NewSimpleClientset(secret), authz) + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, authz) resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: tc.actorSpiffeID}) if tc.wantCode != codes.OK { + if len(client.Actions()) != 0 { + t.Fatal("denied request reached Kubernetes") + } if status.Code(err) != tc.wantCode { t.Fatalf("code = %v, want %v (err=%v)", status.Code(err), tc.wantCode, err) } @@ -180,14 +200,14 @@ func TestFetchSecretAuthorization(t *testing.T) { }) } - // With no authorizer configured, enforcement is bypassed entirely. - t.Run("nil authorizer bypasses", func(t *testing.T) { + // A missing authorizer must fail closed. + t.Run("nil authorizer denies", func(t *testing.T) { srv := NewServer(fake.NewSimpleClientset(secret), nil) if _, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{ Uri: "ate-secret://k8s.io/default/ns1/example-api/token", - ActorSpiffeId: "not-a-spiffe-uri", - }); err != nil { - t.Fatalf("nil authorizer should not enforce, got %v", err) + ActorSpiffeId: teamAURI, + }); status.Code(err) != codes.PermissionDenied { + t.Fatalf("nil authorizer should deny, got %v", err) } }) } @@ -199,6 +219,14 @@ func TestFetchSecret(t *testing.T) { "token": []byte("s3cr3t"), }, } + multiKey := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "multi", Namespace: "ns1"}, + Data: map[string][]byte{ + "a": []byte("aa"), + "b": []byte("bb"), + }, + } + tests := []struct { name string uri string @@ -210,6 +238,16 @@ func TestFetchSecret(t *testing.T) { uri: "ate-secret://k8s.io/default/ns1/example-api/token", want: "s3cr3t", }, + { + name: "key required", + uri: "ate-secret://k8s.io/default/ns1/example-api", + wantCode: codes.InvalidArgument, + }, + { + name: "no key, multiple keys", + uri: "ate-secret://k8s.io/default/ns1/multi", + wantCode: codes.InvalidArgument, + }, { name: "missing key", uri: "ate-secret://k8s.io/default/ns1/example-api/nope", @@ -220,22 +258,17 @@ func TestFetchSecret(t *testing.T) { uri: "ate-secret://k8s.io/default/ns1/absent/token", wantCode: codes.NotFound, }, - { - name: "remote form rejected", - uri: "ate-secret://k8s.io/cluster/remote-east/ns1/example-api/token", - wantCode: codes.InvalidArgument, - }, { name: "bad uri", - uri: "ate-secret://vault.io/default/ns1/example-api", + uri: "ate-secret://vault.io/ns1/example-api", wantCode: codes.InvalidArgument, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - client := fake.NewSimpleClientset(secret) - srv := NewServer(client, nil) - resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri}) + client := fake.NewSimpleClientset(secret, multiKey) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + resp, err := srv.FetchSecret(context.Background(), &credproviderpb.FetchSecretRequest{Uri: tc.uri, ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/my-actor"}) if tc.wantCode != codes.OK { if status.Code(err) != tc.wantCode { t.Fatalf("FetchSecret(%q) code = %v, want %v (err=%v)", tc.uri, status.Code(err), tc.wantCode, err) @@ -251,3 +284,80 @@ func TestFetchSecret(t *testing.T) { }) } } + +func TestLoadNamespaceAuthorizer(t *testing.T) { + for _, tc := range []struct { + name, policy string + wantErr bool + }{ + {name: "valid", policy: "policies:\n- atespace: team-a\n allowedNamespaces: [ns1]\n"}, + {name: "empty", policy: "policies: []"}, + {name: "unknown field", policy: "polices: []", wantErr: true}, + {name: "duplicate field", policy: "policies: []\npolicies: []", wantErr: true}, + {name: "missing atespace", policy: "policies: [{allowedNamespaces: [ns1]}]", wantErr: true}, + {name: "invalid namespace", policy: "policies: [{atespace: team-a, allowedNamespaces: ['*']}]", wantErr: true}, + {name: "malformed", policy: "policies: [", wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "policy.yaml") + if err := os.WriteFile(path, []byte(tc.policy), 0600); err != nil { + t.Fatal(err) + } + auth, err := LoadNamespaceAuthorizer(path) + if (err != nil) != tc.wantErr { + t.Fatalf("LoadNamespaceAuthorizer: %v", err) + } + if err == nil && auth.Allowed("team-a", "ns1") != (tc.name == "valid") { + t.Fatal("unexpected namespace grant") + } + }) + } + if _, err := LoadNamespaceAuthorizer(filepath.Join(t.TempDir(), "absent")); err == nil { + t.Fatal("missing policy accepted") + } +} + +func TestFetchSecretKubernetesErrors(t *testing.T) { + for _, tc := range []struct { + name string + err error + code codes.Code + }{ + {"forbidden", k8serrors.NewForbidden(schema.GroupResource{Resource: "secrets"}, "api", errors.New("RBAC")), codes.PermissionDenied}, + {"unavailable", errors.New("upstream response body should stay private"), codes.Unavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + client := fake.NewSimpleClientset() + client.PrependReactor("get", "secrets", func(k8stesting.Action) (bool, runtime.Object, error) { return true, nil, tc.err }) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + _, err := srv.FetchSecret(t.Context(), &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if status.Code(err) != tc.code { + t.Fatalf("FetchSecret: %v, want %v", err, tc.code) + } + if strings.Contains(err.Error(), "stay private") { + t.Fatal("Kubernetes response body exposed") + } + }) + } +} + +func TestFetchSecretObservesRotation(t *testing.T) { + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("first")}} + client := fake.NewSimpleClientset(secret) + srv := NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}}) + req := &credproviderpb.FetchSecretRequest{Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a"} + first, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(first.GetOpaqueBytes()) != "first" { + t.Fatalf("first fetch: %v, %v", first, err) + } + secret.Data["token"] = []byte("rotated") + if _, err := client.CoreV1().Secrets("ns1").Update(t.Context(), secret, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + next, err := srv.FetchSecret(t.Context(), req) + if err != nil || string(next.GetOpaqueBytes()) != "rotated" { + t.Fatalf("fetch after rotation: %v, %v", next, err) + } +} diff --git a/cmd/credential-provider/kubernetes-secrets/main.go b/cmd/credential-provider/kubernetes-secrets/main.go index eb04cfff2a..c03e1eb8d7 100644 --- a/cmd/credential-provider/kubernetes-secrets/main.go +++ b/cmd/credential-provider/kubernetes-secrets/main.go @@ -25,7 +25,9 @@ import ( "fmt" "log/slog" "net" + "net/url" "os/signal" + "strings" "syscall" "time" @@ -179,6 +181,11 @@ func buildServerCreds(ctx context.Context) (credentials.TransportCredentials, er return nil, fmt.Errorf("--client-ca-file is required") } + id, err := url.Parse(*injectorIdentity) + if err != nil || id.Scheme != "spiffe" || id.Host == "" || id.Path == "" || id.User != nil || id.RawQuery != "" || id.ForceQuery || strings.Contains(*injectorIdentity, "#") { + return nil, fmt.Errorf("--injector-identity must be a SPIFFE URI") + } + // Load the client CA pool once so a missing or empty projection fails the // pod promptly; GetConfigForClient below reloads it for every connection. loadPool := credbundle.PoolLoader(*clientCAFile) diff --git a/cmd/credential-provider/kubernetes-secrets/main_test.go b/cmd/credential-provider/kubernetes-secrets/main_test.go index cb444305b2..2b918b6e97 100644 --- a/cmd/credential-provider/kubernetes-secrets/main_test.go +++ b/cmd/credential-provider/kubernetes-secrets/main_test.go @@ -32,7 +32,13 @@ import ( "time" "github.com/agent-substrate/substrate/internal/installdefaults" + "github.com/agent-substrate/substrate/internal/localca" + "github.com/agent-substrate/substrate/pkg/proto/credproviderpb" + "google.golang.org/grpc" "google.golang.org/grpc/credentials" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" ) func certWithURIs(t *testing.T, uris ...string) *x509.Certificate { @@ -295,3 +301,116 @@ func writeFileWithMtime(t *testing.T, path string, data []byte, mtime time.Time) t.Fatalf("chtimes %s: %v", path, err) } } + +func TestProviderMTLS(t *testing.T) { + ca, err := localca.GenerateCA("trusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + untrustedCA, err := localca.GenerateCA("untrusted", localca.KeyTypeECDSAP256, time.Hour) + if err != nil { + t.Fatal(err) + } + servingCert := issueCertificate(t, ca, "") + dir := t.TempDir() + oldBundle, oldCAFile, oldInjector := *serverBundle, *clientCAFile, *injectorIdentity + t.Cleanup(func() { *serverBundle, *clientCAFile, *injectorIdentity = oldBundle, oldCAFile, oldInjector }) + *serverBundle, *clientCAFile = filepath.Join(dir, "server.pem"), filepath.Join(dir, "ca.pem") + *injectorIdentity = "spiffe://cluster.local/ns/custom/sa/release-atenet-egress" + key, err := x509.MarshalPKCS8PrivateKey(servingCert.PrivateKey) + if err != nil { + t.Fatal(err) + } + bundle := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: key}) + bundle = append(bundle, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: servingCert.Certificate[0]})...) + if err := os.WriteFile(*serverBundle, bundle, 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(*clientCAFile, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: ca.RootCertificate.Raw}), 0600); err != nil { + t.Fatal(err) + } + creds, err := buildServerCreds(t.Context()) + if err != nil { + t.Fatal(err) + } + client := fake.NewSimpleClientset(&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "api", Namespace: "ns1"}, Data: map[string][]byte{"token": []byte("credential")}, + }) + srv := grpc.NewServer(grpc.Creds(creds)) + credproviderpb.RegisterCredentialProviderServer(srv, NewServer(client, &NamespaceAuthorizer{allowed: map[string]map[string]struct{}{"team-a": {"ns1": {}}}})) + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + roots := x509.NewCertPool() + roots.AddCert(ca.RootCertificate) + for _, tc := range []struct { + name string + certs []tls.Certificate + allowed bool + }{ + {"injector", []tls.Certificate{issueCertificate(t, ca, *injectorIdentity)}, true}, + {"other workload", []tls.Certificate{issueCertificate(t, ca, "spiffe://cluster.local/ns/custom/sa/other")}, false}, + {"missing certificate", nil, false}, + {"untrusted injector", []tls.Certificate{issueCertificate(t, untrustedCA, *injectorIdentity)}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + conn, err := grpc.NewClient(lis.Addr().String(), grpc.WithTransportCredentials(credentials.NewTLS(&tls.Config{ + RootCAs: roots, ServerName: "api.ate-system.svc", Certificates: tc.certs, MinVersion: tls.VersionTLS13, + }))) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + before := len(client.Actions()) + resp, err := credproviderpb.NewCredentialProviderClient(conn).FetchSecret(ctx, &credproviderpb.FetchSecretRequest{ + Uri: "ate-secret://k8s.io/default/ns1/api/token", ActorSpiffeId: "spiffe://substrate-actor.local/atespace/team-a/actor/a", + }) + if tc.allowed { + if err != nil || string(resp.GetOpaqueBytes()) != "credential" { + t.Fatalf("FetchSecret: %v, %v", resp, err) + } + } else { + if err == nil { + t.Fatal("unauthorized peer received credentials") + } + if len(client.Actions()) != before { + t.Fatal("unauthorized peer reached Kubernetes") + } + } + }) + } +} + +func issueCertificate(t *testing.T, ca *localca.CA, uri string) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + t.Fatal(err) + } + template := &x509.Certificate{ + SerialNumber: serial, NotBefore: time.Now().Add(-time.Minute), NotAfter: time.Now().Add(time.Hour), + DNSNames: []string{"api.ate-system.svc", "localhost"}, IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + } + if uri != "" { + parsed, err := url.Parse(uri) + if err != nil { + t.Fatal(err) + } + template.URIs = []*url.URL{parsed} + } + der, err := x509.CreateCertificate(rand.Reader, template, ca.RootCertificate, &key.PublicKey, ca.SigningKey) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go new file mode 100644 index 0000000000..0f478af3cd --- /dev/null +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -0,0 +1,309 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "errors" + "io" + "os/exec" + "reflect" + "slices" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/yaml" +) + +func TestProviderManifests(t *testing.T) { + for _, tc := range []struct { + name, tool, namespace, prefix string + image string + args []string + }{ + {name: "default", tool: "helm", namespace: "ate-system", args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", namespace: "custom", prefix: "test-", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom", "--set", "credentialProvider.namespacePolicies[0].atespace=team-a", "--set", "credentialProvider.namespacePolicies[0].allowedNamespaces[0]=ns1"}}, + {name: "kustomize", tool: "kubectl", namespace: "ate-system", + args: []string{"kustomize", "../../../manifests/egress-credential-injection"}}, + {name: "CI images", tool: "helm", namespace: "ate-system", + image: "localhost:5001/kagent-dev/substrate/kubernetes-secrets:helm-e2e", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", "--set", "image.registry=localhost:5001", "--set", "image.tag=helm-e2e"}}, + {name: "global images", tool: "helm", namespace: "ate-system", + image: "mirror.example/custom/substrate/kubernetes-secrets:test", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system", + "--set", "image.repository=custom/substrate", "--set", "image.tag=test", "--set", "global.imageRegistry=mirror.example", + "--set", "imagePullSecrets[0].name=local", "--set", "global.imagePullSecrets[0].name=global", "--set", "global.imagePullPolicy=Always"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + var providerFound, portFound, policyFound, accountFound bool + var roleFound, bindingFound bool + for { + var doc struct { + Kind string + Metadata metav1.ObjectMeta + Spec struct { + Template corev1.PodTemplateSpec + Ports []corev1.ServicePort + } + Data map[string]string + Rules []rbacv1.PolicyRule + RoleRef rbacv1.RoleRef + Subjects []rbacv1.Subject + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + switch doc.Kind { + case "ServiceAccount": + if doc.Metadata.Name == tc.prefix+"k8s-credential-provider" { + accountFound = true + } + case "Deployment": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { + continue + } + pod := doc.Spec.Template.Spec + if pod.ServiceAccountName != tc.prefix+"k8s-credential-provider" { + t.Fatalf("unexpected ServiceAccount %q", pod.ServiceAccountName) + } + for _, container := range pod.Containers { + if container.Name != "k8s-credential-provider" { + continue + } + providerFound = true + if tc.image != "" && container.Image != tc.image { + t.Errorf("image = %q, want %q", container.Image, tc.image) + } + if tc.name == "global images" { + if container.ImagePullPolicy != corev1.PullAlways { + t.Errorf("imagePullPolicy = %q, want Always", container.ImagePullPolicy) + } + for _, name := range []string{"local", "global"} { + if !slices.Contains(pod.ImagePullSecrets, corev1.LocalObjectReference{Name: name}) { + t.Errorf("missing imagePullSecret %q", name) + } + } + } + args := strings.Join(container.Args, " ") + for _, required := range []string{ + "--listen-address=:50051", "--metrics-address=:9090", + "--injector-identity=spiffe://cluster.local/ns/" + tc.namespace + "/sa/" + tc.prefix + "atenet-egress", + "--server-cred-bundle=/run/servicedns.podcert.ate.dev/credential-bundle.pem", + "--client-ca-file=/run/podidentity.podcert.ate.dev/trust-bundle.pem", + } { + if tc.tool == "kubectl" && strings.HasPrefix(required, "--injector-identity=") { + continue + } + if !strings.Contains(args, required) { + t.Errorf("provider missing %s", required) + } + } + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet.Port.StrVal != "metrics" { + t.Fatal("missing dedicated readiness probe") + } + } + case "Service": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider" { + continue + } + for _, port := range doc.Spec.Ports { + if port.Port == 50051 && port.TargetPort.StrVal == "grpc" { + portFound = true + } + } + case "ConfigMap": + if !strings.HasPrefix(doc.Metadata.Name, tc.prefix+"k8s-credential-provider-namespace-policy") { + continue + } + policyFound = true + var policy namespacePolicyFile + if err := yaml.UnmarshalStrict([]byte(doc.Data["namespace-policy.yaml"]), &policy); err != nil { + t.Fatal(err) + } + auth, err := newNamespaceAuthorizer(policy) + if err != nil { + t.Fatal(err) + } + if auth.Allowed("team-a", "ns1") != (tc.name == "custom release") { + t.Fatal("unexpected namespace policy") + } + case "ClusterRole": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { + continue + } + roleFound = true + want := []rbacv1.PolicyRule{{APIGroups: []string{""}, Resources: []string{"secrets"}, Verbs: []string{"get"}}} + if !reflect.DeepEqual(doc.Rules, want) { + t.Fatalf("provider rules = %#v, want get-only Secret access", doc.Rules) + } + case "ClusterRoleBinding": + if doc.Metadata.Name != tc.prefix+"k8s-credential-provider-secret-reader" { + continue + } + bindingFound = true + wantRef := rbacv1.RoleRef{APIGroup: rbacv1.GroupName, Kind: "ClusterRole", Name: tc.prefix + "k8s-credential-provider-secret-reader"} + wantSubjects := []rbacv1.Subject{{Kind: "ServiceAccount", Name: tc.prefix + "k8s-credential-provider", Namespace: tc.namespace}} + if doc.RoleRef != wantRef || !reflect.DeepEqual(doc.Subjects, wantSubjects) { + t.Fatalf("unexpected provider binding: roleRef=%+v subjects=%+v", doc.RoleRef, doc.Subjects) + } + } + } + if !providerFound || !portFound || !policyFound || !accountFound { + t.Fatalf("provider=%v port=%v policy=%v account=%v", providerFound, portFound, policyFound, accountFound) + } + if !roleFound || !bindingFound { + t.Fatalf("role=%v binding=%v", roleFound, bindingFound) + } + }) + } +} + +func TestAgentgatewayCredentialConfiguration(t *testing.T) { + for _, tc := range []struct { + name, tool, host, roots string + args []string + }{ + {name: "default", tool: "helm", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "substrate", "../../../charts/substrate", "-n", "ate-system"}}, + {name: "custom release", tool: "helm", host: "test-k8s-credential-provider.custom.svc:50051", roots: "/run/servicedns.podcert.ate.dev/trust-bundle.pem", + args: []string{"template", "test", "../../../charts/substrate", "-n", "custom"}}, + {name: "kustomize", tool: "kubectl", host: "k8s-credential-provider.ate-system.svc:50051", roots: "/run/servicedns-ca/trust-bundle.pem", + args: []string{"kustomize", "--load-restrictor=LoadRestrictionsNone", "../../../manifests/ate-install/agentgateway-egress-mitm"}}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := exec.LookPath(tc.tool); err != nil { + t.Skipf("%s is not installed", tc.tool) + } + data, err := exec.CommandContext(t.Context(), tc.tool, tc.args...).CombinedOutput() + if err != nil { + t.Fatalf("render: %v\n%s", err, data) + } + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(data), 4096) + providers := map[string]int{} + mitmMounts, mitmVolumes, passthroughListeners := 0, 0, 0 + for { + var doc struct { + Kind string + Data map[string]string + Spec struct{ Template corev1.PodTemplateSpec } + } + if err := decoder.Decode(&doc); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + if doc.Kind == "Deployment" { + for _, volume := range doc.Spec.Template.Spec.Volumes { + if volume.Secret != nil && volume.Secret.SecretName == "egress-mitm-ca-pool" { + mitmVolumes++ + } + } + for _, container := range doc.Spec.Template.Spec.Containers { + if container.Name != "agentgateway" { + continue + } + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/run/egress-mitm" { + mitmMounts++ + } + } + } + } + if doc.Kind != "ConfigMap" { + continue + } + var config struct { + Binds []struct { + Listeners []struct { + Protocol string + TLS struct{ Mode, Cert, Key string } + Routes []struct { + Backends []struct { + Dynamic map[string]any + Policies struct{ BackendTLS map[string]any } + } + Policies struct { + SubstrateEgress struct { + CredentialProviders []struct { + URIAuthority string `json:"uriAuthority"` + Target struct { + Host string + Policies struct { + BackendTLS struct{ Cert, Key, Root string } + } + } + } + } + } + } + } + } + } + if err := yaml.Unmarshal([]byte(doc.Data["config.yaml"]), &config); err != nil { + t.Fatal(err) + } + for _, bind := range config.Binds { + for _, listener := range bind.Listeners { + if listener.Protocol == "TLS" { + passthroughListeners++ + } + for _, route := range listener.Routes { + for _, provider := range route.Policies.SubstrateEgress.CredentialProviders { + providers[listener.Protocol]++ + if listener.Protocol == "HTTPS" { + if listener.TLS.Mode != "dynamicCa" || listener.TLS.Cert != "/run/egress-mitm/tls.crt" || listener.TLS.Key != "/run/egress-mitm/tls.key" { + t.Fatal("incorrect MITM configuration") + } + if len(route.Backends) != 1 || route.Backends[0].Dynamic == nil || len(route.Backends[0].Dynamic) != 0 || route.Backends[0].Policies.BackendTLS == nil || len(route.Backends[0].Policies.BackendTLS) != 0 { + t.Fatal("HTTPS must use a dynamic destination with default public TLS trust") + } + } else if listener.Protocol != "HTTP" { + t.Fatalf("credentials enabled on unexpected protocol %q", listener.Protocol) + } + if provider.URIAuthority != "k8s.io" || provider.Target.Host != tc.host { + t.Fatalf("incorrect provider: %+v", provider) + } + tls := provider.Target.Policies.BackendTLS + if tls.Root != tc.roots || tls.Cert != "/run/podidentity.podcert.ate.dev/credential-bundle.pem" || tls.Key != tls.Cert { + t.Fatalf("incorrect provider mTLS: %+v", tls) + } + } + } + } + } + } + if providers["HTTP"] != 1 || providers["HTTPS"] != 1 { + t.Fatalf("providers=%v, want one per HTTP/HTTPS route", providers) + } + if mitmMounts != 1 || mitmVolumes != 1 || passthroughListeners != 0 { + t.Fatalf("MITM mounts=%d volumes=%d passthrough listeners=%d", mitmMounts, mitmVolumes, passthroughListeners) + } + }) + } +} diff --git a/cmd/credential-provider/kubernetes-secrets/nsauthz.go b/cmd/credential-provider/kubernetes-secrets/nsauthz.go index d5ffe13348..c648220898 100644 --- a/cmd/credential-provider/kubernetes-secrets/nsauthz.go +++ b/cmd/credential-provider/kubernetes-secrets/nsauthz.go @@ -18,6 +18,9 @@ import ( "fmt" "os" + "github.com/agent-substrate/substrate/internal/resources" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/yaml" ) @@ -48,7 +51,7 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { return nil, fmt.Errorf("reading namespace policy file %q: %w", path, err) } var file namespacePolicyFile - if err := yaml.Unmarshal(data, &file); err != nil { + if err := yaml.UnmarshalStrict(data, &file); err != nil { return nil, fmt.Errorf("parsing namespace policy file %q: %w", path, err) } return newNamespaceAuthorizer(file) @@ -59,8 +62,8 @@ func LoadNamespaceAuthorizer(path string) (*NamespaceAuthorizer, error) { func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, error) { allowed := make(map[string]map[string]struct{}) for i, p := range file.Policies { - if p.Atespace == "" { - return nil, fmt.Errorf("namespace policy %d: atespace is required", i) + if !resources.IsValidResourceName(p.Atespace) { + return nil, fmt.Errorf("namespace policy %d: valid atespace is required", i) } set := allowed[p.Atespace] if set == nil { @@ -68,6 +71,9 @@ func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, err allowed[p.Atespace] = set } for _, ns := range p.AllowedNamespaces { + if len(validation.IsDNS1123Label(ns)) != 0 { + return nil, fmt.Errorf("namespace policy %d: invalid namespace %q", i, ns) + } set[ns] = struct{}{} } } @@ -78,6 +84,9 @@ func newNamespaceAuthorizer(file namespacePolicyFile) (*NamespaceAuthorizer, err // deny: an atespace absent from the mapping, or a namespace not in its list, is // refused. func (a *NamespaceAuthorizer) Allowed(atespace, namespace string) bool { + if a == nil { + return false + } set, ok := a.allowed[atespace] if !ok { return false diff --git a/docs/kubernetes-credential-provider.md b/docs/kubernetes-credential-provider.md new file mode 100644 index 0000000000..aba95669fd --- /dev/null +++ b/docs/kubernetes-credential-provider.md @@ -0,0 +1,102 @@ +# Kubernetes credential provider + +The `k8s-credential-provider` Deployment follows the provider from +[upstream](https://github.com/agent-substrate/substrate/pull/1335). It serves +`CredentialProvider.FetchSecret` at `k8s-credential-provider.ate-system.svc:50051` +with its own ServiceAccount and projected serving certificate. AGW calls it +directly over mTLS to inject credentials into HTTP and intercepted HTTPS requests. + +`ate-secret://k8s.io/default/team-a-secrets/example-api/token` resolves the `token` +entry in that Kubernetes Secret. The `default` locator and an explicit key are required. +The provider reads Kubernetes on every fetch and never persists or logs values. +AGW caches successful credentials per actor and URI for five minutes, so rotation +can take that long to reach injected requests. + +Each request requires a trusted injector certificate with the configured SPIFFE +identity, an explicit atespace-to-namespace grant for the attested actor, and +Kubernetes `get` permission for the provider's ServiceAccount. Both installers +include the upstream get-only Secret ClusterRole and bind it to that ServiceAccount. +The provider can read Secrets across namespaces; its namespace policy controls +which namespaces each actor may use. Empty policies deny all requests. + +## Configure the provider + +Keep the pinned `images.agentgateway` image. It includes the +[protocol update](https://github.com/agentgateway/agentgateway/pull/3524) from +[this build](https://github.com/agentgateway/agentgateway/actions/runs/35238449333) +and implements the current [FetchSecret contract](../pkg/proto/credproviderpb/credprovider.proto). + +Create the MITM CA Secret using the existing installation tooling: + +```sh +hack/install-ate-kind.sh --create-egress-mitm-ca-pool-secret +``` + +The gateway needs `egress-mitm-ca-pool` with `tls.crt` and `tls.key` in its namespace. +Actors making HTTPS requests must trust this CA; see the +[MITM trust bundle guide](egress-trust-bundle.md). + +For Helm, add these values to your release configuration: + +```yaml +credentialProvider: + namespacePolicies: + - atespace: team-a + allowedNamespaces: [team-a-secrets] +``` + +The Helm chart always deploys the provider and configures AGW's HTTP route and +HTTPS interception route. Namespace grants default to an empty list. +Policy changes roll the provider's Pods. Resource names and the injector identity +follow the release: release `demo` in namespace `platform` uses ServiceAccount +`demo-k8s-credential-provider`, endpoint +`demo-k8s-credential-provider.platform.svc:50051`, and injector identity +`spiffe://cluster.local/ns/platform/sa/demo-atenet-egress`. + +HTTPS uses a dynamic backend: AGW selects the destination from the request and +validates its certificate using the system CA roots (`backendTLS: {}`). Public +APIs such as OpenAI and Anthropic need no per-backend certificates. The single +MITM CA lets AGW generate actor-facing certificates as needed. HTTP also travels +through the authenticated CONNECT tunnel, then leaves AGW over plaintext HTTP. + +For the manifest installer, set your grants in +`manifests/egress-credential-injection/namespace-policy.yaml`, then deploy: + +```sh +kubectl kustomize manifests/egress-credential-injection | ko apply -f - +hack/install-ate.sh --deploy-atenet \ + --atenet-dataplane=agentgateway --experimental-use-sdsmint +``` + +The policy file uses `policies:` with the same list of grants as the Helm values. +Its generated ConfigMap name changes with the policy, rolling the provider on +reapplication. Direct policy ConfigMap edits require a rollout restart. Client CA +bundles and serving certificates reload automatically for new TLS connections. + +## Configure injection + +Create the Secret and set an actor's egress policy header injection to use credential URI +`ate-secret://k8s.io/default/team-a-secrets/example-api/token`, header +`authorization`, and prefix `Bearer `. Namespace grants alone do not create an +egress policy. No ext_proc injector is needed. + +## Tests + +The Helm PR workflow installs the provider and MITM gateway from the start and +runs `internal/e2e/suites/credentials` alongside the standard suites with real actors, +Secrets, chart-managed RBAC, AGW, and the deployed provider. It checks +the exact injected token, an unauthenticated-origin control, namespace-policy +denial and cache isolation between atespaces. The local origin serves HTTP; +the suite uses the installed gateway configuration without modifying ConfigMaps. + +Include `-f internal/e2e/suites/credentials/values.yaml` in the initial Helm +installation to grant the test atespace access. After deploying the standard +MITM egress fixtures, run the suites together: + +```sh +E2E_ATENET_DATAPLANE=agentgateway E2E_CREDENTIAL_PROVIDER=1 E2E_EGRESS_MITM=1 \ + hack/run-e2e-kind.sh -v -args --no-color +``` + +The credential suite tests HTTP injection. The existing MITM suite checks HTTPS +interception and actor trust against a public HTTPS origin using the same install. diff --git a/go.mod b/go.mod index a8429602a3..633cbf441b 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spiffe/go-spiffe/v2 v2.7.0 + github.com/stretchr/testify v1.12.1 github.com/testcontainers/testcontainers-go v0.44.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 github.com/vishvananda/netlink v1.3.1 @@ -221,7 +222,6 @@ require ( github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/viper v1.20.1 // indirect - github.com/stretchr/testify v1.12.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect diff --git a/hack/render-manifests.sh b/hack/render-manifests.sh index 2187044970..e75108a1cb 100755 --- a/hack/render-manifests.sh +++ b/hack/render-manifests.sh @@ -38,6 +38,8 @@ PRESERVED_FILES=( atenet-egress-with-sdsmint.yaml atenet-router.yaml atenet-router-monitoring.yaml + # The provider's upstream manifest lives in manifests/egress-credential-injection. + k8s-credential-provider.yaml pod-certificate-controller.yaml postgres.yaml sandboxconfig-gvisor.yaml diff --git a/internal/e2e/fixture.go b/internal/e2e/fixture.go index 956ca53045..e793ac63b2 100644 --- a/internal/e2e/fixture.go +++ b/internal/e2e/fixture.go @@ -147,6 +147,16 @@ func DeploySubstrateFixture(t *testing.T, ctx context.Context, clients *Clients, t.Fatalf("fixture %s declares templates in different atespaces (%q and %q)", manifests.Template, atespace, got) } } + t.Cleanup(func() { + // Remove workers before the namespace so its controller does not wait + // on their one-hour termination grace estimate after the Pods exit. + delArgs := []string{"delete", "workerpools", "--all", "--namespace=" + atespace, + "--ignore-not-found", "--cascade=foreground", "--timeout=2m"} + if KubeContext != "" { + delArgs = append([]string{"--context=" + KubeContext}, delArgs...) + } + RunCmd(t, "kubectl", delArgs...) + }) if _, err := clients.SubstrateAPI.CreateAtespace(ctx, &ateapipb.CreateAtespaceRequest{Atespace: &ateapipb.Atespace{Metadata: &ateapipb.ResourceMetadata{Name: atespace}}}); err != nil && status.Code(err) != codes.AlreadyExists { t.Fatalf("failed to create atespace %q: %v", atespace, err) diff --git a/internal/e2e/fixtures/testserver/http.go b/internal/e2e/fixtures/testserver/http.go index 0ae303baa8..5eaf72806d 100644 --- a/internal/e2e/fixtures/testserver/http.go +++ b/internal/e2e/fixtures/testserver/http.go @@ -17,19 +17,17 @@ package main import ( "log" "net/http" + "os" "time" "github.com/spf13/cobra" ) // newHTTPCmd is a plain HTTP/1.1 origin an Actor's egress lands on. It exists so -// a test can assert the destination port is recovered from SO_ORIGINAL_DST -// rather than defaulted from the URL scheme: the actor fetches its /healthz on a -// non-standard port, and the gateway's access log is expected to carry that -// port. There is nothing to serve beyond readiness, so /healthz is all it -// answers. +// a test can assert the destination port is recovered from SO_ORIGINAL_DST. +// It can also verify an injected Authorization header against a mounted token. func newHTTPCmd() *cobra.Command { - var listenAddress string + var listenAddress, authorizationFile string cmd := &cobra.Command{ Use: "http", Short: "Serve a plain HTTP/1.1 origin answering /healthz.", @@ -39,6 +37,9 @@ func newHTTPCmd() *cobra.Command { mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + if authorizationFile != "" { + mux.HandleFunc("/credential", credentialHandler(authorizationFile)) + } server := &http.Server{ Addr: listenAddress, @@ -51,5 +52,21 @@ func newHTTPCmd() *cobra.Command { }, } cmd.Flags().StringVar(&listenAddress, "listen", ":8080", "Address the HTTP origin listens on.") + cmd.Flags().StringVar(&authorizationFile, "authorization-file", "", "Enable /credential, requiring a Bearer token matching this file.") return cmd } + +func credentialHandler(path string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + token, err := os.ReadFile(path) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + if len(token) == 0 || r.Header.Get("Authorization") != "Bearer "+string(token) { + w.WriteHeader(http.StatusUnauthorized) + return + } + w.WriteHeader(http.StatusNoContent) + } +} diff --git a/internal/e2e/fixtures/testserver/http_test.go b/internal/e2e/fixtures/testserver/http_test.go new file mode 100644 index 0000000000..b42c292677 --- /dev/null +++ b/internal/e2e/fixtures/testserver/http_test.go @@ -0,0 +1,51 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestCredentialHandler(t *testing.T) { + path := filepath.Join(t.TempDir(), "token") + if err := os.WriteFile(path, []byte("expected-token"), 0600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + header string + status int + }{ + {"", http.StatusUnauthorized}, + {"Bearer wrong-token", http.StatusUnauthorized}, + {"Bearer expected-token", http.StatusNoContent}, + } { + req := httptest.NewRequest(http.MethodGet, "/credential", nil) + req.Header.Set("Authorization", tc.header) + resp := httptest.NewRecorder() + credentialHandler(path)(resp, req) + if resp.Code != tc.status || resp.Body.Len() != 0 { + t.Errorf("header %q: status=%d body=%q, want status=%d and no body", tc.header, resp.Code, resp.Body.String(), tc.status) + } + } + resp := httptest.NewRecorder() + credentialHandler(path+"-missing")(resp, httptest.NewRequest(http.MethodGet, "/credential", nil)) + if resp.Code != http.StatusInternalServerError { + t.Fatalf("unreadable credential file: status=%d, want 500", resp.Code) + } +} diff --git a/internal/e2e/suites/credentials/credentials_test.go b/internal/e2e/suites/credentials/credentials_test.go new file mode 100644 index 0000000000..2e39fc128e --- /dev/null +++ b/internal/e2e/suites/credentials/credentials_test.go @@ -0,0 +1,147 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "testing" + "time" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/agent-substrate/substrate/internal/e2e" + "github.com/agent-substrate/substrate/internal/resources" + "github.com/agent-substrate/substrate/pkg/proto/ateapipb" +) + +// TestKubernetesCredentialInjection uses the Helm values in values.yaml, real +// actors, and a local HTTP origin with the installed gateway configuration and RBAC. +func TestKubernetesCredentialInjection(t *testing.T) { + if os.Getenv("E2E_CREDENTIAL_PROVIDER") == "" { + t.Skip("requires credential E2E namespace grants and E2E_CREDENTIAL_PROVIDER=1") + } + env, err := e2e.CheckEnv("BUCKET_NAME", "KO_DOCKER_REPO") + require.NoError(t, err) + ctx := t.Context() + clients := e2e.GetClients() + namespace, template := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials") + deniedAtespace, deniedTemplate := e2e.DeployProbe(t, env["BUCKET_NAME"], "credentials-denied") + otherNamespace := e2e.CreateNamespace(t).Name + + for _, secret := range []struct{ namespace, name string }{ + {namespace, "allowed"}, {otherNamespace, "allowed"}, + } { + _, err := clients.K8s.CoreV1().Secrets(secret.namespace).Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: secret.name}, + Data: map[string][]byte{"token": []byte("e2e-credential-token")}, + }, metav1.CreateOptions{}) + require.NoError(t, err) + } + + e2e.DeployServerPod(t, ctx, e2e.ServerPod{ + Name: "credential-origin", Namespace: namespace, + ImportPath: "github.com/agent-substrate/substrate/internal/e2e/fixtures/testserver", + Args: []string{"http", "--authorization-file=/run/token/token"}, + Port: 80, TargetPort: 8080, + Volumes: []corev1.Volume{ + {Name: "token", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "allowed"}}}, + }, + VolumeMounts: []corev1.VolumeMount{{Name: "token", MountPath: "/run/token", ReadOnly: true}}, + }) + host := "credential-origin." + namespace + ".svc" + router, err := e2e.NewRouterClient(ctx) + require.NoError(t, err) + t.Cleanup(router.Close) + // Keep the successful actor alive through the cache-isolation check. + suite := t + for _, tc := range []struct { + name, secretNamespace, secret, want string + }{ + {"without-injection", "", "", "401"}, + {"allowed", namespace, "allowed", "204"}, + {"atespace-denied", namespace, "allowed", "403"}, + {"namespace-denied", otherNamespace, "allowed", "403"}, + } { + t.Run(tc.name, func(t *testing.T) { + atespace, actorTemplate := namespace, template + actorName := tc.name + if tc.name == "atespace-denied" { + // Use the already-fetched URI from an ungranted atespace so an + // incorrectly shared gateway cache cannot bypass authorization. + atespace, actorTemplate = deniedAtespace, deniedTemplate + actorName = "allowed" + } + actor := &ateapipb.ObjectRef{Atespace: atespace, Name: actorName} + _, _ = clients.SubstrateAPI.SuspendActor(ctx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, _ = clients.SubstrateAPI.DeleteActor(ctx, &ateapipb.DeleteActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.CreateActor(ctx, &ateapipb.CreateActorRequest{Actor: &ateapipb.Actor{ + Metadata: &ateapipb.ResourceMetadata{Atespace: atespace, Name: actorName}, + ActorTemplate: &ateapipb.ObjectRef{Atespace: atespace, Name: actorTemplate.GetMetadata().GetName()}, + }}) + require.NoError(t, err) + cleanupTest := t + if tc.name == "allowed" { + cleanupTest = suite + } + cleanupTest.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, _ = clients.SubstrateAPI.SuspendActor(cleanupCtx, &ateapipb.SuspendActorRequest{Actor: actor}) + _, err := clients.SubstrateAPI.DeleteActor(cleanupCtx, &ateapipb.DeleteActorRequest{Actor: actor}) + if err != nil { + cleanupTest.Errorf("delete actor %s/%s: %v", atespace, actorName, err) + } + }) + rule := e2e.EgressAllowHostnames(host) + if tc.secret != "" { + rule.Hostnames.Effects = &ateapipb.EgressRuleEffects{InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{ + Header: "authorization", Prefix: "Bearer ", + CredentialUri: fmt.Sprintf("ate-secret://k8s.io/default/%s/%s/token", tc.secretNamespace, tc.secret), + }}} + } + e2e.EnsureEgressPolicy(t, ctx, clients, actor, rule) + _, err = clients.SubstrateAPI.ResumeActor(ctx, &ateapipb.ResumeActorRequest{Actor: actor}) + require.NoError(t, err) + path := "/fetch?roots=system&url=" + url.QueryEscape("http://"+host+"/credential") + // Route discovery is asynchronous. Retries require the precise status; + // transport errors never pass. + deadline := time.Now().Add(90 * time.Second) + for { + resp, err := router.Get(ctx, resources.ActorRef{Atespace: atespace, Name: actorName}, path) + var body []byte + if err == nil { + body, err = io.ReadAll(resp.Body) + resp.Body.Close() + var result struct{ Status, Error string } + if err == nil && resp.StatusCode == http.StatusOK && json.Unmarshal(body, &result) == nil && result.Error == "" && result.Status == tc.want { + break + } + } + if time.Now().After(deadline) { + t.Fatalf("want origin status %s; last response: %s; error: %v", tc.want, body, err) + } + time.Sleep(2 * time.Second) + } + }) + } +} diff --git a/internal/e2e/suites/credentials/testmain_test.go b/internal/e2e/suites/credentials/testmain_test.go new file mode 100644 index 0000000000..6b8549a17a --- /dev/null +++ b/internal/e2e/suites/credentials/testmain_test.go @@ -0,0 +1,24 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package credentials + +import ( + "os" + "testing" + + "github.com/agent-substrate/substrate/internal/e2e" +) + +func TestMain(m *testing.M) { os.Exit(e2e.RunTestMain(m)) } diff --git a/internal/e2e/suites/credentials/values.yaml b/internal/e2e/suites/credentials/values.yaml new file mode 100644 index 0000000000..27ff95125a --- /dev/null +++ b/internal/e2e/suites/credentials/values.yaml @@ -0,0 +1,18 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +credentialProvider: + namespacePolicies: + - atespace: ate-e2e-probe-credentials + allowedNamespaces: [ate-e2e-probe-credentials] diff --git a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml index b90274819d..6b72ca6de0 100644 --- a/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway-egress-mitm/kustomization.yaml @@ -72,6 +72,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: k8s-credential-provider.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: HTTP routes: - backends: @@ -85,6 +94,15 @@ patches: cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem key: /run/podidentity.podcert.ate.dev/credential-bundle.pem root: /run/servicedns-ca/trust-bundle.pem + credentialProviders: + - uriAuthority: k8s.io + target: + host: k8s-credential-provider.ate-system.svc:50051 + policies: + backendTLS: + cert: /run/podidentity.podcert.ate.dev/credential-bundle.pem + key: /run/podidentity.podcert.ate.dev/credential-bundle.pem + root: /run/servicedns-ca/trust-bundle.pem - protocol: TCP tcpRoutes: - backends: diff --git a/manifests/ate-install/components/agentgateway/kustomization.yaml b/manifests/ate-install/components/agentgateway/kustomization.yaml index fe062e4447..cdff87553f 100644 --- a/manifests/ate-install/components/agentgateway/kustomization.yaml +++ b/manifests/ate-install/components/agentgateway/kustomization.yaml @@ -42,7 +42,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml @@ -118,7 +118,7 @@ patches: path: /spec/template/spec/containers/0 value: name: agentgateway - image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.9f9744cf + image: ghcr.io/agentgateway/agentgateway:v0.0.0-alpha.8dba3989@sha256:fdde26d4b0ea11d3e740dc19905dfe9b26e88f40fa8b9985f94ed1d8420e389e args: - -f - /etc/agentgateway/config.yaml diff --git a/manifests/egress-credential-injection/k8s-credential-provider.yaml b/manifests/egress-credential-injection/k8s-credential-provider.yaml index 5144d818fe..52eae32afe 100644 --- a/manifests/egress-credential-injection/k8s-credential-provider.yaml +++ b/manifests/egress-credential-injection/k8s-credential-provider.yaml @@ -22,9 +22,7 @@ metadata: name: k8s-credential-provider namespace: ate-system --- -# POC scope note: this grants read on Secrets cluster-wide so a policy can name a -# Secret in any namespace. A production deployment would scope this to the -# namespaces a provider instance is allowed to serve. +# The provider checks the actor's atespace-to-namespace grant before reading. apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/manifests/egress-credential-injection/kustomization.yaml b/manifests/egress-credential-injection/kustomization.yaml new file mode 100644 index 0000000000..19bf054fef --- /dev/null +++ b/manifests/egress-credential-injection/kustomization.yaml @@ -0,0 +1,25 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- k8s-credential-provider.yaml + +configMapGenerator: +- name: k8s-credential-provider-namespace-policy + namespace: ate-system + files: + - namespace-policy.yaml diff --git a/manifests/egress-credential-injection/namespace-policy.yaml b/manifests/egress-credential-injection/namespace-policy.yaml index d98d1ae76b..78747a0d26 100644 --- a/manifests/egress-credential-injection/namespace-policy.yaml +++ b/manifests/egress-credential-injection/namespace-policy.yaml @@ -12,23 +12,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -# The atespace→namespace authorization policy k8s-credential-provider enforces: an actor's -# atespace may only resolve Secrets whose URI namespace is listed for it here. -# Enforcement is default-deny — an atespace absent from this file resolves -# nothing. -# -# TODO: k8s-credential-provider loads this once at startup, so editing this ConfigMap -# requires restarting the k8s-credential-provider Deployment. Make it reload dynamically. -apiVersion: v1 -kind: ConfigMap -metadata: - name: k8s-credential-provider-namespace-policy - namespace: ate-system -data: - namespace-policy.yaml: | - # atespace "team-a" may resolve secrets in namespace "ns1" (matches the - # sample policy's ate-secret://k8s.io/default/ns1/example-api/token). - policies: - - atespace: team-a - allowedNamespaces: - - ns1 +# Default-deny atespace-to-namespace grants. Secret RBAC is configured separately. +policies: [] From c0f9b51dfc68d3833de91d3c3c99756f01dca662 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Fri, 18 Sep 2026 06:48:03 -0400 Subject: [PATCH 19/22] Publish the Kubernetes credential provider in releases Build the nested provider package with the release images and publish it under the kubernetes-secrets basename expected by the Helm chart. Signed-off-by: Eitan Yarmush --- .github/workflows/release.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 4a26a5cbe9..5ccb325ad5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,8 +88,8 @@ jobs: run: | set -o errexit -o nounset -o pipefail - for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet; do - KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component}" \ + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet credential-provider/kubernetes-secrets; do + KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component##*/}" \ ./hack/run-tool.sh ko build \ --tags "${IMAGE_TAGS}" \ --platform linux/amd64,linux/arm64 \ From c384d7eaebc04b955b80dfa3f128dc3cc965cf05 Mon Sep 17 00:00:00 2001 From: Krisztian F <103492698+krisztianfekete@users.noreply.github.com> Date: Mon, 21 Sep 2026 17:49:33 +0200 Subject: [PATCH 20/22] Allow Helm deployments to enable actor lifecycle events Signed-off-by: Eitan Yarmush --- charts/substrate/README.md | 4 ++-- charts/substrate/templates/_helpers.tpl | 8 ++++++++ charts/substrate/values.yaml | 3 ++- docs/dev/best-practices/otel-collector.md | 3 ++- docs/observability.md | 4 ++-- 5 files changed, 16 insertions(+), 6 deletions(-) diff --git a/charts/substrate/README.md b/charts/substrate/README.md index 2803a5aa63..7fd7b09ed1 100644 --- a/charts/substrate/README.md +++ b/charts/substrate/README.md @@ -49,11 +49,11 @@ See `values.yaml` for the full set; the important keys: | `atelet.gcpAuthForImagePulls` | `false` | Enable only when using GCP registry auth | | `credentialProvider.namespacePolicies` | `[]` | Default-deny atespace-to-namespace grants; the chart includes get-only Secret RBAC for the provider | | `ateApi.extraArgs` | `[]` | Additional command-line arguments appended to the ateapi defaults | -| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics and the router access log | +| `otel.endpoint` | `""` | Set to an OTLP endpoint to export traces, metrics, the actor lifecycle events and the router access log | | `otel.traces.enabled` | `true` | Set to `false` to export no traces from the router; the Go components do not honor this yet | | `otel.traces.endpoint` | `""` | OTLP endpoint for traces, overriding `otel.endpoint` | | `otel.traces.samplingRatio` | `0.01` | Fraction of parentless requests that start a trace, applied to the Go components and the router | | `otel.metrics.enabled` | `true` | Sets the OTLP metrics exporter to `none`; the Go components do not honor this yet | | `otel.metrics.endpoint` | `""` | OTLP endpoint for metrics, overriding `otel.endpoint` | -| `otel.logs.enabled` | `true` | Set to `false` to export no logs; the router access log is the only OTLP log source today | +| `otel.logs.enabled` | `true` | Set to `false` to export no logs. Gates both OTLP log sources: ateapi's actor lifecycle events and the router access log | | `otel.logs.endpoint` | `""` | OTLP endpoint for logs, overriding `otel.endpoint` | diff --git a/charts/substrate/templates/_helpers.tpl b/charts/substrate/templates/_helpers.tpl index 3b81dc9c0b..45184413fb 100644 --- a/charts/substrate/templates/_helpers.tpl +++ b/charts/substrate/templates/_helpers.tpl @@ -124,6 +124,14 @@ Usage: value: {{ $cfg.endpoint | quote }} {{- end }} {{- end }} +{{- if include "substrate.otel.signalEndpoint" (list "logs" .) }} +{{- /* Only logs need turning on: serverboot defaults the component to none, so + an enabled signal exports nothing without this. Traces and metrics always + export, so they need no such branch -- keep this out of the range above. + Gated on an endpoint, since otlp without one retries localhost:4317. */}} +- name: OTEL_LOGS_EXPORTER + value: otlp +{{- end }} {{- if include "substrate.otel.signalEndpoint" (list "traces" .) }} - name: OTEL_TRACES_SAMPLER value: parentbased_traceidratio diff --git a/charts/substrate/values.yaml b/charts/substrate/values.yaml index fef293a355..618be970b1 100644 --- a/charts/substrate/values.yaml +++ b/charts/substrate/values.yaml @@ -77,7 +77,8 @@ ateApiServerEnvVarsConfigMap: ate-api-server-envvars # traces.enabled and metrics.enabled set OTEL__EXPORTER=none on the # Go components, which do not read it yet and keep exporting; the setting # takes effect on the router's agentgateway only. logs.enabled works in full, -# since the agentgateway access log is the only OTLP log source. +# and gates both OTLP log sources: ateapi's actor lifecycle events and the +# router's agentgateway access log. otel: endpoint: "" traces: diff --git a/docs/dev/best-practices/otel-collector.md b/docs/dev/best-practices/otel-collector.md index d01fb81a8a..26254edf0d 100644 --- a/docs/dev/best-practices/otel-collector.md +++ b/docs/dev/best-practices/otel-collector.md @@ -435,7 +435,8 @@ for a worked example. **Substrate exports one thing over OTLP: the actor lifecycle events**, from ateapi, through `serverboot.InitLogging`. They are off unless -`OTEL_LOGS_EXPORTER=otlp` is set, which only the kind overlay does today. See +`OTEL_LOGS_EXPORTER=otlp` is set. The kind overlay sets it, and the Helm chart +sets it from `otel.logs.enabled` once `otel.endpoint` resolves. See [the same records over OTLP](../../observability.md#the-same-records-over-otlp). Everything else is stdout. `serverboot.InitLogger` writes structured JSON there, diff --git a/docs/observability.md b/docs/observability.md index c0f1a832ce..9127d4b4b7 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -178,7 +178,7 @@ The counter carries the same reason but no actor identity, so this record is the #### The same records over OTLP -Both records also go out as OTLP log events, so a collector reads them without knowing substrate's stdout envelope. Set `OTEL_LOGS_EXPORTER=otlp` to turn it on; unset means `none`, which is what every environment but kind uses today. Only ateapi has a LoggerProvider today; [the ateom relay](#the-ateom-otlp-relay) carries logs, traces, and metrics, so an ateom exports log records the same way once it has one. +Both records also go out as OTLP log events, so a collector reads them without knowing substrate's stdout envelope. Set `OTEL_LOGS_EXPORTER=otlp` to turn it on; unset means `none`. The kind overlay sets it, and a chart install gets it from `otel.logs.enabled` once `otel.endpoint` resolves. Only ateapi has a LoggerProvider today; [the ateom relay](#the-ateom-otlp-relay) carries logs, traces, and metrics, so an ateom exports log records the same way once it has one. Two `event.name` values, which is the OTLP LogRecord's own field rather than an attribute: @@ -399,7 +399,7 @@ Telemetry is emitted the same way everywhere; only the backend differs between a | Path | service → in-cluster `opentelemetry-collector` | service → Google Managed Prometheus (GMP) | | Metrics | collector Prometheus exporter on `:8889` | Google Cloud Monitoring | | Traces | Jaeger UI | Google Cloud Trace | -| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs: `OTEL_LOGS_EXPORTER` is unset | +| Logs | pod stdout; ateapi's [actor lifecycle events](#the-same-records-over-otlp) also to the collector's `debug` exporter | pod stdout. No OTLP logs unless `OTEL_LOGS_EXPORTER` is set | | Dashboards | Not supported | Google Cloud Monitoring (see [Dashboards](#5-dashboards)) | > In Kind, `ateapi`, `atelet`, `ate-controller`, and `atenet-router` are pointed at the in-cluster collector, and the controller propagates the endpoint to the ateom worker pods it creates, so all component telemetry lands locally. From 08c930d656a85d9e4fc80fd175ecfa9f00462688 Mon Sep 17 00:00:00 2001 From: Eitan Yarmush Date: Wed, 23 Sep 2026 12:11:25 +0000 Subject: [PATCH 21/22] Configure Helm identities for renamed deployments Pass the resolved atelet and router ServiceAccounts to the control plane, expose the controller pod namespace, and configure router Service discovery. Cover canonical and custom Helm releases in manifest tests. Signed-off-by: Eitan Yarmush --- .../substrate/templates/ate-api-server.yaml | 1 + .../substrate/templates/ate-controller.yaml | 8 +++++- charts/substrate/templates/atenet-router.yaml | 1 + .../kubernetes-secrets/manifests_test.go | 27 +++++++++++++++++++ 4 files changed, 36 insertions(+), 1 deletion(-) diff --git a/charts/substrate/templates/ate-api-server.yaml b/charts/substrate/templates/ate-api-server.yaml index b93dbf4207..fc8e728e3d 100644 --- a/charts/substrate/templates/ate-api-server.yaml +++ b/charts/substrate/templates/ate-api-server.yaml @@ -90,6 +90,7 @@ spec: - "--actor-id-jwt-pool=/run/actor-id-jwt-pool/pool.json" - "--actor-id-ca-pool=/run/actor-id-ca-pool/pool.json" - "--egress-gateway-address={{ include "substrate.fullname" (list "atenet-egress" .) }}.{{ .Release.Namespace }}.svc:443" + - '--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}' - "--atelet-client-cred-bundle=/run/podidentity.podcert.ate.dev/credential-bundle.pem" - "--pod-identity-ca-certs=/run/podidentity.podcert.ate.dev/trust-bundle.pem" - "--drain-delay=13s" diff --git a/charts/substrate/templates/ate-controller.yaml b/charts/substrate/templates/ate-controller.yaml index 9e8588ceb2..b3c5b4521c 100644 --- a/charts/substrate/templates/ate-controller.yaml +++ b/charts/substrate/templates/ate-controller.yaml @@ -78,10 +78,16 @@ spec: # "ate-system"). Pass the chart-resolved Service so the controller # dials the right backend when substrate is installed as a subchart. - "--ateapi-conn-spec=dns:///{{ include "substrate.fullname" (list "api" .) }}.{{ .Release.Namespace }}.svc:443" + - '--atelet-service-account={{ include "substrate.fullname" (list "atelet" .) }}' + - '--router-service-account={{ include "substrate.fullname" (list "atenet-router" .) }}' - "--ateapi-ca-file=/run/servicedns-ca/trust-bundle.pem" - "--ateapi-client-cert=/run/podidentity.podcert.ate.dev/credential-bundle.pem" -{{- with include "substrate.otel.env" . }} env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace +{{- with include "substrate.otel.env" . }} {{- . | trim | nindent 8 }} {{- end }} ports: diff --git a/charts/substrate/templates/atenet-router.yaml b/charts/substrate/templates/atenet-router.yaml index 8d14854925..9ebd429d2f 100644 --- a/charts/substrate/templates/atenet-router.yaml +++ b/charts/substrate/templates/atenet-router.yaml @@ -221,6 +221,7 @@ spec: - "--mode=ingress" - "--atenet-dataplane=agentgateway" - "--namespace={{ .Release.Namespace }}" + - '--router-service-name={{ include "substrate.fullname" (list "atenet-router" .) }}' - "--port-http=8080" - "--port-extproc=50051" - "--extproc-address=127.0.0.1" diff --git a/cmd/credential-provider/kubernetes-secrets/manifests_test.go b/cmd/credential-provider/kubernetes-secrets/manifests_test.go index 0f478af3cd..0908335934 100644 --- a/cmd/credential-provider/kubernetes-secrets/manifests_test.go +++ b/cmd/credential-provider/kubernetes-secrets/manifests_test.go @@ -79,6 +79,33 @@ func TestProviderManifests(t *testing.T) { } else if err != nil { t.Fatal(err) } + if tc.tool == "helm" && doc.Kind == "Deployment" { + for _, container := range doc.Spec.Template.Spec.Containers { + var required []string + switch container.Name { + case "ate-api-server": + required = []string{"--atelet-service-account=" + tc.prefix + "atelet"} + case "ate-controller": + required = []string{ + "--atelet-service-account=" + tc.prefix + "atelet", + "--router-service-account=" + tc.prefix + "atenet-router", + } + if !slices.ContainsFunc(container.Env, func(env corev1.EnvVar) bool { + return env.Name == "POD_NAMESPACE" && env.ValueFrom != nil && + env.ValueFrom.FieldRef != nil && env.ValueFrom.FieldRef.FieldPath == "metadata.namespace" + }) { + t.Error("controller must resolve worker identities from its pod namespace") + } + case "atenet-router": + required = []string{"--router-service-name=" + tc.prefix + "atenet-router"} + } + for _, arg := range required { + if !slices.Contains(container.Args, arg) { + t.Errorf("%s missing %s", container.Name, arg) + } + } + } + } switch doc.Kind { case "ServiceAccount": if doc.Metadata.Name == tc.prefix+"k8s-credential-provider" { From fae094e6147abda35887e25ec1855532b508bc1e Mon Sep 17 00:00:00 2001 From: Christopher Boyd <6323077+cpboyd@users.noreply.github.com> Date: Thu, 24 Sep 2026 23:30:52 -0400 Subject: [PATCH 22/22] ci(release): improve support for forks --- .github/workflows/release.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 5ccb325ad5..b804752838 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -88,6 +88,9 @@ jobs: run: | set -o errexit -o nounset -o pipefail + # ghcr requires lowercase + export IMAGE_REPOSITORY="${IMAGE_REPOSITORY,,}" + for component in ateapi atecontroller atelet ateom-gvisor ateom-microvm podcertcontroller atenet credential-provider/kubernetes-secrets; do KO_DOCKER_REPO="${IMAGE_REPOSITORY}/${component##*/}" \ ./hack/run-tool.sh ko build \ @@ -101,10 +104,13 @@ jobs: if: inputs.create_release env: HELM_EXPERIMENTAL_OCI: "1" - CHART_REPOSITORY: oci://ghcr.io/kagent-dev/substrate/helm + CHART_REPOSITORY: oci://ghcr.io/${{ github.repository }}/helm run: | set -o errexit -o nounset -o pipefail + # ghcr requires lowercase + export CHART_REPOSITORY="${CHART_REPOSITORY,,}" + tag="${{ steps.tag.outputs.value }}" chart_version="${tag#v}" package_dir="${RUNNER_TEMP}/helm-packages"