From bdeef5c38d4aec801c9de2c4e57c53c8f878102a Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Wed, 7 Oct 2026 12:44:42 +0100 Subject: [PATCH 01/26] CARRY: feat(odh): unify shared and managed E2E runs Drive local and image tiers from one validated phase plan. Deploy shared and managed gateway modes with the canonical OpenShift overlays, check the deployed configuration before testing, and import provider profiles per phase. Merge phase JUnit reports into one tier report while preserving cleanup on test, deployment, and signal failures. Document the shared runner and managed-mode workflow. Signed-off-by: Ignas Baranauskas --- deploy/konflux/e2e-odh/README.md | 35 +++- e2e/rust/Cargo.toml | 2 +- e2e/rust/tests/odh/AGENTS.md | 18 +- e2e/rust/tests/odh/README.md | 192 +++++++++++------ e2e/rust/tests/odh/run-odh-test-tier.sh | 155 +++++--------- e2e/rust/tests/odh/tier_plan.py | 187 +++++++++++++++++ e2e/rust/tests/odh/tiers.toml | 53 +++-- odh/scripts/e2e-odh-entrypoint.sh | 192 ++++++++++++----- odh/scripts/merge-e2e-odh-reports.py | 85 ++++++++ odh/scripts/openshell-deploy-from-quay.sh | 240 ++++++++++++++++++---- odh/values-openshift-e2e.yaml | 25 +++ tasks/test-odh.toml | 25 ++- 12 files changed, 901 insertions(+), 308 deletions(-) create mode 100644 e2e/rust/tests/odh/tier_plan.py create mode 100755 odh/scripts/merge-e2e-odh-reports.py create mode 100644 odh/values-openshift-e2e.yaml diff --git a/deploy/konflux/e2e-odh/README.md b/deploy/konflux/e2e-odh/README.md index 43babcebcc..2b092abc6e 100644 --- a/deploy/konflux/e2e-odh/README.md +++ b/deploy/konflux/e2e-odh/README.md @@ -7,15 +7,32 @@ RPMs, and cluster tools prefetched by Hermeto for a network-isolated build. The archive, Cargo and nextest executables, cluster tools, the SSH client needed by sandbox lifecycle tests, and Git for repository workloads. -At runtime, the image entrypoint uses the Quay deployment script to create a -gateway on the target OpenShift cluster, runs one selected test tier, writes -JUnit and HTML reports to a mounted results directory, and tears down the -deployment. Cleanup is armed before deployment and tracks namespace creation -separately from local gateway registration, so rollout failures remove the -owned namespace and validation failures preserve local registration. Signals -are forwarded through the tier runner to nextest before teardown. Extracted -client mTLS material is stored in a mode `0700` directory with mode `0600` -files. The Rust tests consume the deployed gateway. +At runtime, the image entrypoint reads the fork-local tier plan, deploys each +phase's gateway mode through the shared Quay deployment script, runs the +phase-specific nextest selection from the precompiled archive, and tears down +that deployment before moving to the next phase. Local `mise run e2e:odh*` +tasks use the same plan, entrypoint, and deployment path. Tier 2, `odh`, and +`full` run shared mode followed by managed-workspace mode; each logical tier +runs image provenance once in the shared phase. The entrypoint preserves +cleanup on failures and signals, continues after a test-phase failure, and +merges the ordered JUnit files into one `e2e-odh-.xml` plus its HTML +companion. Phase reports remain available as +`e2e-odh--.xml`. The no-deploy option is supported only for +single-phase tiers. Extracted client mTLS material is stored in a mode `0700` +directory with mode `0600` files. + +Each deployed invocation has a DNS-safe run ID. During the temporary upstream +managed-test compatibility period, the default namespace and Helm release are +both `openshell`; the gateway registration and Route host remain run-scoped. +Do not run ODH E2E invocations concurrently against the same cluster. Explicit +resource-name overrides remain exact. The entrypoint pins phase CLI commands to +that invocation's gateway and exports the run identity for follow-up test modes. + +Gateway modes and tier phases are data in +`e2e/rust/tests/odh/tiers.toml`. Mode overlays are validated repository paths, +and the deploy script checks the deployed Helm values before the tests start. +Image repository, tag, and digest overrides can vary between local and image +runs; phase selection and gateway configuration come from the shared plan. The image also carries a fully prefetched Python 3.11 test environment and generated protobuf bindings, so later named Python test modes can execute diff --git a/e2e/rust/Cargo.toml b/e2e/rust/Cargo.toml index 77e85d214f..058302bb17 100644 --- a/e2e/rust/Cargo.toml +++ b/e2e/rust/Cargo.toml @@ -41,7 +41,7 @@ e2e-vm = ["e2e", "e2e-host-gateway"] # opendatahub-io fork: downstream-only ODH/RHOAI e2e tests (see # e2e/rust/tests/odh/ and TESTING-odh.md). Inherits e2e-kubernetes so # upstream harness improvements propagate automatically. -e2e-odh = ["e2e-kubernetes"] +e2e-odh = ["e2e-kubernetes", "e2e-kubernetes-workspace-managed"] [[test]] name = "policy_activation" diff --git a/e2e/rust/tests/odh/AGENTS.md b/e2e/rust/tests/odh/AGENTS.md index dcd376d1d2..e32e02d4e9 100644 --- a/e2e/rust/tests/odh/AGENTS.md +++ b/e2e/rust/tests/odh/AGENTS.md @@ -10,10 +10,10 @@ Before adding or changing any test here, read `README.md` in this directory. It is the source of truth for: - the directory layout and how tiers map to test modules; -- `tiers.toml` (tier → upstream `[[test]]` binaries + ODH module filter); +- `tiers.toml` (validated gateway modes and ordered tier phases); - how to run each `mise run e2e:odh:*` task and the single-test invocation; - the image-provenance check and its required env vars - (`ALLOWED_IMAGE_REGISTRY_PREFIXES`, `NAMESPACE`, `RELEASE`, `SKIP_IMAGE_PROVENANCE`); + (`ALLOWED_IMAGE_REGISTRY_PREFIXES`, `NAMESPACE`, `RELEASE`); - the `KUBECONFIG` isolation gotcha for `mise` tasks; - the Konflux e2e image, its gateway lifecycle, and report artifacts; - rebase guidance for keeping this fork-only directory additive against @@ -44,19 +44,21 @@ a rule and the README disagree, fix the drift rather than guessing. - Sign off every commit for DCO (`git commit --signoff`) and never reference AI agents in the message, per the root `AGENTS.md`. -## Write verification as Rust tests, not bash +## Keep verification in the right layer - Implement test and verification logic as Rust tests in the tier modules (`smoke/`, `tier1/`, `tier2/`, `tier3/`). Do not add bespoke bash runners to carry verification logic. +- Infrastructure contract tests for the plan resolver, deploy script, phase + entrypoint, and report merger belong in `odh/scripts/tests/`. - Shelling out to `oc`/`kubectl` from a test is fine and already established (see `smoke/image_provenance.rs`). Node-level checks such as `oc debug node ... chroot /host ausearch` work the same from Rust — put them in a shared helper (see below), not a shell script. - Cluster/deployment setup (Helm values, in-cluster fixtures, proxies) is - environment setup, not a test. Keep it out of test bodies and the tier - runner. The e2e image entrypoint may deploy and tear down the gateway; - tests still assume an already-deployed, working gateway. + environment setup, not a test. Keep it out of test bodies and the phase test + runner. The shared local/image entrypoint deploys and tears down each mode; + Rust tests still assume an already-deployed, working gateway. ## Shared helpers @@ -83,6 +85,10 @@ a rule and the README disagree, fix the drift rather than guessing. (`e2e:odh:smoke|tier1|tier2|tier3`, `e2e:odh`, and `e2e:odh:full`) stable. The OpenShift AI Shift-Left pipeline consumes those tiers as quality gates; the image entrypoint handles deployment around them. +- Route local tasks and the image through + `odh/scripts/e2e-odh-entrypoint.sh`. Extend gateway modes with validated data + in `tiers.toml`; never add executable shell fragments to the plan. Only + single-phase tiers support `OPENSHELL_E2E_DEPLOY_GATEWAY=0`. ## Gate environment-specific tests, don't fork the task diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index 0382901ebd..c4fec369c8 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -31,8 +31,9 @@ e2e/rust/tests/odh/ │ └── mod.rs # empty — no scenarios yet ├── tier3/ # Tier 3: negative and destructive tests │ └── mod.rs # empty — no scenarios yet -├── tiers.toml # tier → upstream test binaries + ODH module filter -└── run-odh-test-tier.sh # runs one tier against a deployed gateway +├── tiers.toml # data-only gateway modes and ordered tier phases +├── tier_plan.py # validates modes, overlays, and test selectors +└── run-odh-test-tier.sh # runs one resolved phase against its gateway ``` All ODH test functions compile into a single `odh` test binary @@ -83,46 +84,73 @@ duplicating logic across tier files. Two rules keep this rebase-safe: | ODH | All ODH tests | No limit | | Full | Every feature-enabled upstream and ODH test, except listed exclusions | No limit | -`tiers.toml` maps each tier to upstream Cargo test binaries (explicit or -auto-discovered) and the ODH module filter that belong to it: +`tiers.toml` keeps the familiar top-level tier tables. Shared is the default, +so each tier lists only its shared tests. A nested `[tier.managed]` table adds +the managed tests to that same tier; it does not create another tier. The +runner executes the base shared phase and then the managed extension: ```toml -[smoke] -upstream_tests = [] -odh_filter = "smoke::" -``` +[modes.shared] +expected_helm_values = { server = { drivers = { kubernetes = { workspaceMode = "shared" } } } } + +[tier2] +upstream_tests = ["workspace_lifecycle"] +odh_filter = "tier2::" -The upstream test assignments in `tiers.toml` are currently placeholders — -they should be revisited based on measured execution time and actual test -criticality, not just copied as-is. +[tier2.managed] +upstream_tests = ["workspace_namespace_managed"] +``` Smoke covers gateway reachability, sandbox lifecycle, and image provenance. Tier 1 checks the process-supervisor SELinux label and its mapped upstream -tests. Tier 2 and Tier 3 have no ODH scenarios yet; they run their mapped -upstream tests and the image provenance check. Add scenarios by creating a -`.rs` file under the tier's directory and declaring it with a `mod` line in -that tier's `mod.rs`. +tests. Tier 2 runs `workspace_lifecycle`, ODH Tier 2, and image provenance in +shared mode, then runs `workspace_namespace_managed` against a separate +managed-mode deployment. The `odh` and `full` tiers use the same shared then +managed split. Image provenance runs once in each tier's shared phase. Smoke +and Tier 1/3 use only shared mode. Add Rust scenarios by creating a `.rs` file +under the tier's directory and declaring it with a `mod` line in that tier's +`mod.rs`; add its selector to the corresponding tier table or mode extension. + +To add a Helm-overlay mode, define its validated values paths and expected +deployed values under `[modes.]`. Shared is implicit; add a named mode +table such as `[tier2.managed]` only when that tier needs mode-specific tests. +The resolver rejects unknown modes, unsafe overlay paths, invalid selectors, +and empty phases before any cluster change. Configuration is data-only; it +cannot run commands. The nextest `e2e-odh` profile runs the serial sandbox lifecycle tests and `tier1::selinux` tests in one group with `max-threads = 1`, preserving serialization across nextest's separate test processes. Tier 1 and Full include the previously quarantined sandbox lifecycle tests. -Full applies its other exact upstream exclusions from -`[full.upstream_test_exclusions]`. Excluded tests do not appear as passes or -skips in the JUnit report. +The upstream managed-workspace tests currently hard-code gateway identity +assumptions, so the entrypoint temporarily defaults the gateway namespace and +Helm release to `openshell`. The +`managed_gateway_cannot_read_or_modify_workspace_secrets` test also checks +generic Secret access although the source Secret Role is restricted to named +Secrets. `managed_supervisor_reads_client_tls_from_its_bootstrap_secret` expects +a client certificate in the supervisor bootstrap Secret, while the driver +intentionally stages only the CA certificate. Both assertions are excluded +globally until corrected upstream. Do not run ODH E2E invocations concurrently +against the same cluster during this workaround. +`tiers.toml`'s root `[global_test_exclusions]` table keeps exact known upstream +failures out of every tier and gateway phase; remove an entry when its underlying +test failure is fixed. +Full runs every other feature-enabled binary in shared mode and excludes the +managed-only binary there. It also keeps the existing exact exclusions for +`initial_sparse_policy_is_acknowledged_as_loaded`, +`sandbox_reaches_localhost_without_proxy_environment`, and `port_forward_echo`. +Excluded tests do not appear as passes or skips in the JUnit report. ## Prerequisites -- An OpenShift cluster with ODH/RHOAI installed, and the OpenShell gateway - already deployed to it (via Helm or otherwise). +- An OpenShift cluster with ODH/RHOAI installed. The normal tier tasks deploy + the selected Quay images and tear down each phase deployment. - `oc` CLI authenticated to that cluster. - Rust toolchain and `mise` installed. - Python 3.11 or newer, `cargo-nextest`, and `xsltproc` installed for tiered runs and HTML reports. -- The `openshell` CLI binary built (`cargo build -p openshell-cli`) and its - active gateway pointed at the deployed OpenShell instance (`openshell - gateway add ...` / `openshell gateway select ...`) — the harness shells out - to this binary and relies on its persisted config, not on any env var. +- The local task builds `openshell-cli` when `OPENSHELL_BIN` is unset. Set + `OPENSHELL_BIN` to use another CLI binary. ### The `KUBECONFIG` gotcha @@ -154,7 +182,7 @@ context you happen to have active elsewhere. This means: | Command | What runs | |---|---| -| `mise run e2e:odh` | All ODH-specific tests (all tiers, `odh` binary only), including image provenance | +| `mise run e2e:odh` | All ODH-specific scenarios plus the managed-workspace binary, including image provenance | | `mise run e2e:odh:full` | All upstream e2e + e2e-kubernetes + ODH tests (see caveat below), including image provenance | | `mise run e2e:odh:smoke` | Smoke tier: mapped upstream tests + ODH `smoke::` (includes image provenance) | | `mise run e2e:odh:tier1` | Tier 1: mapped upstream tests + ODH `tier1::` + image provenance | @@ -172,13 +200,30 @@ ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/" \ mise run e2e:odh:smoke ``` -`ALLOWED_IMAGE_REGISTRY_PREFIXES` is required by the image provenance test — -see below. For RHOAI images, use -`registry.redhat.io/,nvcr.io/nvidia/base/` instead. Set `NAMESPACE`/`RELEASE` -too if your deployment doesn't use the defaults (`openshell`/`openshell`). -The Quay deployment script sets -`sandbox.image.pullPolicy=IfNotPresent`; other deployments must -configure it themselves. +`IMAGE_TAG` and `ALLOWED_IMAGE_REGISTRY_PREFIXES` are required by deployment +and the image provenance test — see below. For RHOAI images, use +`registry.redhat.io/,nvcr.io/nvidia/base/` instead. The shared entrypoint +generates a DNS-safe `OPENSHELL_E2E_RUN_ID` for each deployed invocation unless +you provide one. The namespace and Helm release default to `openshell` for +both shared and managed phases to preserve managed-test compatibility; gateway +registration and Route host remain run-scoped. Do not run ODH E2E invocations +concurrently against the same cluster. Set `NAMESPACE`, `RELEASE`, +`GATEWAY_NAME`, or `ROUTE_HOST` to preserve an exact custom value. Managed tests +currently require the default `openshell` namespace and release. +`OPENSHELL_GATEWAY` is set for phase commands so they target that invocation's +gateway. Shared and managed phases in a tier reuse the same run identity and run +sequentially, with teardown between phases. The shared deployment script applies +the OpenShift SCC and Route/TLS overlays for local and image runs, then layers +the ODH overrides: `IfNotPresent` pull policies and disabled unauthenticated +dev access. User RPCs use mTLS authentication at the application layer; TLS +still permits certificate-less sandbox clients using bearer JWTs. The deploy +preflight checks that a certificate-less `ListSandboxes` RPC is rejected; +health endpoints such as `/readyz` remain unauthenticated. +Set `OPENSHELL_E2E_RUN_ID` explicitly when you need a reproducible run-scoped +name; valid values are 1–24 lowercase DNS-safe letters, digits, or hyphens, +starting and ending with a letter or digit. +Set `OPENSHELL_E2E_DEPLOY_GATEWAY=0` only to attach to a compatible existing +gateway; this option works only for single-phase tiers. ### SELinux-enforcing OCP validation @@ -201,20 +246,20 @@ SELinux-specific proxy fixture. ### Why `e2e:odh` / `e2e:odh:full` run more than you might expect -`e2e-odh` activates the upstream `e2e-kubernetes` and `e2e` features. -`run-odh-test-tier.sh` selects tests with one nextest filter per tier. -`e2e:odh` selects the `odh` binary; `e2e:odh:full` selects every test binary -enabled by those features. Both produce JUnit XML and HTML in `results/`. +`e2e-odh` activates the upstream `e2e-kubernetes`, `e2e`, and managed-workspace +features. The shared entrypoint reads the same tier plan locally and in the +image, deploying and running one phase at a time. `e2e:odh` runs all ODH +scenarios; `e2e:odh:full` runs all feature-enabled binaries except the +managed-only binary in its shared phase, followed by that binary in managed +mode. Phase JUnit files are named `e2e-odh--.xml`; the entrypoint +merges them into `e2e-odh-.xml` and the matching HTML report in +`results/`. ### Every `e2e:odh*` task runs the image provenance check -`smoke::image_provenance::test_sandbox_gateway_supervisor_images` is a -regular test in the `odh` binary. The smoke, ODH, and full filters include it. -Tier 1–3 filters include it alongside their assigned tests so one nextest run -and one JUnit report cover the whole tier. `SKIP_IMAGE_PROVENANCE=1` excludes -it from Tier 1–3 when a local deployment cannot meet the image requirements. -An empty tier fails instead of passing with an empty report. Tier 3 currently -has no scenarios, so skipping image provenance makes it fail. +`smoke::image_provenance::test_sandbox_gateway_supervisor_images` runs in the +shared phase of every logical tier. An empty phase fails instead of passing +with an empty report. ## Image provenance verification @@ -246,19 +291,21 @@ ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/" \ A prefix without a trailing `/` is rejected outright, since it could otherwise match a lookalike host (e.g. `registry.redhat.io` would also match `registry.redhat.io.attacker.example/image`). -- `NAMESPACE`/`RELEASE` env vars default to `openshell`/`openshell`. +- For deployed runs, `NAMESPACE` and `RELEASE` default to `openshell` across + shared and managed phases; this preserves managed-test compatibility. + `GATEWAY_NAME` and `ROUTE_HOST` are run-scoped. Explicit values are used + exactly as supplied; do not run concurrent invocations against the same + cluster while using the default namespace and release. - The check is a registry-prefix allowlist, not an exact image/digest match. It checks each observed image against the configured allowed prefixes. - The `imagePullPolicy: IfNotPresent` check applies to every container, - including the sandbox pod's. The Quay deployment script sets - `sandbox.image.pullPolicy=IfNotPresent`; other deployments must set it - explicitly. + including the sandbox pod's. The shared Quay deployment script sets the + gateway, supervisor, and sandbox pull policies to `IfNotPresent`; manually + attached gateways must use the same values. - Requires the `oc` CLI in PATH with a kubeconfig targeting the cluster (same as the rest of this suite). When `OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE` is set, all provenance queries use that context; otherwise they use the kubeconfig's current context. -- Skip it locally with `SKIP_IMAGE_PROVENANCE=1 mise run e2e:odh:tier1` when - testing a local deployment. Smoke, ODH, and full always include it. ## E2E container image @@ -297,30 +344,49 @@ podman run --rm \ quay.io/opendatahub/odh-openshell-e2e:odh-stable smoke ``` -By default, the entrypoint calls -`odh/scripts/openshell-deploy-from-quay.sh deploy --yes`, runs the tier, -then calls `teardown --yes` even if tests fail. The deploy script refuses to -replace an existing namespace unless +The entrypoint assigns each invocation a DNS-safe run ID. For compatibility +with managed-workspace tests, the namespace and release default to `openshell` +across both shared and managed phases; the gateway name and Route host are +run-scoped. Explicit `NAMESPACE`, `RELEASE`, `GATEWAY_NAME`, and `ROUTE_HOST` +values remain exact. Do not run concurrent invocations against the same cluster +with the default namespace and release. Concurrent invocations also need +separate `OPENSHELL_E2E_RESULTS_DIR` directories to keep their tier reports +isolated. +The entrypoint pins the phase CLI to that gateway, then deploys each declared +mode, runs that phase, and tears it down before starting the next phase; shared +and managed phases do not run in parallel. It calls teardown after deployment, +test, or signal failures, then continues to later phases after a test failure. +Managed mode creates the attachable `e2e-regcred` source Secret required by +managed image-pull Secret staging before installing the gateway. +The deploy script refuses to replace an existing namespace unless `OPENSHELL_E2E_REPLACE_EXISTING=1` is set and the namespace has the `openshell.nvidia.com/deployed-by=odh-e2e` label. Failed deployments are -cleaned up after namespace creation; validation failures preserve local -gateway registration. SIGTERM is forwarded to nextest before teardown. +cleaned up only when a state marker confirms this script created the +namespace. SIGTERM is forwarded through the phase runner before teardown. The kubeconfig must permit namespace creation, Helm deployment, and the OpenShift operations used by -the tests. Set `OPENSHELL_E2E_DEPLOY_GATEWAY=0` only when a gateway is -already deployed and configured for the CLI inside the container. +the tests. Set `OPENSHELL_E2E_DEPLOY_GATEWAY=0` only when a single-phase tier +uses an already deployed gateway configured for the CLI inside the container; +multi-phase tiers reject this option before cluster access. The environment file must set `IMAGE_TAG` for the gateway, supervisor, and sandbox images, plus `ALLOWED_IMAGE_REGISTRY_PREFIXES` for the provenance test. By default, the deployment uses the `quay.io/opendatahub/odh-openshell-*` midstream repositories. Override `QUAY_NAMESPACE`, `GATEWAY_IMAGE`, `SUPERVISOR_IMAGE`, or -`SANDBOX_IMAGE` when testing another repository. `NAMESPACE`, `RELEASE`, -`ROUTE_HOST`, and `GATEWAY_NAME` control the deployment. The image writes -`e2e-odh-.xml` and -`e2e-odh-.html` to `results/`. Set -`OPENSHELL_E2E_REPORT_NAME` to choose another basename. The JUnit report -is produced by one nextest invocation per tier. +`SANDBOX_IMAGE` when testing another repository. Image variables accept a bare +repository (which inherits `IMAGE_TAG`) or a complete tagged/digest reference. +`GATEWAY_IMAGE_DIGEST` remains available to override the gateway digest. +`NAMESPACE`, `RELEASE`, `ROUTE_HOST`, and `GATEWAY_NAME` control the deployment +and are exact when set; +use distinct values across parallel invocations. The entrypoint exports +`OPENSHELL_E2E_RUN_ID` and the derived names so future Python test modes can +reuse the same invocation identity. The image writes +phase reports named `e2e-odh--.xml`, then merges them into +`e2e-odh-.xml` and `e2e-odh-.html` in `results/`. +Missing phase reports are skipped, so reports from completed phases remain +available after a later deployment or preflight failure; the task still exits +with the failure status. Konflux automation creates the Tekton YAML files for the e2e image. diff --git a/e2e/rust/tests/odh/run-odh-test-tier.sh b/e2e/rust/tests/odh/run-odh-test-tier.sh index d767c6825a..2080554248 100755 --- a/e2e/rust/tests/odh/run-odh-test-tier.sh +++ b/e2e/rust/tests/odh/run-odh-test-tier.sh @@ -1,118 +1,76 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Run one ODH test tier against an already deployed gateway. The image -# entrypoint handles gateway setup and teardown; this script only runs tests -# and writes a JUnit XML report with an HTML companion. - +# Run exactly one validated ODH tier phase. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${SCRIPT_DIR}/../../../.." && pwd)" -TIERS_FILE="${SCRIPT_DIR}/tiers.toml" +if [[ "${OPENSHELL_BIN:-}" == */* && "${OPENSHELL_BIN}" != /* ]]; then + OPENSHELL_BIN="${ROOT}/${OPENSHELL_BIN}" + export OPENSHELL_BIN +fi +if [[ -z "${OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE:-}" ]]; then + OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE="$(oc config current-context)" + [[ -n "${OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE}" ]] || { + echo "ERROR: no active Kubernetes context for ODH tests" >&2 + exit 1 + } + export OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE +fi PYTHON_BIN="${PYTHON_BIN:-python3}" TIER="${1:-}" - -if [[ -z "${TIER}" || $# -ne 1 ]]; then - echo "Usage: $0 " >&2 +PHASE="${2:-}" +REPORT="${3:-}" +if [[ -z "${TIER}" || -z "${PHASE}" || -z "${REPORT}" || $# -ne 3 ]]; then + echo "Usage: $0 " >&2 exit 2 fi -# Build one nextest filter for the selected tier. Running every selected test -# in one nextest invocation gives Jenkins a complete report for the tier. -filter="$("${PYTHON_BIN}" - "${TIERS_FILE}" "${TIER}" "${SKIP_IMAGE_PROVENANCE:-0}" <<'PY' -import re -import sys -import tomllib - -with open(sys.argv[1], "rb") as stream: - tiers = tomllib.load(stream) - -tier = sys.argv[2] -if tier not in tiers: - sys.exit(f"unknown tier {tier!r}; available: {', '.join(sorted(tiers))}") - -config = tiers[tier] -exclusions = config.get("upstream_test_exclusions", {}) -for binary, tests in exclusions.items(): - if not re.fullmatch(r"[A-Za-z0-9_-]+", binary): - sys.exit(f"invalid upstream test binary: {binary!r}") - for test in tests: - if not re.fullmatch(r"[A-Za-z0-9_:]+", test): - sys.exit(f"invalid excluded upstream test: {test!r}") - -if config.get("odh_only"): - print("binary(=odh)") -elif config.get("all_binaries"): - excluded = " | ".join( - f"(binary(={binary}) & test(={test}))" - for binary, tests in exclusions.items() - for test in tests - ) - print(f"(all() - ({excluded}))" if excluded else "all()") -else: - terms = [] - odh_filter = config.get("odh_filter", "") - if odh_filter: - if not re.fullmatch(r"[A-Za-z0-9_:]+", odh_filter): - sys.exit(f"invalid ODH test filter: {odh_filter!r}") - terms.append(f"(binary(=odh) & test(~{odh_filter}))") - - provenance = "smoke::image_provenance::" - if sys.argv[3] != "1" and not (odh_filter and provenance.startswith(odh_filter)): - terms.append(f"(binary(=odh) & test(~{provenance}))") - - upstream_tests = config.get("upstream_tests", []) - unknown_binaries = exclusions.keys() - set(upstream_tests) - if unknown_binaries: - sys.exit(f"excluded tests reference unselected binaries: {sorted(unknown_binaries)}") - - for binary in upstream_tests: - if not re.fullmatch(r"[A-Za-z0-9_-]+", binary): - sys.exit(f"invalid upstream test binary: {binary!r}") - selection = f"binary(={binary})" - for test in exclusions.get(binary, []): - selection = f"({selection} - test(={test}))" - terms.append(selection) +resolved="$("${PYTHON_BIN}" "${SCRIPT_DIR}/tier_plan.py" "${TIER}" "${SCRIPT_DIR}/tiers.toml" "${PHASE}")" +filter="$(printf '%s' "${resolved}" | "${PYTHON_BIN}" -c 'import json,sys; print(json.load(sys.stdin)["filter"])')" +results_dir="${OPENSHELL_E2E_RESULTS_DIR:-${ROOT}/results}" +mkdir -p "${results_dir}" +results_dir="$(cd "${results_dir}" && pwd)" +export OPENSHELL_E2E_RESULTS_DIR="${results_dir}" - if not terms: - sys.exit(f"tier {tier!r} selects no tests") - print(" | ".join(terms)) -PY -)" - -name="${OPENSHELL_E2E_REPORT_NAME:-e2e-odh-${TIER}}" +report_parent="$(cd "$(dirname "${REPORT}")" 2>/dev/null && pwd)" || { + echo "ERROR: phase report parent directory does not exist" >&2 + exit 2 +} +report_name="${REPORT##*/}" +if [[ "${report_parent}" != "${results_dir}" || "${report_name}" != *.xml ]]; then + echo "ERROR: phase report must be an XML file directly inside ${results_dir}" >&2 + exit 2 +fi +name="${report_name%.xml}" +report="${results_dir}/${report_name}" if [[ ! "${name}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]*$ ]]; then echo "ERROR: invalid report name: ${name}" >&2 exit 2 fi -mkdir -p "${ROOT}/results" -junit_xml="${ROOT}/results/e2e-odh.xml" -report="${ROOT}/results/${name}.xml" +junit_xml="${results_dir}/e2e-odh.xml" rm -f "${junit_xml}" "${report}" "${report%.xml}.html" - -nextest_args=( - --profile e2e-odh - --config-file "${ROOT}/.config/nextest.toml" - --no-tests fail - -E "${filter}" -) +config_file="$(mktemp)" +trap 'rm -f "${config_file}"' EXIT +"${PYTHON_BIN}" - "${ROOT}/.config/nextest.toml" "${config_file}" "${junit_xml}" <<'PY' +import pathlib, sys +source, destination, report = map(pathlib.Path, sys.argv[1:]) +text = source.read_text() +old = 'path = "../../../../../results/e2e-odh.xml"' +if old not in text: + raise SystemExit("ODH nextest JUnit path setting not found") +destination.write_text(text.replace(old, f'path = "{report}"')) +PY +nextest_args=(--profile e2e-odh --config-file "${config_file}" --no-tests fail -E "${filter}") if [[ -n "${OPENSHELL_E2E_NEXTEST_ARCHIVE:-}" ]]; then - nextest_args+=( - --archive-file "${OPENSHELL_E2E_NEXTEST_ARCHIVE}" - --workspace-remap "${ROOT}/e2e/rust" - ) + nextest_args+=(--archive-file "${OPENSHELL_E2E_NEXTEST_ARCHIVE}" --workspace-remap "${ROOT}/e2e/rust") else - nextest_args+=( - --manifest-path "${ROOT}/e2e/rust/Cargo.toml" - --target-dir "${ROOT}/e2e/rust/target" - --features e2e-odh - ) + nextest_args+=(--manifest-path "${ROOT}/e2e/rust/Cargo.toml" --target-dir "${ROOT}/e2e/rust/target" --features e2e-odh) fi -echo "==> Running ODH tier ${TIER}: ${filter}" +echo "==> Running ODH ${TIER}/${PHASE}: ${filter}" status=0 nextest_pid="" stop_nextest() { @@ -131,23 +89,16 @@ nextest_pid=$! wait "${nextest_pid}" || status=$? nextest_pid="" trap - INT TERM - if [[ -f "${junit_xml}" ]]; then - if [[ "${report}" != "${junit_xml}" ]]; then - mv -f "${junit_xml}" "${report}" - fi - echo "JUnit report: ${report}" + mv -f "${junit_xml}" "${report}" if command -v xsltproc >/dev/null 2>&1; then - xsltproc --stringparam title "${name}" \ - "${ROOT}/scripts/junit-to-html.xsl" "${report}" \ - > "${report%.xml}.html" \ - || echo "WARNING: failed to render HTML report" >&2 + xsltproc --stringparam title "${name}" "${ROOT}/scripts/junit-to-html.xsl" "${report}" > "${report%.xml}.html" || echo "WARNING: failed to render HTML report" >&2 else echo "WARNING: xsltproc not found; HTML report unavailable" >&2 fi + printf 'JUnit report: %s\n' "${report}" else echo "ERROR: cargo-nextest did not write a JUnit report" >&2 [[ "${status}" != 0 ]] || status=1 fi - exit "${status}" diff --git a/e2e/rust/tests/odh/tier_plan.py b/e2e/rust/tests/odh/tier_plan.py new file mode 100644 index 0000000000..3f27a635e1 --- /dev/null +++ b/e2e/rust/tests/odh/tier_plan.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python3 +"""Validated, data-only resolver for downstream ODH gateway phases.""" + +from __future__ import annotations + +import json +import re +import sys +import tomllib +from pathlib import Path + +PLAN_FILE = Path(__file__).with_name("tiers.toml") +REPO_ROOT = PLAN_FILE.parents[4] +GLOBAL_EXCLUSIONS_KEY = "global_test_exclusions" +_BINARY = re.compile(r"[A-Za-z0-9_-]+\Z") +_FILTER = re.compile(r"[A-Za-z0-9_:]+\Z") + + +def _validate_mode(mode_name: str, mode: dict) -> dict: + if not isinstance(mode, dict): + raise ValueError(f"mode {mode_name!r} must be a table") + expected = mode.get("expected_helm_values") + if not isinstance(expected, dict): + raise ValueError(f"mode {mode_name!r} expected_helm_values must be a table") + overlays = mode.get("helm_values", []) + if not isinstance(overlays, list): + raise ValueError(f"mode {mode_name!r} helm_values must be an array") + for overlay in overlays: + if not isinstance(overlay, str): + raise ValueError(f"mode {mode_name!r} overlay path must be a string") + candidate = (REPO_ROOT / overlay).resolve() + if Path(overlay).is_absolute() or not candidate.is_relative_to(REPO_ROOT) or not candidate.is_file(): + raise ValueError(f"unsafe or missing Helm overlay path: {overlay!r}") + return {**mode, "helm_values": overlays} + + +def load_plan(tier: str, plan_path: Path = PLAN_FILE) -> dict: + """Load and validate a tier, including all phase and mode references.""" + plan_path = Path(plan_path).resolve() + with plan_path.open("rb") as stream: + raw = tomllib.load(stream) + modes = raw.get("modes", {}) + global_exclusions = raw.get(GLOBAL_EXCLUSIONS_KEY, {}) + tiers = { + key: value + for key, value in raw.items() + if key not in {"modes", GLOBAL_EXCLUSIONS_KEY} + } + if not isinstance(modes, dict) or not isinstance(tiers, dict): + raise ValueError("tier plan must define mode and tier tables") + if not isinstance(global_exclusions, dict) or any( + not isinstance(binary, str) or not _BINARY.fullmatch(binary) + for binary in global_exclusions + ): + raise ValueError("global_test_exclusions must map valid binary names to test arrays") + for tests in global_exclusions.values(): + if not isinstance(tests, list) or any( + not isinstance(test, str) or not _FILTER.fullmatch(test) + for test in tests + ): + raise ValueError("global_test_exclusions must map valid binary names to test arrays") + for mode_name, mode in modes.items(): + _validate_mode(mode_name, mode) + if tier not in tiers: + raise ValueError(f"unknown tier {tier!r}; available: {', '.join(sorted(tiers))}") + entry = tiers[tier] + if not isinstance(entry, dict): + raise ValueError(f"tier {tier!r} must be a table") + base_mode = entry.get("mode", "shared") + phase_tables = [(base_mode, entry)] + phase_tables.extend( + (name, config) + for name, config in entry.items() + if name in modes and name != base_mode + ) + names: set[str] = set() + resolved = [] + for phase_name, phase_config in phase_tables: + phase = phase_config + if not isinstance(phase, dict): + raise ValueError(f"phase {phase_name!r} in tier {tier!r} must be a table") + name, mode_name = phase_name, phase.get("mode", phase_name) + if not isinstance(name, str) or not _BINARY.fullmatch(name) or name in names: + raise ValueError(f"invalid or duplicate phase name: {name!r}") + names.add(name) + if not isinstance(mode_name, str) or mode_name not in modes: + raise ValueError(f"unknown mode {mode_name!r} in tier {tier!r}") + binaries = phase.get("upstream_tests", []) + exclusions = phase.get("upstream_test_exclusions", {}) + excluded_binaries = phase.get("excluded_binaries", []) + if not isinstance(excluded_binaries, list) or any(not isinstance(b, str) or not _BINARY.fullmatch(b) for b in excluded_binaries): + raise ValueError(f"invalid excluded binary in phase {name!r}") + if excluded_binaries and not phase.get("all_binaries"): + raise ValueError(f"phase {name!r} may exclude binaries only when selecting all binaries") + if not isinstance(binaries, list) or any(not isinstance(b, str) or not _BINARY.fullmatch(b) for b in binaries): + raise ValueError(f"invalid upstream test binary in phase {name!r}") + if not isinstance(exclusions, dict) or any(not isinstance(binary, str) or not _BINARY.fullmatch(binary) for binary in exclusions): + raise ValueError(f"invalid excluded test binary in phase {name!r}") + if not phase.get("all_binaries") and set(exclusions) - set(binaries): + raise ValueError(f"exclusions reference unselected binaries in phase {name!r}") + for tests in exclusions.values(): + if not isinstance(tests, list) or any(not isinstance(t, str) or not _FILTER.fullmatch(t) for t in tests): + raise ValueError(f"invalid upstream test exclusion in phase {name!r}") + odh_filter = phase.get("odh_filter", "") + if not isinstance(odh_filter, str) or (odh_filter and not _FILTER.fullmatch(odh_filter)): + raise ValueError(f"invalid ODH test filter in phase {name!r}") + for flag in ("all_binaries", "all_odh_tests", "include_image_provenance"): + if flag in phase and not isinstance(phase[flag], bool): + raise ValueError(f"phase {name!r} {flag} must be a boolean") + if not binaries and not odh_filter and not phase.get("all_binaries") and not phase.get("all_odh_tests") and not phase.get("odh_only") and not phase.get("include_image_provenance"): + raise ValueError(f"phase {name!r} selects no tests") + if "odh_only" in phase and not isinstance(phase["odh_only"], bool): + raise ValueError(f"phase {name!r} odh_only must be a boolean") + resolved.append({**phase, "name": name, "mode": mode_name, "upstream_tests": binaries, + "upstream_test_exclusions": exclusions, "odh_filter": odh_filter, + "global_test_exclusions": global_exclusions, + "all_odh_tests": phase.get("all_odh_tests", phase.get("odh_only", False)), + "include_image_provenance": phase.get("include_image_provenance", mode_name == "shared")}) + return {"tier": tier, "phases": resolved, "modes": modes} + + +def resolve_phase(plan: dict, phase_name: str) -> dict: + for phase in plan["phases"]: + if phase["name"] == phase_name: + return phase + raise ValueError(f"unknown phase {phase_name!r} for tier {plan['tier']!r}") + + +def load_mode(mode_name: str, plan_path: Path = PLAN_FILE) -> dict: + """Resolve a validated named mode for the deployment helper.""" + with Path(plan_path).open("rb") as stream: + raw = tomllib.load(stream) + modes = raw.get("modes", {}) + if not isinstance(modes, dict): + raise ValueError("tier plan must define a modes table") + if mode_name not in modes: + raise ValueError(f"unknown mode {mode_name!r}; available: {', '.join(sorted(modes))}") + return _validate_mode(mode_name, modes[mode_name]) + + +def nextest_filter(phase: dict) -> str: + terms = [] + odh_filter = phase["odh_filter"] + if odh_filter: + terms.append(f"(binary(=odh) & test(~{odh_filter}))") + elif phase.get("all_odh_tests"): + terms.append("binary(=odh)") + if phase.get("all_binaries"): + exclusions = phase.get("upstream_test_exclusions", {}) + excluded = " | ".join([*(f"binary(={binary})" for binary in phase.get("excluded_binaries", [])), + *(f"(binary(={binary}) & test(={test}))" for binary, tests in exclusions.items() for test in tests)]) + terms.append(f"(all() - ({excluded}))" if excluded else "all()") + else: + for binary in phase["upstream_tests"]: + selection = f"binary(={binary})" + for test in phase["upstream_test_exclusions"].get(binary, []): + selection = f"({selection} - test(={test}))" + terms.append(selection) + provenance = "smoke::image_provenance::" + covered = bool(odh_filter) and provenance.startswith(odh_filter) + if phase.get("include_image_provenance") and not phase.get("all_binaries") and not phase.get("all_odh_tests") and not covered: + terms.append(f"(binary(=odh) & test(~{provenance}))") + if not terms: + raise ValueError(f"phase {phase['name']!r} selects no tests") + selection = " | ".join(terms) + global_exclusions = phase.get("global_test_exclusions", {}) + excluded = " | ".join( + f"(binary(={binary}) & test(={test}))" + for binary, tests in global_exclusions.items() + for test in tests + ) + return f"({selection}) - ({excluded})" if excluded else selection + + +if __name__ == "__main__": + try: + if sys.argv[1] == "--mode": + print(json.dumps(load_mode(sys.argv[2], Path(sys.argv[3]) if len(sys.argv) > 3 else PLAN_FILE))) + raise SystemExit(0) + plan = load_plan(sys.argv[1], Path(sys.argv[2]) if len(sys.argv) > 2 else PLAN_FILE) + if len(sys.argv) > 3: + phase = resolve_phase(plan, sys.argv[3]) + print(json.dumps({"phase": phase, "filter": nextest_filter(phase)})) + else: + print(json.dumps(plan)) + except (ValueError, OSError, tomllib.TOMLDecodeError) as exc: + sys.exit(str(exc)) diff --git a/e2e/rust/tests/odh/tiers.toml b/e2e/rust/tests/odh/tiers.toml index 4784241bc0..0425b1a896 100644 --- a/e2e/rust/tests/odh/tiers.toml +++ b/e2e/rust/tests/odh/tiers.toml @@ -1,26 +1,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Maps each test tier to the upstream e2e test binaries and the ODH test -# module filter that belong to it. Read at runtime by run-odh-test-tier.sh. -# -# `upstream_tests` names explicit or auto-discovered Cargo test binaries. -# `upstream_test_exclusions` removes named tests from a selected binary. -# `odh_filter` matches module-path-prefixed test names in the `odh` binary -# (e.g. "smoke::" runs every test under tests/odh/smoke/). -# -# The upstream test assignments below are placeholders — fill them in based -# on actual test criticality and measured execution time. +# This upstream assertion expects unrestricted Secret reads in openshell, but +# managed-mode RBAC deliberately limits reads to named source Secrets. +# This assertion also expects a client certificate in the supervisor bootstrap +# Secret, but the driver intentionally stages only the CA certificate there. +# Remove each exclusion when its assertion matches the intended contract. +[global_test_exclusions] +workspace_namespace_managed = [ + "managed_gateway_cannot_read_or_modify_workspace_secrets", + "managed_supervisor_reads_client_tls_from_its_bootstrap_secret", +] + +[modes.shared] +expected_helm_values = { server = { auth = { allowUnauthenticatedUsers = false }, drivers = { kubernetes = { workspaceMode = "shared" } } } } + +[modes.managed] +helm_values = ["deploy/helm/openshell/ci/values-workspace-managed.yaml"] +expected_helm_values = { server = { auth = { allowUnauthenticatedUsers = false }, drivers = { kubernetes = { workspaceMode = "managed" } } } } [smoke] description = "Component-level critical tests (5 min target)" -# Upstream removed the standalone `smoke` test binary (commit bb704618, -# "run conformance in gateway lanes") in favor of a portable CLI conformance -# scenario run via the standalone `openshell-conformance` binary -# (`mise run e2e:cli-conformance`) — not a `cargo test --test ` target, -# so it doesn't fit this list's shape. Left empty rather than pointing at a -# nonexistent test target; the ODH `smoke::` filter below still covers this -# tier's own gateway/sandbox/image-provenance checks. +# The standalone smoke binary was removed in favor of the portable CLI +# conformance scenario. The ODH smoke filter covers this tier's own scenarios. upstream_tests = [] odh_filter = "smoke::" @@ -35,6 +37,10 @@ description = "Medium/low priority positive tests" upstream_tests = ["workspace_lifecycle"] odh_filter = "tier2::" +[tier2.managed] +description = "Managed-workspace lifecycle tests" +upstream_tests = ["workspace_namespace_managed"] + [tier3] description = "Negative and destructive tests" upstream_tests = [] @@ -42,15 +48,21 @@ odh_filter = "tier3::" [odh] description = "All ODH-specific tests" +odh_only = true upstream_tests = [] odh_filter = "" -odh_only = true + +[odh.managed] +description = "Managed-workspace tests" +upstream_tests = ["workspace_namespace_managed"] +odh_filter = "" [full] description = "All feature-enabled upstream and ODH tests except listed exclusions" upstream_tests = [] odh_filter = "" all_binaries = true +excluded_binaries = ["workspace_namespace_managed"] [full.upstream_test_exclusions] # Full OpenShift runs identified the failures and timeouts below. @@ -58,3 +70,8 @@ all_binaries = true live_policy_update = ["initial_sparse_policy_is_acknowledged_as_loaded"] no_proxy = ["sandbox_reaches_localhost_without_proxy_environment"] port_forward = ["port_forward_echo"] + +[full.managed] +description = "Managed-workspace tests" +upstream_tests = ["workspace_namespace_managed"] +odh_filter = "" diff --git a/odh/scripts/e2e-odh-entrypoint.sh b/odh/scripts/e2e-odh-entrypoint.sh index e92bacfc9d..906e7d3b40 100755 --- a/odh/scripts/e2e-odh-entrypoint.sh +++ b/odh/scripts/e2e-odh-entrypoint.sh @@ -1,58 +1,97 @@ #!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Container entrypoint for the Shift-Left ODH e2e image. Deploy the selected -# Quay images, run one test tier, and remove that deployment on every exit. - +# Shared local and image entrypoint for ODH tier phases. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" -DEPLOY_SCRIPT="${SCRIPT_DIR}/openshell-deploy-from-quay.sh" -TEST_SCRIPT="${ROOT}/e2e/rust/tests/odh/run-odh-test-tier.sh" -TIER="${1:-smoke}" - -if [[ $# -gt 1 ]]; then - echo "Usage: $0 [smoke|tier1|tier2|tier3|odh|full]" >&2 +PLAN_RESOLVER="${ROOT}/e2e/rust/tests/odh/tier_plan.py" +TIERS_FILE="${ROOT}/e2e/rust/tests/odh/tiers.toml" +DEPLOY_SCRIPT="${OPENSHELL_E2E_DEPLOY_SCRIPT:-${SCRIPT_DIR}/openshell-deploy-from-quay.sh}" +TEST_SCRIPT="${OPENSHELL_E2E_PHASE_RUNNER:-${ROOT}/e2e/rust/tests/odh/run-odh-test-tier.sh}" +MERGE_SCRIPT="${OPENSHELL_E2E_REPORT_MERGER:-${SCRIPT_DIR}/merge-e2e-odh-reports.py}" +PYTHON_BIN="${PYTHON_BIN:-python3}" +TIER="${1:-}" +if [[ -z "${TIER}" || $# -ne 1 ]]; then + echo "Usage: $0 " >&2 exit 2 fi -case "${TIER}" in - smoke|tier1|tier2|tier3|odh|full) ;; - *) echo "ERROR: unknown test tier: ${TIER}" >&2; exit 2 ;; -esac -if [[ -z "${KUBECONFIG:-}" ]]; then - KUBECONFIG="${ROOT}/.kube/config" - export KUBECONFIG +plan_json="$("${PYTHON_BIN}" "${PLAN_RESOLVER}" "${TIER}" "${TIERS_FILE}")" +phase_lines="$(printf '%s' "${plan_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(p["name"]+"|"+p["mode"] for p in json.load(sys.stdin)["phases"]))')" +phase_count=0 +while IFS= read -r _; do ((phase_count+=1)); done <<< "${phase_lines}" +deploy_enabled="${OPENSHELL_E2E_DEPLOY_GATEWAY:-1}" +if [[ "${deploy_enabled}" == 0 && "${phase_count}" -gt 1 ]]; then + echo "ERROR: OPENSHELL_E2E_DEPLOY_GATEWAY=0 only supports single-phase tiers; this tier uses multiple gateway modes" >&2 + exit 2 fi -if [[ ! -f "${KUBECONFIG}" ]]; then - echo "ERROR: kubeconfig not found at ${KUBECONFIG}" >&2 - exit 1 + +if [[ "${deploy_enabled}" != 0 ]]; then + if [[ -z "${OPENSHELL_E2E_RUN_ID:-}" ]]; then + OPENSHELL_E2E_RUN_ID="$("${PYTHON_BIN}" -c 'import secrets; print(secrets.token_hex(8))')" + fi + if [[ ! "${OPENSHELL_E2E_RUN_ID}" =~ ^[a-z0-9]([a-z0-9-]{0,22}[a-z0-9])?$ ]]; then + echo "ERROR: OPENSHELL_E2E_RUN_ID must be 1-24 lowercase DNS-safe characters" >&2 + exit 2 + fi + export OPENSHELL_E2E_RUN_ID + # Temporary compatibility with upstream managed-workspace tests, which + # currently assume the gateway namespace, release, and ID are all openshell. + NAMESPACE="${NAMESPACE:-openshell}" + RELEASE="${RELEASE:-openshell}" + GATEWAY_NAME="${GATEWAY_NAME:-openshift-${OPENSHELL_E2E_RUN_ID}}" + OPENSHELL_GATEWAY="${GATEWAY_NAME}" + # A direct endpoint takes precedence over the selected gateway in the CLI. + unset OPENSHELL_GATEWAY_ENDPOINT + export NAMESPACE RELEASE GATEWAY_NAME OPENSHELL_GATEWAY + + if [[ -z "${KUBECONFIG:-}" ]]; then + KUBECONFIG="${ROOT}/.kube/config" + export KUBECONFIG + fi + if [[ ! -f "${KUBECONFIG}" ]]; then + echo "ERROR: kubeconfig not found at ${KUBECONFIG}" >&2 + exit 1 + fi fi -cleanup_enabled=0 +status=0 +active_state="" +active_mode="" child_pid="" -deploy_state_dir="" +results_dir="${OPENSHELL_E2E_RESULTS_DIR:-${ROOT}/results}" +mkdir -p "${results_dir}" +results_dir="$(cd "${results_dir}" && pwd)" +export OPENSHELL_E2E_RESULTS_DIR="${results_dir}" +phase_reports=() +for phase_line in ${phase_lines}; do + phase="${phase_line%%|*}" + phase_report="${results_dir}/e2e-odh-${TIER}-${phase}.xml" + phase_reports+=("${phase_report}") + rm -f "${phase_report}" "${phase_report%.xml}.html" +done +merged_report="${results_dir}/e2e-odh-${TIER}.xml" +rm -f "${merged_report}" "${merged_report%.xml}.html" + cleanup() { - local status=$? - local cleanup_status=0 + local result=$? trap - EXIT - if [[ "${cleanup_enabled}" == 1 && -f "${deploy_state_dir}/namespace" ]]; then - teardown_args=(teardown --yes) - if [[ ! -f "${deploy_state_dir}/gateway" ]]; then - teardown_args+=(--keep-local-gateway) - fi - "${DEPLOY_SCRIPT}" "${teardown_args[@]}" || cleanup_status=$? - if [[ "${cleanup_status}" != 0 ]]; then - echo "ERROR: OpenShell teardown failed (${cleanup_status})" >&2 - [[ "${status}" != 0 ]] || status="${cleanup_status}" + trap '' INT TERM + if [[ -n "${active_state}" ]]; then + if [[ -f "${active_state}/namespace" ]]; then + teardown_args=(teardown --yes --mode "${active_mode}") + if [[ ! -f "${active_state}/gateway" ]]; then teardown_args+=(--keep-local-gateway); fi + "${DEPLOY_SCRIPT}" "${teardown_args[@]}" || { + cleanup_status=$? + echo "ERROR: OpenShell teardown failed (${cleanup_status})" >&2 + [[ "${result}" != 0 ]] || result="${cleanup_status}" + } fi + rm -rf "${active_state}" fi - if [[ -n "${deploy_state_dir}" ]]; then - rm -rf "${deploy_state_dir}" - fi - exit "${status}" + exit "${result}" } stop_child() { local signal="$1" code="$2" @@ -60,34 +99,79 @@ stop_child() { if [[ -n "${child_pid}" ]]; then kill -"${signal}" "${child_pid}" 2>/dev/null || true wait "${child_pid}" || true + child_pid="" fi exit "${code}" } run_child() { - local status=0 + local result=0 "$@" & child_pid=$! - wait "${child_pid}" || status=$? + wait "${child_pid}" || result=$? child_pid="" - return "${status}" + return "${result}" } - trap cleanup EXIT -# Bash background jobs inherit SIGINT ignored; use TERM to stop the child. trap 'stop_child TERM 130' INT trap 'stop_child TERM 143' TERM +phase_index=0 +for phase_line in ${phase_lines}; do + phase="${phase_line%%|*}" + mode="${phase_line#*|}" + phase_report="${phase_reports[phase_index]}" + phase_deploy_status=0 + phase_test_status=0 + teardown_failed=0 + if [[ "${deploy_enabled}" != 0 ]]; then + active_state="$(mktemp -d)" + active_mode="${mode}" + export OPENSHELL_E2E_DEPLOY_STATE_DIR="${active_state}" + deploy_args=(deploy --yes --mode "${mode}") + if [[ "${OPENSHELL_E2E_REPLACE_EXISTING:-0}" == 1 ]]; then deploy_args+=(--replace-existing); fi + run_child "${DEPLOY_SCRIPT}" "${deploy_args[@]}" || phase_deploy_status=$? + if [[ "${phase_deploy_status}" == 0 ]]; then + run_child "${OPENSHELL_BIN:-openshell}" provider profile import --from "${ROOT}/providers" --global || phase_deploy_status=$? + fi + fi + if [[ "${phase_deploy_status}" == 0 ]]; then + run_child "${TEST_SCRIPT}" "${TIER}" "${phase}" "${phase_report}" || phase_test_status=$? + else + echo "ERROR: deployment/preflight failed for ${TIER}/${phase} (${phase_deploy_status})" >&2 + fi + if [[ -n "${active_state}" ]]; then + if [[ -f "${active_state}/namespace" ]]; then + teardown_args=(teardown --yes --mode "${mode}") + if [[ ! -f "${active_state}/gateway" ]]; then teardown_args+=(--keep-local-gateway); fi + run_child "${DEPLOY_SCRIPT}" "${teardown_args[@]}" || { + teardown_status=$? + echo "ERROR: teardown failed for ${TIER}/${phase} (${teardown_status})" >&2 + [[ "${phase_deploy_status}" != 0 ]] || phase_deploy_status="${teardown_status}" + teardown_failed=1 + } + fi + if [[ "${teardown_failed}" != 1 ]]; then + rm -rf "${active_state}" + active_state="" + active_mode="" + unset OPENSHELL_E2E_DEPLOY_STATE_DIR + fi + fi + if [[ "${status}" == 0 && "${phase_deploy_status}" != 0 ]]; then status="${phase_deploy_status}"; fi + if [[ "${status}" == 0 && "${phase_test_status}" != 0 ]]; then status="${phase_test_status}"; fi + if [[ "${teardown_failed}" == 1 ]]; then break; fi + ((phase_index+=1)) +done -if [[ "${OPENSHELL_E2E_DEPLOY_GATEWAY:-1}" != 0 ]]; then - deploy_args=(deploy --yes) - if [[ "${OPENSHELL_E2E_REPLACE_EXISTING:-0}" == 1 ]]; then - deploy_args+=(--replace-existing) +run_child "${PYTHON_BIN}" "${MERGE_SCRIPT}" "${TIER}" "${phase_reports[@]}" || { + merge_status=$? + echo "ERROR: failed to merge ODH phase reports" >&2 + [[ "${status}" != 0 ]] || status="${merge_status}" +} +if [[ -f "${merged_report}" ]]; then + if command -v xsltproc >/dev/null 2>&1; then + xsltproc --stringparam title "e2e-odh-${TIER}" "${ROOT}/scripts/junit-to-html.xsl" "${merged_report}" > "${merged_report%.xml}.html" || echo "WARNING: failed to render HTML report" >&2 + else + echo "WARNING: xsltproc not found; HTML report unavailable" >&2 fi - deploy_state_dir="$(mktemp -d)" - export OPENSHELL_E2E_DEPLOY_STATE_DIR="${deploy_state_dir}" - cleanup_enabled=1 - run_child "${DEPLOY_SCRIPT}" "${deploy_args[@]}" - echo ">> Importing example provider profiles" - run_child "${OPENSHELL_BIN:-openshell}" provider profile import --from "${ROOT}/providers" --global fi - -run_child "${TEST_SCRIPT}" "${TIER}" +exit "${status}" diff --git a/odh/scripts/merge-e2e-odh-reports.py b/odh/scripts/merge-e2e-odh-reports.py new file mode 100755 index 0000000000..68226d00e5 --- /dev/null +++ b/odh/scripts/merge-e2e-odh-reports.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Merge ordered phase JUnit files into one tier report and HTML summary.""" + +from __future__ import annotations + +import os +import sys +import tempfile +import xml.etree.ElementTree as ET +from copy import deepcopy +from pathlib import Path + + +def merge(tier: str, paths: list[Path], results_dir: Path) -> Path: + if not paths: + raise ValueError("at least one phase report is required") + suites: list[ET.Element] = [] + cases: list[ET.Element] = [] + for path in paths: + if not path.is_file(): + print(f"WARNING: skipping missing phase report: {path}", file=sys.stderr) + continue + root = ET.parse(path).getroot() + if root.tag == "testsuite": + suites.append(deepcopy(root)) + elif root.tag == "testsuites": + children = [child for child in root if child.tag == "testsuite"] + if not children: + raise ValueError(f"JUnit report has no testsuite elements: {path}") + suites.extend(deepcopy(child) for child in children) + else: + raise ValueError(f"unsupported JUnit root {root.tag!r}: {path}") + if not suites: + raise ValueError("no phase reports were available to merge") + for suite in suites: + cases.extend(suite.iter("testcase")) + totals = { + "tests": len(cases), + "failures": sum(1 for case in cases for child in case if child.tag == "failure"), + "errors": sum(1 for case in cases for child in case if child.tag == "error"), + "skipped": sum(1 for case in cases for child in case if child.tag == "skipped"), + "time": f"{sum(float(case.get('time', '0')) for case in cases):.6f}", + } + root = ET.Element("testsuites", {key: str(value) for key, value in totals.items()}) + for suite in suites: + root.append(suite) + xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=True) + results_dir.mkdir(parents=True, exist_ok=True) + xml_path = results_dir / f"e2e-odh-{tier}.xml" + fd, temporary = tempfile.mkstemp(prefix=f".{xml_path.name}.", dir=results_dir) + try: + with os.fdopen(fd, "wb") as stream: + stream.write(xml_bytes) + os.replace(temporary, xml_path) + except BaseException: + try: + os.unlink(temporary) + except FileNotFoundError: + pass + raise + return xml_path + + +def main(argv: list[str]) -> int: + if len(argv) < 3: + print("Usage: merge-e2e-odh-reports.py ...", file=sys.stderr) + return 2 + tier = argv[1] + if not tier.replace("-", "").replace("_", "").isalnum(): + print(f"invalid tier name: {tier!r}", file=sys.stderr) + return 2 + results_dir = Path(os.environ.get("OPENSHELL_E2E_RESULTS_DIR", Path(__file__).resolve().parents[2] / "results")) + try: + xml_path = merge(tier, [Path(arg) for arg in argv[2:]], results_dir) + except (OSError, ET.ParseError, ValueError) as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + print(f"JUnit report: {xml_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/odh/scripts/openshell-deploy-from-quay.sh b/odh/scripts/openshell-deploy-from-quay.sh index a35246b8e7..e0fef032ab 100755 --- a/odh/scripts/openshell-deploy-from-quay.sh +++ b/odh/scripts/openshell-deploy-from-quay.sh @@ -6,7 +6,7 @@ # pushed to Quay, or remove a deployment created by this script. # # Usage: -# ./openshell-deploy-from-quay.sh [deploy] [--yes] [--replace-existing] +# ./openshell-deploy-from-quay.sh [deploy] [--yes] [--replace-existing] [--mode shared|managed] # ./openshell-deploy-from-quay.sh teardown [--yes] # # The e2e image entrypoint can run `deploy --yes`, run the tests, then call @@ -22,14 +22,15 @@ # Env overrides: # IMAGE_TAG image tag to deploy (required without Git branch metadata) # QUAY_NAMESPACE quay.io namespace (default: opendatahub) -# GATEWAY_IMAGE full gateway image repository override -# GATEWAY_IMAGE_DIGEST optional gateway digest, overriding IMAGE_TAG for Helm -# SUPERVISOR_IMAGE full supervisor image repository override -# SANDBOX_IMAGE full sandbox image repository override +# GATEWAY_IMAGE gateway image reference (bare repository uses IMAGE_TAG) +# GATEWAY_IMAGE_DIGEST optional gateway digest override +# SUPERVISOR_IMAGE supervisor image reference (bare repository uses IMAGE_TAG) +# SANDBOX_IMAGE sandbox image reference (bare repository uses IMAGE_TAG) # NAMESPACE target k8s namespace (default: openshell) # RELEASE Helm release name (default: openshell) # ROUTE_HOST gateway Route hostname (default: openshell.) # GATEWAY_NAME local CLI gateway name (default: openshift) +# OPENSHELL_E2E_RUN_ID run-scoped suffix used by the shared test entrypoint set -euo pipefail @@ -42,6 +43,7 @@ Usage: openshell-deploy-from-quay.sh [deploy|teardown] [--yes] [--replace-existi --yes Skip the confirmation prompt for automation. --replace-existing Permit --yes to replace an existing namespace during deploy. --keep-local-gateway Preserve local CLI registration during teardown. + --mode NAME Gateway profile declared in tiers.toml (deploy only). USAGE } @@ -54,6 +56,7 @@ ACTION=deploy ASSUME_YES=0 REPLACE_EXISTING=0 KEEP_LOCAL_GATEWAY=0 +MODE=shared if [[ "${1:-}" == deploy || "${1:-}" == teardown ]]; then ACTION="$1" @@ -65,6 +68,7 @@ while (( $# > 0 )); do --yes) ASSUME_YES=1 ;; --replace-existing) REPLACE_EXISTING=1 ;; --keep-local-gateway) KEEP_LOCAL_GATEWAY=1 ;; + --mode) shift; [[ $# -gt 0 ]] || fail "--mode requires a value"; MODE="$1" ;; -h|--help) usage; exit 0 ;; *) usage >&2; fail "unknown argument: $1" ;; esac @@ -89,12 +93,19 @@ confirm_action() { SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +source "${REPO_ROOT}/e2e/support/gateway-common.sh" CHART="${REPO_ROOT}/deploy/helm/openshell" +PLAN_RESOLVER="${REPO_ROOT}/e2e/rust/tests/odh/tier_plan.py" +TIERS_FILE="${REPO_ROOT}/e2e/rust/tests/odh/tiers.toml" +PYTHON_BIN="${PYTHON_BIN:-python3}" +OPEN_SHELL="${OPENSHELL_BIN:-openshell}" QUAY_NAMESPACE="${QUAY_NAMESPACE:-opendatahub}" NAMESPACE="${NAMESPACE:-openshell}" RELEASE="${RELEASE:-openshell}" GATEWAY_NAME="${GATEWAY_NAME:-openshift}" +XDG_CONFIG_DIR="${XDG_CONFIG_HOME-${HOME}/.config}" +MTLS_DIR="${XDG_CONFIG_DIR:+${XDG_CONFIG_DIR}/}openshell/gateways/${GATEWAY_NAME}/mtls" QNS="quay.io/${QUAY_NAMESPACE}" DEPLOY_LABEL="openshell.nvidia.com/deployed-by" DEPLOY_LABEL_VALUE="odh-e2e" @@ -106,12 +117,14 @@ else fi HELM_FULLNAME="${HELM_FULLNAME%-}" -for tool in oc helm openshell; do +for tool in oc helm "${OPEN_SHELL}" "${PYTHON_BIN}"; do command -v "${tool}" >/dev/null 2>&1 || fail "${tool} is required" done if [[ "${ACTION}" == deploy ]]; then command -v base64 >/dev/null 2>&1 || fail "base64 is required" + command -v curl >/dev/null 2>&1 || fail "curl is required" [[ -d "${CHART}" ]] || fail "Helm chart not found at ${CHART}" + mode_json="$("${PYTHON_BIN}" "${PLAN_RESOLVER}" --mode "${MODE}" "${TIERS_FILE}")" fi echo ">> Verifying oc session" @@ -145,7 +158,15 @@ if [[ "${ACTION}" == teardown ]]; then fi if [[ "${KEEP_LOCAL_GATEWAY}" != 1 ]]; then echo ">> Removing local gateway '${GATEWAY_NAME}'" - openshell gateway remove "${GATEWAY_NAME}" 2>/dev/null || true + "${OPEN_SHELL}" gateway remove "${GATEWAY_NAME}" 2>/dev/null || true + fi + if [[ -n "${OPENSHELL_E2E_DEPLOY_STATE_DIR:-}" ]] && + [[ -f "${OPENSHELL_E2E_DEPLOY_STATE_DIR}/mtls-created" ]]; then + rm -f -- "${MTLS_DIR}/ca.crt" "${MTLS_DIR}/tls.crt" "${MTLS_DIR}/tls.key" + rmdir -- "${MTLS_DIR}" 2>/dev/null || true + if [[ -f "${OPENSHELL_E2E_DEPLOY_STATE_DIR}/gateway-dir-created" ]]; then + rmdir -- "${MTLS_DIR%/mtls}" 2>/dev/null || true + fi fi if [[ "${teardown_status}" != 0 ]]; then fail "namespace ${NAMESPACE} could not be removed" @@ -166,26 +187,25 @@ fi GATEWAY_IMAGE="${GATEWAY_IMAGE:-${QNS}/odh-openshell-gateway}" SUPERVISOR_IMAGE="${SUPERVISOR_IMAGE:-${QNS}/odh-openshell-supervisor}" SANDBOX_IMAGE="${SANDBOX_IMAGE:-${QNS}/odh-openshell-sandbox}" -for image in "${GATEWAY_IMAGE}" "${SUPERVISOR_IMAGE}" "${SANDBOX_IMAGE}"; do - [[ "${image}" == */* ]] || fail "image repository must include a registry: ${image}" -done -GATEWAY_REGISTRY="${GATEWAY_IMAGE%%/*}" -GATEWAY_REPOSITORY="${GATEWAY_IMAGE#*/}" -SUPERVISOR_REGISTRY="${SUPERVISOR_IMAGE%%/*}" -SUPERVISOR_REPOSITORY="${SUPERVISOR_IMAGE#*/}" -SANDBOX_REGISTRY="${SANDBOX_IMAGE%%/*}" -SANDBOX_REPOSITORY="${SANDBOX_IMAGE#*/}" -IMAGE_DIGEST_ARGS=() -GATEWAY_IMAGE_REF="${GATEWAY_IMAGE}:${IMAGE_TAG}" +GATEWAY_IMAGE_REF="$(e2e_resolve_image_reference "${GATEWAY_IMAGE}" "${IMAGE_TAG}")" +SUPERVISOR_IMAGE_REF="$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE}" "${IMAGE_TAG}")" +SANDBOX_IMAGE_REF="$(e2e_resolve_image_reference "${SANDBOX_IMAGE}" "${IMAGE_TAG}")" if [[ -n "${GATEWAY_IMAGE_DIGEST:-}" ]]; then [[ "${GATEWAY_IMAGE_DIGEST}" == sha256:* ]] || fail "GATEWAY_IMAGE_DIGEST must start with sha256:" - IMAGE_DIGEST_ARGS+=(--set-string "gateway.image.digest=${GATEWAY_IMAGE_DIGEST}") - GATEWAY_IMAGE_REF="${GATEWAY_IMAGE}@${GATEWAY_IMAGE_DIGEST}" + GATEWAY_IMAGE_REF="$(e2e_image_reference_repository "${GATEWAY_IMAGE_REF}")@${GATEWAY_IMAGE_DIGEST}" fi +for image_ref in "${GATEWAY_IMAGE_REF}" "${SUPERVISOR_IMAGE_REF}" "${SANDBOX_IMAGE_REF}"; do + [[ -n "$(e2e_image_reference_registry "${image_ref}")" ]] || + fail "image reference must include a registry: ${image_ref}" +done if [[ -z "${ROUTE_HOST:-}" ]]; then APPS="$(oc get ingresses.config/cluster -o jsonpath='{.spec.domain}')" - ROUTE_HOST="openshell.${APPS}" + if [[ -n "${OPENSHELL_E2E_RUN_ID:-}" ]]; then + ROUTE_HOST="openshell-${OPENSHELL_E2E_RUN_ID}.${APPS}" + else + ROUTE_HOST="openshell.${APPS}" + fi fi echo @@ -200,8 +220,8 @@ echo " Gateway name: ${GATEWAY_NAME}" echo " Route host: ${ROUTE_HOST}" echo " Images:" echo " - ${GATEWAY_IMAGE_REF}" -echo " - ${SUPERVISOR_IMAGE}:${IMAGE_TAG}" -echo " - ${SANDBOX_IMAGE}:${IMAGE_TAG}" +echo " - ${SUPERVISOR_IMAGE_REF}" +echo " - ${SANDBOX_IMAGE_REF}" echo " Actions:" echo " 1. Replace namespace '${NAMESPACE}' if it exists" echo " 2. Create namespace" @@ -232,7 +252,7 @@ if [[ -n "${namespace_resource}" ]]; then fi echo ">> Removing local gateway '${GATEWAY_NAME}' (if present)" -openshell gateway remove "${GATEWAY_NAME}" 2>/dev/null || true +"${OPEN_SHELL}" gateway remove "${GATEWAY_NAME}" 2>/dev/null || true echo ">> Creating namespace '${NAMESPACE}'" # Apply the ownership label in the create request, so a failed label update @@ -244,36 +264,67 @@ oc create namespace "${NAMESPACE}" --dry-run=client -o yaml | oc label --local -f - "${DEPLOY_LABEL}=${DEPLOY_LABEL_VALUE}" -o yaml | oc create -f - +if [[ "${MODE}" == managed ]]; then + echo ">> Creating managed-workspace image-pull Secret" + oc -n "${NAMESPACE}" create secret docker-registry e2e-regcred \ + --docker-server=registry.example.test \ + --docker-username=e2e-user \ + --docker-password=e2e-password + oc -n "${NAMESPACE}" label secret e2e-regcred \ + openshell.ai/sandbox-attachable=true +fi + echo ">> Deploying OpenShell via Helm" +helm_value_args=() +helm_value_args+=(--values "${REPO_ROOT}/deploy/helm/openshell/ci/values-openshift-scc.yaml") +helm_value_args+=(--values "${REPO_ROOT}/deploy/helm/openshell/ci/values-openshift-e2e.yaml") +helm_value_args+=(--values "${REPO_ROOT}/odh/values-openshift-e2e.yaml") +overlay_lines="$(printf '%s' "${mode_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(json.load(sys.stdin)["helm_values"]))')" +if [[ -n "${overlay_lines}" ]]; then + while IFS= read -r value_file; do + helm_value_args+=(--values "${REPO_ROOT}/${value_file}") + done <<< "${overlay_lines}" +fi +image_helm_args=() +append_image_helm_args() { + local component="$1" image_ref="$2" value + for value in \ + "registry=$(e2e_image_reference_registry "${image_ref}")" \ + "repository=$(e2e_image_reference_repository_path "${image_ref}")" \ + "tag=$(e2e_image_reference_tag "${image_ref}")" \ + "digest=$(e2e_image_reference_digest "${image_ref}")"; do + image_helm_args+=(--set-string "${component}.image.${value}") + done +} +append_image_helm_args gateway "${GATEWAY_IMAGE_REF}" +append_image_helm_args supervisor "${SUPERVISOR_IMAGE_REF}" +append_image_helm_args sandboxRuntime "${SANDBOX_IMAGE_REF}" helm upgrade --install "${RELEASE}" "${CHART}" \ --namespace "${NAMESPACE}" --create-namespace \ - --set-string "gateway.image.registry=${GATEWAY_REGISTRY}" \ - --set-string "gateway.image.repository=${GATEWAY_REPOSITORY}" \ - --set-string "gateway.image.tag=${IMAGE_TAG}" \ - --set-string "supervisor.image.registry=${SUPERVISOR_REGISTRY}" \ - --set-string "supervisor.image.repository=${SUPERVISOR_REPOSITORY}" \ - --set-string "supervisor.image.tag=${IMAGE_TAG}" \ - --set-string "sandboxRuntime.image.registry=${SANDBOX_REGISTRY}" \ - --set-string "sandboxRuntime.image.repository=${SANDBOX_REPOSITORY}" \ - --set-string "sandboxRuntime.image.tag=${IMAGE_TAG}" \ - --set "sandbox.image.pullPolicy=IfNotPresent" \ - --set "podSecurityContext.fsGroup=null" \ - --set "securityContext.runAsUser=null" \ - --set "openshiftRoute.enabled=true" \ + "${helm_value_args[@]}" \ + "${image_helm_args[@]}" \ --set "openshiftRoute.host=${ROUTE_HOST}" \ - --set "pkiInitJob.serverDnsNames[0]=${ROUTE_HOST}" \ - --set "server.auth.allowUnauthenticatedUsers=true" \ - "${IMAGE_DIGEST_ARGS[@]}" + --set "pkiInitJob.serverDnsNames[0]=${ROUTE_HOST}" echo ">> Waiting for gateway rollout" if ! oc -n "${NAMESPACE}" rollout status "statefulset/${HELM_FULLNAME}" --timeout=300s; then oc -n "${NAMESPACE}" get pods fail "gateway did not become ready" fi +oc -n "${NAMESPACE}" wait "route/${HELM_FULLNAME}" \ + --for='jsonpath={.status.ingress[0].conditions[?(@.type=="Admitted")].status}=True' \ + --timeout=120s # Extract client mTLS materials so the CLI can reach the mandatory-mTLS gateway. -MTLS_DIR="${HOME}/.config/openshell/gateways/${GATEWAY_NAME}/mtls" echo ">> Writing client mTLS materials to ${MTLS_DIR}" +if [[ -n "${OPENSHELL_E2E_DEPLOY_STATE_DIR:-}" ]] && + [[ ! -e "${MTLS_DIR%/mtls}" && ! -L "${MTLS_DIR%/mtls}" ]]; then + : > "${OPENSHELL_E2E_DEPLOY_STATE_DIR}/gateway-dir-created" +fi +if [[ -n "${OPENSHELL_E2E_DEPLOY_STATE_DIR:-}" ]] && + [[ ! -e "${MTLS_DIR}" && ! -L "${MTLS_DIR}" ]]; then + : > "${OPENSHELL_E2E_DEPLOY_STATE_DIR}/mtls-created" +fi umask 077 mkdir -p "${MTLS_DIR}" chmod 0700 "${MTLS_DIR}" @@ -288,11 +339,116 @@ oc -n "${NAMESPACE}" get secret openshell-client-tls \ oc -n "${NAMESPACE}" get secret openshell-client-tls \ -o jsonpath='{.data.tls\.key}' | base64 -d > "${MTLS_DIR}/tls.key" +auth_check_dir="" +cleanup_auth_check() { + if [[ -n "${auth_check_dir}" ]]; then + rm -rf "${auth_check_dir}" + fi +} +trap cleanup_auth_check EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +auth_check_dir="$(mktemp -d)" +auth_check_request="${auth_check_dir}/request" +auth_check_headers="${auth_check_dir}/headers" +auth_check_status="${auth_check_dir}/status" +auth_check_error="${auth_check_dir}/error" +printf '\000\000\000\000\000' > "${auth_check_request}" + +probe_list_sandboxes() { + local curl_timeout="$1" + curl --disable --silent --show-error --noproxy '*' --http2 \ + --cacert "${MTLS_DIR}/ca.crt" \ + --output /dev/null --dump-header "${auth_check_headers}" \ + --write-out '%{http_code}' --connect-timeout 10 --max-time "${curl_timeout}" \ + --header 'content-type: application/grpc' --header 'te: trailers' \ + --data-binary "@${auth_check_request}" \ + "https://${ROUTE_HOST}/openshell.v1.OpenShell/ListSandboxes" \ + > "${auth_check_status}" 2> "${auth_check_error}" +} + + echo ">> Waiting for OpenShift Route to reject unauthenticated ListSandboxes RPCs" +route_deadline=$((SECONDS + 180)) +route_ready=0 +route_probe_error="" +while (( SECONDS < route_deadline )); do + remaining=$((route_deadline - SECONDS)) + (( remaining > 0 )) || break + curl_timeout=10 + if (( remaining < curl_timeout )); then curl_timeout="${remaining}"; fi + if probe_list_sandboxes "${curl_timeout}"; then + auth_check_http_status="$(cat "${auth_check_status}")" + route_grpc_status="" + if [[ -f "${auth_check_headers}" ]]; then + route_grpc_status="$(awk 'tolower($1) == "grpc-status:" { gsub("\\r", "", $2); status = $2 } END { print status }' "${auth_check_headers}")" + fi + if [[ "${auth_check_http_status}" == 200 ]] && + [[ "${route_grpc_status}" == 16 ]]; then + route_ready=1 + break + fi + if [[ "${auth_check_http_status}" == 503 || "${route_grpc_status}" == 14 ]]; then + route_probe_error="HTTP ${auth_check_http_status:-unknown}" + if [[ "${route_grpc_status}" == 14 ]]; then + route_probe_error+=", grpc-status 14" + fi + else + rm -rf "${auth_check_dir}" + fail "unauthenticated ListSandboxes RPC was not rejected (HTTP ${auth_check_http_status:-unknown}, grpc-status ${route_grpc_status:-missing})" + fi + else + curl_status=$? + route_probe_error="$(cat "${auth_check_error}")" + case "${curl_status}" in + 5|6|7|16|18|28|35|52|55|56|92) + route_probe_error="curl exit ${curl_status}: ${route_probe_error}" + ;; + *) + rm -rf "${auth_check_dir}" + fail "client-certificate route probe failed with curl exit ${curl_status}: ${route_probe_error}" + ;; + esac + fi + remaining=$((route_deadline - SECONDS)) + (( remaining > 0 )) || break + sleep_seconds=3 + if (( remaining < sleep_seconds )); then sleep_seconds="${remaining}"; fi + sleep "${sleep_seconds}" +done +if [[ "${route_ready}" != 1 ]]; then + rm -rf "${auth_check_dir}" + fail "OpenShift Route did not reject unauthenticated ListSandboxes RPCs within 180s: ${route_probe_error}" +fi + +rm -rf "${auth_check_dir}" + echo ">> Registering gateway '${GATEWAY_NAME}' in the local CLI" if [[ -n "${OPENSHELL_E2E_DEPLOY_STATE_DIR:-}" ]]; then : > "${OPENSHELL_E2E_DEPLOY_STATE_DIR}/gateway" fi -openshell gateway add "https://${ROUTE_HOST}" --local --name "${GATEWAY_NAME}" +"${OPEN_SHELL}" gateway add "https://${ROUTE_HOST}" --local --name "${GATEWAY_NAME}" + +echo ">> Checking deployed gateway mode '${MODE}'" +expected_values="$(printf '%s' "${mode_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(json.dumps(json.load(sys.stdin)["expected_helm_values"]))')" +helm get values "${RELEASE}" --namespace "${NAMESPACE}" --all -o json | + "${PYTHON_BIN}" -c ' +import json, sys +expected = json.loads(sys.argv[1]) +actual = json.load(sys.stdin) +def subset(want, got, path=""): + for key, value in want.items(): + current = f"{path}.{key}" if path else key + if key not in got: + raise SystemExit(f"mode preflight missing Helm value {current}") + if isinstance(value, dict): + if not isinstance(got[key], dict): + raise SystemExit(f"mode preflight mismatch for {current}") + subset(value, got[key], current) + elif got[key] != value: + raise SystemExit(f"mode preflight mismatch for {current}: expected {value!r}, got {got[key]!r}") +subset(expected, actual) +' "${expected_values}" echo echo ">> Done. Verify with:" diff --git a/odh/values-openshift-e2e.yaml b/odh/values-openshift-e2e.yaml new file mode 100644 index 0000000000..907eaa4b27 --- /dev/null +++ b/odh/values-openshift-e2e.yaml @@ -0,0 +1,25 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# ODH e2e overrides. The shared OpenShift overlay enables unauthenticated dev +# access; ODH uses the chart's default mTLS user authentication instead. +server: + auth: + allowUnauthenticatedUsers: false + +# Upstream mutable-image tests use Always; ODH provenance requires IfNotPresent. +gateway: + image: + pullPolicy: IfNotPresent + +supervisor: + image: + pullPolicy: IfNotPresent + +sandboxRuntime: + image: + pullPolicy: IfNotPresent + +sandbox: + image: + pullPolicy: IfNotPresent diff --git a/tasks/test-odh.toml b/tasks/test-odh.toml index c004776b66..d50e3b93da 100644 --- a/tasks/test-odh.toml +++ b/tasks/test-odh.toml @@ -4,32 +4,31 @@ # opendatahub-io fork: downstream-only ODH/RHOAI e2e tasks. # Fork-only file — see e2e/rust/tests/odh/README.md. # -# All tasks require oc CLI authenticated to an OpenShift cluster with -# ODH/RHOAI installed and the OpenShell gateway deployed. All tasks also -# require ALLOWED_IMAGE_REGISTRY_PREFIXES for the image provenance test -# (smoke::image_provenance, see e2e/rust/tests/odh/smoke/image_provenance.rs) -# — it runs automatically whenever the odh binary runs unfiltered. +# Normal runs require oc authenticated to an OpenShift cluster with ODH/RHOAI +# installed, IMAGE_TAG for the Quay images, and ALLOWED_IMAGE_REGISTRY_PREFIXES +# for the image provenance test. The shared entrypoint deploys and tears down +# each declared mode; see e2e/rust/tests/odh/README.md for no-deploy limits. ["e2e:odh"] -description = "Run all ODH-specific tests (all tiers, ODH binary only), including image provenance" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh odh" +description = "Run ODH scenarios and managed-workspace tests in their gateway modes" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh odh" ["e2e:odh:full"] -description = "Run all tiers (upstream e2e + e2e-kubernetes + ODH), including image provenance" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh full" +description = "Run all feature-enabled e2e binaries across shared and managed gateway modes" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh full" ["e2e:odh:smoke"] description = "Run Smoke tier: upstream + ODH (5 min target)" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh smoke" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh smoke" ["e2e:odh:tier1"] description = "Run Tier 1: upstream + ODH, high-priority (15 min target)" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh tier1" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier1" ["e2e:odh:tier2"] description = "Run Tier 2: upstream + ODH, medium/low priority positive tests" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh tier2" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier2" ["e2e:odh:tier3"] description = "Run Tier 3: upstream + ODH, negative and destructive tests" -run = "e2e/rust/tests/odh/run-odh-test-tier.sh tier3" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier3" From f557da289588a8814f3707f9fe6bdb6e1b84e62e Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Thu, 8 Oct 2026 21:27:56 +0100 Subject: [PATCH 02/26] CARRY: fix(odh): clean up managed namespaces after test exits --- e2e/rust/tests/odh/README.md | 5 +++ odh/scripts/e2e-odh-entrypoint.sh | 57 ++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index c4fec369c8..8804c1a5f8 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -363,6 +363,11 @@ The deploy script refuses to replace an existing namespace unless `openshell.nvidia.com/deployed-by=odh-e2e` label. Failed deployments are cleaned up only when a state marker confirms this script created the namespace. SIGTERM is forwarded through the phase runner before teardown. +Before a managed phase, the entrypoint snapshots namespaces labelled for the +test gateway. It removes only matching namespaces that were absent from that +snapshot, including after test failure or termination, before tearing down the +gateway namespace. This cleanup relies on the documented no-overlapping-runs +constraint until upstream managed tests support a run-specific gateway identity. The kubeconfig must permit namespace creation, Helm deployment, and the OpenShift operations used by the tests. Set `OPENSHELL_E2E_DEPLOY_GATEWAY=0` only when a single-phase tier diff --git a/odh/scripts/e2e-odh-entrypoint.sh b/odh/scripts/e2e-odh-entrypoint.sh index 906e7d3b40..091a5f690f 100755 --- a/odh/scripts/e2e-odh-entrypoint.sh +++ b/odh/scripts/e2e-odh-entrypoint.sh @@ -75,11 +75,52 @@ done merged_report="${results_dir}/e2e-odh-${TIER}.xml" rm -f "${merged_report}" "${merged_report%.xml}.html" +managed_namespace_selector='openshell.ai/managed-by=openshell,openshell.ai/gateway-id=openshell' +snapshot_managed_namespaces() { + local snapshot_file="$1" + local temporary_file="${snapshot_file}.tmp" + if ! oc get namespaces -l "${managed_namespace_selector}" -o name > "${temporary_file}"; then + rm -f -- "${temporary_file}" + return 1 + fi + mv -- "${temporary_file}" "${snapshot_file}" +} +cleanup_managed_namespaces() { + local state_dir="$1" + local baseline_file="${state_dir}/managed-namespaces-before" + local current_file="${state_dir}/managed-namespaces-current" + local resource namespace cleanup_status=0 + [[ -f "${baseline_file}" ]] || return 0 + if ! oc get namespaces -l "${managed_namespace_selector}" -o name > "${current_file}"; then + echo "ERROR: could not list managed-workspace namespaces for cleanup" >&2 + return 1 + fi + while IFS= read -r resource; do + [[ -n "${resource}" ]] || continue + if grep -Fqx -- "${resource}" "${baseline_file}"; then + continue + fi + namespace="${resource#namespace/}" + if [[ "${namespace}" == "${resource}" || -z "${namespace}" ]]; then + echo "ERROR: unexpected namespace resource from oc: ${resource}" >&2 + cleanup_status=1 + continue + fi + echo ">> Removing managed-workspace namespace '${namespace}' created during this phase" + oc delete namespace "${namespace}" --ignore-not-found --wait=false || cleanup_status=1 + done < "${current_file}" + return "${cleanup_status}" +} + cleanup() { local result=$? trap - EXIT trap '' INT TERM if [[ -n "${active_state}" ]]; then + if ! cleanup_managed_namespaces "${active_state}"; then + echo "ERROR: managed-workspace namespace cleanup failed" >&2 + [[ "${result}" != 0 ]] || result=1 + fi if [[ -f "${active_state}/namespace" ]]; then teardown_args=(teardown --yes --mode "${active_mode}") if [[ ! -f "${active_state}/gateway" ]]; then teardown_args+=(--keep-local-gateway); fi @@ -121,6 +162,7 @@ for phase_line in ${phase_lines}; do phase_report="${phase_reports[phase_index]}" phase_deploy_status=0 phase_test_status=0 + phase_managed_cleanup_status=0 teardown_failed=0 if [[ "${deploy_enabled}" != 0 ]]; then active_state="$(mktemp -d)" @@ -134,11 +176,23 @@ for phase_line in ${phase_lines}; do fi fi if [[ "${phase_deploy_status}" == 0 ]]; then - run_child "${TEST_SCRIPT}" "${TIER}" "${phase}" "${phase_report}" || phase_test_status=$? + if [[ -n "${active_state}" && "${mode}" == managed ]]; then + if ! snapshot_managed_namespaces "${active_state}/managed-namespaces-before"; then + echo "ERROR: could not snapshot managed-workspace namespaces before ${TIER}/${phase}" >&2 + phase_test_status=1 + fi + fi + if [[ "${phase_test_status}" == 0 ]]; then + run_child "${TEST_SCRIPT}" "${TIER}" "${phase}" "${phase_report}" || phase_test_status=$? + fi else echo "ERROR: deployment/preflight failed for ${TIER}/${phase} (${phase_deploy_status})" >&2 fi if [[ -n "${active_state}" ]]; then + if ! cleanup_managed_namespaces "${active_state}"; then + echo "ERROR: managed-workspace namespace cleanup failed for ${TIER}/${phase}" >&2 + phase_managed_cleanup_status=1 + fi if [[ -f "${active_state}/namespace" ]]; then teardown_args=(teardown --yes --mode "${mode}") if [[ ! -f "${active_state}/gateway" ]]; then teardown_args+=(--keep-local-gateway); fi @@ -158,6 +212,7 @@ for phase_line in ${phase_lines}; do fi if [[ "${status}" == 0 && "${phase_deploy_status}" != 0 ]]; then status="${phase_deploy_status}"; fi if [[ "${status}" == 0 && "${phase_test_status}" != 0 ]]; then status="${phase_test_status}"; fi + if [[ "${status}" == 0 && "${phase_managed_cleanup_status}" != 0 ]]; then status="${phase_managed_cleanup_status}"; fi if [[ "${teardown_failed}" == 1 ]]; then break; fi ((phase_index+=1)) done From a42a5a20739e2b370e559adc50f3912fc6b37b36 Mon Sep 17 00:00:00 2001 From: Ignas Baranauskas Date: Mon, 12 Oct 2026 09:56:32 +0100 Subject: [PATCH 03/26] CARRY: fix(odh): align gateway values and streamline e2e runner --- e2e/rust/tests/odh/tiers.toml | 4 ++-- odh/scripts/e2e-odh-entrypoint.sh | 1 + odh/values-openshift-e2e.yaml | 31 +++++++++++++------------------ tasks/test-odh.toml | 23 +++++++++++++++++------ 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/e2e/rust/tests/odh/tiers.toml b/e2e/rust/tests/odh/tiers.toml index 0425b1a896..9d2e774427 100644 --- a/e2e/rust/tests/odh/tiers.toml +++ b/e2e/rust/tests/odh/tiers.toml @@ -13,11 +13,11 @@ workspace_namespace_managed = [ ] [modes.shared] -expected_helm_values = { server = { auth = { allowUnauthenticatedUsers = false }, drivers = { kubernetes = { workspaceMode = "shared" } } } } +expected_helm_values = { gatewayConfig = { "openshell.gateway.auth" = { allow_unauthenticated_users = false }, "openshell.drivers.kubernetes" = { workspace_mode = "shared", image_pull_policy = "IfNotPresent", sandbox_runtime_image_pull_policy = "IfNotPresent", supervisor_image_pull_policy = "IfNotPresent" } } } [modes.managed] helm_values = ["deploy/helm/openshell/ci/values-workspace-managed.yaml"] -expected_helm_values = { server = { auth = { allowUnauthenticatedUsers = false }, drivers = { kubernetes = { workspaceMode = "managed" } } } } +expected_helm_values = { gatewayConfig = { "openshell.gateway.auth" = { allow_unauthenticated_users = false }, "openshell.drivers.kubernetes" = { workspace_mode = "managed", image_pull_policy = "IfNotPresent", sandbox_runtime_image_pull_policy = "IfNotPresent", supervisor_image_pull_policy = "IfNotPresent" } } } [smoke] description = "Component-level critical tests (5 min target)" diff --git a/odh/scripts/e2e-odh-entrypoint.sh b/odh/scripts/e2e-odh-entrypoint.sh index 091a5f690f..c5d5f84304 100755 --- a/odh/scripts/e2e-odh-entrypoint.sh +++ b/odh/scripts/e2e-odh-entrypoint.sh @@ -6,6 +6,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +export OPENSHELL_BIN="${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" PLAN_RESOLVER="${ROOT}/e2e/rust/tests/odh/tier_plan.py" TIERS_FILE="${ROOT}/e2e/rust/tests/odh/tiers.toml" DEPLOY_SCRIPT="${OPENSHELL_E2E_DEPLOY_SCRIPT:-${SCRIPT_DIR}/openshell-deploy-from-quay.sh}" diff --git a/odh/values-openshift-e2e.yaml b/odh/values-openshift-e2e.yaml index 907eaa4b27..71c63a0cd1 100644 --- a/odh/values-openshift-e2e.yaml +++ b/odh/values-openshift-e2e.yaml @@ -1,25 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# ODH e2e overrides. The shared OpenShift overlay enables unauthenticated dev -# access; ODH uses the chart's default mTLS user authentication instead. -server: - auth: - allowUnauthenticatedUsers: false +# ODH e2e overrides for the effective gateway configuration. The shared +# OpenShift e2e overlay uses dev auth and mutable image tags; ODH requires mTLS +# user authentication and stable image pull behavior. +gatewayConfig: + openshell.gateway.auth: + allow_unauthenticated_users: false + openshell.drivers.kubernetes: + image_pull_policy: IfNotPresent + sandbox_runtime_image_pull_policy: IfNotPresent + supervisor_image_pull_policy: IfNotPresent + workspace_mode: shared -# Upstream mutable-image tests use Always; ODH provenance requires IfNotPresent. +# Upstream mutable-image tests use Always for the gateway pod; ODH provenance +# requires IfNotPresent. gateway: image: pullPolicy: IfNotPresent - -supervisor: - image: - pullPolicy: IfNotPresent - -sandboxRuntime: - image: - pullPolicy: IfNotPresent - -sandbox: - image: - pullPolicy: IfNotPresent diff --git a/tasks/test-odh.toml b/tasks/test-odh.toml index d50e3b93da..26a735209a 100644 --- a/tasks/test-odh.toml +++ b/tasks/test-odh.toml @@ -9,26 +9,37 @@ # for the image provenance test. The shared entrypoint deploys and tears down # each declared mode; see e2e/rust/tests/odh/README.md for no-deploy limits. +["build:openshell-cli"] +description = "Build the OpenShell CLI for ODH e2e tests" +run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi" +hide = true + ["e2e:odh"] description = "Run ODH scenarios and managed-workspace tests in their gateway modes" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh odh" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh odh" ["e2e:odh:full"] description = "Run all feature-enabled e2e binaries across shared and managed gateway modes" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh full" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh full" ["e2e:odh:smoke"] description = "Run Smoke tier: upstream + ODH (5 min target)" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh smoke" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh smoke" ["e2e:odh:tier1"] description = "Run Tier 1: upstream + ODH, high-priority (15 min target)" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier1" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh tier1" ["e2e:odh:tier2"] description = "Run Tier 2: upstream + ODH, medium/low priority positive tests" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier2" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh tier2" ["e2e:odh:tier3"] description = "Run Tier 3: upstream + ODH, negative and destructive tests" -run = "if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli || exit $?; export OPENSHELL_BIN=target/debug/openshell; fi; odh/scripts/e2e-odh-entrypoint.sh tier3" +depends = ["build:openshell-cli"] +run = "odh/scripts/e2e-odh-entrypoint.sh tier3" From ca187123dd807f5e0f1cbeae64dcbf32245c296f Mon Sep 17 00:00:00 2001 From: Krzysztof Malczuk Date: Mon, 12 Oct 2026 11:14:31 +0000 Subject: [PATCH 04/26] feat(gateway): add gRPC server reflection (#3161) * feat(gateway): add gRPC server reflection Signed-off-by: Krzysztof Malczuk * fix(gateway): rate limit gRPC reflection queries Signed-off-by: Krzysztof Malczuk * fix(gateway): comply with gRPC reflection protocol Signed-off-by: Krzysztof Malczuk * fix(gateway): complete reflection symbol lookups Signed-off-by: Krzysztof Malczuk --------- Signed-off-by: Krzysztof Malczuk --- Cargo.lock | 15 + Cargo.toml | 1 + README.md | 17 + crates/openshell-server/Cargo.toml | 1 + crates/openshell-server/src/auth/oidc.rs | 10 +- crates/openshell-server/src/lib.rs | 69 +++ crates/openshell-server/src/multiplex.rs | 539 +++++++++++++++++- crates/openshell-server/src/reflection.rs | 449 +++++++++++++++ docs/how-it-works/gateways/authentication.mdx | 33 ++ docs/how-it-works/gateways/configuration.mdx | 1 + 10 files changed, 1129 insertions(+), 6 deletions(-) create mode 100644 crates/openshell-server/src/reflection.rs diff --git a/Cargo.lock b/Cargo.lock index 407213cf88..414e242b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4977,6 +4977,7 @@ dependencies = [ "toml", "tonic", "tonic-prost-build", + "tonic-reflection", "tower", "tower-http", "tracing", @@ -8276,6 +8277,20 @@ dependencies = [ "tonic-build", ] +[[package]] +name = "tonic-reflection" +version = "0.14.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acccd136a4bf19810a1fde9c74edc6129b42a66b44d0c1c8aaa67aeb49a146a7" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tokio-stream", + "tonic", + "tonic-prost", +] + [[package]] name = "tonic-types" version = "0.14.6" diff --git a/Cargo.toml b/Cargo.toml index c6136bdc26..56f9e8b58c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ tokio = { version = "1.43", features = ["full"] } # gRPC/Protobuf tonic = "0.14" tonic-types = "0.14" +tonic-reflection = "0.14" tonic-prost = "0.14" tonic-prost-build = "0.14" prost = "0.14" diff --git a/README.md b/README.md index e0aaa43641..43f887bb95 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,23 @@ The installer sets up the CLI and a local gateway. The default sandbox image is - [Tutorials](https://docs.nvidia.com/openshell/latest/tutorials/first-network-policy): step-by-step policy and agent walkthroughs. - [Prerelease and development builds](https://docs.nvidia.com/openshell/latest/about/installation#prerelease-and-development-builds): try an upcoming release or the latest commit on `main`. +### Test the gRPC API with grpcurl + +The gateway serves the gRPC reflection v1 protocol. After starting a local +plaintext gateway, use `grpcurl` without checking out or supplying the proto +files: + +```shell +grpcurl -plaintext localhost:18080 list +grpcurl -plaintext localhost:18080 describe openshell.v1.OpenShell +grpcurl -plaintext -d '{}' localhost:18080 openshell.v1.OpenShell/Health +``` + +The service list contains the public `openshell.v1.OpenShell` API. Reflection +does not advertise the gateway's internal compute-driver, credential-driver, +interceptor, or middleware services. For a TLS gateway, omit `-plaintext` and +supply the CA and client certificate options required by the deployment. + ## Agent Skills Install the public OpenShell skills for your coding agent: diff --git a/crates/openshell-server/Cargo.toml b/crates/openshell-server/Cargo.toml index 0b34c233b7..d3bed5daff 100644 --- a/crates/openshell-server/Cargo.toml +++ b/crates/openshell-server/Cargo.toml @@ -39,6 +39,7 @@ libc = "0.2" # gRPC tonic = { workspace = true, features = ["channel", "tls-native-roots"] } +tonic-reflection = { workspace = true } prost = { workspace = true } prost-reflect = { workspace = true } prost-types = { workspace = true } diff --git a/crates/openshell-server/src/auth/oidc.rs b/crates/openshell-server/src/auth/oidc.rs index eb02f1ef7d..5e876826b0 100644 --- a/crates/openshell-server/src/auth/oidc.rs +++ b/crates/openshell-server/src/auth/oidc.rs @@ -32,7 +32,8 @@ use tracing::{debug, error, info, warn}; /// These are structural bypasses for gRPC infrastructure that doesn't map to a /// single RPC method. Per-method bypasses (e.g. `Health`) are declared at the /// handler with `auth_mode: "unauthenticated"` in the proto annotation. -const UNAUTHENTICATED_PREFIXES: &[&str] = &["/grpc.reflection.", "/grpc.health."]; +const UNAUTHENTICATED_PREFIXES: &[&str] = + &[crate::multiplex::REFLECTION_PATH_PREFIX, "/grpc.health."]; /// Returns `true` if the method needs no authentication at all. pub fn is_unauthenticated_method(path: &str) -> bool { @@ -1205,10 +1206,13 @@ mod tests { #[test] fn reflection_is_unauthenticated() { assert!(is_unauthenticated_method( + "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo" + )); + assert!(!is_unauthenticated_method( "/grpc.reflection.v1alpha.ServerReflection/ServerReflectionInfo" )); - assert!(is_unauthenticated_method( - "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo" + assert!(!is_unauthenticated_method( + "/grpc.reflection.v2.ServerReflection/ServerReflectionInfo" )); } diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 8ab2c8815d..da6a74c216 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -38,6 +38,7 @@ pub(crate) mod policy_store; mod provider_profile_sources; mod provider_refresh; mod readiness; +mod reflection; mod sandbox_index; mod sandbox_watch; mod service_routing; @@ -349,6 +350,9 @@ pub struct ServerState { /// Gateway-wide gRPC request rate limiter shared by every multiplex path. pub(crate) grpc_rate_limiter: Option, + /// Immutable public reflection index and its per-query rate limiter. + pub(crate) reflection_service: reflection::GatewayReflectionServer, + /// Per-sandbox bound on extension credential minting, which resolves the /// caller's effective policy on every request. pub(crate) extension_mint_limiter: auth::extension_mint_limit::ExtensionMintLimiter, @@ -426,6 +430,8 @@ impl ServerState { compute.configure_ssh_identities(credentials.clone()); let peer_endpoint = derive_peer_endpoint(&config); let grpc_rate_limiter = multiplex::GrpcRateLimiter::from_config(&config); + let reflection_service = reflection::build_gateway_reflection_service(&config) + .expect("compiled public gateway descriptors must be valid"); let admin_role = config .oidc .as_ref() @@ -458,6 +464,7 @@ impl ServerState { compute_driver_authenticator: None, peer_authenticator: None, grpc_rate_limiter, + reflection_service, gateway_interceptors: None, provider_profile_sources: provider_profile_sources::ProviderProfileSources::with_default_sources(), @@ -2219,6 +2226,68 @@ mod tests { (listen_addr, shutdown_tx, handle, tls_dir) } + async fn start_plaintext_gateway_listener() + -> (SocketAddr, watch::Sender, tokio::task::JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind test listener"); + let listen_addr = listener.local_addr().expect("failed to read local addr"); + let state = test_state(listen_addr, false).await; + let service = MultiplexService::new(state); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let handle = tokio::spawn(serve_gateway_listener( + BoundGatewayListener { + listener, + address: listen_addr, + }, + service, + None, + false, + shutdown_rx, + )); + (listen_addr, shutdown_tx, handle) + } + + #[tokio::test] + async fn production_gateway_listener_serves_reflection() { + use tonic_reflection::pb::v1::{ + ServerReflectionRequest, server_reflection_client::ServerReflectionClient, + server_reflection_request::MessageRequest, server_reflection_response::MessageResponse, + }; + + let (addr, shutdown, handle) = start_plaintext_gateway_listener().await; + let channel = tonic::transport::Channel::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = ServerReflectionClient::new(channel); + let request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::ListServices(String::new())), + }; + let mut responses = client + .server_reflection_info(tokio_stream::iter([request])) + .await + .unwrap() + .into_inner(); + let response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::ListServicesResponse(services)) = response.message_response + else { + panic!("expected a reflection list-services response"); + }; + assert_eq!( + services + .service + .iter() + .map(|service| service.name.as_str()) + .collect::>(), + ["openshell.v1.OpenShell"] + ); + + stop_listener(shutdown, handle).await; + } + async fn send_plain_http(addr: SocketAddr, request: String) -> String { let connect_addr: SocketAddr = format!("127.0.0.1:{}", addr.port()) .parse() diff --git a/crates/openshell-server/src/multiplex.rs b/crates/openshell-server/src/multiplex.rs index a383f75c4a..d1b26220b4 100644 --- a/crates/openshell-server/src/multiplex.rs +++ b/crates/openshell-server/src/multiplex.rs @@ -215,7 +215,6 @@ macro_rules! request_id_middleware { /// the largest payload and well within this cap under normal use. const MAX_GRPC_DECODE_SIZE: usize = 1_048_576; const MAX_INTERCEPTED_GRPC_BODY_SIZE: usize = MAX_GRPC_DECODE_SIZE + 5; - /// Concurrent HTTP/2 streams allowed per connection. Sits above the /// per-replica pending relay budget so pooled peer connections are bounded by /// the relay caps rather than by the transport. @@ -288,6 +287,7 @@ impl MultiplexService { self.state.gateway_interceptors.clone(), Some(self.state.clone()), ); + let reflection = self.state.reflection_service.clone(); let authz_policy = self.state.config.oidc.as_ref().map(|oidc| AuthzPolicy { admin_role: oidc.admin_role.clone(), user_role: oidc.user_role.clone(), @@ -295,7 +295,7 @@ impl MultiplexService { }); let authenticator_chain = build_authenticator_chain(&self.state); let grpc_service = AuthGrpcRouter::with_peer_identity( - openshell, + GrpcRouter::new(openshell, reflection), authenticator_chain, authz_policy, self.state @@ -790,7 +790,7 @@ impl GrpcRateLimiter { }) } - fn allow(&self) -> bool { + pub(crate) fn allow(&self) -> bool { let now = Instant::now(); let mut state = self .state @@ -911,6 +911,56 @@ where } } +/// Combined gRPC service that routes between `OpenShell` and reflection. +#[derive(Clone)] +pub struct GrpcRouter { + openshell: N, + reflection: R, +} + +impl GrpcRouter { + fn new(openshell: N, reflection: R) -> Self { + Self { + openshell, + reflection, + } + } +} + +pub const REFLECTION_PATH_PREFIX: &str = "/grpc.reflection.v1."; + +impl tower::Service> for GrpcRouter +where + N: tower::Service> + Clone + Send + 'static, + N::Response: Send, + N::Future: Send, + N::Error: Send, + R: tower::Service, Response = N::Response, Error = N::Error> + + Clone + + Send + + 'static, + R::Future: Send, + B: Send + 'static, +{ + type Response = N::Response; + type Error = N::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, req: Request) -> Self::Future { + if req.uri().path().starts_with(REFLECTION_PATH_PREFIX) { + let mut svc = self.reflection.clone(); + Box::pin(async move { svc.ready().await?.call(req).await }) + } else { + let mut svc = self.openshell.clone(); + Box::pin(async move { svc.ready().await?.call(req).await }) + } + } +} + /// Assemble the authenticator chain for the gateway. /// /// Chain order (first-match-wins): @@ -2653,6 +2703,464 @@ mod tests { assert_eq!(grpc_method_from_path(""), ""); } + #[tokio::test] + async fn grpc_router_dispatches_gateway_and_reflection_paths() { + #[derive(Clone)] + struct RouteRecorder { + name: &'static str, + calls: Arc>>, + } + + impl Service> for RouteRecorder { + type Response = Response; + type Error = Infallible; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, _req: Request) -> Self::Future { + self.calls.lock().unwrap().push(self.name); + Box::pin(async { Ok(Response::new(tonic::body::Body::empty())) }) + } + } + + let calls = Arc::new(Mutex::new(Vec::new())); + let service = |name| RouteRecorder { + name, + calls: calls.clone(), + }; + let mut router = GrpcRouter::new(service("openshell"), service("reflection")); + + for path in [ + "/openshell.v1.OpenShell/Health", + "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", + ] { + router + .call( + Request::builder() + .uri(path) + .body(Empty::::new()) + .unwrap(), + ) + .await + .unwrap(); + } + + assert_eq!(*calls.lock().unwrap(), vec!["openshell", "reflection"]); + } + + #[tokio::test] + async fn reflection_protocol_serves_complete_public_descriptors_and_recovers_from_errors() { + use crate::auth::authenticator::test_support::MockAuthenticator; + use prost_reflect::{DescriptorPool, Value}; + use tonic_reflection::pb::v1::{ + ServerReflectionRequest, server_reflection_client::ServerReflectionClient, + server_reflection_request::MessageRequest, server_reflection_response::MessageResponse, + }; + + let reflection = + crate::reflection::build_gateway_reflection_service(&Config::new(None)).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let unrouted = tower::service_fn(|_request: Request| async { + Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http()) + }); + let rejecting_oidc = Arc::new(MockAuthenticator::returning(Err( + tonic::Status::unauthenticated("OIDC credentials required"), + ))); + let grpc = AuthGrpcRouter::with_peer_identity( + GrpcRouter::new(unrouted, reflection), + Some(AuthenticatorChain::new(vec![rejecting_oidc])), + None, + None, + true, + false, + ); + let service = MultiplexedService::new(grpc, unrouted); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let service = service.clone(); + tokio::spawn(async move { + Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await + .unwrap(); + }); + } + }); + + let channel = tonic::transport::Channel::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = ServerReflectionClient::new(channel); + let list_request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::ListServices(String::new())), + }; + let descriptor_request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::FileContainingSymbol( + "openshell.v1.OpenShell".to_string(), + )), + }; + let missing_request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::FileContainingSymbol( + "openshell.v1.DoesNotExist".to_string(), + )), + }; + let list_after_error_request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::ListServices(String::new())), + }; + let mut responses = client + .server_reflection_info(tokio_stream::iter([ + list_request, + descriptor_request, + missing_request, + list_after_error_request, + ])) + .await + .unwrap() + .into_inner(); + let response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::ListServicesResponse(response)) = response.message_response + else { + panic!("expected a reflection list-services response"); + }; + let mut names: Vec<_> = response + .service + .into_iter() + .map(|service| service.name) + .collect(); + names.sort(); + + assert_eq!(names, vec!["openshell.v1.OpenShell"]); + + let descriptor_response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::FileDescriptorResponse(response)) = + descriptor_response.message_response + else { + panic!("expected a reflection file-descriptor response"); + }; + let mut reflected_pool = DescriptorPool::new(); + for descriptor in &response.file_descriptor_proto { + reflected_pool + .decode_file_descriptor_proto(descriptor.as_slice()) + .unwrap(); + } + assert!( + reflected_pool + .get_service_by_name("openshell.v1.OpenShell") + .is_some(), + "a fresh descriptor pool must resolve the advertised service" + ); + assert!( + reflected_pool + .get_message_by_name("openshell.v1.HealthRequest") + .is_some(), + "the response must include imported public message descriptors" + ); + + let source_pool = DescriptorPool::decode(openshell_core::FILE_DESCRIPTOR_SET).unwrap(); + let source_auth = source_pool + .get_extension_by_name("openshell.options.v1.authorization") + .unwrap(); + let reflected_auth = reflected_pool + .get_extension_by_name("openshell.options.v1.authorization") + .unwrap(); + let source_health = source_pool + .get_service_by_name("openshell.v1.OpenShell") + .unwrap() + .methods() + .find(|method| method.name() == "Health") + .unwrap(); + let reflected_health = reflected_pool + .get_service_by_name("openshell.v1.OpenShell") + .unwrap() + .methods() + .find(|method| method.name() == "Health") + .unwrap(); + let source_options = source_health.options(); + let reflected_options = reflected_health.options(); + let Value::Message(source_auth_value) = &*source_options.get_extension(&source_auth) else { + panic!("source authorization option must be a message"); + }; + let Value::Message(reflected_auth_value) = + &*reflected_options.get_extension(&reflected_auth) + else { + panic!("reflected authorization option must be a message"); + }; + assert_eq!( + source_auth_value.encode_to_vec(), + reflected_auth_value.encode_to_vec() + ); + + let source_secret = source_pool + .get_extension_by_name("openshell.options.v1.secret") + .unwrap(); + let reflected_secret = reflected_pool + .get_extension_by_name("openshell.options.v1.secret") + .unwrap(); + let source_field = source_pool + .get_message_by_name("openshell.v1.TcpForwardInit") + .unwrap() + .get_field_by_name("authorization_token") + .unwrap(); + let reflected_field = reflected_pool + .get_message_by_name("openshell.v1.TcpForwardInit") + .unwrap() + .get_field_by_name("authorization_token") + .unwrap(); + assert_eq!( + source_field.options().get_extension(&source_secret), + reflected_field.options().get_extension(&reflected_secret), + "secret-field annotations must retain their original value" + ); + + let error_response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::ErrorResponse(error)) = error_response.message_response else { + panic!("expected an in-band reflection error response"); + }; + assert_eq!(error.error_code, tonic::Code::NotFound as i32); + + let response_after_error = responses.message().await.unwrap().unwrap(); + assert!(matches!( + response_after_error.message_response, + Some(MessageResponse::ListServicesResponse(_)) + )); + server.abort(); + } + + #[tokio::test] + async fn reflection_protocol_resolves_canonical_enum_values_and_extensions() { + use prost::Message; + use tonic_reflection::pb::v1::{ + ExtensionRequest, ServerReflectionRequest, + server_reflection_client::ServerReflectionClient, + server_reflection_request::MessageRequest, server_reflection_response::MessageResponse, + }; + + let reflection = + crate::reflection::build_gateway_reflection_service(&Config::new(None)).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let unrouted = tower::service_fn(|_request: Request| async { + Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http()) + }); + let service = MultiplexedService::new(GrpcRouter::new(unrouted, reflection), unrouted); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let service = service.clone(); + tokio::spawn(async move { + Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await + .unwrap(); + }); + } + }); + + let channel = tonic::transport::Channel::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = ServerReflectionClient::new(channel); + + for (symbol, expected_file) in [ + ("openshell.v1.SANDBOX_PHASE_READY", "openshell.proto"), + ( + "google.protobuf.FieldDescriptorProto.TYPE_DOUBLE", + "google/protobuf/descriptor.proto", + ), + ] { + let request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::FileContainingSymbol(symbol.to_string())), + }; + let mut responses = client + .server_reflection_info(tokio_stream::iter([request])) + .await + .unwrap() + .into_inner(); + let response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::FileDescriptorResponse(response)) = response.message_response + else { + panic!("expected a descriptor response for canonical enum value {symbol}"); + }; + let names: std::collections::BTreeSet<_> = response + .file_descriptor_proto + .iter() + .map(|descriptor| { + prost_types::FileDescriptorProto::decode(descriptor.as_slice()) + .unwrap() + .name + .unwrap() + }) + .collect(); + assert!(names.contains(expected_file)); + } + + for (containing_type, extension_number) in [ + ("google.protobuf.MethodOptions", 50_000), + ("google.protobuf.FieldOptions", 50_001), + ] { + let request = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::FileContainingExtension(ExtensionRequest { + containing_type: containing_type.to_string(), + extension_number, + })), + }; + let mut responses = client + .server_reflection_info(tokio_stream::iter([request])) + .await + .unwrap() + .into_inner(); + let response = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::FileDescriptorResponse(response)) = response.message_response + else { + panic!("expected a descriptor response for extension {extension_number}"); + }; + let names: std::collections::BTreeSet<_> = response + .file_descriptor_proto + .iter() + .map(|descriptor| { + prost_types::FileDescriptorProto::decode(descriptor.as_slice()) + .unwrap() + .name + .unwrap() + }) + .collect(); + assert!(names.contains("options.proto")); + } + + let requests = [ + ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::AllExtensionNumbersOfType( + "google.protobuf.MethodOptions".to_string(), + )), + }, + ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::FileContainingExtension(ExtensionRequest { + containing_type: "google.protobuf.MethodOptions".to_string(), + extension_number: 49_999, + })), + }, + ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::ListServices(String::new())), + }, + ]; + let mut responses = client + .server_reflection_info(tokio_stream::iter(requests)) + .await + .unwrap() + .into_inner(); + let extension_numbers = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::AllExtensionNumbersResponse(extension_numbers)) = + extension_numbers.message_response + else { + panic!("expected an extension-number response"); + }; + assert_eq!( + extension_numbers.base_type_name, + "google.protobuf.MethodOptions" + ); + assert_eq!(extension_numbers.extension_number, [50_000]); + + let unknown_extension = responses.message().await.unwrap().unwrap(); + let Some(MessageResponse::ErrorResponse(error)) = unknown_extension.message_response else { + panic!("expected an in-band error for an unknown extension"); + }; + assert_eq!(error.error_code, tonic::Code::NotFound as i32); + assert!(matches!( + responses.message().await.unwrap().unwrap().message_response, + Some(MessageResponse::ListServicesResponse(_)) + )); + + server.abort(); + } + + #[tokio::test] + async fn reflection_rate_limit_charges_each_query_on_one_stream() { + use tonic::Code; + use tonic_reflection::pb::v1::{ + ServerReflectionRequest, server_reflection_client::ServerReflectionClient, + server_reflection_request::MessageRequest, + }; + + let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60)); + let reflection = crate::reflection::build_gateway_reflection_service(&config).unwrap(); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let unrouted = tower::service_fn(|_request: Request| async { + Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http()) + }); + let service = MultiplexedService::new(GrpcRouter::new(unrouted, reflection), unrouted); + let server = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let service = service.clone(); + tokio::spawn(async move { + Builder::new(TokioExecutor::new()) + .serve_connection(TokioIo::new(stream), service) + .await + .unwrap(); + }); + } + }); + + let channel = tonic::transport::Channel::from_shared(format!("http://{addr}")) + .unwrap() + .connect() + .await + .unwrap(); + let mut client = ServerReflectionClient::new(channel); + let query = ServerReflectionRequest { + host: String::new(), + message_request: Some(MessageRequest::ListServices(String::new())), + }; + let mut responses = client + .server_reflection_info(tokio_stream::iter([query.clone(), query])) + .await + .unwrap() + .into_inner(); + + assert!(responses.message().await.unwrap().is_some()); + let status = responses + .message() + .await + .expect_err("second query on the same stream must be rate limited"); + assert_eq!(status.code(), Code::ResourceExhausted); + server.abort(); + } + + #[test] + fn reflection_descriptor_excludes_internal_service_protos() { + let descriptors = crate::reflection::gateway_reflection_descriptors().unwrap(); + let names: std::collections::BTreeSet<_> = descriptors + .iter() + .map(prost_reflect::FileDescriptor::name) + .collect(); + + assert!(names.contains("openshell.proto")); + assert!(names.contains("sandbox.proto")); + assert!(!names.contains("compute_driver.proto")); + assert!(!names.contains("credential_driver.proto")); + assert!(!names.contains("gateway_interceptor.proto")); + assert!(!names.contains("supervisor_middleware.proto")); + } + #[test] fn normalize_ws_tunnel() { assert_eq!(normalize_http_path("/_ws_tunnel"), "/_ws_tunnel"); @@ -2988,6 +3496,31 @@ mod tests { assert_eq!(grpc_status(&res).as_deref(), Some("16")); } + #[tokio::test] + async fn reflection_bypasses_oidc_and_mtls_user_authentication() { + let oidc = Arc::new(MockAuthenticator::returning(Err( + tonic::Status::unauthenticated("OIDC credentials required"), + ))); + let chain = AuthenticatorChain::new(vec![oidc]); + let (recorder, seen) = PrincipalRecorder::new(); + let mut router = + AuthGrpcRouter::with_peer_identity(recorder, Some(chain), None, None, true, false); + + let res = router + .call(empty_request( + "/grpc.reflection.v1.ServerReflection/ServerReflectionInfo", + )) + .await + .unwrap(); + + assert_eq!(res.status(), 200); + assert_eq!(grpc_status(&res), None); + assert!( + seen.lock().unwrap().is_none(), + "reflection must not receive an authenticated user principal" + ); + } + #[tokio::test] async fn unauthenticated_dev_user_fills_missing_principal_when_enabled() { let mock = Arc::new(MockAuthenticator::returning(Ok(None))); diff --git a/crates/openshell-server/src/reflection.rs b/crates/openshell-server/src/reflection.rs new file mode 100644 index 0000000000..140b53659e --- /dev/null +++ b/crates/openshell-server/src/reflection.rs @@ -0,0 +1,449 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Public gateway gRPC reflection service. + +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +use prost_reflect::{DescriptorError, DescriptorPool, FileDescriptor}; +use prost_types::{ + DescriptorProto, EnumDescriptorProto, FieldDescriptorProto, FileDescriptorProto, +}; +use tokio::sync::mpsc; +use tokio_stream::{Stream, StreamExt}; +use tonic::{Request, Response, Status, Streaming}; +use tonic_reflection::pb::v1::server_reflection_request::MessageRequest; +use tonic_reflection::pb::v1::server_reflection_response::MessageResponse; +use tonic_reflection::pb::v1::server_reflection_server::{ + ServerReflection, ServerReflectionServer, +}; +use tonic_reflection::pb::v1::{ + ErrorResponse, ExtensionNumberResponse, FileDescriptorResponse, ListServiceResponse, + ServerReflectionRequest, ServerReflectionResponse, ServiceResponse, +}; + +use openshell_core::Config; + +use crate::multiplex::GrpcRateLimiter; + +const REFLECTED_PROTO_ROOTS: &[&str] = &["openshell.proto"]; +const ADVERTISED_SERVICES: &[&str] = &["openshell.v1.OpenShell"]; + +pub type GatewayReflectionServer = ServerReflectionServer; + +/// Decode and filter the compiled descriptors to the public gateway schema. +pub fn gateway_reflection_descriptors() -> Result, DescriptorError> { + let descriptor_pool = DescriptorPool::decode(openshell_core::FILE_DESCRIPTOR_SET)?; + let mut included: BTreeSet = REFLECTED_PROTO_ROOTS + .iter() + .map(|name| (*name).to_string()) + .collect(); + + loop { + let before = included.len(); + for file in descriptor_pool.files() { + if included.contains(file.name()) { + included.extend( + file.dependencies() + .map(|dependency| dependency.name().to_string()), + ); + } + } + if included.len() == before { + break; + } + } + + Ok(descriptor_pool + .files() + .filter(|file| included.contains(file.name())) + .collect()) +} + +/// Build the immutable reflection index once at gateway service startup. +pub fn build_gateway_reflection_service( + config: &Config, +) -> Result { + let mut descriptors = gateway_reflection_descriptors()?; + descriptors + .extend(DescriptorPool::decode(tonic_reflection::pb::v1::FILE_DESCRIPTOR_SET)?.files()); + + let state = ReflectionState::new(descriptors); + Ok(ServerReflectionServer::new(GatewayReflectionService { + state: Arc::new(state), + limiter: GrpcRateLimiter::from_config(config), + })) +} + +#[derive(Debug)] +struct ReflectionState { + files: HashMap>, + symbols: HashMap>, + extensions: HashMap<(String, i32), Arc>, + extension_numbers: HashMap>, +} + +#[derive(Debug)] +struct ReflectionFile { + descriptor: FileDescriptorProto, + encoded: Vec, +} + +impl ReflectionState { + fn new(descriptors: Vec) -> Self { + let mut state = Self { + files: HashMap::new(), + symbols: HashMap::new(), + extensions: HashMap::new(), + extension_numbers: HashMap::new(), + }; + for descriptor in descriptors { + let name = descriptor.name().to_string(); + let descriptor = Arc::new(ReflectionFile { + descriptor: descriptor.file_descriptor_proto().clone(), + encoded: descriptor.encode_to_vec(), + }); + state.process_file(descriptor.clone()); + state.files.insert(name, descriptor); + } + state + } + + fn process_file(&mut self, file: Arc) { + let prefix = file + .descriptor + .package + .as_deref() + .unwrap_or_default() + .to_string(); + for message in &file.descriptor.message_type { + self.process_message(file.clone(), &prefix, message); + } + for enumeration in &file.descriptor.enum_type { + self.process_enum(file.clone(), &prefix, enumeration); + } + for extension in &file.descriptor.extension { + self.process_extension(file.clone(), &prefix, extension); + } + for service in &file.descriptor.service { + let Some(name) = service.name.as_deref() else { + continue; + }; + let service_name = qualified_name(&prefix, name); + self.symbols.insert(service_name.clone(), file.clone()); + for method in &service.method { + if let Some(name) = method.name.as_deref() { + self.symbols + .insert(qualified_name(&service_name, name), file.clone()); + } + } + } + } + + fn process_message( + &mut self, + file: Arc, + prefix: &str, + message: &DescriptorProto, + ) { + let Some(name) = message.name.as_deref() else { + return; + }; + let message_name = qualified_name(prefix, name); + self.symbols.insert(message_name.clone(), file.clone()); + for nested in &message.nested_type { + self.process_message(file.clone(), &message_name, nested); + } + for enumeration in &message.enum_type { + self.process_enum(file.clone(), &message_name, enumeration); + } + for extension in &message.extension { + self.process_extension(file.clone(), &message_name, extension); + } + for field in &message.field { + if let Some(name) = field.name.as_deref() { + self.symbols + .insert(qualified_name(&message_name, name), file.clone()); + } + } + for oneof in &message.oneof_decl { + if let Some(name) = oneof.name.as_deref() { + self.symbols + .insert(qualified_name(&message_name, name), file.clone()); + } + } + } + + fn process_enum( + &mut self, + file: Arc, + prefix: &str, + enumeration: &EnumDescriptorProto, + ) { + let Some(name) = enumeration.name.as_deref() else { + return; + }; + let enum_name = qualified_name(prefix, name); + self.symbols.insert(enum_name, file.clone()); + for value in &enumeration.value { + if let Some(name) = value.name.as_deref() { + self.symbols + .insert(qualified_name(prefix, name), file.clone()); + } + } + } + + fn process_extension( + &mut self, + file: Arc, + prefix: &str, + extension: &FieldDescriptorProto, + ) { + if let Some(name) = extension.name.as_deref() { + self.symbols + .insert(qualified_name(prefix, name), file.clone()); + } + + let (Some(containing_type), Some(number)) = + (extension.extendee.as_deref(), extension.number) + else { + return; + }; + let containing_type = normalize_type_name(containing_type).to_string(); + self.extensions + .insert((containing_type.clone(), number), file); + self.extension_numbers + .entry(containing_type) + .or_default() + .insert(number); + } + + fn file_by_name(&self, name: &str) -> Result, Status> { + self.files + .get(name) + .cloned() + .ok_or_else(|| Status::not_found(format!("file '{name}' not found"))) + } + + fn file_by_symbol(&self, symbol: &str) -> Result, Status> { + self.symbols + .get(symbol) + .cloned() + .ok_or_else(|| Status::not_found(format!("symbol '{symbol}' not found"))) + } + + fn file_by_extension( + &self, + containing_type: &str, + extension_number: i32, + ) -> Result, Status> { + let containing_type = normalize_type_name(containing_type); + self.extensions + .get(&(containing_type.to_string(), extension_number)) + .cloned() + .ok_or_else(|| { + Status::not_found(format!( + "extension '{containing_type}:{extension_number}' not found" + )) + }) + } + + fn all_extension_numbers(&self, containing_type: &str) -> Result, Status> { + let containing_type = normalize_type_name(containing_type); + if !self.symbols.contains_key(containing_type) { + return Err(Status::not_found(format!( + "type '{containing_type}' not found" + ))); + } + Ok(self + .extension_numbers + .get(containing_type) + .map_or_else(Vec::new, |numbers| numbers.iter().copied().collect())) + } + + fn file_with_dependencies( + &self, + file: Arc, + sent: &mut BTreeSet, + ) -> Result>, Status> { + let mut descriptors = Vec::new(); + self.collect_file_with_dependencies(file, sent, &mut descriptors)?; + Ok(descriptors) + } + + fn collect_file_with_dependencies( + &self, + file: Arc, + sent: &mut BTreeSet, + descriptors: &mut Vec>, + ) -> Result<(), Status> { + let name = file + .descriptor + .name + .as_deref() + .ok_or_else(|| Status::internal("reflection descriptor is missing its filename"))?; + if !sent.insert(name.to_string()) { + return Ok(()); + } + + for dependency in &file.descriptor.dependency { + let dependency = self.files.get(dependency).cloned().ok_or_else(|| { + Status::internal(format!( + "reflection descriptor '{name}' has unavailable dependency '{dependency}'" + )) + })?; + self.collect_file_with_dependencies(dependency, sent, descriptors)?; + } + descriptors.push(file.encoded.clone()); + Ok(()) + } +} + +fn qualified_name(prefix: &str, name: &str) -> String { + if prefix.is_empty() { + name.to_string() + } else { + format!("{prefix}.{name}") + } +} + +fn normalize_type_name(name: &str) -> &str { + name.strip_prefix('.').unwrap_or(name) +} + +/// Reflection implementation with a quota charged for every stream message. +#[derive(Clone, Debug)] +pub struct GatewayReflectionService { + state: Arc, + limiter: Option, +} + +#[tonic::async_trait] +impl ServerReflection for GatewayReflectionService { + type ServerReflectionInfoStream = ReflectionResponseStream; + + async fn server_reflection_info( + &self, + request: Request>, + ) -> Result, Status> { + let mut requests = request.into_inner(); + let (responses_tx, responses_rx) = mpsc::channel(1); + let state = self.state.clone(); + let limiter = self.limiter.clone(); + + tokio::spawn(async move { + let mut sent_descriptors = BTreeSet::new(); + while let Some(request) = requests.next().await { + let Ok(request) = request else { + return; + }; + if limiter.as_ref().is_some_and(|limiter| !limiter.allow()) { + let _ = responses_tx + .send(Err(Status::resource_exhausted( + "gRPC reflection query rate limit exceeded", + ))) + .await; + return; + } + + let response = match request.message_request.as_ref() { + Some(MessageRequest::FileByFilename(name)) => state + .file_by_name(name) + .and_then(|descriptor| { + state.file_with_dependencies(descriptor, &mut sent_descriptors) + }) + .map(|descriptors| { + MessageResponse::FileDescriptorResponse(FileDescriptorResponse { + file_descriptor_proto: descriptors, + }) + }), + Some(MessageRequest::FileContainingSymbol(symbol)) => state + .file_by_symbol(symbol) + .and_then(|descriptor| { + state.file_with_dependencies(descriptor, &mut sent_descriptors) + }) + .map(|descriptors| { + MessageResponse::FileDescriptorResponse(FileDescriptorResponse { + file_descriptor_proto: descriptors, + }) + }), + Some(MessageRequest::FileContainingExtension(extension)) => state + .file_by_extension(&extension.containing_type, extension.extension_number) + .and_then(|descriptor| { + state.file_with_dependencies(descriptor, &mut sent_descriptors) + }) + .map(|descriptors| { + MessageResponse::FileDescriptorResponse(FileDescriptorResponse { + file_descriptor_proto: descriptors, + }) + }), + Some(MessageRequest::AllExtensionNumbersOfType(containing_type)) => state + .all_extension_numbers(containing_type) + .map(|extension_number| { + MessageResponse::AllExtensionNumbersResponse(ExtensionNumberResponse { + base_type_name: normalize_type_name(containing_type).to_string(), + extension_number, + }) + }), + Some(MessageRequest::ListServices(_)) => { + Ok(MessageResponse::ListServicesResponse(ListServiceResponse { + service: ADVERTISED_SERVICES + .iter() + .map(|name| ServiceResponse { + name: (*name).to_string(), + }) + .collect(), + })) + } + None => Err(Status::invalid_argument("invalid MessageRequest")), + }; + + match response { + Ok(message_response) => { + let response = ServerReflectionResponse { + valid_host: request.host.clone(), + original_request: Some(request), + message_response: Some(message_response), + }; + if responses_tx.send(Ok(response)).await.is_err() { + return; + } + } + Err(status) => { + let response = ServerReflectionResponse { + valid_host: request.host.clone(), + original_request: Some(request), + message_response: Some(MessageResponse::ErrorResponse(ErrorResponse { + error_code: status.code() as i32, + error_message: status.message().to_string(), + })), + }; + if responses_tx.send(Ok(response)).await.is_err() { + return; + } + } + } + } + }); + + Ok(Response::new(ReflectionResponseStream { + inner: tokio_stream::wrappers::ReceiverStream::new(responses_rx), + })) + } +} + +pub struct ReflectionResponseStream { + inner: tokio_stream::wrappers::ReceiverStream>, +} + +impl Stream for ReflectionResponseStream { + type Item = Result; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::pin::Pin::new(&mut self.inner).poll_next(cx) + } +} diff --git a/docs/how-it-works/gateways/authentication.mdx b/docs/how-it-works/gateways/authentication.mdx index 6b05e42be9..b764d3fbfe 100644 --- a/docs/how-it-works/gateways/authentication.mdx +++ b/docs/how-it-works/gateways/authentication.mdx @@ -95,6 +95,39 @@ setting. See [Providers](/how-it-works/providers/overview#retrieve-runtime-credentials) for the credential retrieval contract. +### Inspect the API with grpcurl + +The primary gateway listener serves the gRPC reflection v1 protocol. Reflection +does not require application authentication, but the listener's TLS and client +certificate requirements still apply. + +For a local plaintext development gateway: + +```shell +grpcurl -plaintext localhost:18080 list +grpcurl -plaintext localhost:18080 describe openshell.v1.OpenShell +grpcurl -plaintext -d '{}' localhost:18080 openshell.v1.OpenShell/Health +``` + +For an mTLS gateway, use the bundle associated with the gateway: + +```shell +grpcurl \ + -cacert ~/.config/openshell/gateways//mtls/ca.crt \ + -cert ~/.config/openshell/gateways//mtls/tls.crt \ + -key ~/.config/openshell/gateways//mtls/tls.key \ + : list +``` + +Reflection advertises only `openshell.v1.OpenShell`. It does not advertise +internal driver, interceptor, or middleware services. + +When the gateway gRPC rate limit is enabled, reflection uses a separate counter +with the same request count and window. Opening the reflection RPC consumes one +gateway-wide request, and every query sent through that stream consumes one +reflection counter unit. Exceeding the reflection quota closes the stream with +`RESOURCE_EXHAUSTED`. + ### OIDC Gateways can validate OpenID Connect access tokens on gRPC requests. Configure OIDC when you want users or automation to authenticate with an identity provider such as Keycloak, Entra ID, or Okta. diff --git a/docs/how-it-works/gateways/configuration.mdx b/docs/how-it-works/gateways/configuration.mdx index 21eda0a59d..d3910f9fd3 100644 --- a/docs/how-it-works/gateways/configuration.mdx +++ b/docs/how-it-works/gateways/configuration.mdx @@ -200,6 +200,7 @@ disable_tls = false guest_tls_ca = "/etc/openshell/certs/ca.pem" # Optional gRPC rate limit. Both values must be positive to enable the limit. +# Reflection queries use a separate counter with the same count and window. # Set either value to 0, or omit both, to disable rate limiting. grpc_rate_limit_requests = 120 grpc_rate_limit_window_seconds = 60 From e16a359c8c8abca845475c1db903763221d48c68 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:52:01 +0000 Subject: [PATCH 05/26] chore(deps): update registry.access.redhat.com/ubi9/nodejs-24-minimal docker tag to v9.8-1791837113 (#99) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/docker/Dockerfile.konflux.openclaw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile.konflux.openclaw b/deploy/docker/Dockerfile.konflux.openclaw index c3f9283ff8..e4fc08c654 100644 --- a/deploy/docker/Dockerfile.konflux.openclaw +++ b/deploy/docker/Dockerfile.konflux.openclaw @@ -25,7 +25,7 @@ # `&& \` continuations only. # Both stages use the same base and OpenClaw version; bump them only here. -ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791306535@sha256:c23f4bd66ff13561ed7226a8525ddeb259d70f5b4106dc8e289e2db4a3152ed1 +ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791837113@sha256:c0205a5c0bb750ec8a34248da9f87b4013cbfd46e327ec321996104913633fe9 ARG OPENCLAW_VERSION=2026.9.5 # --------------------------------------------------------------------------- From 4a0320b7c5a0744cee024ae706fca91e3d6611fa Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Mon, 12 Oct 2026 11:07:55 +0100 Subject: [PATCH 06/26] CARRY: docs(openclaw): version sandbox image after OpenClaw Signed-off-by: Bobbins228 --- .../odh-openshell-sandbox-openclaw-tag.yaml | 60 ------------------- deploy/konflux/openclaw/README.md | 2 +- 2 files changed, 1 insertion(+), 61 deletions(-) delete mode 100644 .tekton/odh-openshell-sandbox-openclaw-tag.yaml diff --git a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml deleted file mode 100644 index 9144bb0ad9..0000000000 --- a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun -# -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/openshell?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/cancel-in-progress: "false" - pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: | - event == "push" - && target_branch.matches("^refs/tags/v[0-9]+\\.[0-9]+\\.[0-9]+-rhaiv\\.[0-9]+$") - creationTimestamp: null - labels: - appstudio.openshift.io/application: opendatahub-builds - appstudio.openshift.io/component: odh-openshell-sandbox-openclaw-ci - pipelines.appstudio.openshift.io/type: build - name: odh-openshell-sandbox-openclaw-on-tag - namespace: open-data-hub-tenant -spec: - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: output-image - value: quay.io/opendatahub/odh-openshell-sandbox-openclaw:{{git_tag}} - - name: dockerfile - value: deploy/docker/Dockerfile.konflux.openclaw - - name: path-context - value: . - - name: hermetic - value: "true" - - name: build-platforms - value: - - linux/x86_64 - - linux/arm64 - - name: prefetch-input - value: | - [ - {"type": "npm", "path": "deploy/konflux/openclaw"}, - {"type": "rpm", "path": "deploy/konflux/openclaw"} - ] - pipelineRef: - resolver: git - params: - - name: url - value: https://github.com/opendatahub-io/odh-konflux-central.git - - name: revision - value: main - - name: pathInRepo - value: pipeline/multi-arch-container-build.yaml - taskRunTemplate: - serviceAccountName: build-pipeline-odh-openshell-sandbox-openclaw-ci - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} diff --git a/deploy/konflux/openclaw/README.md b/deploy/konflux/openclaw/README.md index 4cc682b405..bde760187a 100644 --- a/deploy/konflux/openclaw/README.md +++ b/deploy/konflux/openclaw/README.md @@ -158,7 +158,7 @@ The npm inputs come from AIPCC agentic commit dc756bc3 (OpenClaw 2026.9.5). AIPC ## Konflux -odh-konflux-central owns the Tekton pipelines. Its onboarder sync opens a PR here that copies `pipelineruns/openshell/odh-openshell-sandbox-openclaw-*.yaml` over `.tekton/`, so change their params there and run the sync. Merges to `main` publish `odh-stable`, and each `v*-rhaiv.*` tag rebuilds the same pinned OpenClaw release under that tag; only the commit named in [Status](#status) is verified. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: +This image is versioned after OpenClaw. Merges to `main` publish `odh-stable`; consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: ```json [{"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"}] From ed9238dfd15a82374f1a32c0dd4a5f6a1852fae4 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 13 Oct 2026 08:46:33 +0100 Subject: [PATCH 07/26] CARRY: ci(openclaw): publish OpenClaw version tags Signed-off-by: Bobbins228 --- .../odh-openshell-sandbox-openclaw-tag.yaml | 63 +++++++++++++++++++ deploy/konflux/openclaw/README.md | 2 +- 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 .tekton/odh-openshell-sandbox-openclaw-tag.yaml diff --git a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml new file mode 100644 index 0000000000..784315665f --- /dev/null +++ b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml @@ -0,0 +1,63 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +# +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/openshell?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/cancel-in-progress: "false" + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: | + event == "push" + && target_branch.matches("^refs/tags/[0-9]+\\.[0-9]+\\.[0-9]+$") + creationTimestamp: null + labels: + appstudio.openshift.io/application: opendatahub-builds + appstudio.openshift.io/component: odh-openshell-sandbox-openclaw-ci + pipelines.appstudio.openshift.io/type: build + name: odh-openshell-sandbox-openclaw-on-tag + namespace: open-data-hub-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/opendatahub/odh-openshell-sandbox-openclaw:{{git_tag}} + - name: dockerfile + value: deploy/docker/Dockerfile.konflux.openclaw + - name: path-context + value: . + - name: hermetic + value: "true" + - name: build-platforms + value: + - linux/x86_64 + - linux/arm64 + - name: prefetch-input + value: | + [ + {"type": "npm", "path": "deploy/konflux/openclaw"}, + {"type": "rpm", "path": "deploy/konflux/openclaw"} + ] + - name: build-args + value: + - "OPENCLAW_VERSION={{git_tag}}" + pipelineRef: + resolver: git + params: + - name: url + value: https://github.com/opendatahub-io/odh-konflux-central.git + - name: revision + value: main + - name: pathInRepo + value: pipeline/multi-arch-container-build.yaml + taskRunTemplate: + serviceAccountName: build-pipeline-odh-openshell-sandbox-openclaw-ci + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/deploy/konflux/openclaw/README.md b/deploy/konflux/openclaw/README.md index bde760187a..e491ff918a 100644 --- a/deploy/konflux/openclaw/README.md +++ b/deploy/konflux/openclaw/README.md @@ -158,7 +158,7 @@ The npm inputs come from AIPCC agentic commit dc756bc3 (OpenClaw 2026.9.5). AIPC ## Konflux -This image is versioned after OpenClaw. Merges to `main` publish `odh-stable`; consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: +An OpenShell Git tag matching the OpenClaw version (for example, `2026.9.5`) triggers a versioned image build and publishes the same Quay tag. The tag pipeline passes that version as `OPENCLAW_VERSION`, and the Dockerfile checks it against the installed OpenClaw package. Merges to `main` publish `odh-stable`; consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: ```json [{"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"}] From 7bbcd630f4ea3f3cba077cb6deaa84a9c7935224 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Tue, 13 Oct 2026 07:17:53 -0700 Subject: [PATCH 08/26] feat(rhoai): install OpenShell gateway and dashboard on OpenShift Signed-off-by: Varsha Prasad Narsing --- deploy/rhoai/README.md | 203 +++++++++ deploy/rhoai/config.example.json | 25 ++ deploy/rhoai/install.py | 561 ++++++++++++++++++++++++ deploy/rhoai/test_install.py | 294 +++++++++++++ docs/kubernetes/setup.mdx | 9 + skills/debug-openshell-cluster/SKILL.md | 11 + 6 files changed, 1103 insertions(+) create mode 100644 deploy/rhoai/README.md create mode 100644 deploy/rhoai/config.example.json create mode 100644 deploy/rhoai/install.py create mode 100644 deploy/rhoai/test_install.py diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md new file mode 100644 index 0000000000..1085e8d6d2 --- /dev/null +++ b/deploy/rhoai/README.md @@ -0,0 +1,203 @@ +# OpenShell and Dashboard installer for OpenShift + +This ODH-fork installer creates a dedicated namespace and installs two independent +Helm releases: `openshell` and `openshell-dashboard`. The gateway chart supplies +the gateway and the configuration for launching supervisor and sandbox workloads. +The UI chart remains in the [dashboard repository](https://github.com/opendatahub-io/openshell-dashboard/tree/main/deploy/helm/openshell-dashboard). + +This is an initial installer, not a qualified RHOAI deployment. Gateway chart +`0.1.3` and dashboard chart `0.1.0`, using their default images, passed the live +smoke checks below on OpenShift `4.21.8`. Matching `0.1` release lines alone do +not prove API compatibility. Choose exact published versions and validate them +together. The script intentionally has no default release versions. + +## Live validation + +On October 13, 2026, the installer was exercised on an OpenShift `4.21.8` cluster +with RHOAI installed, OVN Kubernetes, `gp3-csi` storage, Agent Sandbox `v1.0.6`, and +an existing Red Hat build of Keycloak `26.4.10`. A dedicated test realm supplied +OIDC; existing customer realms were not changed. + +The following checks passed with gateway chart `0.1.3` and dashboard chart `0.1.0`: + +- First installation into a new namespace and a second run with the same inputs. +- Gateway readiness, both dashboard containers ready, and persistent storage bound. +- Public HTTPS routing and HTTP 401 for unauthenticated dashboard API requests. +- Browser OIDC redirect/login/callback flow and authenticated workspace and gateway API calls. +- Manual browser confirmation of the workspaces page, platform-admin identity, + healthy gateway indicator, and active default workspace. +- Dashboard compatibility response reporting the gateway release line as supported. +- Sandbox creation through the dashboard API, workload/supervisor readiness, + authenticated terminal WebSocket execution with exit code 0, and completed deletion. + +An expired, unrelated Kagenti admission webhook initially blocked pod creation. +Removing Kagenti restored both Agent Sandbox and the existing Keycloak instance. +The installer correctly stopped before installing OpenShell while Agent Sandbox +was missing or unhealthy. Diagnose admission webhook failures before changing +OpenShell SCC settings or authentication configuration. + +These checks do not establish FIPS qualification, production certificate renewal, +standard-user cross-workspace isolation, or a full UI interaction test. The live +configuration and credentials are kept outside the repository. + +## Prerequisites + +- `uv`, Helm 3.17.3 or later, and `kubectl` on the administrator's workstation. +- An OpenShift cluster, an explicit kubeconfig context, and permission to create + namespaces and the namespaced/cluster RBAC required by the gateway chart. +- Agent Sandbox CRDs and a running controller installed by the cluster + administrator. The installer checks the supported served Sandbox API and waits + for the controller Deployment. Override `agent_sandbox_namespace` and + `agent_sandbox_deployment` in the JSON if your installation uses different names. +- A CNI enforcing NetworkPolicy, a working default StorageClass, and cluster SCC + configuration appropriate for the pinned gateway's sandbox runtime. +- Customer-managed HTTPS OIDC, a confidential browser client with PKCE S256, + and access tokens whose audience and role claims match the gateway settings. +- Distinct dashboard and gateway DNS names pointing at the OpenShift router. + The dashboard name must be covered by the router's default certificate. +- Registry access from the workstation for chart pulls and from cluster nodes + for all images. Private image pull-secret provisioning is outside this first + installer; use cluster-managed registry credentials if needed. + +The installer does not install an IdP, Agent Sandbox, RHOAI, a modular operator, +or console integration. Administrators retain responsibility for these cluster +prerequisites. It does not change your current kubeconfig context. + +## Configure and install + +Copy the configuration outside the checkout so local credentials cannot be +accidentally committed: + +```shell +mkdir -p "$HOME/.config/openshell-rhoai" +chmod 700 "$HOME/.config/openshell-rhoai" +cp deploy/rhoai/config.example.json "$HOME/.config/openshell-rhoai/config.json" +``` + +Edit the copy. Replace both version placeholders with exact published `0.1.x` +chart versions, and set the namespace, kubeconfig context, DNS names, and OIDC +claims. The two versions need not have the same patch number. The dashboard OCI +reference currently points to its upstream publisher, not an ODH chart registry. +Both chart references are configurable for mirrors. + +Save the browser client's secret to `dashboard-client-secret.txt` beside the JSON, +using a trusted editor or your secret manager. Restrict its permissions to `0600`. +Do not put the secret in the JSON, Helm values, or command arguments. File paths +in `oidc.client_secret_file` and optional `oidc.ca_file` are relative to the JSON. +The latter supplies a PEM CA bundle for a private OIDC issuer to both releases. + +Register `https:///oauth2/callback` with your IdP. The browser +client ID is separate from the gateway API audience. Generic OIDC uses +`allowed_groups`; Keycloak can use `provider: "keycloak-oidc"` and `allowed_roles` +such as `openshell-api:openshell-user`. These are alternatives, not assumptions +about customer claim names. Without either UI filter, any authenticated IdP user +can reach the UI, but the gateway still enforces its roles and workspace access. + +Validate both charts without connecting to the cluster: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --check +``` + +Install: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" +``` + +`--check` still downloads the pinned charts. It discards rendered manifests and +does not validate connectivity, SCC admission, token claims, or runtime behavior. +It reads the client secret file's existence/size, not its contents. Temporary +values files have owner-only permissions and contain no client or cookie secrets. + +The installer creates a labeled namespace, a client/cookie Secret, and an optional +OIDC CA ConfigMap. It installs the gateway first, checks the gateway CA Secret, +and installs the dashboard with that CA and the internal gateway Service URL. +Both releases use OpenShift-assigned UID/GID settings. It waits for Helm readiness +and checks Route admission. Set `--timeout 15m` for a slower cluster. + +## TLS and onboarding + +The gateway uses the chart's PKI bootstrap hook and a passthrough Route. Generated +server SANs include the gateway public hostname and namespace-specific Service +names. This initial mode uses a private gateway CA: export and distribute its +`ca.crt` from `openshell-client-tls` through your trusted administrator process. +Register the gateway using the installed CLI's CA options from +`openshell gateway add --help`; do not disable certificate verification. +TLS client-certificate authentication is disabled for this OIDC-only deployment. +Supervisors retain gateway-minted sandbox JWT authentication. + +The dashboard uses an edge Route and trusts the gateway CA for internal calls. +Its authentication proxy is the only exposed dashboard port. The OpenShift +router must use the `openshift-ingress` namespace network identity; host-network +router configurations require a separately reviewed chart configuration. + +Configure a CLI OIDC registration separately with your IdP. After installation, +verify administrator and standard-user login. Administrators add workspace +memberships using the token's subject ID. Verify denied cross-workspace access, +sandbox creation/deletion, and terminal WebSockets before sharing the deployment. +Readiness and Route admission alone do not prove this flow works. + +## Reruns, upgrades, and failures + +Rerun the same command after correcting a failure. An existing namespace is +accepted only if it has this installer's ownership label. Unrelated namespaces +are rejected, and existing namespace SCC allocation annotations are never copied. +Do not manually label an unrelated namespace to bypass this check. +The original gateway hostname is recorded on the namespace. Changing it is +rejected because the certificate hook preserves existing certificates; migrate +the certificate and endpoint explicitly rather than changing this annotation. + +The installer preserves existing client and cookie secrets. You may omit +`client_secret_file` after the first installation. A provided secret or private +OIDC CA that differs from the existing object causes an error; rotate credentials +explicitly and restart the consuming pods. Removing `ca_file` removes the mounts +on the next upgrade, but does not delete the old ConfigMap. + +Change the pinned versions to upgrade. Both Helm releases receive the generated +values with `--reset-values`, so out-of-band Helm value edits are not preserved. +There is no cross-release transaction or automatic rollback: gateway success +followed by dashboard failure leaves the upgraded gateway in place. Failed +releases, namespaces, Secrets, and PVCs are preserved for diagnosis. + +Inspect the two Helm releases, namespace pods/events, and the `openshell` and +`openshell-dashboard` Routes with the same explicit kubeconfig context. For +sandbox failures, inspect Agent Sandbox, SCC admission, storage, supervisor logs, +and NetworkPolicy enforcement. Namespace deletion is never part of this script. + +## Image overrides + +Optional `gateway.images` accepts `gateway`, `supervisor`, `sandboxRuntime`, and +`sandbox` objects. Optional `dashboard.images` accepts `dashboard` and `authProxy`. +Each object accepts `registry`, `repository`, `tag`, and `digest`, using the chart's +image semantics. For example: + +```json +"images": { + "gateway": { + "registry": "quay.io", + "repository": "your-org/openshell-gateway", + "digest": "sha256:" + } +} +``` + +Pin compatible runtime and UI images as well as chart versions. The default +workload image uses a complete registry/repository path in `sandbox.repository` +(within `gateway.images.sandbox`); its chart does not use a separate registry key. +Image overrides do not establish FIPS qualification. + +## Contributor validation + +```shell +uv run --no-project --python 3.12 deploy/rhoai/test_install.py +``` + +For source-chart rendering, use `--check --gateway-chart-directory +--dashboard-chart-directory `. These overrides are rejected for installation. +Source charts carry version `0.0.0`, so this check renders images with the versions +from your configuration. The gateway source originally selected for this work is +`348a1fc62566892ba5530ac9f35acc983e4d9324`; a source commit is not a published OCI +chart version. Resolve it to the intended release before installing. diff --git a/deploy/rhoai/config.example.json b/deploy/rhoai/config.example.json new file mode 100644 index 0000000000..1ebd306c69 --- /dev/null +++ b/deploy/rhoai/config.example.json @@ -0,0 +1,25 @@ +{ + "namespace": "openshell-team", + "context": "REPLACE_WITH_KUBECONFIG_CONTEXT", + "gateway": { + "chart": "oci://ghcr.io/nvidia/openshell/helm-chart", + "version": "REPLACE_WITH_GATEWAY_0.1_VERSION", + "host": "gateway.apps.example.com" + }, + "dashboard": { + "chart": "oci://ghcr.io/gkrumbach07/openshell-dashboard/helm-chart/openshell-dashboard", + "version": "REPLACE_WITH_DASHBOARD_0.1_VERSION", + "host": "dashboard.apps.example.com" + }, + "oidc": { + "provider": "oidc", + "issuer": "https://identity.example.com/realms/customer", + "client_id": "openshell-dashboard", + "client_secret_file": "dashboard-client-secret.txt", + "audience": "openshell-api", + "roles_claim": "roles", + "admin_role": "openshell-admin", + "user_role": "openshell-user", + "allowed_groups": [] + } +} diff --git a/deploy/rhoai/install.py b/deploy/rhoai/install.py new file mode 100644 index 0000000000..9f0f92f35f --- /dev/null +++ b/deploy/rhoai/install.py @@ -0,0 +1,561 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Install the gateway and dashboard as separate Helm releases on OpenShift.""" + +import argparse +import base64 +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.parse import urlsplit + +OWNER = "openshell.ai/installer" +OWNER_VALUE = "odh-dashboard" +GATEWAY_HOST = "openshell.ai/installer-gateway-host" +OIDC_SECRET = "openshell-dashboard-oidc" +OIDC_CA = "openshell-oidc-ca" +VERSION = re.compile(r"0\.1\.[0-9]+(?:-[0-9A-Za-z.-]+)?") +DNS = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?") + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def fields(obj, required, optional=()): + require(isinstance(obj, dict), "Configuration sections must be JSON objects") + require( + not (set(obj) - set(required) - set(optional)), + f"Unknown configuration fields: {set(obj) - set(required) - set(optional)}", + ) + for key in required: + require(key in obj and obj[key] != "", f"Missing configuration field: {key}") + + +def load_config(path): + config = json.loads(path.read_text()) + fields( + config, + ("namespace", "context", "gateway", "dashboard", "oidc"), + ("agent_sandbox_namespace", "agent_sandbox_deployment"), + ) + ns = config["namespace"] + require( + isinstance(ns, str) + and len(ns) <= 63 + and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", ns), + "Invalid namespace", + ) + require( + isinstance(config["context"], str) and config["context"].strip(), + "context is required", + ) + for name in ("gateway", "dashboard"): + item = config[name] + fields(item, ("chart", "version", "host"), ("images",)) + require( + isinstance(item["chart"], str) and item["chart"].startswith("oci://"), + f"{name}.chart must be an OCI chart reference", + ) + require( + isinstance(item["version"], str) and VERSION.fullmatch(item["version"]), + f"{name}.version must pin a 0.1.x release (no latest or version ranges)", + ) + host = item["host"] + require( + isinstance(host, str) + and len(host) <= 253 + and DNS.fullmatch(host) + and all( + 0 < len(label) <= 63 + and not label.startswith("-") + and not label.endswith("-") + for label in host.split(".") + ), + f"Invalid {name}.host", + ) + components = ( + ("gateway", "supervisor", "sandboxRuntime", "sandbox") + if name == "gateway" + else ("dashboard", "authProxy") + ) + fields(item.get("images", {}), (), components) + for component, image in item.get("images", {}).items(): + keys = ( + ("repository", "tag", "digest") + if component == "sandbox" + else ("registry", "repository", "tag", "digest") + ) + fields(image, (), keys) + require( + all(isinstance(value, str) for value in image.values()), + "Image values must be strings", + ) + require( + config["gateway"]["host"] != config["dashboard"]["host"], + "Gateway and dashboard need distinct hosts", + ) + oidc = config["oidc"] + fields( + oidc, + ("issuer", "client_id", "audience", "roles_claim", "admin_role", "user_role"), + ( + "provider", + "allowed_groups", + "allowed_roles", + "client_secret_file", + "ca_file", + ), + ) + for key in ( + "issuer", + "client_id", + "audience", + "roles_claim", + "admin_role", + "user_role", + ): + require( + isinstance(oidc[key], str) and oidc[key].strip(), + f"oidc.{key} must be a nonempty string", + ) + issuer = urlsplit(oidc["issuer"]) + require( + issuer.scheme == "https" + and issuer.hostname + and not issuer.username + and not issuer.password + and not issuer.query + and not issuer.fragment, + "OIDC issuer must be an HTTPS URL", + ) + require( + oidc["client_id"] != oidc["audience"], + "Browser client ID and gateway audience must be distinct", + ) + require( + oidc["admin_role"] != oidc["user_role"], "Admin and user roles must be distinct" + ) + require( + oidc.get("provider", "oidc") in ("oidc", "keycloak-oidc"), + "Unsupported OIDC provider", + ) + for key in ("allowed_groups", "allowed_roles"): + require( + isinstance(oidc.get(key, []), list) + and all(isinstance(v, str) and v for v in oidc.get(key, [])), + f"oidc.{key} must be a list of strings", + ) + require( + not oidc.get("allowed_roles") or oidc.get("provider") == "keycloak-oidc", + "allowed_roles requires keycloak-oidc; use allowed_groups for generic OIDC", + ) + for key in ("client_secret_file", "ca_file"): + if oidc.get(key): + source = (path.parent / oidc[key]).resolve() + require( + source.is_file() and source.stat().st_size > 0, + f"Missing or empty oidc.{key}", + ) + oidc[key] = str(source) + return config + + +def chart_values(config): + ns, oidc = config["namespace"], config["oidc"] + ca = OIDC_CA if oidc.get("ca_file") else "" + security = {"runAsUser": None, "seccompProfile": {"type": "RuntimeDefault"}} + gateway = { + "fullnameOverride": "openshell", + "podSecurityContext": None, + "securityContext": security, + "replicaCount": 1, + "server": { + "disableTls": False, + "sandboxNamespace": ns, + "auth": {"allowUnauthenticatedUsers": False}, + "tls": { + "enableMtls": False, + "clientCaSecretName": "", + "certSecretName": "openshell-server-tls", + "clientTlsSecretName": "openshell-client-tls", + }, + "oidc": { + "issuer": oidc["issuer"], + "audience": oidc["audience"], + "rolesClaim": oidc["roles_claim"], + "adminRole": oidc["admin_role"], + "userRole": oidc["user_role"], + "caConfigMapName": ca, + }, + }, + "pkiInitJob": { + "enabled": True, + "serverDnsNames": [ + config["gateway"]["host"], + f"openshell.{ns}.svc", + f"openshell.{ns}.svc.cluster.local", + ], + }, + "openshiftRoute": {"enabled": True, "host": config["gateway"]["host"]}, + } + dashboard = { + "fullnameOverride": "openshell-dashboard", + "podSecurityContext": None, + "securityContext": security, + "gateway": { + "url": f"grpcs://openshell.{ns}.svc.cluster.local:8080", + "caSecretName": "openshell-client-tls", + }, + "oidc": { + "provider": oidc.get("provider", "oidc"), + "issuer": oidc["issuer"], + "clientId": oidc["client_id"], + "existingSecret": OIDC_SECRET, + "audience": oidc["audience"], + "rolesClaim": oidc["roles_claim"], + "adminRole": oidc["admin_role"], + "codeChallengeMethod": "S256", + "allowedGroups": oidc.get("allowed_groups", []), + "allowedRoles": oidc.get("allowed_roles", []), + "caConfigMapName": ca, + }, + "openshiftRoute": { + "enabled": True, + "host": config["dashboard"]["host"], + "termination": "edge", + }, + "networkPolicy": {"ingressNamespace": "openshift-ingress"}, + } + for name, values in (("gateway", gateway), ("dashboard", dashboard)): + for component, image in config[name].get("images", {}).items(): + values.setdefault(component, {})["image"] = image + return gateway, dashboard + + +class Installer: + def __init__(self, config, timeout): + self.config = config + self.timeout = timeout + self.kube = ["kubectl", "--context", config["context"], "--request-timeout=30s"] + self.ns = config["namespace"] + + def run(self, args, payload=None): + # Never log stdin: it can contain Secret data. Do not echo commands. + result = subprocess.run( + args, input=payload, text=True, capture_output=True, check=False + ) + if result.returncode: + raise RuntimeError(f"{args[0]} failed: {result.stderr.strip()}") + return result.stdout + + def get(self, kind, name, namespaced=True): + args = self.kube + (["-n", self.ns] if namespaced else []) + output = self.run( + [*args, "get", kind, name, "--ignore-not-found", "-o", "json"] + ) + return json.loads(output) if output.strip() else None + + def create(self, resource): + self.run([*self.kube, "create", "-f", "-"], json.dumps(resource)) + + def preflight(self): + self.run([*self.kube, "get", "--raw", "/apis/route.openshift.io/v1"]) + crd = self.get("crd", "sandboxes.agents.x-k8s.io", False) + require( + crd + and any( + v["served"] and v["name"] in ("v1alpha1", "v1beta1") + for v in crd["spec"]["versions"] + ), + "Install the Agent Sandbox CRDs and controller first", + ) + self.run( + [ + *self.kube, + "-n", + self.config.get("agent_sandbox_namespace", "agent-sandbox-system"), + "rollout", + "status", + "deployment/" + + self.config.get( + "agent_sandbox_deployment", "agent-sandbox-controller" + ), + "--timeout", + self.timeout, + ] + ) + namespace = self.get("namespace", self.ns, False) + require( + not namespace + or namespace["metadata"].get("labels", {}).get(OWNER) == OWNER_VALUE, + "Namespace already exists and is not owned by this installer; choose a new namespace", + ) + require( + not namespace or namespace.get("status", {}).get("phase") != "Terminating", + "Namespace is terminating", + ) + require( + not namespace + or namespace["metadata"].get("annotations", {}).get(GATEWAY_HOST) + == self.config["gateway"]["host"], + "Gateway hostname differs from the original install; certificate migration is required", + ) + existing = self.get("secret", OIDC_SECRET) if namespace else None + require( + existing or self.config["oidc"].get("client_secret_file"), + "oidc.client_secret_file is required for the first install", + ) + return namespace + + def credentials(self): + oidc = self.config["oidc"] + existing = self.get("secret", OIDC_SECRET) + supplied = ( + Path(oidc["client_secret_file"]).read_bytes().rstrip(b"\r\n") + if oidc.get("client_secret_file") + else None + ) + if supplied is not None: + require(supplied, "Client secret file contains no secret") + if existing: + data = existing.get("data", {}) + require( + data.get("client-secret") and data.get("cookie-secret"), + "Existing OIDC Secret is missing required keys", + ) + require( + supplied is None or base64.b64decode(data["client-secret"]) == supplied, + "OIDC Secret differs from file; rotate it explicitly before rerunning", + ) + else: + require(supplied, "Client secret file is required") + self.create( + { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": {"name": OIDC_SECRET, "namespace": self.ns}, + "data": { + "client-secret": base64.b64encode(supplied).decode(), + "cookie-secret": base64.b64encode( + secrets.token_urlsafe(32).encode() + ).decode(), + }, + } + ) + if oidc.get("ca_file"): + ca = Path(oidc["ca_file"]).read_text() + existing_ca = self.get("configmap", OIDC_CA) + require( + not existing_ca or existing_ca.get("data", {}).get("ca.crt") == ca, + "OIDC CA differs; rotate the ConfigMap and restart both releases explicitly", + ) + if not existing_ca: + self.create( + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": OIDC_CA, "namespace": self.ns}, + "data": {"ca.crt": ca}, + } + ) + + def wait_for_route(self, release): + seconds = ( + int(self.timeout[:-1]) * {"s": 1, "m": 60, "h": 3600}[self.timeout[-1]] + ) + deadline = time.monotonic() + seconds + while True: + route = self.get("route", release) + if route and any( + condition.get("type") == "Admitted" + and condition.get("status") == "True" + for ingress in route.get("status", {}).get("ingress", []) + for condition in ingress.get("conditions", []) + ): + return + require( + time.monotonic() < deadline, + f"Route {release} is not admitted; inspect it and rerun", + ) + time.sleep(min(2, max(0, deadline - time.monotonic()))) + + def install(self, charts, values): + namespace = self.preflight() + if not namespace: + self.create( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": self.ns, + "labels": {OWNER: OWNER_VALUE}, + "annotations": {GATEWAY_HOST: self.config["gateway"]["host"]}, + }, + } + ) + self.credentials() + for index, release in enumerate(("openshell", "openshell-dashboard")): + print(f"Installing {release} in {self.ns}...", flush=True) + self.run( + [ + "helm", + "upgrade", + "--install", + release, + str(charts[index]), + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + "--values", + str(values[index]), + "--reset-values", + "--wait", + "--wait-for-jobs", + "--timeout", + self.timeout, + "--history-max", + "5", + ] + ) + if index == 0: + ca = self.get("secret", "openshell-client-tls") + require( + ca and ca.get("data", {}).get("ca.crt"), + "Gateway did not create its CA Secret", + ) + self.wait_for_route(release) + print(f"Dashboard: https://{self.config['dashboard']['host']}") + print(f"Gateway: https://{self.config['gateway']['host']}") + print( + "Register the gateway with the CLI using its CA, then log in through customer OIDC." + ) + print( + "An administrator must add users to workspaces. Verify login, denied access, sandbox creation and terminals." + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--check", + action="store_true", + help="Validate and render charts without accessing the cluster", + ) + parser.add_argument( + "--gateway-chart-directory", + type=Path, + help="Local source chart for --check only", + ) + parser.add_argument( + "--dashboard-chart-directory", + type=Path, + help="Local source chart for --check only", + ) + parser.add_argument( + "--timeout", default="10m", help="Helm/controller wait timeout (default: 10m)" + ) + args = parser.parse_args() + require( + args.check + or not (args.gateway_chart_directory or args.dashboard_chart_directory), + "Local source charts are only supported with --check; install versioned OCI artifacts", + ) + require( + re.fullmatch(r"[1-9][0-9]*[smh]", args.timeout), + "Timeout must be a positive duration such as 10m", + ) + for tool in ["helm"] if args.check else ["helm", "kubectl"]: + require(shutil.which(tool), f"Required tool not found: {tool}") + config = load_config(args.config.resolve()) + installer = Installer(config, args.timeout) + os.umask(0o077) + with tempfile.TemporaryDirectory(prefix="openshell-install-") as temporary: + root = Path(temporary) + charts, files = [], [] + for index, (name, values, local) in enumerate( + zip( + ("gateway", "dashboard"), + chart_values(config), + (args.gateway_chart_directory, args.dashboard_chart_directory), + strict=True, + ) + ): + if local: + chart = local.resolve() + require((chart / "Chart.yaml").is_file(), f"Missing chart: {chart}") + # Source charts carry 0.0.0; render with the requested release image tag. + values.setdefault("global", {})["image"] = { + "tag": config[name]["version"] + } + else: + destination = root / name + destination.mkdir() + installer.run( + [ + "helm", + "pull", + config[name]["chart"], + "--version", + config[name]["version"], + "--destination", + str(destination), + ] + ) + packages = list(destination.glob("*.tgz")) + require(len(packages) == 1, f"Expected one downloaded {name} chart") + chart = packages[0] + value_file = root / f"{name}.json" + value_file.write_text(json.dumps(values)) + release = ("openshell", "openshell-dashboard")[index] + installer.run( + [ + "helm", + "template", + release, + str(chart), + "--namespace", + config["namespace"], + "--values", + str(value_file), + "--api-versions", + "route.openshift.io/v1", + "--api-versions", + "agents.x-k8s.io/v1beta1", + ] + ) + charts.append(chart) + files.append(value_file) + print( + "Both charts rendered successfully. This does not verify cluster or API compatibility.", + flush=True, + ) + if not args.check: + installer.install(charts, files) + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError, KeyError, TypeError) as error: + print( + f"Installation stopped: {error}\nCluster resources were not rolled back or deleted.", + file=sys.stderr, + ) + sys.exit(1) diff --git a/deploy/rhoai/test_install.py b/deploy/rhoai/test_install.py new file mode 100644 index 0000000000..01bba8be57 --- /dev/null +++ b/deploy/rhoai/test_install.py @@ -0,0 +1,294 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +"""Behavioral tests with a simulated cluster; no network or cluster access.""" + +import base64 +import copy +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location( + "installer", Path(__file__).with_name("install.py") +) +installer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(installer) + + +def configuration(): + config = json.loads(Path(__file__).with_name("config.example.json").read_text()) + config["context"] = "test-context" + config["gateway"]["version"] = "0.1.3" + config["dashboard"]["version"] = "0.1.7" + config["oidc"].pop("client_secret_file") + return config + + +class FakeCluster(installer.Installer): + def __init__(self, config): + super().__init__(config, "10m") + self.objects = { + ("crd", "sandboxes.agents.x-k8s.io"): { + "spec": {"versions": [{"name": "v1beta1", "served": True}]} + } + } + self.calls = [] + self.fail_dashboard = False + + def run(self, args, payload=None): + self.calls.append((args, payload)) + if args[0] == "helm": + if self.fail_dashboard and "openshell-dashboard" in args: + raise RuntimeError("dashboard failed") + self.objects[("secret", "openshell-client-tls")] = { + "data": {"ca.crt": "Y2E="} + } + self.objects[("route", args[3])] = { + "status": { + "ingress": [ + {"conditions": [{"type": "Admitted", "status": "True"}]} + ] + } + } + elif "get" in args and "--raw" not in args: + start = args.index("get") + obj = self.objects.get(tuple(args[start + 1 : start + 3])) + return json.dumps(obj) if obj is not None else "" + elif "create" in args: + obj = json.loads(payload) + key = (obj["kind"].lower(), obj["metadata"]["name"]) + if key in self.objects: + raise RuntimeError("already exists") + self.objects[key] = obj + return "" + + +class InstallTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.config = configuration() + + def load(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + return installer.load_config(path) + + def cluster(self): + secret = self.root / "client.txt" + secret.write_text("a-secret-that-must-not-appear-in-argv\n") + self.config["oidc"]["client_secret_file"] = str(secret) + return FakeCluster(self.load()) + + def test_rejects_unpinned_versions_and_unknown_keys(self): + for version in ("latest", "0.0.0", "0.2.1", ">=0.1.0", "REPLACE_ME"): + with self.subTest(version=version): + self.config["gateway"]["version"] = version + with self.assertRaises(ValueError): + self.load() + self.config = configuration() + self.config["oidc"]["client_secret"] = "never-inline" + with self.assertRaisesRegex(ValueError, "Unknown"): + self.load() + + def test_rejects_invalid_oidc_and_namespace(self): + for section, key, value in ( + ("oidc", "issuer", "http://issuer.example"), + ("oidc", "issuer", "https://user:password@issuer.example"), + ("oidc", "audience", "openshell-dashboard"), + ("oidc", "allowed_roles", ["admin"]), + ("gateway", "host", "gateway..example"), + ): + self.config = configuration() + self.config[section][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.load() + self.config = configuration() + self.config["namespace"] = "openshell/another" + with self.assertRaises(ValueError): + self.load() + + def test_generated_connection_and_identity_contract(self): + gateway, dashboard = installer.chart_values(self.load()) + self.assertIn( + "openshell.openshell-team.svc.cluster.local", + gateway["pkiInitJob"]["serverDnsNames"], + ) + self.assertEqual( + dashboard["gateway"]["url"], + "grpcs://openshell.openshell-team.svc.cluster.local:8080", + ) + self.assertEqual( + gateway["server"]["oidc"]["issuer"], dashboard["oidc"]["issuer"] + ) + self.assertFalse(gateway["server"]["tls"]["enableMtls"]) + self.assertIsNone(gateway["securityContext"]["runAsUser"]) + + def test_refuses_foreign_namespace_before_mutation(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {"labels": {}}} + with self.assertRaisesRegex(ValueError, "not owned"): + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + self.assertFalse( + any("create" in args or args[0] == "helm" for args, _ in cluster.calls) + ) + + def test_missing_prerequisite_stops_before_mutation(self): + cluster = self.cluster() + cluster.objects.clear() + with self.assertRaisesRegex(ValueError, "Agent Sandbox"): + cluster.install([], []) + self.assertFalse(any("create" in args for args, _ in cluster.calls)) + + def test_install_order_context_and_secret_transport(self): + cluster = self.cluster() + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + helm = [args for args, _ in cluster.calls if args[0] == "helm"] + self.assertEqual( + [args[3] for args in helm], ["openshell", "openshell-dashboard"] + ) + for args, _ in cluster.calls: + self.assertNotIn("a-secret-that", " ".join(args)) + self.assertIn("test-context", args) + self.assertNotIn("--atomic", args) + self.assertNotIn("delete", args) + data = cluster.objects[("secret", installer.OIDC_SECRET)]["data"] + self.assertEqual( + base64.b64decode(data["client-secret"]), + b"a-secret-that-must-not-appear-in-argv", + ) + cookie = base64.b64decode(data["cookie-secret"]).decode() + self.assertEqual(len(base64.urlsafe_b64decode(cookie + "=")), 32) + + def test_rerun_preserves_secrets_without_input_file(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + before = copy.deepcopy(cluster.objects[("secret", installer.OIDC_SECRET)]) + cluster.config["oidc"].pop("client_secret_file") + cluster.calls.clear() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("secret", installer.OIDC_SECRET)], before) + self.assertFalse(any("create" in args for args, _ in cluster.calls)) + + def test_changed_gateway_host_requires_certificate_migration(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + cluster.config["gateway"]["host"] = "other.apps.example.com" + cluster.calls.clear() + with self.assertRaisesRegex(ValueError, "certificate migration"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertFalse( + any(args[0] == "helm" or "create" in args for args, _ in cluster.calls) + ) + + def test_different_secret_is_not_silently_rotated(self): + cluster = self.cluster() + cluster.credentials() + before = copy.deepcopy(cluster.objects) + Path(cluster.config["oidc"]["client_secret_file"]).write_text("changed") + with self.assertRaisesRegex(ValueError, "rotate"): + cluster.credentials() + self.assertEqual(cluster.objects, before) + + def test_private_ca_and_relative_path(self): + (self.root / "issuer.pem").write_text("test CA") + self.config["oidc"]["ca_file"] = "issuer.pem" + cluster = self.cluster() + cluster.credentials() + gateway, dashboard = installer.chart_values(cluster.config) + self.assertEqual( + gateway["server"]["oidc"]["caConfigMapName"], installer.OIDC_CA + ) + self.assertEqual(dashboard["oidc"]["caConfigMapName"], installer.OIDC_CA) + self.assertEqual( + cluster.objects[("configmap", installer.OIDC_CA)]["data"]["ca.crt"], + "test CA", + ) + + def test_dashboard_failure_keeps_gateway_and_credentials(self): + cluster = self.cluster() + cluster.fail_dashboard = True + with self.assertRaisesRegex(RuntimeError, "dashboard failed"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn(("namespace", cluster.ns), cluster.objects) + self.assertIn(("secret", installer.OIDC_SECRET), cluster.objects) + self.assertIn(("secret", "openshell-client-tls"), cluster.objects) + self.assertFalse( + any("delete" in args or "uninstall" in args for args, _ in cluster.calls) + ) + + def test_route_admission_waits_and_times_out(self): + cluster = self.cluster() + admitted = { + "status": { + "ingress": [{"conditions": [{"type": "Admitted", "status": "True"}]}] + } + } + with ( + patch.object(cluster, "get", side_effect=[{}, admitted]), + patch.object(installer.time, "sleep") as sleep, + ): + cluster.wait_for_route("openshell") + sleep.assert_called_once() + with ( + patch.object(installer.time, "monotonic", side_effect=[0, 601]), + self.assertRaisesRegex(ValueError, "not admitted"), + ): + cluster.wait_for_route("openshell") + + def test_render_failure_prevents_cluster_access(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + commands = [] + + def run(_instance, args, payload=None): + self.assertIsNone(payload) + commands.append(args) + if args[:2] == ["helm", "pull"]: + destination = Path(args[args.index("--destination") + 1]) + (destination / "chart.tgz").write_bytes(b"fake") + return "" + raise RuntimeError("invalid chart values") + + with ( + patch("sys.argv", ["install.py", "--config", str(path)]), + patch.object(installer.shutil, "which", return_value="tool"), + patch.object(installer.Installer, "run", run), + self.assertRaisesRegex(RuntimeError, "invalid chart values"), + ): + installer.main() + self.assertTrue(all(args[0] == "helm" for args in commands)) + + def test_check_only_never_calls_kubectl(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + for chart in ("gateway", "dashboard"): + (self.root / chart).mkdir() + (self.root / chart / "Chart.yaml").write_text("version: 0.0.0\n") + argv = [ + "install.py", + "--config", + str(path), + "--check", + "--gateway-chart-directory", + str(self.root / "gateway"), + "--dashboard-chart-directory", + str(self.root / "dashboard"), + ] + with ( + patch("sys.argv", argv), + patch.object(installer.shutil, "which", return_value="helm"), + patch.object(installer.Installer, "run", return_value="") as run, + ): + installer.main() + self.assertEqual(run.call_count, 2) + for call in run.call_args_list: + self.assertEqual(call.args[0][:2], ["helm", "template"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 53ecd932c5..7932ae6b5c 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -88,6 +88,15 @@ OpenShell detects the served Agent Sandbox `Sandbox` API when the Kubernetes gat ## Install OpenShell +For an administrator-managed OpenShift deployment with a browser UI, the ODH +fork provides a [gateway and dashboard installer](https://github.com/opendatahub-io/openshell/tree/main/deploy/rhoai). +It creates a dedicated namespace and installs separately versioned gateway and +dashboard charts using customer OIDC. Agent Sandbox remains a prerequisite. +The installer includes a render-only check, preserves resources on failure, and +requires explicit chart versions. Validate login, workspace authorization, +sandbox lifecycle, and terminal access on your target cluster before sharing the +deployment; chart rendering does not establish RHOAI qualification. + ## Create the namespace diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 418b67ec51..3eee56e270 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -92,6 +92,17 @@ Use gateway metadata, deployment values, or the user's setup notes to identify t ### Step 3: Check Gateway Startup Dependencies +For an ODH gateway-and-dashboard installation, inspect both Helm releases +(`openshell` and `openshell-dashboard`) in the same namespace. The installer +preserves partial installations and credentials on failure; gateway readiness +does not establish dashboard login or sandbox readiness. Check namespace label +`openshell.ai/installer=odh-dashboard`, the dashboard OIDC Secret, the shared +gateway CA Secret, and admission of both Routes. Do not add the ownership label +to an unrelated namespace to bypass installation checks. Use the published +[Kubernetes setup guide](https://docs.nvidia.com/openshell/latest/kubernetes/setup.md) +for the installation entry point. Diagnose Agent Sandbox availability separately; +the installer does not provision that controller or RHOAI itself. + Before debugging the compute platform, inspect gateway logs for failures in dependencies initialized before the listener becomes ready. The gateway container uses a Distroless Debian runtime. For OS-library From f1a46d3f22b5a95d6749bd1b280211ae8c9dc52b Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Tue, 13 Oct 2026 07:35:53 -0700 Subject: [PATCH 09/26] fix(rhoai): support administrator-selected namespaces Signed-off-by: Varsha Prasad Narsing --- deploy/rhoai/README.md | 53 +++++++-- deploy/rhoai/install.py | 110 +++++++++++++++++-- deploy/rhoai/test_install.py | 140 ++++++++++++++++++++++-- docs/kubernetes/setup.mdx | 7 +- skills/debug-openshell-cluster/SKILL.md | 15 ++- 5 files changed, 288 insertions(+), 37 deletions(-) diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md index 1085e8d6d2..9f5842a555 100644 --- a/deploy/rhoai/README.md +++ b/deploy/rhoai/README.md @@ -1,7 +1,9 @@ # OpenShell and Dashboard installer for OpenShift -This ODH-fork installer creates a dedicated namespace and installs two independent -Helm releases: `openshell` and `openshell-dashboard`. The gateway chart supplies +This ODH-fork installer installs two independent Helm releases, `openshell` and +`openshell-dashboard`, into the administrator-selected namespace. It reuses an +existing namespace or creates it if missing. Each namespace has one gateway and +its dashboard; reruns upgrade the same releases. The gateway chart supplies the gateway and the configuration for launching supervisor and sandbox workloads. The UI chart remains in the [dashboard repository](https://github.com/opendatahub-io/openshell-dashboard/tree/main/deploy/helm/openshell-dashboard). @@ -21,6 +23,9 @@ OIDC; existing customer realms were not changed. The following checks passed with gateway chart `0.1.3` and dashboard chart `0.1.0`: - First installation into a new namespace and a second run with the same inputs. +- Installation and rerun in a separately pre-created namespace selected with + `--namespace`, with both releases reaching revision 2 and the namespace's labels, + annotations, and OpenShift SCC allocations preserved. - Gateway readiness, both dashboard containers ready, and persistent storage bound. - Public HTTPS routing and HTTP 401 for unauthenticated dashboard API requests. - Browser OIDC redirect/login/callback flow and authenticated workspace and gateway API calls. @@ -43,8 +48,9 @@ configuration and credentials are kept outside the repository. ## Prerequisites - `uv`, Helm 3.17.3 or later, and `kubectl` on the administrator's workstation. -- An OpenShift cluster, an explicit kubeconfig context, and permission to create - namespaces and the namespaced/cluster RBAC required by the gateway chart. +- An OpenShift cluster, an explicit kubeconfig context, and permission to manage + the namespaced/cluster resources required by the charts. Permission to create + namespaces is needed only when the selected namespace does not exist. - Agent Sandbox CRDs and a running controller installed by the cluster administrator. The installer checks the supported served Sandbox API and waits for the controller Deployment. Override `agent_sandbox_namespace` and @@ -107,13 +113,30 @@ uv run --script --python 3.12 deploy/rhoai/install.py \ --config "$HOME/.config/openshell-rhoai/config.json" ``` +Set `namespace` in the JSON to your team/project namespace, or override it on the +command line (the override also works with `--check`): + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --namespace research-team +``` + +`--namespace` takes precedence; you may omit the JSON `namespace` when passing +this option. Both charts, Secrets, and sandbox resources target this namespace. +Gateway Service DNS and certificate SANs are derived from the selected name. +Existing namespace labels, annotations, and OpenShift SCC allocations are +preserved. Use a separate configuration with distinct public hostnames for each +gateway namespace, and register each dashboard callback with the IdP. Specifying +a different namespace creates another deployment; it does not move an existing one. + `--check` still downloads the pinned charts. It discards rendered manifests and does not validate connectivity, SCC admission, token claims, or runtime behavior. It reads the client secret file's existence/size, not its contents. Temporary values files have owner-only permissions and contain no client or cookie secrets. -The installer creates a labeled namespace, a client/cookie Secret, and an optional -OIDC CA ConfigMap. It installs the gateway first, checks the gateway CA Secret, +The installer creates the namespace only when missing, then creates an installer +state ConfigMap, a client/cookie Secret, and an optional OIDC CA ConfigMap. +It installs the gateway first, checks the gateway CA Secret, and installs the dashboard with that CA and the internal gateway Service URL. Both releases use OpenShift-assigned UID/GID settings. It waits for Helm readiness and checks Route admission. Set `--timeout 15m` for a slower cluster. @@ -142,13 +165,19 @@ Readiness and Route admission alone do not prove this flow works. ## Reruns, upgrades, and failures -Rerun the same command after correcting a failure. An existing namespace is -accepted only if it has this installer's ownership label. Unrelated namespaces -are rejected, and existing namespace SCC allocation annotations are never copied. -Do not manually label an unrelated namespace to bypass this check. -The original gateway hostname is recorded on the namespace. Changing it is +Rerun the same command after correcting a failure. The administrator selects the +namespace; it does not need an installer ownership label. The fixed release names +provide one gateway/dashboard pair per namespace. An existing gateway chart under +a different release name, or either fixed release without installer state, is +rejected and needs explicit migration. Unmanaged gateway TLS/JWT Secrets are also +rejected so the bootstrap hook cannot silently reuse another gateway's identity. +Unrelated Helm releases in the selected namespace are left alone. + +The original gateway hostname is recorded in the `openshell-installer` ConfigMap. +Earlier installations recorded it on the namespace; reruns migrate that state +without changing namespace metadata. Changing the recorded hostname is rejected because the certificate hook preserves existing certificates; migrate -the certificate and endpoint explicitly rather than changing this annotation. +the certificate and endpoint explicitly rather than editing installer state. The installer preserves existing client and cookie secrets. You may omit `client_secret_file` after the first installation. A provided secret or private diff --git a/deploy/rhoai/install.py b/deploy/rhoai/install.py index 9f0f92f35f..504e8b612b 100644 --- a/deploy/rhoai/install.py +++ b/deploy/rhoai/install.py @@ -24,6 +24,7 @@ OWNER = "openshell.ai/installer" OWNER_VALUE = "odh-dashboard" GATEWAY_HOST = "openshell.ai/installer-gateway-host" +INSTALL_STATE = "openshell-installer" OIDC_SECRET = "openshell-dashboard-oidc" OIDC_CA = "openshell-oidc-ca" VERSION = re.compile(r"0\.1\.[0-9]+(?:-[0-9A-Za-z.-]+)?") @@ -45,8 +46,11 @@ def fields(obj, required, optional=()): require(key in obj and obj[key] != "", f"Missing configuration field: {key}") -def load_config(path): +def load_config(path, namespace=None): config = json.loads(path.read_text()) + if namespace is not None: + require(isinstance(config, dict), "Configuration must be a JSON object") + config["namespace"] = namespace fields( config, ("namespace", "context", "gateway", "dashboard", "oidc"), @@ -272,6 +276,50 @@ def get(self, kind, name, namespaced=True): def create(self, resource): self.run([*self.kube, "create", "-f", "-"], json.dumps(resource)) + def check_releases(self, managed): + # Explicit status flags work with both Helm 3 and Helm 4. Paginate so a + # busy administrator-managed namespace cannot hide a conflicting release. + offset = 0 + while True: + releases = json.loads( + self.run( + [ + "helm", + "list", + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + "--output", + "json", + "--max", + "256", + "--offset", + str(offset), + "--deployed", + "--failed", + "--pending", + "--uninstalling", + "--uninstalled", + "--superseded", + ] + ) + ) + for release in releases: + name = release["name"] + gateway = release.get("chart", "").startswith("helm-chart-") + require( + not gateway or name == "openshell", + f"Namespace {self.ns} already has gateway release {name}; use one gateway per namespace", + ) + require( + managed or name not in ("openshell", "openshell-dashboard"), + f"Release {name} already exists outside this installer; migrate it explicitly", + ) + if len(releases) < 256: + break + offset += len(releases) + def preflight(self): self.run([*self.kube, "get", "--raw", "/apis/route.openshift.io/v1"]) crd = self.get("crd", "sandboxes.agents.x-k8s.io", False) @@ -299,21 +347,45 @@ def preflight(self): ] ) namespace = self.get("namespace", self.ns, False) - require( - not namespace - or namespace["metadata"].get("labels", {}).get(OWNER) == OWNER_VALUE, - "Namespace already exists and is not owned by this installer; choose a new namespace", - ) require( not namespace or namespace.get("status", {}).get("phase") != "Terminating", "Namespace is terminating", ) + state = self.get("configmap", INSTALL_STATE) if namespace else None require( - not namespace - or namespace["metadata"].get("annotations", {}).get(GATEWAY_HOST) - == self.config["gateway"]["host"], + not state or state["metadata"].get("labels", {}).get(OWNER) == OWNER_VALUE, + f"ConfigMap {INSTALL_STATE} already exists outside this installer", + ) + # Preserve compatibility with installations made before state moved off + # the Namespace. Never modify administrator-owned Namespace metadata. + legacy_host = ( + (namespace or {}) + .get("metadata", {}) + .get("annotations", {}) + .get(GATEWAY_HOST) + ) + original_host = ( + state.get("data", {}).get("gateway-host") if state else legacy_host + ) + require( + not (state or legacy_host) + or original_host == self.config["gateway"]["host"], "Gateway hostname differs from the original install; certificate migration is required", ) + if namespace: + self.check_releases(managed=bool(state or legacy_host)) + # The chart's bootstrap hook reuses existing TLS material. Do not + # accidentally use certificates belonging to another gateway. + if not (state or legacy_host): + for secret in ( + "openshell-server-tls", + "openshell-client-tls", + "openshell-jwt-keys", + ): + require( + not self.get("secret", secret), + f"Existing gateway Secret {secret} requires explicit migration", + ) existing = self.get("secret", OIDC_SECRET) if namespace else None require( existing or self.config["oidc"].get("client_secret_file"), @@ -404,10 +476,22 @@ def install(self, charts, values): "metadata": { "name": self.ns, "labels": {OWNER: OWNER_VALUE}, - "annotations": {GATEWAY_HOST: self.config["gateway"]["host"]}, }, } ) + if not self.get("configmap", INSTALL_STATE): + self.create( + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": INSTALL_STATE, + "namespace": self.ns, + "labels": {OWNER: OWNER_VALUE}, + }, + "data": {"gateway-host": self.config["gateway"]["host"]}, + } + ) self.credentials() for index, release in enumerate(("openshell", "openshell-dashboard")): print(f"Installing {release} in {self.ns}...", flush=True) @@ -453,6 +537,10 @@ def install(self, charts, values): def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--namespace", + help="Administrator-selected namespace; overrides config.namespace (created if missing)", + ) parser.add_argument( "--check", action="store_true", @@ -483,7 +571,7 @@ def main(): ) for tool in ["helm"] if args.check else ["helm", "kubectl"]: require(shutil.which(tool), f"Required tool not found: {tool}") - config = load_config(args.config.resolve()) + config = load_config(args.config.resolve(), args.namespace) installer = Installer(config, args.timeout) os.umask(0o077) with tempfile.TemporaryDirectory(prefix="openshell-install-") as temporary: diff --git a/deploy/rhoai/test_install.py b/deploy/rhoai/test_install.py index 01bba8be57..fb2dfd273d 100644 --- a/deploy/rhoai/test_install.py +++ b/deploy/rhoai/test_install.py @@ -36,11 +36,15 @@ def __init__(self, config): } } self.calls = [] + self.releases = [] self.fail_dashboard = False def run(self, args, payload=None): self.calls.append((args, payload)) - if args[0] == "helm": + if args[:2] == ["helm", "list"]: + offset = int(args[args.index("--offset") + 1]) + return json.dumps(self.releases[offset : offset + 256]) + if args[:2] == ["helm", "upgrade"]: if self.fail_dashboard and "openshell-dashboard" in args: raise RuntimeError("dashboard failed") self.objects[("secret", "openshell-client-tls")] = { @@ -53,6 +57,15 @@ def run(self, args, payload=None): ] } } + if not any(r["name"] == args[3] for r in self.releases): + self.releases.append( + { + "name": args[3], + "chart": "helm-chart-0.1.3" + if args[3] == "openshell" + else "openshell-dashboard-0.1.0", + } + ) elif "get" in args and "--raw" not in args: start = args.index("get") obj = self.objects.get(tuple(args[start + 1 : start + 3])) @@ -128,13 +141,122 @@ def test_generated_connection_and_identity_contract(self): self.assertFalse(gateway["server"]["tls"]["enableMtls"]) self.assertIsNone(gateway["securityContext"]["runAsUser"]) - def test_refuses_foreign_namespace_before_mutation(self): + def test_existing_admin_namespace_preserves_metadata_and_targets_releases(self): + self.config["namespace"] = "research-team" cluster = self.cluster() - cluster.objects[("namespace", cluster.ns)] = {"metadata": {"labels": {}}} - with self.assertRaisesRegex(ValueError, "not owned"): - cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) - self.assertFalse( - any("create" in args or args[0] == "helm" for args, _ in cluster.calls) + namespace = { + "metadata": { + "name": cluster.ns, + "labels": {"team": "research"}, + "annotations": {"openshift.io/sa.scc.uid-range": "1000100000/10000"}, + }, + "status": {"phase": "Active"}, + } + cluster.objects[("namespace", cluster.ns)] = copy.deepcopy(namespace) + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("namespace", cluster.ns)], namespace) + self.assertEqual( + {r["name"] for r in cluster.releases}, {"openshell", "openshell-dashboard"} + ) + for args, payload in cluster.calls: + if args[:2] == ["helm", "upgrade"]: + self.assertEqual(args[args.index("--namespace") + 1], "research-team") + if "create" in args: + resource = json.loads(payload) + self.assertNotEqual(resource["kind"], "Namespace") + self.assertEqual(resource["metadata"]["namespace"], "research-team") + gateway, dashboard = installer.chart_values(cluster.config) + self.assertEqual(gateway["server"]["sandboxNamespace"], "research-team") + self.assertIn( + "openshell.research-team.svc", gateway["pkiInitJob"]["serverDnsNames"] + ) + self.assertEqual( + dashboard["gateway"]["url"], + "grpcs://openshell.research-team.svc.cluster.local:8080", + ) + + def test_namespace_override_validated_and_config_namespace_optional_with_override( + self, + ): + self.config.pop("namespace") + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + self.assertEqual( + installer.load_config(path, "admin-project")["namespace"], "admin-project" + ) + with self.assertRaisesRegex(ValueError, "Invalid namespace"): + installer.load_config(path, "invalid/namespace") + with self.assertRaisesRegex(ValueError, "namespace"): + installer.load_config(path) + + def test_existing_gateway_or_foreign_release_rejected_before_mutation(self): + for release in ( + {"name": "another-gateway", "chart": "helm-chart-0.1.3"}, + {"name": "openshell", "chart": "helm-chart-0.1.3"}, + {"name": "openshell-dashboard", "chart": "other-chart-1.0.0"}, + ): + with self.subTest(release=release): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [release] + with self.assertRaises(ValueError): + cluster.install([], []) + self.assertFalse( + any( + "create" in args or args[:2] == ["helm", "upgrade"] + for args, _ in cluster.calls + ) + ) + + def test_conflict_on_later_release_page_is_detected(self): + cluster = self.cluster() + cluster.releases = [ + {"name": f"other-{i}", "chart": "unrelated-1.0.0"} for i in range(256) + ] + cluster.releases.append( + {"name": "another-gateway", "chart": "helm-chart-0.1.3"} + ) + with self.assertRaisesRegex(ValueError, "one gateway"): + cluster.check_releases(managed=True) + + def test_terminating_namespace_is_rejected(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = { + "metadata": {}, + "status": {"phase": "Terminating"}, + } + with self.assertRaisesRegex(ValueError, "terminating"): + cluster.install([], []) + + def test_preexisting_gateway_certificate_is_not_reused(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.objects[("secret", "openshell-server-tls")] = { + "data": {"tls.crt": "old"} + } + with self.assertRaisesRegex(ValueError, "explicit migration"): + cluster.install([], []) + + def test_legacy_install_migrates_without_modifying_namespace(self): + cluster = self.cluster() + namespace = { + "metadata": { + "labels": {installer.OWNER: installer.OWNER_VALUE}, + "annotations": { + installer.GATEWAY_HOST: cluster.config["gateway"]["host"] + }, + } + } + cluster.objects[("namespace", cluster.ns)] = copy.deepcopy(namespace) + cluster.releases = [{"name": "openshell", "chart": "helm-chart-0.1.3"}] + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("namespace", cluster.ns)], namespace) + self.assertEqual( + cluster.objects[("configmap", installer.INSTALL_STATE)]["data"][ + "gateway-host" + ], + cluster.config["gateway"]["host"], ) def test_missing_prerequisite_stops_before_mutation(self): @@ -274,6 +396,8 @@ def test_check_only_never_calls_kubectl(self): "--config", str(path), "--check", + "--namespace", + "render-project", "--gateway-chart-directory", str(self.root / "gateway"), "--dashboard-chart-directory", @@ -288,6 +412,8 @@ def test_check_only_never_calls_kubectl(self): self.assertEqual(run.call_count, 2) for call in run.call_args_list: self.assertEqual(call.args[0][:2], ["helm", "template"]) + args = call.args[0] + self.assertEqual(args[args.index("--namespace") + 1], "render-project") if __name__ == "__main__": diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 7932ae6b5c..c071f0556e 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -90,8 +90,11 @@ OpenShell detects the served Agent Sandbox `Sandbox` API when the Kubernetes gat For an administrator-managed OpenShift deployment with a browser UI, the ODH fork provides a [gateway and dashboard installer](https://github.com/opendatahub-io/openshell/tree/main/deploy/rhoai). -It creates a dedicated namespace and installs separately versioned gateway and -dashboard charts using customer OIDC. Agent Sandbox remains a prerequisite. +It installs separately versioned gateway and dashboard charts using customer OIDC +in the administrator-selected namespace (`namespace` in the configuration or +`--namespace` on the command line), creating it only if missing. Existing namespace +metadata is preserved. Each namespace has one gateway/dashboard pair, which reruns +upgrade in place. Agent Sandbox remains a prerequisite. The installer includes a render-only check, preserves resources on failure, and requires explicit chart versions. Validate login, workspace authorization, sandbox lifecycle, and terminal access on your target cluster before sharing the diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 3eee56e270..8ec766c8ca 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -93,12 +93,17 @@ Use gateway metadata, deployment values, or the user's setup notes to identify t ### Step 3: Check Gateway Startup Dependencies For an ODH gateway-and-dashboard installation, inspect both Helm releases -(`openshell` and `openshell-dashboard`) in the same namespace. The installer +(`openshell` and `openshell-dashboard`) in the administrator-selected namespace +(`namespace` in the configuration or the `--namespace` override). Existing +namespaces do not need an installer ownership label. The installer preserves partial installations and credentials on failure; gateway readiness -does not establish dashboard login or sandbox readiness. Check namespace label -`openshell.ai/installer=odh-dashboard`, the dashboard OIDC Secret, the shared -gateway CA Secret, and admission of both Routes. Do not add the ownership label -to an unrelated namespace to bypass installation checks. Use the published +does not establish dashboard login or sandbox readiness. Check the +`openshell-installer` state ConfigMap, the dashboard OIDC Secret, the shared +gateway CA Secret, and admission of both Routes. A gateway chart under another +release name or an existing fixed release without installer state requires +explicit migration; do not change state to bypass these checks. Earlier +installations used namespace annotations for state, which reruns migrate while +preserving namespace metadata. Use the published [Kubernetes setup guide](https://docs.nvidia.com/openshell/latest/kubernetes/setup.md) for the installation entry point. Diagnose Agent Sandbox availability separately; the installer does not provision that controller or RHOAI itself. From 4644e040af67eedc87f14531675c2d57f4b18876 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Tue, 13 Oct 2026 07:39:01 -0700 Subject: [PATCH 10/26] docs(rhoai): remove unrelated cluster cleanup history Signed-off-by: Varsha Prasad Narsing --- deploy/rhoai/README.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md index 9f5842a555..ce5a967832 100644 --- a/deploy/rhoai/README.md +++ b/deploy/rhoai/README.md @@ -35,12 +35,6 @@ The following checks passed with gateway chart `0.1.3` and dashboard chart `0.1. - Sandbox creation through the dashboard API, workload/supervisor readiness, authenticated terminal WebSocket execution with exit code 0, and completed deletion. -An expired, unrelated Kagenti admission webhook initially blocked pod creation. -Removing Kagenti restored both Agent Sandbox and the existing Keycloak instance. -The installer correctly stopped before installing OpenShell while Agent Sandbox -was missing or unhealthy. Diagnose admission webhook failures before changing -OpenShell SCC settings or authentication configuration. - These checks do not establish FIPS qualification, production certificate renewal, standard-user cross-workspace isolation, or a full UI interaction test. The live configuration and credentials are kept outside the repository. From b99e4c2ee1f7f817fa8f38c8d8c01be520f438b7 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Tue, 13 Oct 2026 09:49:05 -0700 Subject: [PATCH 11/26] feat(rhoai): harden installer configuration and verification Signed-off-by: Varsha Prasad Narsing --- .github/workflows/rhoai-installer.yml | 46 +++ deploy/rhoai/README.md | 134 +++++++- deploy/rhoai/config.example.json | 10 +- deploy/rhoai/dashboard-values.example.yaml | 14 + deploy/rhoai/gateway-values.example.yaml | 14 + deploy/rhoai/install.py | 342 +++++++++++++++++++-- deploy/rhoai/test_install.py | 221 ++++++++++++- deploy/rhoai/test_verify.py | 131 ++++++++ deploy/rhoai/verify.py | 264 ++++++++++++++++ docs/kubernetes/setup.mdx | 4 + skills/debug-openshell-cluster/SKILL.md | 8 + 11 files changed, 1146 insertions(+), 42 deletions(-) create mode 100644 .github/workflows/rhoai-installer.yml create mode 100644 deploy/rhoai/dashboard-values.example.yaml create mode 100644 deploy/rhoai/gateway-values.example.yaml create mode 100644 deploy/rhoai/test_verify.py create mode 100644 deploy/rhoai/verify.py diff --git a/.github/workflows/rhoai-installer.yml b/.github/workflows/rhoai-installer.yml new file mode 100644 index 0000000000..d1512dcb33 --- /dev/null +++ b/.github/workflows/rhoai-installer.yml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 + +name: RHOAI Installer + +on: + pull_request: + paths: + - "deploy/rhoai/**" + - ".github/workflows/rhoai-installer.yml" + - "pyproject.toml" + push: + branches: [main] + paths: + - "deploy/rhoai/**" + - ".github/workflows/rhoai-installer.yml" + - "pyproject.toml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0 + with: + version: "0.10.12" + python-version: "3.12" + - name: Installer regression tests + run: uv run --script --python 3.12 deploy/rhoai/test_install.py + - name: Authorization verifier tests + run: uv run --no-project --python 3.12 deploy/rhoai/test_verify.py + - name: Lint and formatting + run: | + uvx ruff==0.14.14 check deploy/rhoai + uvx ruff==0.14.14 format --check deploy/rhoai diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md index ce5a967832..1c1f5ceaaa 100644 --- a/deploy/rhoai/README.md +++ b/deploy/rhoai/README.md @@ -11,7 +11,9 @@ This is an initial installer, not a qualified RHOAI deployment. Gateway chart `0.1.3` and dashboard chart `0.1.0`, using their default images, passed the live smoke checks below on OpenShift `4.21.8`. Matching `0.1` release lines alone do not prove API compatibility. Choose exact published versions and validate them -together. The script intentionally has no default release versions. +together. The script intentionally has no default release versions. Exact SemVer +prereleases such as `0.0.0-dev.` are accepted for downstream builds; +accepting a version does not establish chart or image compatibility. ## Live validation @@ -34,9 +36,15 @@ The following checks passed with gateway chart `0.1.3` and dashboard chart `0.1. - Dashboard compatibility response reporting the gateway release line as supported. - Sandbox creation through the dashboard API, workload/supervisor readiness, authenticated terminal WebSocket execution with exit code 0, and completed deletion. +- Upgrade using YAML resource overrides and explicit OIDC scopes, with legacy + installer state migrated to record the gateway identity. +- Read-only authorization verification: admin access to both test workspaces, + ordinary-user access to its assigned workspace and HTTP 403 for the other. + A separate live check confirmed access was denied after membership revocation. + Browser login and sandbox/terminal smoke checks also passed after the upgrade. These checks do not establish FIPS qualification, production certificate renewal, -standard-user cross-workspace isolation, or a full UI interaction test. The live +isolation beyond the exercised workspace checks, or a full UI interaction test. The live configuration and credentials are kept outside the repository. ## Prerequisites @@ -44,7 +52,10 @@ configuration and credentials are kept outside the repository. - `uv`, Helm 3.17.3 or later, and `kubectl` on the administrator's workstation. - An OpenShift cluster, an explicit kubeconfig context, and permission to manage the namespaced/cluster resources required by the charts. Permission to create - namespaces is needed only when the selected namespace does not exist. + namespaces is needed only when the selected namespace does not exist. Preflight + also reads StorageClasses, nodes, ClusterVersion, and installer-labeled ConfigMaps + across namespaces. System namespaces such as `default`, `kube-*`, and + `openshift-*` are rejected. - Agent Sandbox CRDs and a running controller installed by the cluster administrator. The installer checks the supported served Sandbox API and waits for the controller Deployment. Override `agent_sandbox_namespace` and @@ -74,12 +85,18 @@ chmod 700 "$HOME/.config/openshell-rhoai" cp deploy/rhoai/config.example.json "$HOME/.config/openshell-rhoai/config.json" ``` -Edit the copy. Replace both version placeholders with exact published `0.1.x` +Edit the copy. Replace both version placeholders with exact published SemVer chart versions, and set the namespace, kubeconfig context, DNS names, and OIDC claims. The two versions need not have the same patch number. The dashboard OCI reference currently points to its upstream publisher, not an ODH chart registry. Both chart references are configurable for mirrors. +For ODH images, select the development chart matching the upstream commit of +the downstream build, then set `gateway.images` to the downstream registry, +repositories, and image tags/digests. The installer does not infer that mapping +or silently select a newer chart. Source version `0.0.0`, floating tags, and +version ranges are rejected for installation. + Save the browser client's secret to `dashboard-client-secret.txt` beside the JSON, using a trusted editor or your secret manager. Restrict its permissions to `0600`. Do not put the secret in the JSON, Helm values, or command arguments. File paths @@ -92,6 +109,10 @@ client ID is separate from the gateway API audience. Generic OIDC uses such as `openshell-api:openshell-user`. These are alternatives, not assumptions about customer claim names. Without either UI filter, any authenticated IdP user can reach the UI, but the gateway still enforces its roles and workspace access. +Optional `oidc.scope` supplies space-separated OAuth scopes including `openid` +(for example `openid email profile offline_access`). Request only scopes your +IdP supports; include `groups` if your provider requires it for group claims. +When omitted, oauth2-proxy keeps its provider defaults. Validate both charts without connecting to the cluster: @@ -123,11 +144,65 @@ preserved. Use a separate configuration with distinct public hostnames for each gateway namespace, and register each dashboard callback with the IdP. Specifying a different namespace creates another deployment; it does not move an existing one. +Provision a distinct API audience and browser client per gateway, even when +gateways share an issuer. The example uses `openshell-api-openshell-team` and +`openshell-dashboard-openshell-team`; change these registrations and the JSON +when selecting another namespace. `--namespace` does not rewrite OIDC identifiers +or create IdP clients. Configure per-gateway role/group mappings at your IdP; +a namespace alone is not an authentication boundary. + +Preflight rejects an issuer/audience or issuer/browser-client pair already +recorded by another installer-managed namespace. It cannot discover registrations +outside this installer, tokens with multiple audiences, or old installation +state that has no identity metadata. Verify those cases with your IdP admin. + `--check` still downloads the pinned charts. It discards rendered manifests and does not validate connectivity, SCC admission, token claims, or runtime behavior. It reads the client secret file's existence/size, not its contents. Temporary values files have owner-only permissions and contain no client or cookie secrets. +Run cluster preflight without installing: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --preflight +``` + +This renders both charts, checks the default and selected workspace storage +classes, checks create/patch permissions for installation resources, verifies +Agent Sandbox readiness, and detects release/identity conflicts. It reports the +OpenShift version and node kernels and flags older kernels for runtime validation. +It does not encode a Red Hat product support matrix or prove storage health, +RBAC bind/escalate permissions, quota, SCC admission, or application compatibility. +Preflight reads supplied credential files to check for unintended rotation; +`--check` does not read their contents. `uv` installs the pinned PyYAML dependency +for parsing values files. + +## Chart values + +Set `gateway.values_files` and/or `dashboard.values_files` to lists of YAML or JSON +files, relative to the configuration file. Files merge in order: maps merge, +later scalars and lists replace earlier ones. Keep the same files for reruns; +the installer passes the merged result on every upgrade. Start with +[gateway-values.example.yaml](gateway-values.example.yaml) and +[dashboard-values.example.yaml](dashboard-values.example.yaml). + +Supported operational overrides are: + +| Chart | Values | +|---|---| +| Both | `resources`, `nodeSelector`, `tolerations`, `affinity`, `imagePullSecrets`, `openshiftRoute.annotations` | +| Gateway | `server.workspaceStorageClass`, `server.workspaceDefaultStorageSize`, `server.defaultRuntimeClassName`, `server.sandboxImagePullSecrets` | +| Dashboard | `authProxy.resources`, `authProxy.cookie.refresh`, `authProxy.upstreamTimeout`, `networkPolicy.ingressNamespace`, `networkPolicy.ingressFrom` | + +For example, add `"values_files": ["gateway-values.yaml"]` to the `gateway` +section. Reference existing image-pull Secrets by name; do not put credentials +in values files. Authentication, TLS, release names, service ports, and namespace +wiring remain owned by the installer. Unsupported overrides fail before cluster +access. Extending this set requires checking the pinned charts' contracts. +The gateway chart's SQLite PVC still requires a default block-backed StorageClass; +`workspaceStorageClass` affects sandbox workspace PVCs only. + The installer creates the namespace only when missing, then creates an installer state ConfigMap, a client/cookie Secret, and an optional OIDC CA ConfigMap. It installs the gateway first, checks the gateway CA Secret, @@ -148,8 +223,9 @@ Supervisors retain gateway-minted sandbox JWT authentication. The dashboard uses an edge Route and trusts the gateway CA for internal calls. Its authentication proxy is the only exposed dashboard port. The OpenShift -router must use the `openshift-ingress` namespace network identity; host-network -router configurations require a separately reviewed chart configuration. +router defaults to the `openshift-ingress` namespace network identity. For a +custom or host-network router, provide reviewed `networkPolicy.ingressFrom` +peers in dashboard values instead; retain ingress restricted to your router. Configure a CLI OIDC registration separately with your IdP. After installation, verify administrator and standard-user login. Administrators add workspace @@ -157,6 +233,37 @@ memberships using the token's subject ID. Verify denied cross-workspace access, sandbox creation/deletion, and terminal WebSockets before sharing the deployment. Readiness and Route admission alone do not prove this flow works. +Use the read-only verifier after provisioning two distinct identities and two +existing workspaces. The admin must access both workspaces; the ordinary user +must belong only to the allowed workspace. Save fresh access tokens in files +with mode `0600` outside the checkout, using your IdP's supported login flow: + +```shell +uv run --script --python 3.12 deploy/rhoai/verify.py \ + --context YOUR_CONTEXT --namespace research-team \ + --dashboard-url https://dashboard.apps.example.com \ + --admin-role openshell-admin \ + --admin-token-file /secure/admin-token.txt \ + --user-token-file /secure/user-token.txt \ + --allowed-workspace team-a --denied-workspace team-b +``` + +The verifier checks public unauthenticated rejection, gateway identity/roles, +admin access to both workspaces, ordinary-user access to one and denial in the +other. It opens a temporary loopback-only port-forward to the BFF to test gateway +authorization with bearer tokens; this requires `pods/portforward` permission. +It performs GET requests only and does not change users, memberships, or workloads. +It closes its port-forward on success or failure and never follows redirects +with credentials or prints tokens/response bodies. + +Optional `--other-gateway-token-file` checks rejection of a token for another +gateway. First confirm that token works on its intended gateway: an expired or +otherwise invalid token cannot prove audience isolation. Optional +`--session-cookie-file` accepts the Cookie header value from an already logged-in +browser and checks the public session endpoint. `--ca-file` adds a private Route +CA to system trust. These checks do not automate browser login, grant/revoke, +sandbox creation/deletion, terminal execution, or network-policy enforcement. + ## Reruns, upgrades, and failures Rerun the same command after correcting a failure. The administrator selects the @@ -166,8 +273,14 @@ a different release name, or either fixed release without installer state, is rejected and needs explicit migration. Unmanaged gateway TLS/JWT Secrets are also rejected so the bootstrap hook cannot silently reuse another gateway's identity. Unrelated Helm releases in the selected namespace are left alone. +Uninstalled releases retained under another name do not count as active gateways; +conflicting remaining TLS/JWT Secrets and fixed release names still require +explicit migration. The original gateway hostname is recorded in the `openshell-installer` ConfigMap. +Issuer, API audience, and browser client ID are recorded too; changing them +requires an explicit identity migration. Legacy state is populated on a successful +preflight followed by an install/rerun without rewriting existing identity values. Earlier installations recorded it on the namespace; reruns migrate that state without changing namespace metadata. Changing the recorded hostname is rejected because the certificate hook preserves existing certificates; migrate @@ -180,7 +293,8 @@ explicitly and restart the consuming pods. Removing `ca_file` removes the mounts on the next upgrade, but does not delete the old ConfigMap. Change the pinned versions to upgrade. Both Helm releases receive the generated -values with `--reset-values`, so out-of-band Helm value edits are not preserved. +values with `--reset-values`, including the configured values files, so out-of-band +Helm value edits are not preserved. There is no cross-release transaction or automatic rollback: gateway success followed by dashboard failure leaves the upgraded gateway in place. Failed releases, namespaces, Secrets, and PVCs are preserved for diagnosis. @@ -214,8 +328,12 @@ Image overrides do not establish FIPS qualification. ## Contributor validation +The `RHOAI Installer` GitHub Actions workflow runs both suites for changes to +this directory. It needs no cluster credentials or IdP. Run them locally with: + ```shell -uv run --no-project --python 3.12 deploy/rhoai/test_install.py +uv run --script --python 3.12 deploy/rhoai/test_install.py +uv run --no-project --python 3.12 deploy/rhoai/test_verify.py ``` For source-chart rendering, use `--check --gateway-chart-directory diff --git a/deploy/rhoai/config.example.json b/deploy/rhoai/config.example.json index 1ebd306c69..c8659ed66b 100644 --- a/deploy/rhoai/config.example.json +++ b/deploy/rhoai/config.example.json @@ -3,20 +3,22 @@ "context": "REPLACE_WITH_KUBECONFIG_CONTEXT", "gateway": { "chart": "oci://ghcr.io/nvidia/openshell/helm-chart", - "version": "REPLACE_WITH_GATEWAY_0.1_VERSION", + "version": "REPLACE_WITH_EXACT_GATEWAY_VERSION", + "values_files": [], "host": "gateway.apps.example.com" }, "dashboard": { "chart": "oci://ghcr.io/gkrumbach07/openshell-dashboard/helm-chart/openshell-dashboard", - "version": "REPLACE_WITH_DASHBOARD_0.1_VERSION", + "version": "REPLACE_WITH_EXACT_DASHBOARD_VERSION", + "values_files": [], "host": "dashboard.apps.example.com" }, "oidc": { "provider": "oidc", "issuer": "https://identity.example.com/realms/customer", - "client_id": "openshell-dashboard", + "client_id": "openshell-dashboard-openshell-team", "client_secret_file": "dashboard-client-secret.txt", - "audience": "openshell-api", + "audience": "openshell-api-openshell-team", "roles_claim": "roles", "admin_role": "openshell-admin", "user_role": "openshell-user", diff --git a/deploy/rhoai/dashboard-values.example.yaml b/deploy/rhoai/dashboard-values.example.yaml new file mode 100644 index 0000000000..fe3a3c59ee --- /dev/null +++ b/deploy/rhoai/dashboard-values.example.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# Optional operational overrides, referenced by dashboard.values_files. +resources: + requests: + cpu: 100m + memory: 128Mi +authProxy: + resources: + requests: + cpu: 50m + memory: 64Mi +# For custom/host-network routers, set networkPolicy.ingressFrom to peers +# reviewed for your ingress controller. Keep the proxy restricted to the router. diff --git a/deploy/rhoai/gateway-values.example.yaml b/deploy/rhoai/gateway-values.example.yaml new file mode 100644 index 0000000000..769087478f --- /dev/null +++ b/deploy/rhoai/gateway-values.example.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# Optional operational overrides. Copy beside your private config.json and add +# its filename to gateway.values_files. Use sizes appropriate for your workload. +resources: + requests: + cpu: 100m + memory: 256Mi +# imagePullSecrets: +# - name: registry-credentials +# server: +# workspaceStorageClass: block-storage +# sandboxImagePullSecrets: +# - name: registry-credentials diff --git a/deploy/rhoai/install.py b/deploy/rhoai/install.py index 504e8b612b..4d76b6d2bb 100644 --- a/deploy/rhoai/install.py +++ b/deploy/rhoai/install.py @@ -3,12 +3,13 @@ # SPDX-License-Identifier: Apache-2.0 # /// script # requires-python = ">=3.11" -# dependencies = [] +# dependencies = ["PyYAML==6.0.3"] # /// """Install the gateway and dashboard as separate Helm releases on OpenShift.""" import argparse import base64 +import copy import json import os import re @@ -21,15 +22,77 @@ from pathlib import Path from urllib.parse import urlsplit +import yaml + OWNER = "openshell.ai/installer" OWNER_VALUE = "odh-dashboard" GATEWAY_HOST = "openshell.ai/installer-gateway-host" INSTALL_STATE = "openshell-installer" OIDC_SECRET = "openshell-dashboard-oidc" OIDC_CA = "openshell-oidc-ca" -VERSION = re.compile(r"0\.1\.[0-9]+(?:-[0-9A-Za-z.-]+)?") +VERSION = re.compile( + r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + r"(?:-(?:0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)" + r"(?:\.(?:0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*))*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" +) DNS = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?") +# Expose operational chart settings without allowing overlays to replace the +# installer's identity, TLS, service naming, or namespace contract. +COMMON_VALUES = dict.fromkeys( + ("resources", "nodeSelector", "tolerations", "affinity", "imagePullSecrets") +) +EXTRA_VALUES = { + "gateway": { + **COMMON_VALUES, + "server": dict.fromkeys( + ( + "workspaceStorageClass", + "workspaceDefaultStorageSize", + "defaultRuntimeClassName", + "sandboxImagePullSecrets", + ) + ), + "openshiftRoute": {"annotations": None}, + }, + "dashboard": { + **COMMON_VALUES, + "authProxy": { + "resources": None, + "cookie": {"refresh": None}, + "upstreamTimeout": None, + }, + "networkPolicy": {"ingressNamespace": None, "ingressFrom": None}, + "openshiftRoute": {"annotations": None}, + }, +} + + +def merge_values(target, overlay, allowed, prefix=""): + require(isinstance(overlay, dict), f"{prefix or 'Values'} must be a mapping") + for key, value in overlay.items(): + require( + key in allowed, f"Unsupported or installer-owned Helm value: {prefix}{key}" + ) + if allowed[key] is not None: + merge_values( + target.setdefault(key, {}), value, allowed[key], f"{prefix}{key}." + ) + elif isinstance(value, dict) and isinstance(target.get(key), dict): + # Helm's map merge semantics; arrays/scalars are replaced. + merge_unrestricted(target[key], value) + else: + target[key] = copy.deepcopy(value) + + +def merge_unrestricted(target, overlay): + for key, value in overlay.items(): + if isinstance(value, dict) and isinstance(target.get(key), dict): + merge_unrestricted(target[key], value) + else: + target[key] = copy.deepcopy(value) + def require(condition, message): if not condition: @@ -63,21 +126,52 @@ def load_config(path, namespace=None): and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", ns), "Invalid namespace", ) + require( + ns not in ("default", "openshift", "kube-system", "kube-public") + and not ns.startswith(("kube-", "openshift-")), + "Choose an application namespace, not a Kubernetes/OpenShift system namespace", + ) + for key in ("agent_sandbox_namespace", "agent_sandbox_deployment"): + if key in config: + require( + isinstance(config[key], str) and DNS.fullmatch(config[key]), + f"Invalid {key}", + ) require( isinstance(config["context"], str) and config["context"].strip(), "context is required", ) for name in ("gateway", "dashboard"): item = config[name] - fields(item, ("chart", "version", "host"), ("images",)) + fields(item, ("chart", "version", "host"), ("images", "values_files")) require( isinstance(item["chart"], str) and item["chart"].startswith("oci://"), f"{name}.chart must be an OCI chart reference", ) require( isinstance(item["version"], str) and VERSION.fullmatch(item["version"]), - f"{name}.version must pin a 0.1.x release (no latest or version ranges)", + f"{name}.version must be exact SemVer (no latest or version ranges)", ) + require( + item["version"] != "0.0.0", + "Use a published release or pinned development version", + ) + sources = item.get("values_files", []) + require( + isinstance(sources, list) + and all(isinstance(p, str) and p for p in sources), + f"{name}.values_files must be a list of file paths", + ) + overrides = {} + for source in sources: + # SafeLoader does not construct Python objects. Do not include file + # contents in diagnostics: user-supplied files may contain secrets. + try: + overlay = yaml.safe_load((path.parent / source).read_text()) + except yaml.YAMLError: + raise ValueError(f"Invalid YAML in {name} values file") from None + merge_values(overrides, overlay, EXTRA_VALUES[name]) + item["values"] = overrides host = item["host"] require( isinstance(host, str) @@ -122,6 +216,7 @@ def load_config(path, namespace=None): "allowed_roles", "client_secret_file", "ca_file", + "scope", ), ) for key in ( @@ -167,6 +262,16 @@ def load_config(path, namespace=None): not oidc.get("allowed_roles") or oidc.get("provider") == "keycloak-oidc", "allowed_roles requires keycloak-oidc; use allowed_groups for generic OIDC", ) + if "scope" in oidc: + require( + isinstance(oidc["scope"], str) + and re.fullmatch( + r"[\x21\x23-\x5B\x5D-\x7E]+(?: [\x21\x23-\x5B\x5D-\x7E]+)*", + oidc["scope"], + ) + and "openid" in oidc["scope"].split(), + "oidc.scope must be space-separated OAuth scopes including openid", + ) for key in ("client_secret_file", "ca_file"): if oidc.get(key): source = (path.parent / oidc[key]).resolve() @@ -236,6 +341,7 @@ def chart_values(config): "allowedGroups": oidc.get("allowed_groups", []), "allowedRoles": oidc.get("allowed_roles", []), "caConfigMapName": ca, + "scope": oidc.get("scope", ""), }, "openshiftRoute": { "enabled": True, @@ -245,6 +351,7 @@ def chart_values(config): "networkPolicy": {"ingressNamespace": "openshift-ingress"}, } for name, values in (("gateway", gateway), ("dashboard", dashboard)): + merge_values(values, config[name].get("values", {}), EXTRA_VALUES[name]) for component, image in config[name].get("images", {}).items(): values.setdefault(component, {})["image"] = image return gateway, dashboard @@ -307,6 +414,13 @@ def check_releases(self, managed): ) for release in releases: name = release["name"] + # --keep-history leaves an uninstalled record, not a gateway. + # Keep fixed-name ownership checks: Helm still owns that history. + if release.get("status") == "uninstalled" and name not in ( + "openshell", + "openshell-dashboard", + ): + continue gateway = release.get("chart", "").startswith("helm-chart-") require( not gateway or name == "openshell", @@ -320,6 +434,160 @@ def check_releases(self, managed): break offset += len(releases) + def cluster_checks(self, namespace): + version = self.get("clusterversion.config.openshift.io", "version", False) + print( + f"OpenShift: {(version or {}).get('status', {}).get('desired', {}).get('version', 'unknown')}" + ) + storage = json.loads( + self.run([*self.kube, "get", "storageclasses", "-o", "json"]) + ) + defaults = [ + item["metadata"]["name"] + for item in storage["items"] + if any( + item["metadata"].get("annotations", {}).get(key) == "true" + for key in ( + "storageclass.kubernetes.io/is-default-class", + "storageclass.beta.kubernetes.io/is-default-class", + ) + ) + ] + require( + defaults, + "No default StorageClass; the gateway chart's SQLite PVC requires one", + ) + print( + f"Default storage: {', '.join(defaults)} (verify block-backed SQLite storage)" + ) + custom = ( + self.config["gateway"] + .get("values", {}) + .get("server", {}) + .get("workspaceStorageClass") + ) + if custom: + require( + any(item["metadata"]["name"] == custom for item in storage["items"]), + "Configured workspaceStorageClass does not exist", + ) + nodes = json.loads(self.run([*self.kube, "get", "nodes", "-o", "json"])) + kernels = sorted( + { + item.get("status", {}) + .get("nodeInfo", {}) + .get("kernelVersion", "unknown") + for item in nodes["items"] + } + ) + print(f"Node kernels: {', '.join(kernels) or 'none reported'}") + if any( + (match := re.match(r"(\d+)\.(\d+)", kernel)) + and tuple(map(int, match.groups())) < (5, 19) + for kernel in kernels + ): + print( + "Note: kernels before 5.19 need an OpenShell build with the legacy-kernel fixes; verify sandbox networking." + ) + # Check actual installation operations instead of requiring wildcard + # cluster-admin. These checks cannot prove RBAC bind/escalate or SCC admission. + scoped = [ + "secrets", + "configmaps", + "services", + "serviceaccounts", + "deployments.apps", + "statefulsets.apps", + "jobs.batch", + "roles.rbac.authorization.k8s.io", + "rolebindings.rbac.authorization.k8s.io", + "routes.route.openshift.io", + "networkpolicies.networking.k8s.io", + ] + operations = [ + (verb, resource, True) + for resource in scoped + for verb in ("create", "patch") + ] + operations += [ + (verb, resource, False) + for resource in ( + "clusterroles.rbac.authorization.k8s.io", + "clusterrolebindings.rbac.authorization.k8s.io", + ) + for verb in ("create", "patch") + ] + if not namespace: + operations.append(("create", "namespaces", False)) + for verb, resource, namespaced in operations: + try: + answer = self.run( + [ + *self.kube, + "auth", + "can-i", + verb, + resource, + *(["-n", self.ns] if namespaced else []), + ] + ) + require(answer.strip() == "yes", "denied") + except (RuntimeError, ValueError): + raise ValueError( + f"Preflight permission check failed: {verb} {resource}" + + (f" in {self.ns}" if namespaced else " cluster-wide") + ) from None + + def identity_state(self): + return { + "gateway-host": self.config["gateway"]["host"], + **{ + f"oidc-{key.replace('_', '-')}": self.config["oidc"][key] + for key in ("issuer", "audience", "client_id") + }, + } + + def check_identity(self, state): + desired = self.identity_state() + for key, value in (state or {}).get("data", {}).items(): + if key in desired: + require( + value == desired[key], + f"Installer identity {key} changed; explicit identity/certificate migration is required", + ) + states = json.loads( + self.run( + [ + *self.kube, + "get", + "configmaps", + "--all-namespaces", + "-l", + f"{OWNER}={OWNER_VALUE}", + "-o", + "json", + ] + ) + ) + for other in states["items"]: + if ( + other["metadata"]["name"] != INSTALL_STATE + or other["metadata"].get("namespace") == self.ns + ): + continue + data = other.get("data", {}) + if "oidc-issuer" not in data: + print( + f"Note: legacy installer state in {other['metadata']['namespace']} has no identity metadata; verify audience isolation manually." + ) + continue + if data["oidc-issuer"].rstrip("/") == desired["oidc-issuer"].rstrip("/"): + require( + data.get("oidc-audience") != desired["oidc-audience"] + and data.get("oidc-client-id") != desired["oidc-client-id"], + f"OIDC audience/browser client already used by namespace {other['metadata']['namespace']}; provision distinct per-gateway registrations", + ) + def preflight(self): self.run([*self.kube, "get", "--raw", "/apis/route.openshift.io/v1"]) crd = self.get("crd", "sandboxes.agents.x-k8s.io", False) @@ -351,6 +619,7 @@ def preflight(self): not namespace or namespace.get("status", {}).get("phase") != "Terminating", "Namespace is terminating", ) + self.cluster_checks(namespace) state = self.get("configmap", INSTALL_STATE) if namespace else None require( not state or state["metadata"].get("labels", {}).get(OWNER) == OWNER_VALUE, @@ -372,6 +641,7 @@ def preflight(self): or original_host == self.config["gateway"]["host"], "Gateway hostname differs from the original install; certificate migration is required", ) + self.check_identity(state) if namespace: self.check_releases(managed=bool(state or legacy_host)) # The chart's bootstrap hook reuses existing TLS material. Do not @@ -391,9 +661,10 @@ def preflight(self): existing or self.config["oidc"].get("client_secret_file"), "oidc.client_secret_file is required for the first install", ) + self.credentials(check_only=True) return namespace - def credentials(self): + def credentials(self, check_only=False): oidc = self.config["oidc"] existing = self.get("secret", OIDC_SECRET) supplied = ( @@ -415,20 +686,21 @@ def credentials(self): ) else: require(supplied, "Client secret file is required") - self.create( - { - "apiVersion": "v1", - "kind": "Secret", - "type": "Opaque", - "metadata": {"name": OIDC_SECRET, "namespace": self.ns}, - "data": { - "client-secret": base64.b64encode(supplied).decode(), - "cookie-secret": base64.b64encode( - secrets.token_urlsafe(32).encode() - ).decode(), - }, - } - ) + if not check_only: + self.create( + { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": {"name": OIDC_SECRET, "namespace": self.ns}, + "data": { + "client-secret": base64.b64encode(supplied).decode(), + "cookie-secret": base64.b64encode( + secrets.token_urlsafe(32).encode() + ).decode(), + }, + } + ) if oidc.get("ca_file"): ca = Path(oidc["ca_file"]).read_text() existing_ca = self.get("configmap", OIDC_CA) @@ -436,7 +708,7 @@ def credentials(self): not existing_ca or existing_ca.get("data", {}).get("ca.crt") == ca, "OIDC CA differs; rotate the ConfigMap and restart both releases explicitly", ) - if not existing_ca: + if not existing_ca and not check_only: self.create( { "apiVersion": "v1", @@ -489,9 +761,24 @@ def install(self, charts, values): "namespace": self.ns, "labels": {OWNER: OWNER_VALUE}, }, - "data": {"gateway-host": self.config["gateway"]["host"]}, + "data": self.identity_state(), } ) + else: + # Add identity metadata to legacy state only after preflight passed. + self.run( + [ + *self.kube, + "-n", + self.ns, + "patch", + "configmap", + INSTALL_STATE, + "--type=merge", + "-p", + json.dumps({"data": self.identity_state()}), + ] + ) self.credentials() for index, release in enumerate(("openshell", "openshell-dashboard")): print(f"Installing {release} in {self.ns}...", flush=True) @@ -541,11 +828,17 @@ def main(): "--namespace", help="Administrator-selected namespace; overrides config.namespace (created if missing)", ) - parser.add_argument( + mode = parser.add_mutually_exclusive_group() + mode.add_argument( "--check", action="store_true", help="Validate and render charts without accessing the cluster", ) + mode.add_argument( + "--preflight", + action="store_true", + help="Render charts and check cluster prerequisites without installing", + ) parser.add_argument( "--gateway-chart-directory", type=Path, @@ -634,7 +927,10 @@ def main(): "Both charts rendered successfully. This does not verify cluster or API compatibility.", flush=True, ) - if not args.check: + if args.preflight: + installer.preflight() + print("Cluster preflight passed; no installation resources were changed.") + elif not args.check: installer.install(charts, files) diff --git a/deploy/rhoai/test_install.py b/deploy/rhoai/test_install.py index fb2dfd273d..48f96d756d 100644 --- a/deploy/rhoai/test_install.py +++ b/deploy/rhoai/test_install.py @@ -1,5 +1,9 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors # SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = ["PyYAML==6.0.3"] +# /// """Behavioral tests with a simulated cluster; no network or cluster access.""" import base64 @@ -38,9 +42,38 @@ def __init__(self, config): self.calls = [] self.releases = [] self.fail_dashboard = False + self.storage = [ + { + "metadata": { + "name": "block", + "annotations": { + "storageclass.kubernetes.io/is-default-class": "true" + }, + } + } + ] + self.other_states = [] + self.denied_permission = None def run(self, args, payload=None): self.calls.append((args, payload)) + if "can-i" in args: + return ( + "no" + if self.denied_permission and self.denied_permission in args + else "yes" + ) + if "storageclasses" in args: + return json.dumps({"items": self.storage}) + if "nodes" in args: + return json.dumps({"items": []}) + if "--all-namespaces" in args: + return json.dumps({"items": self.other_states}) + if "patch" in args: + self.objects[("configmap", installer.INSTALL_STATE)]["data"].update( + json.loads(args[args.index("-p") + 1])["data"] + ) + return "" if args[:2] == ["helm", "list"]: offset = int(args[args.index("--offset") + 1]) return json.dumps(self.releases[offset : offset + 256]) @@ -98,7 +131,14 @@ def cluster(self): return FakeCluster(self.load()) def test_rejects_unpinned_versions_and_unknown_keys(self): - for version in ("latest", "0.0.0", "0.2.1", ">=0.1.0", "REPLACE_ME"): + for version in ( + "latest", + "0.0.0", + "01.2.3", + "1.2.3-01", + ">=0.1.0", + "REPLACE_ME", + ): with self.subTest(version=version): self.config["gateway"]["version"] = version with self.assertRaises(ValueError): @@ -108,11 +148,170 @@ def test_rejects_unpinned_versions_and_unknown_keys(self): with self.assertRaisesRegex(ValueError, "Unknown"): self.load() + def test_pinned_downstream_and_new_release_versions(self): + for version in ("0.0.0-dev.e7fdd6b", "0.2.1", "1.2.3+rhaiv.5"): + self.config["gateway"]["version"] = version + self.assertEqual(self.load()["gateway"]["version"], version) + + def test_yaml_overlays_merge_in_order_and_preserve_managed_values(self): + (self.root / "base.yaml").write_text( + "resources:\n requests:\n cpu: 100m\n memory: 256Mi\n" + ) + (self.root / "site.yaml").write_text( + "resources:\n requests:\n cpu: 200m\nimagePullSecrets:\n - name: registry-creds\n" + ) + self.config["gateway"]["values_files"] = ["base.yaml", "site.yaml"] + self.config["oidc"]["scope"] = "openid email profile offline_access" + gateway, dashboard = installer.chart_values(self.load()) + self.assertEqual( + gateway["resources"]["requests"], {"cpu": "200m", "memory": "256Mi"} + ) + self.assertEqual(gateway["imagePullSecrets"], [{"name": "registry-creds"}]) + self.assertEqual( + gateway["server"]["sandboxNamespace"], self.config["namespace"] + ) + self.assertEqual( + dashboard["oidc"]["scope"], "openid email profile offline_access" + ) + + def test_overlays_cannot_replace_authentication_or_namespace_contract(self): + for name, overlay in ( + ("gateway", "server:\n auth:\n allowUnauthenticatedUsers: true"), + ("gateway", "server: null"), + ("gateway", "fullnameOverride: foreign"), + ("dashboard", "authProxy:\n extraArgs: [--skip-auth-regex=.*]"), + ("dashboard", "networkPolicy:\n enabled: false"), + ("dashboard", "oidc:\n existingSecret: foreign"), + ): + with self.subTest(overlay=overlay): + self.config = configuration() + (self.root / "site.yaml").write_text(overlay) + self.config[name]["values_files"] = ["site.yaml"] + with self.assertRaises(ValueError): + self.load() + + def test_scope_and_system_namespace_validation(self): + for scope in (None, [], "profile email", "openid\nemail"): + self.config["oidc"]["scope"] = scope + with self.assertRaises(ValueError): + self.load() + self.config = configuration() + for ns in ("default", "openshift", "openshift-ingress", "kube-system"): + self.config["namespace"] = ns + with self.assertRaisesRegex(ValueError, "system namespace"): + self.load() + + def test_preflight_storage_and_permissions_fail_before_mutation(self): + for issue in ("storage", "permission", "workspace-storage"): + cluster = self.cluster() + if issue == "storage": + cluster.storage = [] + elif issue == "permission": + cluster.denied_permission = "deployments.apps" + else: + cluster.config["gateway"]["values"] = { + "server": {"workspaceStorageClass": "missing"} + } + with self.subTest(issue=issue), self.assertRaises(ValueError): + cluster.install([], []) + self.assertFalse( + any( + "create" in args and "can-i" not in args + for args, _ in cluster.calls + ) + ) + self.assertFalse( + any(args[:2] == ["helm", "upgrade"] for args, _ in cluster.calls) + ) + + def test_cross_namespace_identity_reuse_is_rejected(self): + for field in ("oidc-audience", "oidc-client-id"): + cluster = self.cluster() + other = { + **cluster.identity_state(), + "oidc-audience": "other-api", + "oidc-client-id": "other-browser", + } + other[field] = cluster.identity_state()[field] + cluster.other_states = [ + { + "metadata": {"name": installer.INSTALL_STATE, "namespace": "other"}, + "data": other, + } + ] + with ( + self.subTest(field=field), + self.assertRaisesRegex(ValueError, "distinct per-gateway"), + ): + cluster.preflight() + cluster.other_states[0]["data"]["oidc-issuer"] = ( + "https://another-issuer.example" + ) + cluster.preflight() + + def test_identity_change_requires_migration(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + cluster.config["oidc"]["audience"] = "different-api" + with self.assertRaisesRegex(ValueError, "identity.*changed"): + cluster.preflight() + + def test_uninstalled_other_release_history_does_not_block_install(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [ + { + "name": "retired-gateway", + "chart": "helm-chart-0.1.3", + "status": "uninstalled", + } + ] + cluster.preflight() + cluster.objects[("secret", "openshell-server-tls")] = {"data": {}} + with self.assertRaisesRegex(ValueError, "explicit migration"): + cluster.preflight() + + def test_preflight_rejects_credential_drift_without_changing_state(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + before = copy.deepcopy(cluster.objects) + Path(cluster.config["oidc"]["client_secret_file"]).write_text("changed") + cluster.calls.clear() + with self.assertRaisesRegex(ValueError, "rotate"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any("patch" in args and "can-i" not in args for args, _ in cluster.calls) + ) + + def test_preflight_mode_renders_and_checks_without_installing(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + + def run(_instance, args, payload=None): + self.assertIsNone(payload) + if args[:2] == ["helm", "pull"]: + (Path(args[args.index("--destination") + 1]) / "chart.tgz").write_bytes( + b"fake" + ) + return "" + + with ( + patch("sys.argv", ["install.py", "--config", str(path), "--preflight"]), + patch.object(installer.shutil, "which", return_value="tool"), + patch.object(installer.Installer, "run", run), + patch.object(installer.Installer, "preflight") as preflight, + patch.object(installer.Installer, "install") as install, + ): + installer.main() + preflight.assert_called_once() + install.assert_not_called() + def test_rejects_invalid_oidc_and_namespace(self): for section, key, value in ( ("oidc", "issuer", "http://issuer.example"), ("oidc", "issuer", "https://user:password@issuer.example"), - ("oidc", "audience", "openshell-dashboard"), + ("oidc", "audience", configuration()["oidc"]["client_id"]), ("oidc", "allowed_roles", ["admin"]), ("gateway", "host", "gateway..example"), ): @@ -162,7 +361,7 @@ def test_existing_admin_namespace_preserves_metadata_and_targets_releases(self): for args, payload in cluster.calls: if args[:2] == ["helm", "upgrade"]: self.assertEqual(args[args.index("--namespace") + 1], "research-team") - if "create" in args: + if "create" in args and "can-i" not in args: resource = json.loads(payload) self.assertNotEqual(resource["kind"], "Namespace") self.assertEqual(resource["metadata"]["namespace"], "research-team") @@ -204,7 +403,8 @@ def test_existing_gateway_or_foreign_release_rejected_before_mutation(self): cluster.install([], []) self.assertFalse( any( - "create" in args or args[:2] == ["helm", "upgrade"] + ("create" in args and "can-i" not in args) + or args[:2] == ["helm", "upgrade"] for args, _ in cluster.calls ) ) @@ -264,7 +464,9 @@ def test_missing_prerequisite_stops_before_mutation(self): cluster.objects.clear() with self.assertRaisesRegex(ValueError, "Agent Sandbox"): cluster.install([], []) - self.assertFalse(any("create" in args for args, _ in cluster.calls)) + self.assertFalse( + any("create" in args and "can-i" not in args for args, _ in cluster.calls) + ) def test_install_order_context_and_secret_transport(self): cluster = self.cluster() @@ -294,7 +496,9 @@ def test_rerun_preserves_secrets_without_input_file(self): cluster.calls.clear() cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) self.assertEqual(cluster.objects[("secret", installer.OIDC_SECRET)], before) - self.assertFalse(any("create" in args for args, _ in cluster.calls)) + self.assertFalse( + any("create" in args and "can-i" not in args for args, _ in cluster.calls) + ) def test_changed_gateway_host_requires_certificate_migration(self): cluster = self.cluster() @@ -304,7 +508,10 @@ def test_changed_gateway_host_requires_certificate_migration(self): with self.assertRaisesRegex(ValueError, "certificate migration"): cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) self.assertFalse( - any(args[0] == "helm" or "create" in args for args, _ in cluster.calls) + any( + args[0] == "helm" or ("create" in args and "can-i" not in args) + for args, _ in cluster.calls + ) ) def test_different_secret_is_not_silently_rotated(self): diff --git a/deploy/rhoai/test_verify.py b/deploy/rhoai/test_verify.py new file mode 100644 index 0000000000..f562169b90 --- /dev/null +++ b/deploy/rhoai/test_verify.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +"""Authorization verifier regression tests; no cluster or identity provider.""" + +import importlib.util +import io +import threading +import unittest +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from unittest.mock import Mock, patch + +spec = importlib.util.spec_from_file_location( + "verifier", Path(__file__).with_name("verify.py") +) +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) + + +class VerifyTests(unittest.TestCase): + def response(self, url, token=None, **_kwargs): + if url.startswith("https:") or not token: + return 401, None + if url.endswith("whoami"): + return 200, { + "subject": token, + "roles": ["admin"] if token == "admin-token" else ["user"], + } + if token == "other-token": + return 401, None + if token == "user-token" and "/private/" in url: + return 403, None + return 200, [] + + def run_verify(self, http): + with redirect_stdout(io.StringIO()) as output: + verifier.verify( + "http://127.0.0.1:1234", + "https://dashboard.example", + "admin-token", + "user-token", + "team", + "private", + "admin", + other="other-token", + http=http, + ) + self.assertNotIn("admin-token", output.getvalue()) + self.assertNotIn("user-token", output.getvalue()) + + def test_allowed_and_denied_access(self): + self.run_verify(self.response) + + def test_false_positive_authorization_results_fail(self): + for failure in ( + "admin", + "user", + "missing-workspace", + "public-redirect", + "missing-subject", + "wrong-audience", + ): + + def response(url, token=None, failure=failure, **kwargs): + if failure == "admin" and token == "admin-token": + return 403, None + if failure == "user" and token == "user-token" and "/private/" in url: + return 200, [] + if failure == "missing-workspace" and "/private/" in url: + return 404, None + if failure == "public-redirect" and url.startswith("https:"): + return 302, None + if failure == "missing-subject" and url.endswith("whoami"): + return 200, {"roles": ["admin"]} + if failure == "wrong-audience" and token == "other-token": + return 200, [] + return self.response(url, token, **kwargs) + + with self.subTest(failure=failure), self.assertRaises(ValueError): + self.run_verify(response) + + def test_redirect_does_not_forward_credentials(self): + received = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + received.append((self.path, self.headers.get("Authorization"))) + self.send_response(302) + self.send_header("Location", "/leak") + self.end_headers() + + def log_message(self, *_args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + status, _ = verifier.request( + f"http://127.0.0.1:{server.server_port}/start", + token="private-token", + ) + self.assertEqual(status, 302) + self.assertEqual(received, [("/start", "Bearer private-token")]) + finally: + server.shutdown() + thread.join() + + def test_port_forward_is_terminated_when_verification_fails(self): + process = Mock() + process.poll.return_value = None + + def start(_args, stdout, **_kwargs): + stdout.write("Forwarding from 127.0.0.1:4567 -> 8080\n") + stdout.flush() + return process + + with ( + patch.object(verifier.subprocess, "Popen", side_effect=start), + self.assertRaisesRegex(ValueError, "test failure"), + verifier.forward("context", "team") as url, + ): + self.assertEqual(url, "http://127.0.0.1:4567") + raise ValueError("test failure") + process.terminate.assert_called_once() + process.wait.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/rhoai/verify.py b/deploy/rhoai/verify.py new file mode 100644 index 0000000000..34df477a90 --- /dev/null +++ b/deploy/rhoai/verify.py @@ -0,0 +1,264 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Read-only gateway authorization checks through the dashboard BFF and public Route.""" + +import argparse +import contextlib +import json +import re +import shutil +import ssl +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit +from urllib.request import ( + HTTPRedirectHandler, + HTTPSHandler, + ProxyHandler, + Request, + build_opener, +) + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, _req, _fp, _code, _msg, _headers, _newurl): + # Never forward an access token or session cookie to a redirect target. + return None + + +def secret_file(path): + value = path.read_text().strip() + if not value or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise ValueError("Credential file must contain one nonempty header value") + return value + + +def request(url, token=None, cookie=None, ca_file=None): + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + if cookie: + headers["Cookie"] = cookie + context = ssl.create_default_context() + if ca_file: + context.load_verify_locations(cafile=str(ca_file)) + # Loopback traffic must never pass through an environment-configured proxy. + handlers = [NoRedirect(), HTTPSHandler(context=context)] + if urlsplit(url).hostname == "127.0.0.1": + handlers.append(ProxyHandler({})) + try: + with build_opener(*handlers).open( + Request(url, headers=headers), timeout=30 + ) as response: + body = response.read(1024 * 1024) + try: + data = json.loads(body) + except ValueError: + data = None + return response.status, data + except HTTPError as error: + error.close() + return error.code, None + except (URLError, OSError): + raise ValueError( + "HTTP connection or TLS verification failed; check endpoint reachability and CA trust" + ) from None + + +@contextlib.contextmanager +def forward(context, namespace): + # kubectl allocates the port. Keep its diagnostics private and never write + # tokens into argv, environment variables, or a subprocess's stdin. + with tempfile.TemporaryFile(mode="w+") as output: + process = subprocess.Popen( + [ + "kubectl", + "--context", + context, + "-n", + namespace, + "port-forward", + "--address=127.0.0.1", + "deployment/openshell-dashboard", + ":8080", + ], + stdout=output, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + output.seek(0) + match = re.search( + r"Forwarding from 127\.0\.0\.1:(\d+) -> 8080", output.read() + ) + if match: + yield f"http://127.0.0.1:{match[1]}" + return + if process.poll() is not None: + break + time.sleep(0.2) + raise ValueError( + "Could not port-forward the dashboard; check context, namespace, pod readiness and pods/portforward permissions" + ) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def verify( + base, + public, + admin, + user, + allowed, + denied, + admin_role, + other=None, + cookie=None, + ca_file=None, + http=request, +): + failures = [] + + def check(label, url, expected, **kwargs): + status, data = http(url, **kwargs) + passed = status in expected + print(f"{'PASS' if passed else 'FAIL'}: {label} (HTTP {status})") + if not passed: + failures.append(label) + return data + + def identity(label, token, is_admin): + data = check(label, f"{base}/api/v1/auth/whoami", (200,), token=token) + valid = ( + isinstance(data, dict) + and bool(data.get("subject")) + and isinstance(data.get("roles"), list) + and (admin_role in data["roles"]) == is_admin + ) + if not valid: + failures.append(f"{label} identity/role") + print(f"FAIL: {label} identity/role") + return data.get("subject") if isinstance(data, dict) else None + + check( + "public API rejects missing session", + f"{public}/api/v1/workspaces", + (401,), + ca_file=ca_file, + ) + check("gateway rejects missing token", f"{base}/api/v1/workspaces", (401,)) + admin_subject = identity("admin token", admin, True) + user_subject = identity("ordinary-user token", user, False) + if admin_subject == user_subject: + failures.append("admin and user must be distinct identities") + for name in (allowed, denied): + path = f"/api/v1/workspaces/{quote(name, safe='')}/sandboxes" + # Establish that BOTH workspaces exist and are accessible to the admin, + # so a missing workspace cannot count as isolation success. + check(f"admin accesses {name}", base + path, (200,), token=admin) + check( + f"ordinary user {'accesses' if name == allowed else 'is denied in'} {name}", + base + path, + (200,) if name == allowed else (403,), + token=user, + ) + if other: + check( + "other-gateway token is rejected", + f"{base}/api/v1/workspaces", + (401, 403), + token=other, + ) + if cookie: + data = check( + "browser session reaches public API", + f"{public}/api/v1/auth/whoami", + (200,), + cookie=cookie, + ca_file=ca_file, + ) + if not isinstance(data, dict) or not data.get("subject"): + failures.append("browser session identity") + if failures: + raise ValueError(f"{len(failures)} authorization check(s) failed") + print( + "Authorization checks passed. Browser login, grant/revoke, sandbox lifecycle and terminal checks require separate validation." + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "context", + "namespace", + "dashboard-url", + "allowed-workspace", + "denied-workspace", + "admin-role", + ): + parser.add_argument(f"--{name}", required=True) + for name in ("admin-token-file", "user-token-file"): + parser.add_argument(f"--{name}", type=Path, required=True) + for name in ("other-gateway-token-file", "session-cookie-file", "ca-file"): + parser.add_argument(f"--{name}", type=Path) + args = parser.parse_args() + url = urlsplit(args.dashboard_url) + if ( + url.scheme != "https" + or not url.hostname + or url.username + or url.password + or url.path not in ("", "/") + or url.query + or url.fragment + ): + parser.error("dashboard-url must be an HTTPS origin without credentials") + if args.allowed_workspace == args.denied_workspace: + parser.error("allowed and denied workspaces must be different") + if not shutil.which("kubectl"): + parser.error("kubectl is required") + admin = secret_file(args.admin_token_file) + user = secret_file(args.user_token_file) + other = ( + secret_file(args.other_gateway_token_file) + if args.other_gateway_token_file + else None + ) + cookie = secret_file(args.session_cookie_file) if args.session_cookie_file else None + with forward(args.context, args.namespace) as base: + verify( + base, + args.dashboard_url.rstrip("/"), + admin, + user, + args.allowed_workspace, + args.denied_workspace, + args.admin_role, + other, + cookie, + args.ca_file, + ) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError) as error: + print(f"Verification stopped: {error}", file=sys.stderr) + sys.exit(1) diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index c071f0556e..ba7033f838 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -99,6 +99,10 @@ The installer includes a render-only check, preserves resources on failure, and requires explicit chart versions. Validate login, workspace authorization, sandbox lifecycle, and terminal access on your target cluster before sharing the deployment; chart rendering does not establish RHOAI qualification. +The installer accepts operational Helm values files and custom OIDC scopes, and +provides a read-only cluster preflight and token-based authorization verifier. +Provision distinct OIDC audiences and browser clients for separate gateways; +namespace separation alone does not establish identity isolation. diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 8ec766c8ca..88de9f3a2f 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -107,6 +107,14 @@ preserving namespace metadata. Use the published [Kubernetes setup guide](https://docs.nvidia.com/openshell/latest/kubernetes/setup.md) for the installation entry point. Diagnose Agent Sandbox availability separately; the installer does not provision that controller or RHOAI itself. +Its read-only preflight checks storage, installation permissions, and recorded +OIDC identities across installer-managed namespaces. Distinct gateways need +distinct audiences and browser clients. Legacy state without OIDC metadata needs +manual review; retained history for another uninstalled release is not an active +gateway. Operational values files are reapplied on every upgrade; authentication, +TLS, naming, and namespace overrides are rejected. The deployment guide also +provides a read-only verifier for admin/ordinary-user workspace authorization; +it does not establish browser login, sandbox lifecycle, or network-policy enforcement. Before debugging the compute platform, inspect gateway logs for failures in dependencies initialized before the listener becomes ready. From 02d545f35535d8c7737e2bc9c02f620038f751a4 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:36:21 -0400 Subject: [PATCH 12/26] chore(deps): refresh rpm lockfiles [SECURITY] (#100) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/cli/rpms.lock.yaml | 40 ++++++++++++------------ deploy/konflux/e2e-odh/rpms.lock.yaml | 40 ++++++++++++------------ deploy/konflux/gateway/rpms.lock.yaml | 20 ++++++------ deploy/konflux/openclaw/rpms.lock.yaml | 20 ++++++------ deploy/konflux/supervisor/rpms.lock.yaml | 20 ++++++------ 5 files changed, 70 insertions(+), 70 deletions(-) diff --git a/deploy/konflux/cli/rpms.lock.yaml b/deploy/konflux/cli/rpms.lock.yaml index a33e768c2c..02a50239c3 100644 --- a/deploy/konflux/cli/rpms.lock.yaml +++ b/deploy/konflux/cli/rpms.lock.yaml @@ -284,13 +284,13 @@ arches: name: libcom_err evr: 1.46.5-8.el9 sourcerpm: e2fsprogs-1.46.5-8.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.7.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.8.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 292483 - checksum: sha256:408a3da221a1d3856a4b9ab7a7c70901430f71521ef9d05f9b847c0deb27b160 + size: 293076 + checksum: sha256:a7ec72530e41371da4984e19c93168de6a911067d2bb597dec1e3bbe48e179a8 name: libcurl - evr: 7.76.1-40.el9_8.7 - sourcerpm: curl-7.76.1-40.el9_8.7.src.rpm + evr: 7.76.1-40.el9_8.8 + sourcerpm: curl-7.76.1-40.el9_8.8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libdb-5.3.28-57.el9_6.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms size: 727417 @@ -599,20 +599,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-6.el9.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 224938 - checksum: sha256:29285f81cef68f73b4f8ff81ee8fdf4ceaa007933302119ed1615e4aa1091613 + size: 227991 + checksum: sha256:40fd0cd92f4cf824fb5186128d360bdc0a8a9f70c9f820b8c0efda4afec75cc8 name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-for-aarch64-baseos-rpms size: 60882 @@ -1023,13 +1023,13 @@ arches: name: libcom_err evr: 1.46.5-8.el9 sourcerpm: e2fsprogs-1.46.5-8.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.7.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 296930 - checksum: sha256:044d7a379f0f0f6ed25b9d41439dedfb55b390108ae5a09d4acc6b3565c39ae1 + size: 297409 + checksum: sha256:cc12ee8e4fb07d93ad55dc59406ff65a3acbb795b3eca30b32195e51b3a12965 name: libcurl - evr: 7.76.1-40.el9_8.7 - sourcerpm: curl-7.76.1-40.el9_8.7.src.rpm + evr: 7.76.1-40.el9_8.8 + sourcerpm: curl-7.76.1-40.el9_8.8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libdb-5.3.28-57.el9_6.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 755192 diff --git a/deploy/konflux/e2e-odh/rpms.lock.yaml b/deploy/konflux/e2e-odh/rpms.lock.yaml index e66135dbfa..7ee2450c15 100644 --- a/deploy/konflux/e2e-odh/rpms.lock.yaml +++ b/deploy/konflux/e2e-odh/rpms.lock.yaml @@ -312,13 +312,13 @@ arches: name: libcom_err evr: 1.46.5-8.el9 sourcerpm: e2fsprogs-1.46.5-8.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.7.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.8.aarch64.rpm repoid: ubi-9-baseos-rpms - size: 292483 - checksum: sha256:408a3da221a1d3856a4b9ab7a7c70901430f71521ef9d05f9b847c0deb27b160 + size: 293076 + checksum: sha256:a7ec72530e41371da4984e19c93168de6a911067d2bb597dec1e3bbe48e179a8 name: libcurl - evr: 7.76.1-40.el9_8.7 - sourcerpm: curl-7.76.1-40.el9_8.7.src.rpm + evr: 7.76.1-40.el9_8.8 + sourcerpm: curl-7.76.1-40.el9_8.8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/l/libdb-5.3.28-57.el9_6.aarch64.rpm repoid: ubi-9-baseos-rpms size: 727417 @@ -641,20 +641,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-6.el9.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.aarch64.rpm repoid: ubi-9-baseos-rpms - size: 224938 - checksum: sha256:29285f81cef68f73b4f8ff81ee8fdf4ceaa007933302119ed1615e4aa1091613 + size: 227991 + checksum: sha256:40fd0cd92f4cf824fb5186128d360bdc0a8a9f70c9f820b8c0efda4afec75cc8 name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-baseos-rpms size: 60882 @@ -1093,13 +1093,13 @@ arches: name: libcom_err evr: 1.46.5-8.el9 sourcerpm: e2fsprogs-1.46.5-8.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.7.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libcurl-7.76.1-40.el9_8.8.x86_64.rpm repoid: ubi-9-baseos-rpms - size: 296930 - checksum: sha256:044d7a379f0f0f6ed25b9d41439dedfb55b390108ae5a09d4acc6b3565c39ae1 + size: 297409 + checksum: sha256:cc12ee8e4fb07d93ad55dc59406ff65a3acbb795b3eca30b32195e51b3a12965 name: libcurl - evr: 7.76.1-40.el9_8.7 - sourcerpm: curl-7.76.1-40.el9_8.7.src.rpm + evr: 7.76.1-40.el9_8.8 + sourcerpm: curl-7.76.1-40.el9_8.8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/l/libdb-5.3.28-57.el9_6.x86_64.rpm repoid: ubi-9-baseos-rpms size: 755192 diff --git a/deploy/konflux/gateway/rpms.lock.yaml b/deploy/konflux/gateway/rpms.lock.yaml index 0a257dda1d..506474bf38 100644 --- a/deploy/konflux/gateway/rpms.lock.yaml +++ b/deploy/konflux/gateway/rpms.lock.yaml @@ -263,20 +263,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-6.el9.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 224938 - checksum: sha256:29285f81cef68f73b4f8ff81ee8fdf4ceaa007933302119ed1615e4aa1091613 + size: 227991 + checksum: sha256:40fd0cd92f4cf824fb5186128d360bdc0a8a9f70c9f820b8c0efda4afec75cc8 name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms size: 33143 diff --git a/deploy/konflux/openclaw/rpms.lock.yaml b/deploy/konflux/openclaw/rpms.lock.yaml index 0a257dda1d..506474bf38 100644 --- a/deploy/konflux/openclaw/rpms.lock.yaml +++ b/deploy/konflux/openclaw/rpms.lock.yaml @@ -263,20 +263,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-6.el9.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 224938 - checksum: sha256:29285f81cef68f73b4f8ff81ee8fdf4ceaa007933302119ed1615e4aa1091613 + size: 227991 + checksum: sha256:40fd0cd92f4cf824fb5186128d360bdc0a8a9f70c9f820b8c0efda4afec75cc8 name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms size: 33143 diff --git a/deploy/konflux/supervisor/rpms.lock.yaml b/deploy/konflux/supervisor/rpms.lock.yaml index 0a257dda1d..506474bf38 100644 --- a/deploy/konflux/supervisor/rpms.lock.yaml +++ b/deploy/konflux/supervisor/rpms.lock.yaml @@ -263,20 +263,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-6.el9.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 224938 - checksum: sha256:29285f81cef68f73b4f8ff81ee8fdf4ceaa007933302119ed1615e4aa1091613 + size: 227991 + checksum: sha256:40fd0cd92f4cf824fb5186128d360bdc0a8a9f70c9f820b8c0efda4afec75cc8 name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-aarch64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.aarch64.rpm repoid: ubi-9-for-aarch64-baseos-rpms size: 33143 From 0b53089275b439b8aaa0e0d4d7726ce791dbbc32 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Tue, 13 Oct 2026 14:37:20 -0400 Subject: [PATCH 13/26] chore(deps): refresh rpm lockfiles (#96) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/sandbox/rpms.lock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/konflux/sandbox/rpms.lock.yaml b/deploy/konflux/sandbox/rpms.lock.yaml index 5eafaec2fa..3a399de171 100644 --- a/deploy/konflux/sandbox/rpms.lock.yaml +++ b/deploy/konflux/sandbox/rpms.lock.yaml @@ -11,13 +11,13 @@ arches: name: glibc-devel evr: 2.34-283.el9_8 sourcerpm: glibc-2.34-283.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/k/kernel-headers-5.14.0-687.56.1.el9_8.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/k/kernel-headers-5.14.0-687.57.1.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-appstream-rpms - size: 2938333 - checksum: sha256:ea29f262c5cc5b180acb285f135820fd984aa7c1b83acc0f25b3562e42fa9221 + size: 2939977 + checksum: sha256:31860419149438987e706dd2929fbc707552cd2e51dbc86547d294f9fed5f204 name: kernel-headers - evr: 5.14.0-687.56.1.el9_8 - sourcerpm: kernel-5.14.0-687.56.1.el9_8.src.rpm + evr: 5.14.0-687.57.1.el9_8 + sourcerpm: kernel-5.14.0-687.57.1.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/l/libxcrypt-devel-4.4.18-3.el9.aarch64.rpm repoid: ubi-9-for-aarch64-appstream-rpms size: 33051 From 99dc72192ed109c83b61878d90797c8d78c0e464 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 05:18:25 +0000 Subject: [PATCH 14/26] chore(deps): update registry.access.redhat.com/ubi9/nodejs-24-minimal docker tag to v9.8-1791925262 (#103) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/docker/Dockerfile.konflux.openclaw | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/Dockerfile.konflux.openclaw b/deploy/docker/Dockerfile.konflux.openclaw index e4fc08c654..1dd8476efb 100644 --- a/deploy/docker/Dockerfile.konflux.openclaw +++ b/deploy/docker/Dockerfile.konflux.openclaw @@ -25,7 +25,7 @@ # `&& \` continuations only. # Both stages use the same base and OpenClaw version; bump them only here. -ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791837113@sha256:c0205a5c0bb750ec8a34248da9f87b4013cbfd46e327ec321996104913633fe9 +ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791925262@sha256:cfca5c8d4d7babc8b74f408db216c2fd4dec872f319c1579e001dcceb51d3058 ARG OPENCLAW_VERSION=2026.9.5 # --------------------------------------------------------------------------- From 635292be5e7c2d57a97b545f33e9178814ab9f47 Mon Sep 17 00:00:00 2001 From: Varsha Prasad Narsing Date: Wed, 14 Oct 2026 05:19:10 -0700 Subject: [PATCH 15/26] fix(rhoai): close installer preflight gaps Signed-off-by: Varsha Prasad Narsing --- deploy/rhoai/README.md | 15 ++- deploy/rhoai/install.py | 63 +++++++++-- deploy/rhoai/test_install.py | 142 ++++++++++++++++++++++++ skills/debug-openshell-cluster/SKILL.md | 9 +- 4 files changed, 214 insertions(+), 15 deletions(-) diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md index 1c1f5ceaaa..c5e488d1b0 100644 --- a/deploy/rhoai/README.md +++ b/deploy/rhoai/README.md @@ -273,6 +273,11 @@ a different release name, or either fixed release without installer state, is rejected and needs explicit migration. Unmanaged gateway TLS/JWT Secrets are also rejected so the bootstrap hook cannot silently reuse another gateway's identity. Unrelated Helm releases in the selected namespace are left alone. +Preflight inspects stored Helm workload manifests for the `openshell-gateway` +container or executable, so renaming the chart or overriding its image does not +bypass gateway detection. Failed releases and scaled-to-zero gateways count too. +Custom packaging must retain one of these markers; workloads outside Helm or +with both markers removed require administrator inspection. Uninstalled releases retained under another name do not count as active gateways; conflicting remaining TLS/JWT Secrets and fixed release names still require explicit migration. @@ -287,9 +292,13 @@ rejected because the certificate hook preserves existing certificates; migrate the certificate and endpoint explicitly rather than editing installer state. The installer preserves existing client and cookie secrets. You may omit -`client_secret_file` after the first installation. A provided secret or private -OIDC CA that differs from the existing object causes an error; rotate credentials -explicitly and restart the consuming pods. Removing `ca_file` removes the mounts +`client_secret_file` only when installer state exists. A first installation requires +the file even if `openshell-dashboard-oidc` already exists; the supplied client +secret must match before that Secret is reused. Preflight validates local +credential files without reading namespaced objects when the namespace is absent. +A provided secret or private OIDC CA that differs from the existing object causes +an error; rotate credentials explicitly and restart the consuming pods. +Removing `ca_file` removes the mounts on the next upgrade, but does not delete the old ConfigMap. Change the pinned versions to upgrade. Both Helm releases receive the generated diff --git a/deploy/rhoai/install.py b/deploy/rhoai/install.py index 4d76b6d2bb..4290f4ac55 100644 --- a/deploy/rhoai/install.py +++ b/deploy/rhoai/install.py @@ -383,6 +383,42 @@ def get(self, kind, name, namespaced=True): def create(self, resource): self.run([*self.kube, "create", "-f", "-"], json.dumps(resource)) + def release_has_gateway(self, release): + # Chart names and images can change in downstream packages. Inspect the + # stored workload instead, including failed releases with no running pod. + manifest = self.run( + [ + "helm", + "get", + "manifest", + release, + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + ] + ) + try: + for resource in yaml.safe_load_all(manifest): + if not isinstance(resource, dict): + continue + if resource.get("kind") == "Pod": + pod = resource + elif resource.get("kind") in ("Deployment", "StatefulSet", "DaemonSet"): + pod = resource.get("spec", {}).get("template", {}) + else: + continue + for container in pod.get("spec", {}).get("containers", []): + if container.get("name") == "openshell-gateway" or any( + Path(arg).name == "openshell-gateway" + for arg in container.get("command", []) + ): + return True + except yaml.YAMLError: + # Helm manifests can contain Secrets; never include parser excerpts. + raise ValueError(f"Cannot inspect manifest for release {release}") from None + return False + def check_releases(self, managed): # Explicit status flags work with both Helm 3 and Helm 4. Paginate so a # busy administrator-managed namespace cannot hide a conflicting release. @@ -421,15 +457,18 @@ def check_releases(self, managed): "openshell-dashboard", ): continue - gateway = release.get("chart", "").startswith("helm-chart-") - require( - not gateway or name == "openshell", - f"Namespace {self.ns} already has gateway release {name}; use one gateway per namespace", - ) require( managed or name not in ("openshell", "openshell-dashboard"), f"Release {name} already exists outside this installer; migrate it explicitly", ) + if name != "openshell": + gateway = release.get("chart", "").startswith( + "helm-chart-" + ) or self.release_has_gateway(name) + require( + not gateway, + f"Namespace {self.ns} already has gateway release {name}; use one gateway per namespace", + ) if len(releases) < 256: break offset += len(releases) @@ -656,17 +695,19 @@ def preflight(self): not self.get("secret", secret), f"Existing gateway Secret {secret} requires explicit migration", ) - existing = self.get("secret", OIDC_SECRET) if namespace else None require( - existing or self.config["oidc"].get("client_secret_file"), + state or legacy_host or self.config["oidc"].get("client_secret_file"), "oidc.client_secret_file is required for the first install", ) - self.credentials(check_only=True) + self.credentials(check_only=True, namespace_exists=bool(namespace)) return namespace - def credentials(self, check_only=False): + def credentials(self, check_only=False, *, namespace_exists=True): + require( + namespace_exists or check_only, "Create the namespace before credentials" + ) oidc = self.config["oidc"] - existing = self.get("secret", OIDC_SECRET) + existing = self.get("secret", OIDC_SECRET) if namespace_exists else None supplied = ( Path(oidc["client_secret_file"]).read_bytes().rstrip(b"\r\n") if oidc.get("client_secret_file") @@ -703,7 +744,7 @@ def credentials(self, check_only=False): ) if oidc.get("ca_file"): ca = Path(oidc["ca_file"]).read_text() - existing_ca = self.get("configmap", OIDC_CA) + existing_ca = self.get("configmap", OIDC_CA) if namespace_exists else None require( not existing_ca or existing_ca.get("data", {}).get("ca.crt") == ca, "OIDC CA differs; rotate the ConfigMap and restart both releases explicitly", diff --git a/deploy/rhoai/test_install.py b/deploy/rhoai/test_install.py index 48f96d756d..ad1b5c66cf 100644 --- a/deploy/rhoai/test_install.py +++ b/deploy/rhoai/test_install.py @@ -41,6 +41,7 @@ def __init__(self, config): } self.calls = [] self.releases = [] + self.manifests = {} self.fail_dashboard = False self.storage = [ { @@ -57,6 +58,13 @@ def __init__(self, config): def run(self, args, payload=None): self.calls.append((args, payload)) + if ( + args[0] == "kubectl" + and "-n" in args + and "get" in args + and ("namespace", args[args.index("-n") + 1]) not in self.objects + ): + raise RuntimeError("namespace not found") if "can-i" in args: return ( "no" @@ -77,6 +85,8 @@ def run(self, args, payload=None): if args[:2] == ["helm", "list"]: offset = int(args[args.index("--offset") + 1]) return json.dumps(self.releases[offset : offset + 256]) + if args[:3] == ["helm", "get", "manifest"]: + return self.manifests.get(args[3], "") if args[:2] == ["helm", "upgrade"]: if self.fail_dashboard and "openshell-dashboard" in args: raise RuntimeError("dashboard failed") @@ -266,7 +276,11 @@ def test_uninstalled_other_release_history_does_not_block_install(self): "status": "uninstalled", } ] + cluster.manifests["retired-gateway"] = self.gateway_manifest() cluster.preflight() + self.assertFalse( + any(args[:3] == ["helm", "get", "manifest"] for args, _ in cluster.calls) + ) cluster.objects[("secret", "openshell-server-tls")] = {"data": {}} with self.assertRaisesRegex(ValueError, "explicit migration"): cluster.preflight() @@ -409,6 +423,74 @@ def test_existing_gateway_or_foreign_release_rejected_before_mutation(self): ) ) + @staticmethod + def gateway_manifest(kind="StatefulSet", container=None): + return json.dumps( + { + "apiVersion": "apps/v1", + "kind": kind, + "metadata": {"name": "renamed-gateway"}, + "spec": { + "replicas": 0, + "template": { + "spec": { + "containers": [ + container + or { + "name": "openshell-gateway", + "image": "registry.example/custom/runtime:1.0", + } + ] + } + }, + }, + } + ) + + def test_repackaged_gateway_rejected_before_mutation(self): + for kind, status in (("StatefulSet", "deployed"), ("Deployment", "failed")): + with self.subTest(kind=kind, status=status): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [ + {"name": "team-gateway", "chart": "custom-1.0.0", "status": status} + ] + cluster.manifests["team-gateway"] = self.gateway_manifest(kind) + before = copy.deepcopy(cluster.objects) + with self.assertRaisesRegex(ValueError, "one gateway per namespace"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any(args[:2] == ["helm", "upgrade"] for args, _ in cluster.calls) + ) + + def test_gateway_executable_detected_with_renamed_container(self): + cluster = self.cluster() + cluster.manifests["custom"] = self.gateway_manifest( + container={ + "name": "server", + "command": ["/usr/local/bin/openshell-gateway"], + } + ) + self.assertTrue(cluster.release_has_gateway("custom")) + + def test_unrelated_release_with_workload_is_preserved(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [{"name": "other-app", "chart": "custom-1.0.0"}] + cluster.manifests["other-app"] = self.gateway_manifest( + "Deployment", {"name": "web", "image": "registry.example/web:1.0"} + ) + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn("other-app", [release["name"] for release in cluster.releases]) + + def test_invalid_release_manifest_stops_without_echoing_content(self): + cluster = self.cluster() + cluster.manifests["custom"] = "secret: [do-not-print-this" + with self.assertRaisesRegex(ValueError, "Cannot inspect manifest") as caught: + cluster.release_has_gateway("custom") + self.assertNotIn("do-not-print-this", str(caught.exception)) + def test_conflict_on_later_release_page_is_detected(self): cluster = self.cluster() cluster.releases = [ @@ -500,6 +582,63 @@ def test_rerun_preserves_secrets_without_input_file(self): any("create" in args and "can-i" not in args for args, _ in cluster.calls) ) + def test_first_install_requires_supplied_secret_even_with_existing_secret(self): + for supplied in (None, "different", "matching"): + with self.subTest(supplied=supplied): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + existing = { + "data": { + "client-secret": base64.b64encode(b"matching").decode(), + "cookie-secret": base64.b64encode(b"existing-cookie").decode(), + } + } + cluster.objects[("secret", installer.OIDC_SECRET)] = copy.deepcopy( + existing + ) + path = Path(cluster.config["oidc"]["client_secret_file"]) + if supplied is None: + cluster.config["oidc"].pop("client_secret_file") + else: + path.write_text(supplied) + before = copy.deepcopy(cluster.objects) + if supplied == "matching": + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual( + cluster.objects[("secret", installer.OIDC_SECRET)], existing + ) + else: + expected = ( + "required for the first install" + if supplied is None + else "rotate" + ) + with self.assertRaisesRegex(ValueError, expected): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + + def test_new_namespace_preflight_checks_files_without_namespaced_reads(self): + (self.root / "issuer.pem").write_text("test CA") + self.config["oidc"]["ca_file"] = "issuer.pem" + cluster = self.cluster() + before = copy.deepcopy(cluster.objects) + self.assertIsNone(cluster.preflight()) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any("-n" in args and "get" in args for args, _ in cluster.calls) + ) + path = Path(cluster.config["oidc"]["client_secret_file"]) + secret = path.read_text() + path.write_text("\n") + with self.assertRaisesRegex(ValueError, "contains no secret"): + cluster.preflight() + self.assertEqual(cluster.objects, before) + path.write_text(secret) + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn(("namespace", cluster.ns), cluster.objects) + self.assertIn(("secret", installer.OIDC_SECRET), cluster.objects) + self.assertIn(("configmap", installer.OIDC_CA), cluster.objects) + def test_changed_gateway_host_requires_certificate_migration(self): cluster = self.cluster() cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) @@ -516,6 +655,7 @@ def test_changed_gateway_host_requires_certificate_migration(self): def test_different_secret_is_not_silently_rotated(self): cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} cluster.credentials() before = copy.deepcopy(cluster.objects) Path(cluster.config["oidc"]["client_secret_file"]).write_text("changed") @@ -527,6 +667,7 @@ def test_private_ca_and_relative_path(self): (self.root / "issuer.pem").write_text("test CA") self.config["oidc"]["ca_file"] = "issuer.pem" cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} cluster.credentials() gateway, dashboard = installer.chart_values(cluster.config) self.assertEqual( @@ -552,6 +693,7 @@ def test_dashboard_failure_keeps_gateway_and_credentials(self): def test_route_admission_waits_and_times_out(self): cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} admitted = { "status": { "ingress": [{"conditions": [{"type": "Admitted", "status": "True"}]}] diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 88de9f3a2f..ae91455fe3 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -101,7 +101,14 @@ does not establish dashboard login or sandbox readiness. Check the `openshell-installer` state ConfigMap, the dashboard OIDC Secret, the shared gateway CA Secret, and admission of both Routes. A gateway chart under another release name or an existing fixed release without installer state requires -explicit migration; do not change state to bypass these checks. Earlier +explicit migration; do not change state to bypass these checks. Detection also +inspects stored Helm workloads for the `openshell-gateway` container or executable, +including failed and scaled-to-zero releases with renamed charts. Custom packaging +that removes both markers and workloads outside Helm require manual inspection. +A first installation requires a client-secret file matching any existing dashboard +OIDC Secret; only an installation with recorded state can omit the file. +For a new namespace, credential preflight checks local files without querying +namespaced objects before namespace creation. Earlier installations used namespace annotations for state, which reruns migrate while preserving namespace metadata. Use the published [Kubernetes setup guide](https://docs.nvidia.com/openshell/latest/kubernetes/setup.md) From 3523fc086a92480a13b32704320bfeb82976b17b Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Wed, 14 Oct 2026 08:26:38 -0400 Subject: [PATCH 16/26] chore(deps): refresh rpm lockfiles [SECURITY] (#104) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/cli/rpms.lock.yaml | 20 ++++++++++---------- deploy/konflux/e2e-odh/rpms.lock.yaml | 20 ++++++++++---------- deploy/konflux/gateway/rpms.lock.yaml | 20 ++++++++++---------- deploy/konflux/openclaw/rpms.lock.yaml | 20 ++++++++++---------- deploy/konflux/supervisor/rpms.lock.yaml | 20 ++++++++++---------- 5 files changed, 50 insertions(+), 50 deletions(-) diff --git a/deploy/konflux/cli/rpms.lock.yaml b/deploy/konflux/cli/rpms.lock.yaml index 02a50239c3..2415182651 100644 --- a/deploy/konflux/cli/rpms.lock.yaml +++ b/deploy/konflux/cli/rpms.lock.yaml @@ -1338,20 +1338,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 60882 diff --git a/deploy/konflux/e2e-odh/rpms.lock.yaml b/deploy/konflux/e2e-odh/rpms.lock.yaml index 7ee2450c15..ccfeff5027 100644 --- a/deploy/konflux/e2e-odh/rpms.lock.yaml +++ b/deploy/konflux/e2e-odh/rpms.lock.yaml @@ -1422,20 +1422,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-baseos-rpms size: 60882 diff --git a/deploy/konflux/gateway/rpms.lock.yaml b/deploy/konflux/gateway/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/gateway/rpms.lock.yaml +++ b/deploy/konflux/gateway/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 diff --git a/deploy/konflux/openclaw/rpms.lock.yaml b/deploy/konflux/openclaw/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/openclaw/rpms.lock.yaml +++ b/deploy/konflux/openclaw/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 diff --git a/deploy/konflux/supervisor/rpms.lock.yaml b/deploy/konflux/supervisor/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/supervisor/rpms.lock.yaml +++ b/deploy/konflux/supervisor/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 From 17bfa4985f372784ecd0e0bdd628f3f8d1c2c415 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 01:34:00 +0000 Subject: [PATCH 17/26] chore(deps): refresh rpm lockfiles [SECURITY] (#105) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> From 9af2b5843ac49bbc305a6629ef9023aa47f48a3d Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Thu, 15 Oct 2026 04:47:10 +0000 Subject: [PATCH 18/26] chore(deps): refresh rpm lockfiles (#106) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/sandbox/rpms.lock.yaml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/konflux/sandbox/rpms.lock.yaml b/deploy/konflux/sandbox/rpms.lock.yaml index 3a399de171..fec0f95960 100644 --- a/deploy/konflux/sandbox/rpms.lock.yaml +++ b/deploy/konflux/sandbox/rpms.lock.yaml @@ -11,13 +11,13 @@ arches: name: glibc-devel evr: 2.34-283.el9_8 sourcerpm: glibc-2.34-283.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/k/kernel-headers-5.14.0-687.57.1.el9_8.aarch64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/k/kernel-headers-5.14.0-687.58.1.el9_8.aarch64.rpm repoid: ubi-9-for-aarch64-appstream-rpms - size: 2939977 - checksum: sha256:31860419149438987e706dd2929fbc707552cd2e51dbc86547d294f9fed5f204 + size: 2942913 + checksum: sha256:ce1098a3559ec136e839c36a3e196251226d818ddb91b529534b8d86c01c3e67 name: kernel-headers - evr: 5.14.0-687.57.1.el9_8 - sourcerpm: kernel-5.14.0-687.57.1.el9_8.src.rpm + evr: 5.14.0-687.58.1.el9_8 + sourcerpm: kernel-5.14.0-687.58.1.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/aarch64/appstream/os/Packages/l/libxcrypt-devel-4.4.18-3.el9.aarch64.rpm repoid: ubi-9-for-aarch64-appstream-rpms size: 33051 @@ -176,13 +176,13 @@ arches: name: glibc-headers evr: 2.34-283.el9_8 sourcerpm: glibc-2.34-283.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.56.1.el9_8.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.57.1.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms - size: 2977605 - checksum: sha256:a797333dff8ff3d82f6e38107e7d9fbf78b60cd8a4d3c2582baaad91309a405c + size: 2979225 + checksum: sha256:c5766b2f8e4531f824575546028de7d8eb9fa95d7246dfafd4883c58312a2ef2 name: kernel-headers - evr: 5.14.0-687.56.1.el9_8 - sourcerpm: kernel-5.14.0-687.56.1.el9_8.src.rpm + evr: 5.14.0-687.57.1.el9_8 + sourcerpm: kernel-5.14.0-687.57.1.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/l/libxcrypt-devel-4.4.18-3.el9.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms size: 33101 From ba5418ac93613c040bf53f44ca7bb7fb3b100e3f Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Thu, 15 Oct 2026 15:40:34 +0100 Subject: [PATCH 19/26] CARRY: fix(odh): align e2e image ownership with pipeline user Signed-off-by: Bobbins228 --- deploy/docker/Dockerfile.konflux.e2e-odh | 40 +++++++++++++----------- deploy/konflux/e2e-odh/README.md | 4 ++- e2e/rust/tests/odh/README.md | 5 +++ 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/deploy/docker/Dockerfile.konflux.e2e-odh b/deploy/docker/Dockerfile.konflux.e2e-odh index ba5e71ab0b..cf9122f4b4 100644 --- a/deploy/docker/Dockerfile.konflux.e2e-odh +++ b/deploy/docker/Dockerfile.konflux.e2e-odh @@ -106,24 +106,26 @@ RUN OC_ARCH=$(case "${TARGETARCH}" in amd64) echo amd64;; arm64) echo arm64;; *) install -m 0555 "/tmp/uv-${UV_ARCH}-unknown-linux-gnu/uv" /usr/local/bin/uv WORKDIR /home/odh/openshell-e2e-odh -COPY --chown=1000:1000 Cargo.toml Cargo.lock ./ -COPY --chown=1000:1000 pyproject.toml uv.lock ./ -COPY --chown=1000:1000 .cargo/ .cargo/ -COPY --chown=1000:1000 .config/ .config/ -COPY --chown=1000:1000 crates/ crates/ -COPY --chown=1000:1000 proto/ proto/ -COPY --from=builder --chown=1000:1000 /home/odh/openshell-e2e-odh/python/ python/ -COPY --chown=1000:1000 providers/ providers/ -COPY --chown=1000:1000 examples/ examples/ -COPY --chown=1000:1000 e2e/ e2e/ -COPY --chown=1000:1000 scripts/ scripts/ -COPY --chown=1000:1000 tasks/ tasks/ -COPY --chown=1000:1000 odh/ odh/ -COPY --chown=1000:1000 deploy/helm/openshell/ deploy/helm/openshell/ -COPY --chown=1000:1000 deploy/helm/openshell-workspace/ deploy/helm/openshell-workspace/ - -RUN install -d -o 1000 -g 1000 results .kube /home/odh/.config/openshell && \ - chown 1000:1000 /home/odh /home/odh/openshell-e2e-odh +# Shift-Left runs with --userns=keep-id --user 1001:1001. Keep the home and +# test workspace writable for that identity, including CLI mTLS storage. +COPY --chown=1001:1001 Cargo.toml Cargo.lock ./ +COPY --chown=1001:1001 pyproject.toml uv.lock ./ +COPY --chown=1001:1001 .cargo/ .cargo/ +COPY --chown=1001:1001 .config/ .config/ +COPY --chown=1001:1001 crates/ crates/ +COPY --chown=1001:1001 proto/ proto/ +COPY --from=builder --chown=1001:1001 /home/odh/openshell-e2e-odh/python/ python/ +COPY --chown=1001:1001 providers/ providers/ +COPY --chown=1001:1001 examples/ examples/ +COPY --chown=1001:1001 e2e/ e2e/ +COPY --chown=1001:1001 scripts/ scripts/ +COPY --chown=1001:1001 tasks/ tasks/ +COPY --chown=1001:1001 odh/ odh/ +COPY --chown=1001:1001 deploy/helm/openshell/ deploy/helm/openshell/ +COPY --chown=1001:1001 deploy/helm/openshell-workspace/ deploy/helm/openshell-workspace/ + +RUN install -d -o 1001 -g 1001 results .kube /home/odh/.config /home/odh/.config/openshell && \ + chown 1001:1001 /home/odh /home/odh/openshell-e2e-odh # Fail the build if an expected test runtime tool is absent. In particular, # sandbox connection uses ssh, and future repository tests need git. cargo @@ -153,6 +155,6 @@ LABEL com.redhat.component="odh-openshell-e2e-container" \ io.k8s.display-name="odh-openshell-e2e" \ io.k8s.description="odh-openshell-e2e" -USER 1000:1000 +USER 1001:1001 ENTRYPOINT ["/home/odh/openshell-e2e-odh/odh/scripts/e2e-odh-entrypoint.sh"] CMD ["smoke"] diff --git a/deploy/konflux/e2e-odh/README.md b/deploy/konflux/e2e-odh/README.md index 2b092abc6e..e4c063edc2 100644 --- a/deploy/konflux/e2e-odh/README.md +++ b/deploy/konflux/e2e-odh/README.md @@ -19,7 +19,9 @@ merges the ordered JUnit files into one `e2e-odh-.xml` plus its HTML companion. Phase reports remain available as `e2e-odh--.xml`. The no-deploy option is supported only for single-phase tiers. Extracted client mTLS material is stored in a mode `0700` -directory with mode `0600` files. +directory with mode `0600` files. The runtime image owns its home and workspace +as UID/GID `1001:1001`, matching the Shift-Left pipeline's +`podman run --userns=keep-id --user 1001:1001`. Each deployed invocation has a DNS-safe run ID. During the temporary upstream managed-test compatibility period, the default namespace and Helm release are diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index 8804c1a5f8..07eb382032 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -338,12 +338,17 @@ oc login --token=XXXXXX \ # Run the e2e tests podman run --rm \ + --userns=keep-id --user 1001:1001 \ -v /path/to/reports:/home/odh/openshell-e2e-odh/results:Z,U \ -v /path/to/kubeconfig:/home/odh/openshell-e2e-odh/.kube/config:ro,Z \ --env-file containerEnvFile \ quay.io/opendatahub/odh-openshell-e2e:odh-stable smoke ``` +The image runs as UID/GID `1001:1001` and owns `/home/odh`, including the +OpenShell CLI configuration directory. Keep the pipeline's `--user` setting +at `1001:1001` so gateway registration and client mTLS material can be written. + The entrypoint assigns each invocation a DNS-safe run ID. For compatibility with managed-workspace tests, the namespace and release default to `openshell` across both shared and managed phases; the gateway name and Route host are From bc9be6b10edaa685870b99294b77f00800d43ec9 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:40:08 +0000 Subject: [PATCH 20/26] chore(deps): refresh rpm lockfiles (#107) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/sandbox/rpms.lock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/konflux/sandbox/rpms.lock.yaml b/deploy/konflux/sandbox/rpms.lock.yaml index fec0f95960..a143e9d7d9 100644 --- a/deploy/konflux/sandbox/rpms.lock.yaml +++ b/deploy/konflux/sandbox/rpms.lock.yaml @@ -176,13 +176,13 @@ arches: name: glibc-headers evr: 2.34-283.el9_8 sourcerpm: glibc-2.34-283.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.57.1.el9_8.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.58.1.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms - size: 2979225 - checksum: sha256:c5766b2f8e4531f824575546028de7d8eb9fa95d7246dfafd4883c58312a2ef2 + size: 2982193 + checksum: sha256:2c29113a9f860f0e2a14c89ec3a399b3e3d0f9ff699d7f5b565ca5e73b0314a9 name: kernel-headers - evr: 5.14.0-687.57.1.el9_8 - sourcerpm: kernel-5.14.0-687.57.1.el9_8.src.rpm + evr: 5.14.0-687.58.1.el9_8 + sourcerpm: kernel-5.14.0-687.58.1.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/l/libxcrypt-devel-4.4.18-3.el9.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms size: 33101 From 79ea2f28cbef71102d69e91fbeae2ffbf650295d Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Fri, 16 Oct 2026 08:50:11 +0100 Subject: [PATCH 21/26] CARRY: ci(openclaw): derive image tag from upstream release Signed-off-by: Bobbins228 --- .../odh-openshell-sandbox-openclaw-push.yaml | 2 + .../odh-openshell-sandbox-openclaw-tag.yaml | 63 ------------------- deploy/konflux/openclaw/README.md | 2 +- 3 files changed, 3 insertions(+), 64 deletions(-) delete mode 100644 .tekton/odh-openshell-sandbox-openclaw-tag.yaml diff --git a/.tekton/odh-openshell-sandbox-openclaw-push.yaml b/.tekton/odh-openshell-sandbox-openclaw-push.yaml index 12b673c8d7..e0f9b5419d 100644 --- a/.tekton/odh-openshell-sandbox-openclaw-push.yaml +++ b/.tekton/odh-openshell-sandbox-openclaw-push.yaml @@ -41,6 +41,8 @@ spec: {"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"} ] + - name: tag-openclaw-release + value: "true" pipelineRef: resolver: git params: diff --git a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml deleted file mode 100644 index 784315665f..0000000000 --- a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: tekton.dev/v1 -kind: PipelineRun -# -metadata: - annotations: - build.appstudio.openshift.io/repo: https://github.com/opendatahub-io/openshell?rev={{revision}} - build.appstudio.redhat.com/commit_sha: '{{revision}}' - build.appstudio.redhat.com/target_branch: '{{target_branch}}' - pipelinesascode.tekton.dev/cancel-in-progress: "false" - pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: | - event == "push" - && target_branch.matches("^refs/tags/[0-9]+\\.[0-9]+\\.[0-9]+$") - creationTimestamp: null - labels: - appstudio.openshift.io/application: opendatahub-builds - appstudio.openshift.io/component: odh-openshell-sandbox-openclaw-ci - pipelines.appstudio.openshift.io/type: build - name: odh-openshell-sandbox-openclaw-on-tag - namespace: open-data-hub-tenant -spec: - params: - - name: git-url - value: '{{source_url}}' - - name: revision - value: '{{revision}}' - - name: output-image - value: quay.io/opendatahub/odh-openshell-sandbox-openclaw:{{git_tag}} - - name: dockerfile - value: deploy/docker/Dockerfile.konflux.openclaw - - name: path-context - value: . - - name: hermetic - value: "true" - - name: build-platforms - value: - - linux/x86_64 - - linux/arm64 - - name: prefetch-input - value: | - [ - {"type": "npm", "path": "deploy/konflux/openclaw"}, - {"type": "rpm", "path": "deploy/konflux/openclaw"} - ] - - name: build-args - value: - - "OPENCLAW_VERSION={{git_tag}}" - pipelineRef: - resolver: git - params: - - name: url - value: https://github.com/opendatahub-io/odh-konflux-central.git - - name: revision - value: main - - name: pathInRepo - value: pipeline/multi-arch-container-build.yaml - taskRunTemplate: - serviceAccountName: build-pipeline-odh-openshell-sandbox-openclaw-ci - workspaces: - - name: git-auth - secret: - secretName: '{{ git_auth_secret }}' -status: {} diff --git a/deploy/konflux/openclaw/README.md b/deploy/konflux/openclaw/README.md index e491ff918a..8645ba4c2b 100644 --- a/deploy/konflux/openclaw/README.md +++ b/deploy/konflux/openclaw/README.md @@ -158,7 +158,7 @@ The npm inputs come from AIPCC agentic commit dc756bc3 (OpenClaw 2026.9.5). AIPC ## Konflux -An OpenShell Git tag matching the OpenClaw version (for example, `2026.9.5`) triggers a versioned image build and publishes the same Quay tag. The tag pipeline passes that version as `OPENCLAW_VERSION`, and the Dockerfile checks it against the installed OpenClaw package. Merges to `main` publish `odh-stable`; consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: +Merges to `main` publish the image as `odh-stable` and with a tag matching the packaged OpenClaw version (currently `2026.9.5`). Before building, the pipeline checks that `package.json`, `package-lock.json`, and the Dockerfile pin agree and that OpenClaw has the matching upstream Git tag (`v2026.9.5`). The version tag is then applied to the same image digest as `odh-stable`. If the pins disagree or the upstream tag cannot be verified, the build stops before publishing. Updating OpenClaw requires updating the npm inputs and Dockerfile pin; the PipelineRun does not need a version edit. Consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: ```json [{"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"}] From ddbaa29767850a7fce1ac8f6fa407a7263391503 Mon Sep 17 00:00:00 2001 From: Bobbins228 Date: Tue, 20 Oct 2026 10:59:56 +0100 Subject: [PATCH 22/26] CARRY: test(odh): exclude tier1 cases and name xunit reports Signed-off-by: Bobbins228 --- deploy/konflux/e2e-odh/README.md | 4 ++-- e2e/rust/tests/odh/README.md | 13 +++++++++---- e2e/rust/tests/odh/tiers.toml | 8 ++++++++ odh/scripts/e2e-odh-entrypoint.sh | 7 ++++--- odh/scripts/merge-e2e-odh-reports.py | 2 +- 5 files changed, 24 insertions(+), 10 deletions(-) diff --git a/deploy/konflux/e2e-odh/README.md b/deploy/konflux/e2e-odh/README.md index e4c063edc2..fd5cf31068 100644 --- a/deploy/konflux/e2e-odh/README.md +++ b/deploy/konflux/e2e-odh/README.md @@ -15,8 +15,8 @@ tasks use the same plan, entrypoint, and deployment path. Tier 2, `odh`, and `full` run shared mode followed by managed-workspace mode; each logical tier runs image provenance once in the shared phase. The entrypoint preserves cleanup on failures and signals, continues after a test-phase failure, and -merges the ordered JUnit files into one `e2e-odh-.xml` plus its HTML -companion. Phase reports remain available as +merges the ordered JUnit files into one `e2e-odh--xunit.xml` for Jenkins +discovery plus `e2e-odh-.html`. Phase reports remain available as `e2e-odh--.xml`. The no-deploy option is supported only for single-phase tiers. Extracted client mTLS material is stored in a mode `0700` directory with mode `0600` files. The runtime image owns its home and workspace diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index 07eb382032..d5c89210a3 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -103,7 +103,12 @@ upstream_tests = ["workspace_namespace_managed"] Smoke covers gateway reachability, sandbox lifecycle, and image provenance. Tier 1 checks the process-supervisor SELinux label and its mapped upstream -tests. Tier 2 runs `workspace_lifecycle`, ODH Tier 2, and image provenance in +tests. It excludes four sandbox lifecycle tests after intermittent OpenShift +failures: `canonical_main_disconnect_reconnect_replays_history_for_same_process`, +`canonical_main_exit_255_is_not_retried_as_transport_failure`, +`on_failure_policy_replaces_runtime_and_preserves_workspace`, and +`canonical_main_and_exec_receive_declared_environment`. Full still includes all four. +Tier 2 runs `workspace_lifecycle`, ODH Tier 2, and image provenance in shared mode, then runs `workspace_namespace_managed` against a separate managed-mode deployment. The `odh` and `full` tiers use the same shared then managed split. Image provenance runs once in each tier's shared phase. Smoke @@ -252,8 +257,8 @@ image, deploying and running one phase at a time. `e2e:odh` runs all ODH scenarios; `e2e:odh:full` runs all feature-enabled binaries except the managed-only binary in its shared phase, followed by that binary in managed mode. Phase JUnit files are named `e2e-odh--.xml`; the entrypoint -merges them into `e2e-odh-.xml` and the matching HTML report in -`results/`. +merges them into `e2e-odh--xunit.xml` for Jenkins discovery and +`e2e-odh-.html` in `results/`. ### Every `e2e:odh*` task runs the image provenance check @@ -393,7 +398,7 @@ use distinct values across parallel invocations. The entrypoint exports `OPENSHELL_E2E_RUN_ID` and the derived names so future Python test modes can reuse the same invocation identity. The image writes phase reports named `e2e-odh--.xml`, then merges them into -`e2e-odh-.xml` and `e2e-odh-.html` in `results/`. +`e2e-odh--xunit.xml` and `e2e-odh-.html` in `results/`. Missing phase reports are skipped, so reports from completed phases remain available after a later deployment or preflight failure; the task still exits with the failure status. diff --git a/e2e/rust/tests/odh/tiers.toml b/e2e/rust/tests/odh/tiers.toml index 9d2e774427..73b3133b22 100644 --- a/e2e/rust/tests/odh/tiers.toml +++ b/e2e/rust/tests/odh/tiers.toml @@ -31,6 +31,14 @@ description = "High-priority tests excluding Smoke (15 min target)" upstream_tests = ["sandbox_lifecycle", "landlock", "user_namespaces"] odh_filter = "tier1::" +[tier1.upstream_test_exclusions] +sandbox_lifecycle = [ + "canonical_main_disconnect_reconnect_replays_history_for_same_process", + "canonical_main_exit_255_is_not_retried_as_transport_failure", + "on_failure_policy_replaces_runtime_and_preserves_workspace", + "canonical_main_and_exec_receive_declared_environment", +] + [tier2] description = "Medium/low priority positive tests" # Corporate proxy needs a cluster-reachable fixture configured before Helm install. diff --git a/odh/scripts/e2e-odh-entrypoint.sh b/odh/scripts/e2e-odh-entrypoint.sh index c5d5f84304..e50b1d0156 100755 --- a/odh/scripts/e2e-odh-entrypoint.sh +++ b/odh/scripts/e2e-odh-entrypoint.sh @@ -73,8 +73,9 @@ for phase_line in ${phase_lines}; do phase_reports+=("${phase_report}") rm -f "${phase_report}" "${phase_report%.xml}.html" done -merged_report="${results_dir}/e2e-odh-${TIER}.xml" -rm -f "${merged_report}" "${merged_report%.xml}.html" +merged_report="${results_dir}/e2e-odh-${TIER}-xunit.xml" +html_report="${results_dir}/e2e-odh-${TIER}.html" +rm -f "${merged_report}" "${html_report}" managed_namespace_selector='openshell.ai/managed-by=openshell,openshell.ai/gateway-id=openshell' snapshot_managed_namespaces() { @@ -225,7 +226,7 @@ run_child "${PYTHON_BIN}" "${MERGE_SCRIPT}" "${TIER}" "${phase_reports[@]}" || { } if [[ -f "${merged_report}" ]]; then if command -v xsltproc >/dev/null 2>&1; then - xsltproc --stringparam title "e2e-odh-${TIER}" "${ROOT}/scripts/junit-to-html.xsl" "${merged_report}" > "${merged_report%.xml}.html" || echo "WARNING: failed to render HTML report" >&2 + xsltproc --stringparam title "e2e-odh-${TIER}" "${ROOT}/scripts/junit-to-html.xsl" "${merged_report}" > "${html_report}" || echo "WARNING: failed to render HTML report" >&2 else echo "WARNING: xsltproc not found; HTML report unavailable" >&2 fi diff --git a/odh/scripts/merge-e2e-odh-reports.py b/odh/scripts/merge-e2e-odh-reports.py index 68226d00e5..738d9d4ed1 100755 --- a/odh/scripts/merge-e2e-odh-reports.py +++ b/odh/scripts/merge-e2e-odh-reports.py @@ -48,7 +48,7 @@ def merge(tier: str, paths: list[Path], results_dir: Path) -> Path: root.append(suite) xml_bytes = ET.tostring(root, encoding="utf-8", xml_declaration=True) results_dir.mkdir(parents=True, exist_ok=True) - xml_path = results_dir / f"e2e-odh-{tier}.xml" + xml_path = results_dir / f"e2e-odh-{tier}-xunit.xml" fd, temporary = tempfile.mkstemp(prefix=f".{xml_path.name}.", dir=results_dir) try: with os.fdopen(fd, "wb") as stream: From f8533ec4ddcd33cd4f53461c94cc3344aaab1be9 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:44:37 +0000 Subject: [PATCH 23/26] chore(deps): refresh rpm lockfiles [SECURITY] (#115) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/cli/rpms.lock.yaml | 20 ++++++++++---------- deploy/konflux/e2e-odh/rpms.lock.yaml | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/deploy/konflux/cli/rpms.lock.yaml b/deploy/konflux/cli/rpms.lock.yaml index 2415182651..8a51220b62 100644 --- a/deploy/konflux/cli/rpms.lock.yaml +++ b/deploy/konflux/cli/rpms.lock.yaml @@ -1268,20 +1268,20 @@ arches: name: openldap evr: 2.6.8-4.el9 sourcerpm: openldap-2.6.8-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-9.9p1-12.el9_8.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-9.9p1-13.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 443141 - checksum: sha256:12db3094d78ed82bd7edc6a7cdd2cbf5198da695e293b47b5d00c31ac142aeb7 + size: 443578 + checksum: sha256:f3a0c1bf130cf8335db06a3b13793b4f590208e5a3f20a1a3391eb8f0162d06a name: openssh - evr: 9.9p1-12.el9_8 - sourcerpm: openssh-9.9p1-12.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-clients-9.9p1-12.el9_8.x86_64.rpm + evr: 9.9p1-13.el9_8 + sourcerpm: openssh-9.9p1-13.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-clients-9.9p1-13.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 800429 - checksum: sha256:a4307b914ec45f69fea0e17b593ae61597db3e2796ce01a8809c55173f30c121 + size: 800352 + checksum: sha256:cea2dde9a34a69071f65e3c623a3ed041320520d34d0edb80ceb74316d2f3ee1 name: openssh-clients - evr: 9.9p1-12.el9_8 - sourcerpm: openssh-9.9p1-12.el9_8.src.rpm + evr: 9.9p1-13.el9_8 + sourcerpm: openssh-9.9p1-13.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssl-3.5.8-2.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 1570074 diff --git a/deploy/konflux/e2e-odh/rpms.lock.yaml b/deploy/konflux/e2e-odh/rpms.lock.yaml index ccfeff5027..31a5c7d4db 100644 --- a/deploy/konflux/e2e-odh/rpms.lock.yaml +++ b/deploy/konflux/e2e-odh/rpms.lock.yaml @@ -1352,20 +1352,20 @@ arches: name: openldap evr: 2.6.8-4.el9 sourcerpm: openldap-2.6.8-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-9.9p1-12.el9_8.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-9.9p1-13.el9_8.x86_64.rpm repoid: ubi-9-baseos-rpms - size: 443141 - checksum: sha256:12db3094d78ed82bd7edc6a7cdd2cbf5198da695e293b47b5d00c31ac142aeb7 + size: 443578 + checksum: sha256:f3a0c1bf130cf8335db06a3b13793b4f590208e5a3f20a1a3391eb8f0162d06a name: openssh - evr: 9.9p1-12.el9_8 - sourcerpm: openssh-9.9p1-12.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-clients-9.9p1-12.el9_8.x86_64.rpm + evr: 9.9p1-13.el9_8 + sourcerpm: openssh-9.9p1-13.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssh-clients-9.9p1-13.el9_8.x86_64.rpm repoid: ubi-9-baseos-rpms - size: 800429 - checksum: sha256:a4307b914ec45f69fea0e17b593ae61597db3e2796ce01a8809c55173f30c121 + size: 800352 + checksum: sha256:cea2dde9a34a69071f65e3c623a3ed041320520d34d0edb80ceb74316d2f3ee1 name: openssh-clients - evr: 9.9p1-12.el9_8 - sourcerpm: openssh-9.9p1-12.el9_8.src.rpm + evr: 9.9p1-13.el9_8 + sourcerpm: openssh-9.9p1-13.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/o/openssl-3.5.8-2.el9_8.x86_64.rpm repoid: ubi-9-baseos-rpms size: 1570074 From 469e16d406a556ae115043b62ebceeb72dd29346 Mon Sep 17 00:00:00 2001 From: "red-hat-konflux[bot]" <126015336+red-hat-konflux[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:45:54 +0000 Subject: [PATCH 24/26] chore(deps): refresh rpm lockfiles (#116) Signed-off-by: red-hat-konflux <126015336+red-hat-konflux[bot]@users.noreply.github.com> Co-authored-by: red-hat-konflux[bot] <126015336+red-hat-konflux[bot]@users.noreply.github.com> --- deploy/konflux/sandbox/rpms.lock.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/deploy/konflux/sandbox/rpms.lock.yaml b/deploy/konflux/sandbox/rpms.lock.yaml index a143e9d7d9..831f6799f1 100644 --- a/deploy/konflux/sandbox/rpms.lock.yaml +++ b/deploy/konflux/sandbox/rpms.lock.yaml @@ -176,13 +176,13 @@ arches: name: glibc-headers evr: 2.34-283.el9_8 sourcerpm: glibc-2.34-283.el9_8.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.58.1.el9_8.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/k/kernel-headers-5.14.0-687.59.1.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms - size: 2982193 - checksum: sha256:2c29113a9f860f0e2a14c89ec3a399b3e3d0f9ff699d7f5b565ca5e73b0314a9 + size: 2985513 + checksum: sha256:fa019cf3b155923a773aacba0f5fc0f0e470480cc0c266cdff2be847d337f189 name: kernel-headers - evr: 5.14.0-687.58.1.el9_8 - sourcerpm: kernel-5.14.0-687.58.1.el9_8.src.rpm + evr: 5.14.0-687.59.1.el9_8 + sourcerpm: kernel-5.14.0-687.59.1.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/appstream/os/Packages/l/libxcrypt-devel-4.4.18-3.el9.x86_64.rpm repoid: ubi-9-for-x86_64-appstream-rpms size: 33101 From 83feba4b380a0ad76efea5ea99dfb06479754013 Mon Sep 17 00:00:00 2001 From: Evie Howard Date: Fri, 18 Sep 2026 09:06:43 +0100 Subject: [PATCH 25/26] CARRY: test(odh): add gateway failover coverage Signed-off-by: Evie Howard --- .config/nextest.toml | 3 + e2e/rust/tests/odh/README.md | 92 +- e2e/rust/tests/odh/odh_harness/mod.rs | 1 + e2e/rust/tests/odh/odh_harness/oc.rs | 262 +++++- e2e/rust/tests/odh/odh_harness/sandbox.rs | 71 ++ e2e/rust/tests/odh/run-odh-test-tier.sh | 14 +- e2e/rust/tests/odh/smoke/image_provenance.rs | 42 +- e2e/rust/tests/odh/tier1/selinux.rs | 4 +- e2e/rust/tests/odh/tier3/gateway_failover.rs | 920 +++++++++++++++++++ e2e/rust/tests/odh/tier3/mod.rs | 2 + 10 files changed, 1367 insertions(+), 44 deletions(-) create mode 100644 e2e/rust/tests/odh/odh_harness/sandbox.rs create mode 100644 e2e/rust/tests/odh/tier3/gateway_failover.rs diff --git a/.config/nextest.toml b/.config/nextest.toml index 3c3047a2d7..ce6825ddba 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -42,6 +42,9 @@ slow-timeout = { period = "60s", terminate-after = 5 } # Relative to the profile store dir (`e2e/rust/target/nextest/e2e-odh/`). [profile.e2e-odh.junit] path = "../../../../../results/e2e-odh.xml" +# Keep ignored destructive ODH tests visible to Jenkins as skipped rather than +# silently omitting them from the report. +report-skipped = "ignored" # serial_test's in-process locks do not span nextest's test processes. # Match names so this config also works without these test binaries. diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index d5c89210a3..b7cfe15d60 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -18,7 +18,9 @@ e2e/rust/tests/odh/ ├── main.rs # crate root, declares helper + tier modules, gated on feature "e2e-odh" ├── odh_harness/ # fork-local shared test helpers (see "Shared test helpers") │ ├── mod.rs -│ └── oc.rs # `oc` command builder (honors active kube context) + JSON runner +│ ├── oc.rs # `oc` command builder (honors active kube context) + JSON runner +│ ├── sandbox.rs # sandbox CR helpers, including pod-selector resolution +│ └── selinux.rs # OpenShift SELinux audit helpers ├── smoke/ # Smoke tier: component-level critical tests │ ├── mod.rs │ ├── gateway.rs # gateway reachability @@ -30,7 +32,8 @@ e2e/rust/tests/odh/ ├── tier2/ # Tier 2: medium/low priority positive tests │ └── mod.rs # empty — no scenarios yet ├── tier3/ # Tier 3: negative and destructive tests -│ └── mod.rs # empty — no scenarios yet +│ ├── mod.rs +│ └── gateway_failover.rs # external-PostgreSQL HA gateway failover ├── tiers.toml # data-only gateway modes and ordered tier phases ├── tier_plan.py # validates modes, overlays, and test selectors └── run-odh-test-tier.sh # runs one resolved phase against its gateway @@ -56,11 +59,15 @@ plain module is enough — no new crate and no workspace change. `tokio::process::Command` for `oc`, injecting `--context` from `OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE` (exported by `e2e/with-kube-gateway.sh`) when it is set, so every ODH test targets the same cluster the upstream - harness does. `oc_json()` runs a query and parses `-o json` output, panicking - with a descriptive message on any failure. + harness does. It also centralizes namespace/release resolution and Pod Ready + checks. `oc_json()` runs a query and parses `-o json` output, panicking with + a descriptive message on any failure. - `odh_harness::selinux::SelinuxAudit` — an opt-in scenario guard that detects OpenShift, requires every Ready worker to report `Enforcing`, records node-local audit cutoffs, and rejects OpenShell AVCs on completion. +- `odh_harness::sandbox::sandbox_pod_selector()` — resolves a sandbox custom + resource's reported workload selector so tests can find its controller-owned + pod. Put ODH-specific shared helpers here (the `oc` builder and node-level SELinux checks), and reuse them rather than @@ -111,10 +118,12 @@ failures: `canonical_main_disconnect_reconnect_replays_history_for_same_process` Tier 2 runs `workspace_lifecycle`, ODH Tier 2, and image provenance in shared mode, then runs `workspace_namespace_managed` against a separate managed-mode deployment. The `odh` and `full` tiers use the same shared then -managed split. Image provenance runs once in each tier's shared phase. Smoke -and Tier 1/3 use only shared mode. Add Rust scenarios by creating a `.rs` file -under the tier's directory and declaring it with a `mod` line in that tier's -`mod.rs`; add its selector to the corresponding tier table or mode extension. +managed split. Tier 3 includes an opt-in destructive external-PostgreSQL HA +gateway failover scenario. Image provenance runs once in each tier's shared +phase. Smoke and Tier 1/3 use only shared mode. Add Rust scenarios by creating +a `.rs` file under the tier's directory and declaring it with a `mod` line in +that tier's `mod.rs`; add its selector to the corresponding tier table or mode +extension. To add a Helm-overlay mode, define its validated values paths and expected deployed values under `[modes.]`. Shared is implicit; add a named mode @@ -156,6 +165,27 @@ Excluded tests do not appear as passes or skips in the JUnit report. runs and HTML reports. - The local task builds `openshell-cli` when `OPENSHELL_BIN` is unset. Set `OPENSHELL_BIN` to use another CLI binary. +- For `tier3::gateway_failover`, a deployment installed with + `deploy/helm/openshell/ci/values-high-availability.yaml`, an external + PostgreSQL Secret, and a configured CLI gateway name. The test discovers two + ready gateway pods and opens its own loopback `oc port-forward` to direct the + initial create and session to one pod. It confirms the session disconnects + when that pod is deleted, then reconnects through a second port-forward to + the pre-existing surviving replica. This pins the check to that replica and + does not exercise the configured Service or Route endpoint. The workload + heartbeat includes a UUID generated at process start; reconnect must report + the same UUID, so a restarted workload does not satisfy the session check. + Set `OPENSHELL_GATEWAY` to the configured gateway name; it also selects the + gateway used by the image-provenance check that runs with every tier. + `OPENSHELL_ODH_HA_GATEWAY_NAME` remains a compatibility fallback when + `OPENSHELL_GATEWAY` is not set. The test identity needs permission to get + the PostgreSQL Secret referenced by `OPENSHELL_DB_URL`, create pod + port-forwards, and delete gateway pods. The test deletes a gateway pod only + when `OPENSHELL_ODH_HA_FAILOVER=1` is set. The test is reported as skipped + when that opt-in is absent. Its default pod selector is the Helm + `app.kubernetes.io/name=openshell,app.kubernetes.io/instance=` + selector; override it with `OPENSHELL_ODH_HA_GATEWAY_SELECTOR` for a renamed + chart deployment. ### The `KUBECONFIG` gotcha @@ -195,6 +225,52 @@ context you happen to have active elsewhere. This means: | `mise run e2e:odh:tier3` | Tier 3: mapped upstream tests + ODH `tier3::` + image provenance | | `cargo nextest run --manifest-path e2e/rust/Cargo.toml --features e2e-odh --test odh -E 'test(=module::test_name)'` | A single ODH test function | +The destructive HA failover scenario is skipped by default. Run it only +against a disposable or explicitly approved HA deployment: + +```bash +OPENSHELL_ODH_HA_FAILOVER=1 mise run e2e:odh:tier3 +``` + +The existing `e2e/with-kube-gateway.sh` wrapper can provision the HA fixture +in an ephemeral namespace: it installs the chart with the HA values overlay, +deploys the PostgreSQL fixture, configures the OpenShift Route and CLI, then +tears down its resources after the test. The Rust test itself assumes an +already-deployed gateway, as do the other ODH tier tests. To run only this +scenario through the wrapper, use a disposable OpenShift context and a built +`target/debug/openshell` CLI. The wrapper's existing-context mode uses one +registry and image tag for gateway, sandbox runtime, and supervisor; deployments +with different image sources for those components still need explicit Helm +overrides or a wrapper extension. + +```bash +OPENSHELL_E2E_KUBE_CONTEXT= \ +OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET=openshell-ha-pg \ +OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-high-availability.yaml \ +OPENSHELL_GATEWAY= \ +OPENSHELL_ODH_HA_FAILOVER=1 \ +e2e/with-kube-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml \ + --features e2e-odh --test odh -- \ + --include-ignored \ + tier3::gateway_failover::gateway_pod_failover_preserves_sandbox_session_and_workspace \ + --exact --nocapture +``` + +Before running failover on an existing deployment, confirm that a normal +`smoke::sandbox::test_create_delete` passes. The failover test bounds sandbox +creation to five minutes and requires the initial session to remain attached +for more than two seconds before deleting any gateway pod. It retries that +preflight with a replacement pod port-forward when `oc port-forward` stalls +the SSH-based `sandbox connect` path. A failure after the retries means +failover has not been exercised. +After sandbox creation, the test awaits a sandbox deletion attempt even when a +later assertion fails. + +The failover scenario explicitly uses the shared E2E workload image, like the +smoke tests. It does not use the chart's default sandbox image: that image can +carry a discovered policy incompatible with the gateway under test, causing +the supervisor to exit before readiness and masking the failover behavior. + Example, running the Smoke tier against a real cluster: ```bash diff --git a/e2e/rust/tests/odh/odh_harness/mod.rs b/e2e/rust/tests/odh/odh_harness/mod.rs index 643594460d..411b0d80a0 100644 --- a/e2e/rust/tests/odh/odh_harness/mod.rs +++ b/e2e/rust/tests/odh/odh_harness/mod.rs @@ -10,4 +10,5 @@ //! collide with the upstream `harness` module. pub mod oc; +pub mod sandbox; pub mod selinux; diff --git a/e2e/rust/tests/odh/odh_harness/oc.rs b/e2e/rust/tests/odh/odh_harness/oc.rs index c8da725fdf..d37c4cb8fe 100644 --- a/e2e/rust/tests/odh/odh_harness/oc.rs +++ b/e2e/rust/tests/odh/odh_harness/oc.rs @@ -8,7 +8,66 @@ //! command construction here keeps every test targeting the same cluster and //! reporting failures the same way. +use std::process::Stdio; +use std::time::Duration; + use serde_json::Value; +use tokio::io::AsyncWriteExt as _; + +/// Maximum duration for one ODH `oc` invocation. +pub const OC_COMMAND_TIMEOUT: Duration = Duration::from_secs(30); + +/// Output from an `oc` invocation. +pub struct OcOutput { + pub success: bool, + exit_code: Option, + stdout: String, + stderr: String, +} + +impl OcOutput { + pub fn from_output(output: &std::process::Output) -> Self { + Self { + success: output.status.success(), + exit_code: output.status.code(), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + } + } + + /// A safe, allowlisted description for assertion failures. + /// + /// Command output may contain Kubernetes objects, credentials, or response + /// bodies. Keep it available only for parsing and never include it in test + /// failure output. + pub fn status_summary(&self) -> String { + match self.exit_code { + Some(code) => format!("exit status: {code}"), + None => "terminated by signal".to_string(), + } + } + + /// Returns whether stdout contains an exact, allowlisted result line. + /// + /// This permits callers to consume a deliberately emitted protocol marker + /// without exposing arbitrary command output in diagnostics. + pub fn has_stdout_line(&self, expected: &str) -> bool { + self.stdout.lines().any(|line| line.trim() == expected) + } + + /// Returns whether `oc` identified this request as an API `NotFound`. + /// + /// `oc get --raw` writes Kubernetes API errors to stderr. Keep that stream + /// private so it cannot leak into assertion output, but retain it to + /// distinguish the expected absence of an API endpoint from real failures. + pub fn is_not_found(&self) -> bool { + !self.success && self.stderr.contains("Error from server (NotFound)") + } + + pub fn json(&self) -> Result { + serde_json::from_str(&self.stdout) + } +} /// Builds an `oc` command targeting the active e2e cluster. /// @@ -27,23 +86,183 @@ pub fn oc_command() -> tokio::process::Command { cmd } +/// Runs `oc `, optionally writing `input` to standard input. +/// +/// Captures command output without emitting it. Retains stdout only for +/// internal parsing; never include command output in test failure messages, +/// since it may contain sensitive cluster data. +pub async fn oc(args: &[&str], input: Option<&str>) -> OcOutput { + oc_with_timeout(args, input, OC_COMMAND_TIMEOUT).await +} + +/// Runs `oc ` with an explicit outer command timeout. +/// +/// Use this for `oc` subcommands which have their own longer timeout, so the +/// harness does not abort them before their requested deadline. Most callers +/// should use [`oc`], which retains the standard short timeout. +pub async fn oc_with_timeout( + args: &[&str], + input: Option<&str>, + command_timeout: Duration, +) -> OcOutput { + let mut cmd = oc_command(); + cmd.args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + if input.is_some() { + cmd.stdin(Stdio::piped()); + } + let output = tokio::time::timeout(command_timeout, async { + let mut child = cmd.spawn().expect( + "failed to run `oc` — required for ODH cluster-state checks; ensure it is in PATH \ + and KUBECONFIG targets the cluster", + ); + if let Some(input) = input { + child + .stdin + .take() + .expect("piped stdin") + .write_all(input.as_bytes()) + .await + .expect("write manifest to oc"); + } + child.wait_with_output().await.expect("wait for oc") + }) + .await + .unwrap_or_else(|_| { + panic!( + "oc {args:?} timed out after {} seconds", + command_timeout.as_secs() + ) + }); + OcOutput::from_output(&output) +} + +/// Returns whether the active cluster exposes the `OpenShift` Route API. +/// +/// ODH-only tests use this to skip cleanly on non-OpenShift clusters while +/// preserving the standard tier entry points. +pub async fn is_openshift() -> Result { + let output = oc(&["get", "--raw", "/apis/route.openshift.io/v1"], None).await; + if output.success { + return Ok(true); + } + if output.is_not_found() { + return Ok(false); + } + Err(format!( + "unable to determine whether the cluster is OpenShift ({})", + output.status_summary() + )) +} + /// Runs `oc ` and parses stdout as JSON. /// /// Panics with a descriptive message if `oc` cannot be launched, exits /// non-zero, or does not return valid JSON — use it for `-o json` queries /// whose failure should fail the test. pub async fn oc_json(args: &[&str]) -> Value { - let output = oc_command().args(args).output().await.expect( - "failed to run `oc` — required for ODH cluster-state checks; ensure it is in PATH \ - and KUBECONFIG targets the cluster", - ); + let output = oc(args, None).await; assert!( - output.status.success(), - "oc {args:?} failed: {}", - String::from_utf8_lossy(&output.stderr) + output.success, + "oc JSON query failed ({})", + output.status_summary() ); - serde_json::from_slice(&output.stdout) - .unwrap_or_else(|e| panic!("oc {args:?} did not return valid JSON: {e}")) + output + .json() + .unwrap_or_else(|_| panic!("oc JSON query returned invalid JSON")) +} + +#[cfg(test)] +mod oc_output_tests { + use super::OcOutput; + + #[test] + fn status_summary_does_not_include_command_output() { + let output = OcOutput { + success: false, + exit_code: Some(1), + stdout: "token=secret".to_string(), + stderr: "Error from server (Forbidden): token=secret".to_string(), + }; + + assert_eq!(output.status_summary(), "exit status: 1"); + } + + #[test] + fn has_stdout_line_requires_an_exact_line() { + let output = OcOutput { + success: true, + exit_code: Some(0), + stdout: "prefix\nRESULT\nRESULT-extra".to_string(), + stderr: String::new(), + }; + + assert!(output.has_stdout_line("RESULT")); + assert!(!output.has_stdout_line("RESULT-extra-extra")); + } + + #[test] + fn recognizes_only_the_oc_not_found_response() { + let not_found = OcOutput { + success: false, + exit_code: Some(1), + stdout: String::new(), + stderr: + "Error from server (NotFound): the server could not find the requested resource" + .to_string(), + }; + let forbidden = OcOutput { + success: false, + exit_code: Some(1), + stdout: String::new(), + stderr: "Error from server (Forbidden): User cannot get path".to_string(), + }; + + assert!(not_found.is_not_found()); + assert!(!forbidden.is_not_found()); + } +} + +const DEFAULT_DEPLOYMENT_NAME: &str = "openshell"; + +/// Resolves the namespace containing gateway resources for the ODH deployment. +/// +/// The names match the standard ODH and Kubernetes e2e configuration +/// variables. +pub fn gateway_namespace() -> String { + std::env::var("NAMESPACE") + .or_else(|_| std::env::var("OPENSHELL_E2E_KUBE_NAMESPACE")) + .unwrap_or_else(|_| DEFAULT_DEPLOYMENT_NAME.to_string()) +} + +/// Resolves the namespace containing Sandbox custom resources and workload Pods. +/// +/// Sandbox resources normally share the gateway namespace. Set +/// `SANDBOX_NAMESPACE` when the compute driver uses a separate namespace. +pub fn sandbox_namespace() -> String { + std::env::var("SANDBOX_NAMESPACE").unwrap_or_else(|_| gateway_namespace()) +} + +/// Resolves the Helm release name used by the ODH deployment under test. +pub fn release() -> String { + std::env::var("RELEASE") + .or_else(|_| std::env::var("OPENSHELL_E2E_KUBE_RELEASE")) + .unwrap_or_else(|_| DEFAULT_DEPLOYMENT_NAME.to_string()) +} + +/// Returns whether a Kubernetes Pod JSON object is Running and Ready. +pub fn pod_is_ready(pod: &Value) -> bool { + pod["status"]["phase"].as_str() == Some("Running") + && pod["status"]["conditions"] + .as_array() + .is_some_and(|conditions| { + conditions.iter().any(|condition| { + condition["type"].as_str() == Some("Ready") + && condition["status"].as_str() == Some("True") + }) + }) } /// Resolve the supervisor Pod paired with a named Sandbox resource. @@ -203,9 +422,32 @@ mod tests { use serde_json::json; use super::{ - pod_node_and_uid, pod_uid_cgroup_form, sandbox_id_from_json, supervisor_pod_from_json, + pod_is_ready, pod_node_and_uid, pod_uid_cgroup_form, sandbox_id_from_json, + supervisor_pod_from_json, }; + #[test] + fn recognizes_only_running_ready_pods() { + assert!(pod_is_ready(&json!({ + "status": { + "phase": "Running", + "conditions": [{"type": "Ready", "status": "True"}] + } + }))); + assert!(!pod_is_ready(&json!({ + "status": { + "phase": "Pending", + "conditions": [{"type": "Ready", "status": "True"}] + } + }))); + assert!(!pod_is_ready(&json!({ + "status": { + "phase": "Running", + "conditions": [{"type": "Ready", "status": "False"}] + } + }))); + } + #[test] fn resolves_sandbox_id_from_named_sandbox() { let sandbox = json!({ diff --git a/e2e/rust/tests/odh/odh_harness/sandbox.rs b/e2e/rust/tests/odh/odh_harness/sandbox.rs new file mode 100644 index 0000000000..bc118c6b85 --- /dev/null +++ b/e2e/rust/tests/odh/odh_harness/sandbox.rs @@ -0,0 +1,71 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Helpers for resolving OpenShell sandbox resources on an ODH cluster. + +use serde_json::Value; + +use super::oc::oc_json; + +/// Returns the pod selector reported by the Sandbox custom resource. +/// +/// The sandbox-agent controller owns pod creation and does not propagate the +/// OpenShell sandbox-name label to its pod. The custom resource's status is +/// therefore the stable way for downstream tests to discover that pod. +pub async fn sandbox_pod_selector(namespace: &str, sandbox_name: &str) -> Option { + let selector = format!("openshell.ai/sandbox-name={sandbox_name}"); + let sandboxes: Value = oc_json(&[ + "get", + "sandboxes.agents.x-k8s.io", + "-n", + namespace, + "-l", + &selector, + "-o", + "json", + ]) + .await; + sandbox_pod_selector_from_json(&sandboxes) +} + +fn sandbox_pod_selector_from_json(sandboxes: &Value) -> Option { + sandboxes + .get("items") + .and_then(Value::as_array) + .and_then(|items| (items.len() == 1).then(|| &items[0])) + .and_then(|sandbox| sandbox["status"]["selector"].as_str()) + .map(str::to_string) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::sandbox_pod_selector_from_json; + + #[test] + fn resolves_selector_from_exactly_one_sandbox() { + let sandboxes = json!({ + "items": [{"status": {"selector": "agents.x-k8s.io/sandbox-name-hash=abc"}}] + }); + + assert_eq!( + sandbox_pod_selector_from_json(&sandboxes), + Some("agents.x-k8s.io/sandbox-name-hash=abc".to_string()) + ); + } + + #[test] + fn rejects_missing_or_ambiguous_sandboxes() { + assert_eq!(sandbox_pod_selector_from_json(&json!({"items": []})), None); + assert_eq!( + sandbox_pod_selector_from_json(&json!({ + "items": [ + {"status": {"selector": "one"}}, + {"status": {"selector": "two"}} + ] + })), + None + ); + } +} diff --git a/e2e/rust/tests/odh/run-odh-test-tier.sh b/e2e/rust/tests/odh/run-odh-test-tier.sh index 2080554248..776f533a73 100755 --- a/e2e/rust/tests/odh/run-odh-test-tier.sh +++ b/e2e/rust/tests/odh/run-odh-test-tier.sh @@ -52,6 +52,7 @@ fi junit_xml="${results_dir}/e2e-odh.xml" rm -f "${junit_xml}" "${report}" "${report%.xml}.html" +<<<<<<< HEAD config_file="$(mktemp)" trap 'rm -f "${config_file}"' EXIT "${PYTHON_BIN}" - "${ROOT}/.config/nextest.toml" "${config_file}" "${junit_xml}" <<'PY' @@ -63,7 +64,18 @@ if old not in text: raise SystemExit("ODH nextest JUnit path setting not found") destination.write_text(text.replace(old, f'path = "{report}"')) PY -nextest_args=(--profile e2e-odh --config-file "${config_file}" --no-tests fail -E "${filter}") +nextest_args=( + --profile e2e-odh + --config-file "${config_file}" + --no-tests fail + -E "${filter}" +) +# The failover test is ignored by default because it deletes a gateway pod. +# Run it only after the caller has explicitly opted into destructive HA +# coverage; `all` preserves the rest of the selected tier. +if [[ "${OPENSHELL_ODH_HA_FAILOVER:-0}" == "1" ]]; then + nextest_args+=(--run-ignored all) +fi if [[ -n "${OPENSHELL_E2E_NEXTEST_ARCHIVE:-}" ]]; then nextest_args+=(--archive-file "${OPENSHELL_E2E_NEXTEST_ARCHIVE}" --workspace-remap "${ROOT}/e2e/rust") else diff --git a/e2e/rust/tests/odh/smoke/image_provenance.rs b/e2e/rust/tests/odh/smoke/image_provenance.rs index 8ec46b1d5f..d40537cac7 100644 --- a/e2e/rust/tests/odh/smoke/image_provenance.rs +++ b/e2e/rust/tests/odh/smoke/image_provenance.rs @@ -18,7 +18,8 @@ use serde_json::Value; use openshell_e2e::harness::sandbox::SandboxGuard; -use crate::odh_harness::oc::{oc_command, oc_json}; +use crate::odh_harness::oc::{gateway_namespace, oc_command, oc_json, release, sandbox_namespace}; +use crate::odh_harness::sandbox::sandbox_pod_selector; fn allowed_prefixes() -> Vec { std::env::var("ALLOWED_IMAGE_REGISTRY_PREFIXES") @@ -108,8 +109,9 @@ fn parse_supervisor_image(gateway_toml: &str) -> Option { #[tokio::test] async fn test_sandbox_gateway_supervisor_images() { - let namespace = std::env::var("NAMESPACE").unwrap_or_else(|_| "openshell".to_string()); - let release = std::env::var("RELEASE").unwrap_or_else(|_| "openshell".to_string()); + let gateway_namespace = gateway_namespace(); + let sandbox_namespace = sandbox_namespace(); + let release = release(); let allowed = allowed_prefixes(); let mut errors = Vec::new(); @@ -143,27 +145,21 @@ async fn test_sandbox_gateway_supervisor_images() { // agents.x-k8s.io/sandbox-name-hash label. So look up the CR first and // follow its reported `status.selector` to find the pod. let sandbox_selector = format!("openshell.ai/sandbox-name={}", sb.name); - let sandbox_crs = oc_json(&[ - "get", - "sandboxes.agents.x-k8s.io", - "-n", - &namespace, - "-l", - &sandbox_selector, - "-o", - "json", - ]) - .await; - let pod_selector = sandbox_crs - .get("items") - .and_then(Value::as_array) - .and_then(|items| items.first()) - .and_then(|cr| cr["status"]["selector"].as_str()) - .map(str::to_string); + let pod_selector = sandbox_pod_selector(&sandbox_namespace, &sb.name).await; match pod_selector { Some(pod_selector) => { - let sandbox_pods = oc_json(&["get", "pods", "-n", &namespace, "-l", &pod_selector, "-o", "json"]).await; + let sandbox_pods = oc_json(&[ + "get", + "pods", + "-n", + &sandbox_namespace, + "-l", + &pod_selector, + "-o", + "json", + ]) + .await; collect_pod_images(&sandbox_pods, &pod_selector, &mut errors, &mut images); } None => errors.push(format!( @@ -177,7 +173,7 @@ async fn test_sandbox_gateway_supervisor_images() { "get", "pods", "-n", - &namespace, + &gateway_namespace, "-l", &gateway_selector, "-o", @@ -196,7 +192,7 @@ async fn test_sandbox_gateway_supervisor_images() { "configmap", &cm_name, "-n", - &namespace, + &gateway_namespace, "-o", "jsonpath={.data.gateway\\.toml}", ]) diff --git a/e2e/rust/tests/odh/tier1/selinux.rs b/e2e/rust/tests/odh/tier1/selinux.rs index 6f5159b84b..eba372eff8 100644 --- a/e2e/rust/tests/odh/tier1/selinux.rs +++ b/e2e/rust/tests/odh/tier1/selinux.rs @@ -9,7 +9,7 @@ use openshell_e2e::harness::sandbox::SandboxGuard; use serial_test::serial; use tempfile::NamedTempFile; -use crate::odh_harness::oc::{paired_supervisor_pod, supervisor_selinux_label}; +use crate::odh_harness::oc::{paired_supervisor_pod, sandbox_namespace, supervisor_selinux_label}; use crate::odh_harness::selinux::SelinuxAudit; /// OCP's container `SELinux` type. The complete label includes per-pod MLS/MCS @@ -58,7 +58,7 @@ async fn selinux_is_enforcing_on_all_worker_nodes() { async fn supervisor_runs_in_container_selinux_domain() { run_audited("supervisor label", || async { let mut sandbox = SandboxGuard::create(&[]).await?; - let namespace = std::env::var("NAMESPACE").unwrap_or_else(|_| "openshell".to_string()); + let namespace = sandbox_namespace(); let supervisor_pod = paired_supervisor_pod(&namespace, &sandbox.name).await?; let label_result = supervisor_selinux_label(&namespace, &supervisor_pod).await; sandbox.cleanup().await; diff --git a/e2e/rust/tests/odh/tier3/gateway_failover.rs b/e2e/rust/tests/odh/tier3/gateway_failover.rs new file mode 100644 index 0000000000..d584f976ed --- /dev/null +++ b/e2e/rust/tests/odh/tier3/gateway_failover.rs @@ -0,0 +1,920 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Gateway failover coverage for an external-PostgreSQL HA deployment. +//! +//! This test is ignored by default because it deletes a gateway pod. The ODH +//! tier runner runs it only with `OPENSHELL_ODH_HA_FAILOVER=1`. The test keeps +//! the client's local endpoint stable while switching its pod port-forward +//! from the deleted replica to a surviving replica, then verifies that the +//! original client reconnects. + +use std::net::TcpListener; +use std::panic::AssertUnwindSafe; +use std::process::{Output, Stdio}; +use std::time::Duration; + +use base64::Engine as _; +use futures_util::FutureExt as _; +use openshell_e2e::harness::binary::openshell_cmd; +use openshell_e2e::harness::sandbox::E2E_WORKLOAD_IMAGE; +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; +use tokio::process::{Child, ChildStdout}; +use tokio::time::{Instant, sleep, timeout}; + +use crate::odh_harness::oc::{ + gateway_namespace, is_openshift, oc_command, oc_json, oc_with_timeout, pod_is_ready, release, + sandbox_namespace, +}; +use crate::odh_harness::sandbox::sandbox_pod_selector; + +// constants +const ENABLE_ENV: &str = "OPENSHELL_ODH_HA_FAILOVER"; +const GATEWAY_SELECTOR_ENV: &str = "OPENSHELL_ODH_HA_GATEWAY_SELECTOR"; +const GATEWAY_NAME_ENV: &str = "OPENSHELL_ODH_HA_GATEWAY_NAME"; +const GATEWAY_PORT: u16 = 8080; +const POD_READY_TIMEOUT: Duration = Duration::from_secs(120); // sets timeouts +const SESSION_TIMEOUT: Duration = Duration::from_secs(60); +const CREATE_TIMEOUT: Duration = Duration::from_secs(300); +const CREATE_RECOVERY_TIMEOUT: Duration = Duration::from_secs(120); +const CLI_COMMAND_TIMEOUT: Duration = Duration::from_secs(120); +const SANDBOX_DELETION_TIMEOUT: Duration = Duration::from_secs(300); +const SESSION_START_ATTEMPTS: usize = 3; +// The CLI only opens its reconnect window after an attachment has survived +// two seconds. Keep a one-second margin so pod deletion cannot race that +// threshold on a slow runner. +const SESSION_ESTABLISHED_DURATION: Duration = Duration::from_secs(3); +const POLL_INTERVAL: Duration = Duration::from_secs(2); +const SENTINEL: &str = "odh-ha-workspace-sentinel"; // marker written to disk to test file persistence +const OUTPUT_MARKER: &str = "odh-ha-session-alive"; // prefix for a process-specific heartbeat used to detect workload restarts + +struct GatewayPod { + name: String, + ready: bool, +} // tracks pod readiness + +struct ActiveSession { + child: Child, + stdout: tokio::io::Lines>, +} // handles running terminal command streams + +struct PodPortForward { + endpoint: String, + port: u16, + child: Child, +} // tracks a local tunnel directly to a single pod + +struct SessionMarker { + process_id: String, + emitted_at: u64, +} + +struct ManagedSandbox { + name: String, + cleaned_up: bool, +} + +impl ManagedSandbox { + fn new(name: String) -> Self { + Self { + name, + cleaned_up: false, + } + } + + async fn cleanup(&mut self, endpoint: &str) -> Result<(), String> { + if self.cleaned_up { + return Ok(()); + } + + let mut command = direct_gateway_command(endpoint); + command + .args(["sandbox", "delete", &self.name]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let output = timeout(CLI_COMMAND_TIMEOUT, command.output()) + .await + .map_err(|_| { + format!( + "delete sandbox {} through {endpoint} timed out after {CLI_COMMAND_TIMEOUT:?}", + self.name + ) + })? + .map_err(|error| { + format!( + "failed to delete sandbox {} through {endpoint}: {error}", + self.name + ) + })?; + if !output.status.success() { + return Err(format!( + "delete sandbox {} through {endpoint} failed with exit {:?}", + self.name, + output.status.code(), + )); + } + + self.cleaned_up = true; + Ok(()) + } +} + +fn gateway_selector(release: &str) -> String { + std::env::var(GATEWAY_SELECTOR_ENV).unwrap_or_else(|_| { + format!("app.kubernetes.io/name=openshell,app.kubernetes.io/instance={release}") + }) +} + +async fn gateway_pods(namespace: &str, selector: &str) -> Vec { + let pods = oc_json(&["get", "pods", "-n", namespace, "-l", selector, "-o", "json"]).await; + pods["items"] + .as_array() + .expect("gateway pod list should contain items") + .iter() + .filter_map(|pod| { + pod["metadata"]["name"].as_str().map(|name| GatewayPod { + name: name.to_string(), + ready: pod_is_ready(pod), + }) + }) + .collect() +} // uses pod_is_ready() on the array of gateway pods + +async fn wait_for_gateway_pod_ready(namespace: &str, selector: &str, pod_name: &str) { + let result = timeout(POD_READY_TIMEOUT, async { + loop { + if gateway_pods(namespace, selector) + .await + .iter() + .any(|pod| pod.name == pod_name && pod.ready) + { + return; + } + sleep(POLL_INTERVAL).await; + } + }) + .await; + assert!( + result.is_ok(), + "gateway pod {pod_name} did not become Ready within {POD_READY_TIMEOUT:?}" + ); +} // continuously polls OpenShift until a specific pre-existing gateway is ready + +fn direct_gateway_command(endpoint: &str) -> tokio::process::Command { + let gateway_name = std::env::var("OPENSHELL_GATEWAY") + .or_else(|_| std::env::var(GATEWAY_NAME_ENV)) + .unwrap_or_else(|_| { + panic!( + "set OPENSHELL_GATEWAY or {GATEWAY_NAME_ENV} to the configured mTLS gateway name" + ) + }); + let mut command = openshell_cmd(); + // The port-forward reaches the gateway directly, so use TLS rather than + // plaintext HTTP. Keep the configured gateway name so the CLI finds its + // mTLS client certificate instead of looking under a port-forward URL. + command + .args(["--gateway", &gateway_name]) + .arg("--gateway-endpoint") + .arg(endpoint); + command +} + +async fn assert_ha_deployment(namespace: &str, selector: &str) { + let deployments = oc_json(&[ + "get", + "deployments", + "-n", + namespace, + "-l", + selector, + "-o", + "json", + ]) + .await; + let items = deployments["items"] + .as_array() + .expect("gateway deployment list should contain items"); + assert_eq!( + items.len(), + 1, + "HA test requires exactly one gateway Deployment selected by '{selector}'" + ); + let deployment = &items[0]; + assert!( + deployment["spec"]["replicas"] + .as_u64() + .is_some_and(|n| n >= 2), + "HA test requires a gateway Deployment configured for at least two replicas" + ); + let containers = deployment["spec"]["template"]["spec"]["containers"] + .as_array() + .expect("gateway Deployment should contain containers"); + let secret_name = containers + .iter() + .find_map(|container| { + container["env"].as_array().and_then(|env| { + env.iter().find_map(|entry| { + (entry["name"].as_str() == Some("OPENSHELL_DB_URL") + && entry["valueFrom"]["secretKeyRef"]["key"].as_str() == Some("uri")) + .then(|| entry["valueFrom"]["secretKeyRef"]["name"].as_str()) + .flatten() + .filter(|name| !name.is_empty()) + }) + }) + }) + .expect("HA test requires OPENSHELL_DB_URL to reference a nonempty uri key in a Secret"); + let secret = oc_json(&["get", "secret", secret_name, "-n", namespace, "-o", "json"]).await; + let encoded_url = secret["data"]["uri"] + .as_str() + .expect("HA test requires the referenced Secret to contain data.uri"); + let url = base64::engine::general_purpose::STANDARD + .decode(encoded_url) + .expect("HA test requires the referenced Secret data.uri to be valid base64"); + let url = std::str::from_utf8(&url) + .expect("HA test requires the referenced Secret data.uri to be valid UTF-8"); + assert!( + url.starts_with("postgres://") || url.starts_with("postgresql://"), + "HA test requires the referenced Secret data.uri to contain a PostgreSQL URL" + ); +} + +fn reserve_loopback_port() -> u16 { + let listener = TcpListener::bind("127.0.0.1:0").expect("reserve a loopback port"); + listener.local_addr().expect("read reserved port").port() +} + +async fn port_forward_gateway_pod(namespace: &str, pod: &str, port: u16) -> PodPortForward { + let target = format!("{port}:{GATEWAY_PORT}"); + let mut command = oc_command(); + command + .args([ + "port-forward", + "-n", + namespace, + &format!("pod/{pod}"), + &target, + "--address=127.0.0.1", + ]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let mut child = command.spawn().expect("start gateway pod port-forward"); + let result = timeout(SESSION_TIMEOUT, async { + loop { + if tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .is_ok() + { + return; + } + if let Some(status) = child.try_wait().expect("poll gateway pod port-forward") { + panic!("gateway pod port-forward exited before becoming ready: {status}"); + } + sleep(POLL_INTERVAL).await; + } + }) + .await; + assert!( + result.is_ok(), + "gateway pod port-forward did not become reachable within {SESSION_TIMEOUT:?}" + ); + PodPortForward { + // The test deployment includes localhost in the server certificate + // SANs. `localhost` therefore keeps certificate validation intact + // while the port-forward still targets exactly one gateway pod. + endpoint: format!("https://localhost:{port}"), + port, + child, + } +} // bypasses the default load balancer and targets one gateway pod + +async fn stop_port_forward(forward: &mut PodPortForward) { + let _ = forward.child.kill().await; + let _ = forward.child.wait().await; +} + +async fn wait_for_created_sandbox(name: &str, endpoint: &str) -> Result<(), String> { + timeout(CREATE_RECOVERY_TIMEOUT, async { + loop { + let mut command = direct_gateway_command(endpoint); + command + .args(["sandbox", "get", name, "--output", "json"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let output = command + .output() + .await + .map_err(|error| format!("failed to check sandbox {name} after create: {error}"))?; + if output.status.success() { + let sandbox: Value = serde_json::from_slice(&output.stdout).map_err(|error| { + format!("sandbox get {name} returned invalid JSON after create: {error}") + })?; + match sandbox["phase"].as_str() { + Some("Ready") => return Ok(()), + Some("Error") => { + return Err(format!( + "sandbox {name} entered Error after the create connection dropped" + )); + } + _ => {} + } + } + sleep(POLL_INTERVAL).await; + } + }) + .await + .map_err(|_| { + format!("sandbox {name} did not become Ready within {CREATE_RECOVERY_TIMEOUT:?}") + })? +} + +async fn create_on_initial_gateway(endpoint: &str, script: &str) -> ManagedSandbox { + // Six random hex characters, together with the hexadecimal process ID, + // keep this below the 19-character sandbox-name limit while separating + // independent CI runs that happen to reuse a process ID. + let name = format!( + "ha-{:x}-{:06x}", + std::process::id(), + rand::random::() & 0x00ff_ffff + ); + let mut command = direct_gateway_command(endpoint); + // A failed create may still leave a retained sandbox behind. Register its + // cleanup before running the CLI so failure paths attempt to remove it. + let mut sandbox = ManagedSandbox::new(name.clone()); + command + .args([ + "sandbox", + "create", + "--detach", + "--name", + &name, + "--from", + E2E_WORKLOAD_IMAGE, + "--", + ]) + .args(["sh", "-lc", script]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let mut child = command.spawn().expect("spawn sandbox create"); + let mut stdout = child + .stdout + .take() + .expect("sandbox create stdout must be piped"); + let mut stderr = child + .stderr + .take() + .expect("sandbox create stderr must be piped"); + let stdout_reader = tokio::spawn(async move { + let mut output = Vec::new(); + stdout.read_to_end(&mut output).await.map(|_| output) + }); + let stderr_reader = tokio::spawn(async move { + let mut output = Vec::new(); + stderr.read_to_end(&mut output).await.map(|_| output) + }); + let status = match timeout(CREATE_TIMEOUT, child.wait()).await { + Ok(Ok(status)) => status, + Ok(Err(error)) => { + let _ = stdout_reader.await; + let _ = stderr_reader.await; + if let Err(cleanup_error) = sandbox.cleanup(endpoint).await { + eprintln!("{cleanup_error}"); + } + panic!("failed to run sandbox create through the initial gateway: {error}"); + } + Err(_) => { + // Retain the child handle through timeout so it is explicitly + // killed and reaped before cleanup can race a still-running CLI. + let _ = child.kill().await; + let _ = child.wait().await; + let _ = stdout_reader.await; + let _ = stderr_reader.await; + if let Err(cleanup_error) = sandbox.cleanup(endpoint).await { + eprintln!("{cleanup_error}"); + } + panic!("sandbox create timed out after {CREATE_TIMEOUT:?}; failover was not exercised"); + } + }; + let output = Output { + status, + stdout: stdout_reader + .await + .expect("sandbox create stdout reader should not panic") + .expect("read sandbox create stdout"), + stderr: stderr_reader + .await + .expect("sandbox create stderr reader should not panic") + .expect("read sandbox create stderr"), + }; + if !output.status.success() { + let stdout = String::from_utf8_lossy(&output.stdout); + let stderr = String::from_utf8_lossy(&output.stderr); + if stdout.contains(&format!("Created sandbox: {name}")) + && stderr.contains("peer closed connection without sending TLS close_notify") + { + match wait_for_created_sandbox(&name, endpoint).await { + Ok(()) => { + eprintln!( + "create watch through pod port-forward disconnected; sandbox {name} became Ready through the initial gateway pod" + ); + return sandbox; + } + Err(error) => { + if let Err(cleanup_error) = sandbox.cleanup(endpoint).await { + eprintln!("{cleanup_error}"); + } + panic!("sandbox create through the initial gateway failed to recover: {error}"); + } + } + } + if let Err(cleanup_error) = sandbox.cleanup(endpoint).await { + eprintln!("{cleanup_error}"); + } + panic!( + "create sandbox through initial gateway failed with exit {:?}", + output.status.code() + ); + } + sandbox +} // create a sandbox via replica-0 direct endpoint, execute a background script that writes sentinel text to /sandbox/.odh-ha-sentinel +// and enters an infinite loop printing OUTPUT_MARKER every second + +async fn start_session(sandbox_name: &str, gateway_endpoint: Option<&str>) -> ActiveSession { + let mut command = gateway_endpoint.map_or_else(openshell_cmd, direct_gateway_command); + command + .args(["sandbox", "connect", sandbox_name]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .kill_on_drop(true); + let mut child = command.spawn().expect("spawn sandbox connect"); + let stdout = child + .stdout + .take() + .expect("sandbox connect stdout must be piped"); + ActiveSession { + child, + stdout: BufReader::new(stdout).lines(), + } +} // opens a streaming terminal session (openshell sandbox connect) + +async fn exec_on_gateway( + endpoint: &str, + sandbox_name: &str, + argv: &[&str], +) -> Result { + let mut command = direct_gateway_command(endpoint); + command + .args(["sandbox", "exec", "--name", sandbox_name, "--no-tty", "--"]) + .args(argv) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let output = timeout(CLI_COMMAND_TIMEOUT, command.output()) + .await + .map_err(|_| { + format!("sandbox exec through {endpoint} timed out after {CLI_COMMAND_TIMEOUT:?}") + })? + .map_err(|error| format!("failed to run sandbox exec through {endpoint}: {error}"))?; + if !output.status.success() { + return Err(format!( + "sandbox exec through {endpoint} failed with exit {:?}", + output.status.code() + )); + } + String::from_utf8(output.stdout) + .map_err(|error| format!("sandbox exec through {endpoint} returned invalid UTF-8: {error}")) +} + +fn session_marker(line: &str) -> Option { + let (prefix_and_process_id, emitted_at) = line.rsplit_once(':')?; + let process_id = prefix_and_process_id.strip_prefix(&format!("{OUTPUT_MARKER}:"))?; + Some(SessionMarker { + process_id: process_id.to_string(), + emitted_at: emitted_at.parse().ok()?, + }) +} + +async fn session_marker_after( + session: &mut ActiveSession, + strictly_after: Option, +) -> Result { + timeout(SESSION_TIMEOUT, async { + loop { + let line = session + .stdout + .next_line() + .await + .map_err(|error| format!("cannot read sandbox connect output: {error}"))? + .ok_or_else(|| { + "sandbox connect exited before receiving session output".to_string() + })?; + if let Some(marker) = session_marker(&line) + && strictly_after.is_none_or(|minimum| marker.emitted_at > minimum) + { + return Ok(marker); + } + } + }) + .await + .map_err(|_| format!("did not receive '{OUTPUT_MARKER}' within {SESSION_TIMEOUT:?}"))? +} // returns a heartbeat emitted after the requested time, excluding buffered output + +async fn session_marker_after_elapsed( + session: &mut ActiveSession, + minimum_elapsed: Instant, +) -> Result { + timeout(SESSION_TIMEOUT, async { + loop { + let line = session + .stdout + .next_line() + .await + .map_err(|error| format!("cannot read sandbox connect output: {error}"))? + .ok_or_else(|| { + "sandbox connect exited before receiving session output".to_string() + })?; + if let Some(marker) = session_marker(&line) + && Instant::now() > minimum_elapsed + { + return Ok(marker); + } + } + }) + .await + .map_err(|_| format!("did not receive '{OUTPUT_MARKER}' within {SESSION_TIMEOUT:?}"))? +} + +async fn start_initial_session( + namespace: &str, + initial_pod: &str, + forward: &mut PodPortForward, + sandbox_name: &str, +) -> (ActiveSession, SessionMarker) { + let mut failures = Vec::new(); + for attempt in 1..=SESSION_START_ATTEMPTS { + let mut session = start_session(sandbox_name, Some(&forward.endpoint)).await; + match session_marker_after(&mut session, None).await { + Ok(_) => match session_marker_after_elapsed( + &mut session, + Instant::now() + SESSION_ESTABLISHED_DURATION, + ) + .await + { + Ok(marker) => return (session, marker), + Err(error) => { + failures.push(format!( + "attempt {attempt}: session did not remain attached for longer than {SESSION_ESTABLISHED_DURATION:?}: {error}" + )); + stop_session(&mut session).await; + if attempt < SESSION_START_ATTEMPTS { + stop_port_forward(forward).await; + *forward = + port_forward_gateway_pod(namespace, initial_pod, forward.port).await; + } + } + }, + Err(error) => { + failures.push(format!("attempt {attempt}: {error}")); + stop_session(&mut session).await; + if attempt < SESSION_START_ATTEMPTS { + // `oc port-forward` can accept TCP connections while its + // SSH stream is stalled. Recreate the tunnel before + // retrying so this remains a transport preflight, before + // the destructive failover action. + stop_port_forward(forward).await; + *forward = port_forward_gateway_pod(namespace, initial_pod, forward.port).await; + } + } + } + } + panic!( + "initial session through pod port-forward could not produce a heartbeat after {SESSION_START_ATTEMPTS} attempts; failover was not exercised:\n{}", + failures.join("\n") + ); +} + +async fn stop_session(session: &mut ActiveSession) { + let _ = session.child.kill().await; + let _ = session.child.wait().await; +} + +async fn assert_sandbox_workload_is_live(namespace: &str, sandbox_name: &str) -> String { + let selector = sandbox_pod_selector(namespace, sandbox_name) + .await + .expect("sandbox CR should retain its workload selector after failover"); + let pods = oc_json(&[ + "get", "pods", "-n", namespace, "-l", &selector, "-o", "json", + ]) + .await; + let items = pods["items"] + .as_array() + .expect("sandbox pod list should contain items"); + assert!( + !items.is_empty() && items.iter().all(pod_is_ready), + "sandbox workload is orphaned or not ready after gateway failover: {:?}", + items + .iter() + .map(|pod| ( + pod["metadata"]["name"].as_str().unwrap_or(""), + pod_is_ready(pod), + )) + .collect::>(), + ); + selector +} // checks k8s to make sure the underlying container running the workload didn't crash or get orphaned during failover + +async fn assert_sandbox_deleted(endpoint: &str, namespace: &str, name: &str, pod_selector: &str) { + let cr_selector = format!("openshell.ai/sandbox-name={name}"); + let mut last_observation = None; + let result = timeout(SANDBOX_DELETION_TIMEOUT, async { + loop { + let mut command = direct_gateway_command(endpoint); + command.args(["sandbox", "list", "--names"]); + let output = command + .output() + .await + .expect("list sandboxes after cleanup"); + assert!( + output.status.success(), + "could not verify sandbox deletion through the gateway" + ); + let still_listed = String::from_utf8_lossy(&output.stdout) + .lines() + .any(|line| line.trim() == name); + let crs = oc_json(&[ + "get", + "sandboxes.agents.x-k8s.io", + "-n", + namespace, + "-l", + &cr_selector, + "-o", + "json", + ]) + .await; + let pods = oc_json(&[ + "get", + "pods", + "-n", + namespace, + "-l", + pod_selector, + "-o", + "json", + ]) + .await; + let cr_count = crs["items"] + .as_array() + .expect("Sandbox CR list items") + .len(); + let pod_count = pods["items"] + .as_array() + .expect("sandbox pod list items") + .len(); + last_observation = Some(format!( + "sandbox listed: {still_listed}, custom resources: {cr_count}, workload pods: {pod_count}" + )); + if !still_listed && cr_count == 0 && pod_count == 0 { + return; + } + sleep(POLL_INTERVAL).await; + } + }) + .await; + assert!( + result.is_ok(), + "sandbox {name}, its custom resource, or its pod remained after cleanup for {SANDBOX_DELETION_TIMEOUT:?}; last observation: {}", + last_observation.unwrap_or_else(|| "no deletion state observed".to_string()) + ); +} + +async fn wait_for_gateway_pod_deleted(namespace: &str, selector: &str, pod_name: &str) { + let result = timeout(POD_READY_TIMEOUT, async { + loop { + if gateway_pods(namespace, selector) + .await + .iter() + .all(|pod| pod.name != pod_name) + { + return; + } + sleep(POLL_INTERVAL).await; + } + }) + .await; + assert!( + result.is_ok(), + "gateway pod {pod_name} was not deleted within {POD_READY_TIMEOUT:?}" + ); +} + +async fn run_failover_scenario( + gateway_namespace: &str, + sandbox_namespace: &str, + selector: &str, + initial_pod: &str, + surviving_pod: &str, + initial_forward: &mut PodPortForward, + sandbox: &ManagedSandbox, +) -> String { + let (mut initial_session, initial_marker) = start_initial_session( + gateway_namespace, + initial_pod, + initial_forward, + &sandbox.name, + ) + .await; + + // Ensure the surviving replica is ready before dropping the client's + // transport, then submit deletion without spending its reconnect window + // waiting for Kubernetes to finish terminating the initial pod. + wait_for_gateway_pod_ready(gateway_namespace, selector, surviving_pod).await; + let deleted = oc_with_timeout( + &[ + "delete", + "pod", + initial_pod, + "-n", + gateway_namespace, + "--wait=false", + "--timeout=150s", + ], + None, + Duration::from_secs(180), + ) + .await; + assert!( + deleted.success, + "delete gateway pod {initial_pod} failed ({})", + deleted.status_summary() + ); + + // Keep the endpoint stable while replacing its direct backend. This lets + // the original client exercise its bounded reconnect behavior. + stop_port_forward(initial_forward).await; + let mut reconnect_forward = + port_forward_gateway_pod(gateway_namespace, surviving_pod, initial_forward.port).await; + // Read the baseline from the sandbox clock. Heartbeat timestamps use the + // same clock, so buffered pre-failover lines cannot satisfy the filter. + let reconnect_started_at: u64 = + exec_on_gateway(&reconnect_forward.endpoint, &sandbox.name, &["date", "+%s"]) + .await + .expect("read sandbox clock after failover") + .trim() + .parse() + .expect("sandbox date +%s should be an integer"); + let reconnected_marker = session_marker_after(&mut initial_session, Some(reconnect_started_at)) + .await + .unwrap_or_else(|error| { + panic!("session did not reconnect through the surviving gateway pod: {error}") + }); + wait_for_gateway_pod_deleted(gateway_namespace, selector, initial_pod).await; + assert!( + reconnected_marker.process_id == initial_marker.process_id, + "sandbox workload process changed while the original client reconnected after gateway failover" + ); + stop_session(&mut initial_session).await; + + let sentinel = exec_on_gateway( + &reconnect_forward.endpoint, + &sandbox.name, + &["cat", "/sandbox/.odh-ha-sentinel"], + ) + .await + .expect("sandbox should remain executable after gateway failover"); + assert!( + sentinel.contains(SENTINEL), + "workspace sentinel was lost after gateway failover: {sentinel}" + ); + stop_port_forward(&mut reconnect_forward).await; + assert_sandbox_workload_is_live(sandbox_namespace, &sandbox.name).await +} + +#[tokio::test] +#[ignore = "destructive HA failover; set OPENSHELL_ODH_HA_FAILOVER=1 to run"] +async fn gateway_pod_failover_preserves_sandbox_session_and_workspace() { + // preflight checks + assert!( + matches!(std::env::var(ENABLE_ENV).as_deref(), Ok("1")), + "set {ENABLE_ENV}=1 before explicitly running this destructive HA failover test" + ); + if !is_openshift().await.unwrap_or_else(|error| { + panic!("cannot determine whether the active cluster is OpenShift: {error}") + }) { + eprintln!("skipping HA failover test; the active cluster is not OpenShift"); + return; + } + + // replica discovery + let gateway_namespace = gateway_namespace(); + let sandbox_namespace = sandbox_namespace(); + let release = release(); + let selector = gateway_selector(&release); + assert_ha_deployment(&gateway_namespace, &selector).await; + let pods = gateway_pods(&gateway_namespace, &selector).await; // scan for running gateways + assert!( + pods.len() >= 2 && pods.iter().filter(|pod| pod.ready).count() >= 2, + "HA test requires two ready gateway pods selected by '{selector}', found: {:?}", + pods.iter() + .map(|pod| (&pod.name, pod.ready)) + .collect::>(), + ); // asserts at least 2 ready replicas exist + let initial_pod = pods + .iter() + .find(|pod| pod.ready) + .expect("one ready gateway pod") + .name + .clone(); // pick an initial_pod + let surviving_pod = pods + .iter() + .find(|pod| pod.ready && pod.name != initial_pod) + .expect("a second pre-existing ready gateway pod") + .name + .clone(); + let port = reserve_loopback_port(); + let mut initial_forward = + port_forward_gateway_pod(&gateway_namespace, &initial_pod, port).await; + // Retain an independent connection to the replica that will survive the + // failure. It is deliberately separate from the stable client endpoint so + // cleanup and its verification never fall back to a Service or Route. + let cleanup_port = reserve_loopback_port(); + let mut surviving_forward = + port_forward_gateway_pod(&gateway_namespace, &surviving_pod, cleanup_port).await; + + let script = format!( + "printf '%s\\n' '{SENTINEL}' > /sandbox/.odh-ha-sentinel; session_id=$(cat /proc/sys/kernel/random/uuid); while :; do printf '%s:%s:%s\\n' '{OUTPUT_MARKER}' \"$session_id\" \"$(date +%s)\"; sleep 1; done" + ); // persist a workspace sentinel and stream a process-specific heartbeat + let mut sandbox = create_on_initial_gateway(&initial_forward.endpoint, &script).await; // make sandbox w/sentinel file + // Keep ownership of the sandbox here so every assertion failure still + // reaches awaited cleanup before the test unwinds. + let scenario = AssertUnwindSafe(run_failover_scenario( + &gateway_namespace, + &sandbox_namespace, + &selector, + &initial_pod, + &surviving_pod, + &mut initial_forward, + &sandbox, + )) + .catch_unwind() + .await; + + // Do not reuse the tunnel that sat idle during failover: `oc port-forward` + // can exit or stall without affecting the session path. Keep it alive + // while creating a fresh tunnel on a separate local port, so it remains a + // best-effort cleanup path if the replacement cannot be established. + let refreshed = AssertUnwindSafe(port_forward_gateway_pod( + &gateway_namespace, + &surviving_pod, + reserve_loopback_port(), + )) + .catch_unwind() + .await; + let cleanup = match refreshed { + Ok(forward) => { + stop_port_forward(&mut surviving_forward).await; + surviving_forward = forward; + sandbox.cleanup(&surviving_forward.endpoint).await + } + Err(_) => { + eprintln!( + "could not establish a fresh port-forward to {surviving_pod} for cleanup; \ + attempting cleanup through the original tunnel" + ); + match sandbox.cleanup(&surviving_forward.endpoint).await { + Ok(()) => Err(format!( + "could not establish a fresh port-forward to {surviving_pod} for cleanup; \ + sandbox was deleted through the original tunnel" + )), + Err(error) => Err(format!( + "could not establish a fresh port-forward to {surviving_pod} for cleanup; \ + cleanup through the original tunnel also failed: {error}" + )), + } + } + }; + match scenario { + Ok(pod_selector) => { + cleanup.expect("delete sandbox through the surviving gateway pod"); + assert_sandbox_deleted( + &surviving_forward.endpoint, + &sandbox_namespace, + &sandbox.name, + &pod_selector, + ) + .await; + } + Err(failure) => { + if let Err(cleanup_error) = cleanup { + eprintln!("{cleanup_error}"); + } + stop_port_forward(&mut surviving_forward).await; + std::panic::resume_unwind(failure); + } + } + stop_port_forward(&mut surviving_forward).await; +} diff --git a/e2e/rust/tests/odh/tier3/mod.rs b/e2e/rust/tests/odh/tier3/mod.rs index 9b74f53bd2..c41c4433ef 100644 --- a/e2e/rust/tests/odh/tier3/mod.rs +++ b/e2e/rust/tests/odh/tier3/mod.rs @@ -2,3 +2,5 @@ // SPDX-License-Identifier: Apache-2.0 //! Tier 3: negative and destructive tests (no time limit). + +mod gateway_failover; From 4d46dd5f4f2aa412b0522af5e4a00732cb606afe Mon Sep 17 00:00:00 2001 From: Evie Howard Date: Fri, 16 Oct 2026 16:24:09 +0100 Subject: [PATCH 26/26] test(odh): automate HA failover deployment Signed-off-by: Evie Howard --- e2e/rust/tests/odh/README.md | 62 ++++++++----- e2e/rust/tests/odh/run-odh-test-tier.sh | 7 +- e2e/rust/tests/odh/tier3/gateway_failover.rs | 98 ++++++++++++++++++-- e2e/rust/tests/odh/tiers.toml | 22 +++++ odh/scripts/e2e-odh-entrypoint.sh | 40 +++++++- odh/scripts/openshell-deploy-from-quay.sh | 26 +++++- 6 files changed, 214 insertions(+), 41 deletions(-) diff --git a/e2e/rust/tests/odh/README.md b/e2e/rust/tests/odh/README.md index b7cfe15d60..352b10cd88 100644 --- a/e2e/rust/tests/odh/README.md +++ b/e2e/rust/tests/odh/README.md @@ -118,11 +118,17 @@ failures: `canonical_main_disconnect_reconnect_replays_history_for_same_process` Tier 2 runs `workspace_lifecycle`, ODH Tier 2, and image provenance in shared mode, then runs `workspace_namespace_managed` against a separate managed-mode deployment. The `odh` and `full` tiers use the same shared then -managed split. Tier 3 includes an opt-in destructive external-PostgreSQL HA -gateway failover scenario. Image provenance runs once in each tier's shared -phase. Smoke and Tier 1/3 use only shared mode. Add Rust scenarios by creating -a `.rs` file under the tier's directory and declaring it with a `mod` line in -that tier's `mod.rs`; add its selector to the corresponding tier table or mode +managed split. Tier 3 uses shared mode by default; when +`OPENSHELL_ODH_HA_FAILOVER=1`, it selects its destructive external-PostgreSQL +HA phase instead. The `odh` and `full` tiers append that HA phase after their +normal phases when the same opt-in is set. The destructive ignored test runs +only in this HA phase, never against the shared or managed deployments. Image +provenance remains in the normal shared phases, including default Tier 3, +and is disabled in destructive HA phases to avoid overlapping sandbox creation +with gateway pod deletion. +Smoke and Tier 1 use only shared mode. Add Rust scenarios by creating a `.rs` +file under the tier's directory and declaring it with a `mod` line in that +tier's `mod.rs`; add its selector to the corresponding tier table or mode extension. To add a Helm-overlay mode, define its validated values paths and expected @@ -165,16 +171,19 @@ Excluded tests do not appear as passes or skips in the JUnit report. runs and HTML reports. - The local task builds `openshell-cli` when `OPENSHELL_BIN` is unset. Set `OPENSHELL_BIN` to use another CLI binary. -- For `tier3::gateway_failover`, a deployment installed with - `deploy/helm/openshell/ci/values-high-availability.yaml`, an external - PostgreSQL Secret, and a configured CLI gateway name. The test discovers two - ready gateway pods and opens its own loopback `oc port-forward` to direct the +- For `tier3::gateway_failover`, permission to create the fixture PostgreSQL + deployment and its Secret, and delete gateway pods. With + `OPENSHELL_ODH_HA_FAILOVER=1`, the Tier 3 runner + deploys the external-PostgreSQL HA gateway automatically. The test discovers + two ready gateway pods and opens its own loopback `oc port-forward` to direct the initial create and session to one pod. It confirms the session disconnects - when that pod is deleted, then reconnects through a second port-forward to - the pre-existing surviving replica. This pins the check to that replica and - does not exercise the configured Service or Route endpoint. The workload - heartbeat includes a UUID generated at process start; reconnect must report - the same UUID, so a restarted workload does not satisfy the session check. + when that pod is deleted by waiting up to 45 seconds for its port-forward + to exit naturally, then reconnects through a second port-forward to + the pre-existing surviving replica. This pins the recovery check to that + replica. Separate authenticated `openshell status` assertions use the + configured Route before and after the failure. The workload heartbeat includes + a UUID generated at process start; reconnect must report the same UUID, so a + restarted workload does not satisfy the session check. Set `OPENSHELL_GATEWAY` to the configured gateway name; it also selects the gateway used by the image-provenance check that runs with every tier. `OPENSHELL_ODH_HA_GATEWAY_NAME` remains a compatibility fallback when @@ -225,13 +234,18 @@ context you happen to have active elsewhere. This means: | `mise run e2e:odh:tier3` | Tier 3: mapped upstream tests + ODH `tier3::` + image provenance | | `cargo nextest run --manifest-path e2e/rust/Cargo.toml --features e2e-odh --test odh -E 'test(=module::test_name)'` | A single ODH test function | -The destructive HA failover scenario is skipped by default. Run it only -against a disposable or explicitly approved HA deployment: +The destructive HA failover scenario is skipped by default. With +`OPENSHELL_ODH_HA_FAILOVER=1`, Tier 3 selects its HA phase, creates the +external PostgreSQL fixture, deploys two gateway replicas, runs the test, and +tears down the namespace and fixture afterward: ```bash OPENSHELL_ODH_HA_FAILOVER=1 mise run e2e:odh:tier3 ``` +To attach to an already deployed compatible HA gateway instead, set +`OPENSHELL_E2E_DEPLOY_GATEWAY=0` as well. + The existing `e2e/with-kube-gateway.sh` wrapper can provision the HA fixture in an ephemeral namespace: it installs the chart with the HA values overlay, deploys the PostgreSQL fixture, configures the OpenShift Route and CLI, then @@ -277,13 +291,13 @@ Example, running the Smoke tier against a real cluster: umask 077 oc --kubeconfig ~/.kube/config config view --minify --flatten > kubeconfig chmod 600 kubeconfig -ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/" \ +ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/,ghcr.io/astral-sh/" \ mise run e2e:odh:smoke ``` `IMAGE_TAG` and `ALLOWED_IMAGE_REGISTRY_PREFIXES` are required by deployment and the image provenance test — see below. For RHOAI images, use -`registry.redhat.io/,nvcr.io/nvidia/base/` instead. The shared entrypoint +`registry.redhat.io/,nvcr.io/nvidia/base/,ghcr.io/astral-sh/` instead. The shared entrypoint generates a DNS-safe `OPENSHELL_E2E_RUN_ID` for each deployed invocation unless you provide one. The namespace and Helm release default to `openshell` for both shared and managed phases to preserve managed-test compatibility; gateway @@ -332,9 +346,11 @@ features. The shared entrypoint reads the same tier plan locally and in the image, deploying and running one phase at a time. `e2e:odh` runs all ODH scenarios; `e2e:odh:full` runs all feature-enabled binaries except the managed-only binary in its shared phase, followed by that binary in managed -mode. Phase JUnit files are named `e2e-odh--.xml`; the entrypoint -merges them into `e2e-odh--xunit.xml` for Jenkins discovery and -`e2e-odh-.html` in `results/`. +mode. Setting `OPENSHELL_ODH_HA_FAILOVER=1` appends a dedicated HA phase to +either command, where it runs the destructive failover scenario. Phase JUnit +files are named `e2e-odh--.xml`; the entrypoint merges them into +`e2e-odh--xunit.xml` for Jenkins discovery and `e2e-odh-.html` in +`results/`.s ### Every `e2e:odh*` task runs the image provenance check @@ -352,7 +368,7 @@ registry, and that no container — including ephemeral containers — has regressed away from `imagePullPolicy: IfNotPresent`. ```bash -ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/" \ +ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/,ghcr.io/astral-sh/" \ cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-odh --test odh \ -- smoke::image_provenance:: ``` @@ -368,6 +384,8 @@ ALLOWED_IMAGE_REGISTRY_PREFIXES="quay.io/opendatahub/,nvcr.io/nvidia/base/" \ - `nvcr.io/nvidia/base/` — the Helm chart's current default workload image (`server.sandboxImage`). Include the registry prefix for any other workload image selected by the deployment. + - `ghcr.io/astral-sh/` — the Agent Sandbox Operator's `agent` and + `workspace-init` containers. A prefix without a trailing `/` is rejected outright, since it could otherwise match a lookalike host (e.g. `registry.redhat.io` would also diff --git a/e2e/rust/tests/odh/run-odh-test-tier.sh b/e2e/rust/tests/odh/run-odh-test-tier.sh index 776f533a73..be665dffeb 100755 --- a/e2e/rust/tests/odh/run-odh-test-tier.sh +++ b/e2e/rust/tests/odh/run-odh-test-tier.sh @@ -52,7 +52,6 @@ fi junit_xml="${results_dir}/e2e-odh.xml" rm -f "${junit_xml}" "${report}" "${report%.xml}.html" -<<<<<<< HEAD config_file="$(mktemp)" trap 'rm -f "${config_file}"' EXIT "${PYTHON_BIN}" - "${ROOT}/.config/nextest.toml" "${config_file}" "${junit_xml}" <<'PY' @@ -71,9 +70,9 @@ nextest_args=( -E "${filter}" ) # The failover test is ignored by default because it deletes a gateway pod. -# Run it only after the caller has explicitly opted into destructive HA -# coverage; `all` preserves the rest of the selected tier. -if [[ "${OPENSHELL_ODH_HA_FAILOVER:-0}" == "1" ]]; then +# Only its dedicated HA phase may enable ignored tests: all-tests shared and +# managed phases must not run it against their non-HA gateway deployments. +if [[ "${PHASE}" == "ha" && "${OPENSHELL_ODH_HA_FAILOVER:-0}" == "1" ]]; then nextest_args+=(--run-ignored all) fi if [[ -n "${OPENSHELL_E2E_NEXTEST_ARCHIVE:-}" ]]; then diff --git a/e2e/rust/tests/odh/tier3/gateway_failover.rs b/e2e/rust/tests/odh/tier3/gateway_failover.rs index d584f976ed..68e1f2001f 100644 --- a/e2e/rust/tests/odh/tier3/gateway_failover.rs +++ b/e2e/rust/tests/odh/tier3/gateway_failover.rs @@ -39,6 +39,10 @@ const SESSION_TIMEOUT: Duration = Duration::from_secs(60); const CREATE_TIMEOUT: Duration = Duration::from_secs(300); const CREATE_RECOVERY_TIMEOUT: Duration = Duration::from_secs(120); const CLI_COMMAND_TIMEOUT: Duration = Duration::from_secs(120); +const ROUTE_STATUS_TIMEOUT: Duration = Duration::from_secs(120); +// Allow the pod's default 30-second termination grace period while staying +// below the CLI's one-minute reconnect window. +const PORT_FORWARD_EXIT_TIMEOUT: Duration = Duration::from_secs(45); const SANDBOX_DELETION_TIMEOUT: Duration = Duration::from_secs(300); const SESSION_START_ATTEMPTS: usize = 3; // The CLI only opens its reconnect window after an attachment has survived @@ -162,14 +166,18 @@ async fn wait_for_gateway_pod_ready(namespace: &str, selector: &str, pod_name: & ); } // continuously polls OpenShift until a specific pre-existing gateway is ready -fn direct_gateway_command(endpoint: &str) -> tokio::process::Command { - let gateway_name = std::env::var("OPENSHELL_GATEWAY") +fn configured_gateway_name() -> String { + std::env::var("OPENSHELL_GATEWAY") .or_else(|_| std::env::var(GATEWAY_NAME_ENV)) .unwrap_or_else(|_| { panic!( "set OPENSHELL_GATEWAY or {GATEWAY_NAME_ENV} to the configured mTLS gateway name" ) - }); + }) +} + +fn direct_gateway_command(endpoint: &str) -> tokio::process::Command { + let gateway_name = configured_gateway_name(); let mut command = openshell_cmd(); // The port-forward reaches the gateway directly, so use TLS rather than // plaintext HTTP. Keep the configured gateway name so the CLI finds its @@ -181,6 +189,56 @@ fn direct_gateway_command(endpoint: &str) -> tokio::process::Command { command } +async fn assert_route_client_connected(phase: &str) { + let mut last_observation = "no status response received".to_string(); + let result = timeout(ROUTE_STATUS_TIMEOUT, async { + loop { + let gateway_name = configured_gateway_name(); + let mut command = openshell_cmd(); + command + .args(["--gateway", &gateway_name, "status", "--output", "json"]) + // A direct endpoint would bypass the Route configured for this + // gateway and make this assertion meaningless. + .env_remove("OPENSHELL_GATEWAY_ENDPOINT") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true); + let output = command + .output() + .await + .expect("run openshell status through the configured Route"); + match serde_json::from_slice::(&output.stdout) { + Ok(status) => { + let connection = status["status"].as_str(); + let authentication = status["authentication"]["status"].as_str(); + if output.status.success() + && connection == Some("connected") + && authentication == Some("authenticated") + { + return; + } + last_observation = format!( + "exit {:?}, status {connection:?}, authentication {authentication:?}", + output.status.code() + ); + } + Err(error) => { + last_observation = format!( + "exit {:?}, invalid status JSON: {error}", + output.status.code() + ); + } + } + sleep(POLL_INTERVAL).await; + } + }) + .await; + assert!( + result.is_ok(), + "authenticated openshell status through the configured Route did not connect {phase} gateway failover within {ROUTE_STATUS_TIMEOUT:?}; last observation: {last_observation}" + ); +} + async fn assert_ha_deployment(namespace: &str, selector: &str) { let deployments = oc_json(&[ "get", @@ -729,10 +787,17 @@ async fn run_failover_scenario( ) .await; - // Ensure the surviving replica is ready before dropping the client's - // transport, then submit deletion without spending its reconnect window - // waiting for Kubernetes to finish terminating the initial pod. + // Ensure the surviving replica and original tunnel are live before + // submitting the pod deletion that must cause the transport loss. wait_for_gateway_pod_ready(gateway_namespace, selector, surviving_pod).await; + assert!( + initial_forward + .child + .try_wait() + .expect("poll initial gateway pod port-forward before deletion") + .is_none(), + "initial gateway pod port-forward exited before pod deletion; failover was not exercised" + ); let deleted = oc_with_timeout( &[ "delete", @@ -753,9 +818,17 @@ async fn run_failover_scenario( deleted.status_summary() ); - // Keep the endpoint stable while replacing its direct backend. This lets - // the original client exercise its bounded reconnect behavior. - stop_port_forward(initial_forward).await; + // Let pod termination close the original transport. Killing our own + // forward here would allow a tunnel switch to masquerade as pod failover. + // Bound the wait so a stalled forward fails rather than consuming the + // client's entire reconnect window. + timeout(PORT_FORWARD_EXIT_TIMEOUT, initial_forward.child.wait()) + .await + .expect( + "initial pod port-forward did not exit after pod deletion within the reconnect window", + ) + .expect("wait for initial pod port-forward to exit after pod deletion"); + // Reuse the same endpoint only after the old forward exits naturally. let mut reconnect_forward = port_forward_gateway_pod(gateway_namespace, surviving_pod, initial_forward.port).await; // Read the baseline from the sandbox clock. Heartbeat timestamps use the @@ -791,6 +864,10 @@ async fn run_failover_scenario( "workspace sentinel was lost after gateway failover: {sentinel}" ); stop_port_forward(&mut reconnect_forward).await; + // This uses the registered gateway without a direct endpoint override, so + // it verifies authenticated CLI traffic through the configured Route after + // the failed replica has been replaced. + assert_route_client_connected("after").await; assert_sandbox_workload_is_live(sandbox_namespace, &sandbox.name).await } @@ -823,6 +900,9 @@ async fn gateway_pod_failover_preserves_sandbox_session_and_workspace() { .map(|pod| (&pod.name, pod.ready)) .collect::>(), ); // asserts at least 2 ready replicas exist + // This is intentionally separate from the port-forward path below: it + // checks the registered, authenticated CLI route before failover. + assert_route_client_connected("before").await; let initial_pod = pods .iter() .find(|pod| pod.ready) diff --git a/e2e/rust/tests/odh/tiers.toml b/e2e/rust/tests/odh/tiers.toml index 73b3133b22..8e19b2f571 100644 --- a/e2e/rust/tests/odh/tiers.toml +++ b/e2e/rust/tests/odh/tiers.toml @@ -19,6 +19,10 @@ expected_helm_values = { gatewayConfig = { "openshell.gateway.auth" = { allow_un helm_values = ["deploy/helm/openshell/ci/values-workspace-managed.yaml"] expected_helm_values = { gatewayConfig = { "openshell.gateway.auth" = { allow_unauthenticated_users = false }, "openshell.drivers.kubernetes" = { workspace_mode = "managed", image_pull_policy = "IfNotPresent", sandbox_runtime_image_pull_policy = "IfNotPresent", supervisor_image_pull_policy = "IfNotPresent" } } } +[modes.ha] +helm_values = ["deploy/helm/openshell/ci/values-high-availability.yaml"] +expected_helm_values = { replicaCount = 2, workload = { kind = "deployment" }, server = { externalDbSecret = "openshell-ha-pg" }, gatewayConfig = { "openshell.gateway.auth" = { allow_unauthenticated_users = false }, "openshell.drivers.kubernetes" = { workspace_mode = "shared", image_pull_policy = "IfNotPresent", sandbox_runtime_image_pull_policy = "IfNotPresent", supervisor_image_pull_policy = "IfNotPresent" } } } + [smoke] description = "Component-level critical tests (5 min target)" # The standalone smoke binary was removed in favor of the portable CLI @@ -54,6 +58,12 @@ description = "Negative and destructive tests" upstream_tests = [] odh_filter = "tier3::" +[tier3.ha] +description = "Destructive HA gateway failover coverage" +mode = "ha" +odh_filter = "tier3::" +include_image_provenance = false + [odh] description = "All ODH-specific tests" odh_only = true @@ -65,6 +75,12 @@ description = "Managed-workspace tests" upstream_tests = ["workspace_namespace_managed"] odh_filter = "" +[odh.ha] +description = "Destructive HA gateway failover coverage" +mode = "ha" +odh_filter = "tier3::" +include_image_provenance = false + [full] description = "All feature-enabled upstream and ODH tests except listed exclusions" upstream_tests = [] @@ -83,3 +99,9 @@ port_forward = ["port_forward_echo"] description = "Managed-workspace tests" upstream_tests = ["workspace_namespace_managed"] odh_filter = "" + +[full.ha] +description = "Destructive HA gateway failover coverage" +mode = "ha" +odh_filter = "tier3::" +include_image_provenance = false diff --git a/odh/scripts/e2e-odh-entrypoint.sh b/odh/scripts/e2e-odh-entrypoint.sh index e50b1d0156..38e1663eb1 100755 --- a/odh/scripts/e2e-odh-entrypoint.sh +++ b/odh/scripts/e2e-odh-entrypoint.sh @@ -21,6 +21,26 @@ fi plan_json="$("${PYTHON_BIN}" "${PLAN_RESOLVER}" "${TIER}" "${TIERS_FILE}")" phase_lines="$(printf '%s' "${plan_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(p["name"]+"|"+p["mode"] for p in json.load(sys.stdin)["phases"]))')" +case "${TIER}" in + tier3) + if [[ "${OPENSHELL_ODH_HA_FAILOVER:-0}" == 1 ]]; then + phase_lines="$(printf '%s' "${plan_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(p["name"]+"|"+p["mode"] for p in json.load(sys.stdin)["phases"] if p["name"] == "ha"))')" + else + phase_lines="$(printf '%s' "${plan_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(p["name"]+"|"+p["mode"] for p in json.load(sys.stdin)["phases"] if p["name"] != "ha"))')" + fi + ;; + odh|full) + # The normal all-tests phases must not run the ignored destructive test. + # Add its dedicated HA deployment only when explicitly requested. + if [[ "${OPENSHELL_ODH_HA_FAILOVER:-0}" != 1 ]]; then + phase_lines="$(printf '%s' "${plan_json}" | "${PYTHON_BIN}" -c 'import json,sys; print(chr(10).join(p["name"]+"|"+p["mode"] for p in json.load(sys.stdin)["phases"] if p["name"] != "ha"))')" + fi + ;; +esac +[[ -n "${phase_lines}" ]] || { + echo "ERROR: ${TIER} did not resolve the required phase" >&2 + exit 1 +} phase_count=0 while IFS= read -r _; do ((phase_count+=1)); done <<< "${phase_lines}" deploy_enabled="${OPENSHELL_E2E_DEPLOY_GATEWAY:-1}" @@ -219,11 +239,21 @@ for phase_line in ${phase_lines}; do ((phase_index+=1)) done -run_child "${PYTHON_BIN}" "${MERGE_SCRIPT}" "${TIER}" "${phase_reports[@]}" || { - merge_status=$? - echo "ERROR: failed to merge ODH phase reports" >&2 - [[ "${status}" != 0 ]] || status="${merge_status}" -} +available_phase_reports=() +for phase_report in "${phase_reports[@]}"; do + if [[ -f "${phase_report}" ]]; then + available_phase_reports+=("${phase_report}") + fi +done +if (( ${#available_phase_reports[@]} > 0 )); then + run_child "${PYTHON_BIN}" "${MERGE_SCRIPT}" "${TIER}" "${available_phase_reports[@]}" || { + merge_status=$? + echo "ERROR: failed to merge ODH phase reports" >&2 + [[ "${status}" != 0 ]] || status="${merge_status}" + } +else + echo "WARNING: no phase reports were produced; skipping report merge" >&2 +fi if [[ -f "${merged_report}" ]]; then if command -v xsltproc >/dev/null 2>&1; then xsltproc --stringparam title "e2e-odh-${TIER}" "${ROOT}/scripts/junit-to-html.xsl" "${merged_report}" > "${html_report}" || echo "WARNING: failed to render HTML report" >&2 diff --git a/odh/scripts/openshell-deploy-from-quay.sh b/odh/scripts/openshell-deploy-from-quay.sh index e0fef032ab..038dab75b5 100755 --- a/odh/scripts/openshell-deploy-from-quay.sh +++ b/odh/scripts/openshell-deploy-from-quay.sh @@ -97,6 +97,11 @@ source "${REPO_ROOT}/e2e/support/gateway-common.sh" CHART="${REPO_ROOT}/deploy/helm/openshell" PLAN_RESOLVER="${REPO_ROOT}/e2e/rust/tests/odh/tier_plan.py" TIERS_FILE="${REPO_ROOT}/e2e/rust/tests/odh/tiers.toml" +POSTGRES_FIXTURE_MANIFEST="${REPO_ROOT}/e2e/kubernetes/postgres-fixture.yaml" +POSTGRES_FIXTURE_SERVICE="openshell-e2e-postgres" +POSTGRES_FIXTURE_SECRET="openshell-ha-pg" +POSTGRES_FIXTURE_USER="openshell" +POSTGRES_FIXTURE_DATABASE="openshell" PYTHON_BIN="${PYTHON_BIN:-python3}" OPEN_SHELL="${OPENSHELL_BIN:-openshell}" @@ -274,6 +279,18 @@ if [[ "${MODE}" == managed ]]; then openshell.ai/sandbox-attachable=true fi +if [[ "${MODE}" == ha ]]; then + echo ">> Creating external PostgreSQL fixture for HA gateway" + oc -n "${NAMESPACE}" apply -f "${POSTGRES_FIXTURE_MANIFEST}" + oc -n "${NAMESPACE}" rollout status "deployment/${POSTGRES_FIXTURE_SERVICE}" --timeout=120s + postgres_password="$(oc -n "${NAMESPACE}" get secret openshell-e2e-postgres-credentials \ + -o jsonpath='{.data.password}' | base64 -d)" + [[ -n "${postgres_password}" ]] || fail "external PostgreSQL fixture password is empty" + postgres_uri="postgresql://${POSTGRES_FIXTURE_USER}:${postgres_password}@${POSTGRES_FIXTURE_SERVICE}.${NAMESPACE}.svc.cluster.local:5432/${POSTGRES_FIXTURE_DATABASE}" + oc -n "${NAMESPACE}" create secret generic "${POSTGRES_FIXTURE_SECRET}" \ + --from-literal=uri="${postgres_uri}" --dry-run=client -o yaml | oc apply -f - +fi + echo ">> Deploying OpenShell via Helm" helm_value_args=() helm_value_args+=(--values "${REPO_ROOT}/deploy/helm/openshell/ci/values-openshift-scc.yaml") @@ -307,7 +324,14 @@ helm upgrade --install "${RELEASE}" "${CHART}" \ --set "pkiInitJob.serverDnsNames[0]=${ROUTE_HOST}" echo ">> Waiting for gateway rollout" -if ! oc -n "${NAMESPACE}" rollout status "statefulset/${HELM_FULLNAME}" --timeout=300s; then +gateway_workload="" +for candidate in "statefulset/${HELM_FULLNAME}" "deployment/${HELM_FULLNAME}"; do + if oc -n "${NAMESPACE}" get "${candidate}" >/dev/null 2>&1; then + gateway_workload="${candidate}" + break + fi +done +if [[ -z "${gateway_workload}" ]] || ! oc -n "${NAMESPACE}" rollout status "${gateway_workload}" --timeout=300s; then oc -n "${NAMESPACE}" get pods fail "gateway did not become ready" fi