diff --git a/.github/workflows/rhoai-installer.yml b/.github/workflows/rhoai-installer.yml new file mode 100644 index 0000000000..d1512dcb33 --- /dev/null +++ b/.github/workflows/rhoai-installer.yml @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 + +name: RHOAI Installer + +on: + pull_request: + paths: + - "deploy/rhoai/**" + - ".github/workflows/rhoai-installer.yml" + - "pyproject.toml" + push: + branches: [main] + paths: + - "deploy/rhoai/**" + - ".github/workflows/rhoai-installer.yml" + - "pyproject.toml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@ae62891fec2bb8e7d6c99fc78c9fec3a63790f8d # v10.0.0 + with: + version: "0.10.12" + python-version: "3.12" + - name: Installer regression tests + run: uv run --script --python 3.12 deploy/rhoai/test_install.py + - name: Authorization verifier tests + run: uv run --no-project --python 3.12 deploy/rhoai/test_verify.py + - name: Lint and formatting + run: | + uvx ruff==0.14.14 check deploy/rhoai + uvx ruff==0.14.14 format --check deploy/rhoai diff --git a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml index 9144bb0ad9..784315665f 100644 --- a/.tekton/odh-openshell-sandbox-openclaw-tag.yaml +++ b/.tekton/odh-openshell-sandbox-openclaw-tag.yaml @@ -10,7 +10,7 @@ metadata: pipelinesascode.tekton.dev/max-keep-runs: "3" pipelinesascode.tekton.dev/on-cel-expression: | event == "push" - && target_branch.matches("^refs/tags/v[0-9]+\\.[0-9]+\\.[0-9]+-rhaiv\\.[0-9]+$") + && target_branch.matches("^refs/tags/[0-9]+\\.[0-9]+\\.[0-9]+$") creationTimestamp: null labels: appstudio.openshift.io/application: opendatahub-builds @@ -42,6 +42,9 @@ spec: {"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"} ] + - name: build-args + value: + - "OPENCLAW_VERSION={{git_tag}}" pipelineRef: resolver: git params: diff --git a/deploy/docker/Dockerfile.konflux.openclaw b/deploy/docker/Dockerfile.konflux.openclaw index e4fc08c654..1dd8476efb 100644 --- a/deploy/docker/Dockerfile.konflux.openclaw +++ b/deploy/docker/Dockerfile.konflux.openclaw @@ -25,7 +25,7 @@ # `&& \` continuations only. # Both stages use the same base and OpenClaw version; bump them only here. -ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791837113@sha256:c0205a5c0bb750ec8a34248da9f87b4013cbfd46e327ec321996104913633fe9 +ARG NODEJS_IMAGE=registry.access.redhat.com/ubi9/nodejs-24-minimal:9.8-1791925262@sha256:cfca5c8d4d7babc8b74f408db216c2fd4dec872f319c1579e001dcceb51d3058 ARG OPENCLAW_VERSION=2026.9.5 # --------------------------------------------------------------------------- diff --git a/deploy/konflux/cli/rpms.lock.yaml b/deploy/konflux/cli/rpms.lock.yaml index 02a50239c3..2415182651 100644 --- a/deploy/konflux/cli/rpms.lock.yaml +++ b/deploy/konflux/cli/rpms.lock.yaml @@ -1338,20 +1338,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 60882 diff --git a/deploy/konflux/e2e-odh/rpms.lock.yaml b/deploy/konflux/e2e-odh/rpms.lock.yaml index 7ee2450c15..ccfeff5027 100644 --- a/deploy/konflux/e2e-odh/rpms.lock.yaml +++ b/deploy/konflux/e2e-odh/rpms.lock.yaml @@ -1422,20 +1422,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/publicsuffix-list-dafsa-20210518-3.el9.noarch.rpm repoid: ubi-9-baseos-rpms size: 60882 diff --git a/deploy/konflux/gateway/rpms.lock.yaml b/deploy/konflux/gateway/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/gateway/rpms.lock.yaml +++ b/deploy/konflux/gateway/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 diff --git a/deploy/konflux/openclaw/README.md b/deploy/konflux/openclaw/README.md index 4cc682b405..e491ff918a 100644 --- a/deploy/konflux/openclaw/README.md +++ b/deploy/konflux/openclaw/README.md @@ -158,7 +158,7 @@ The npm inputs come from AIPCC agentic commit dc756bc3 (OpenClaw 2026.9.5). AIPC ## Konflux -odh-konflux-central owns the Tekton pipelines. Its onboarder sync opens a PR here that copies `pipelineruns/openshell/odh-openshell-sandbox-openclaw-*.yaml` over `.tekton/`, so change their params there and run the sync. Merges to `main` publish `odh-stable`, and each `v*-rhaiv.*` tag rebuilds the same pinned OpenClaw release under that tag; only the commit named in [Status](#status) is verified. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: +An OpenShell Git tag matching the OpenClaw version (for example, `2026.9.5`) triggers a versioned image build and publishes the same Quay tag. The tag pipeline passes that version as `OPENCLAW_VERSION`, and the Dockerfile checks it against the installed OpenClaw package. Merges to `main` publish `odh-stable`; consumers should pin by digest for reproducible deployments, or use `odh-stable` for the latest build. The build is hermetic for linux/x86_64 and linux/arm64 with this prefetch input: ```json [{"type": "npm", "path": "deploy/konflux/openclaw"}, {"type": "rpm", "path": "deploy/konflux/openclaw"}] diff --git a/deploy/konflux/openclaw/rpms.lock.yaml b/deploy/konflux/openclaw/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/openclaw/rpms.lock.yaml +++ b/deploy/konflux/openclaw/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 diff --git a/deploy/konflux/supervisor/rpms.lock.yaml b/deploy/konflux/supervisor/rpms.lock.yaml index 506474bf38..e23aec5a0c 100644 --- a/deploy/konflux/supervisor/rpms.lock.yaml +++ b/deploy/konflux/supervisor/rpms.lock.yaml @@ -624,20 +624,20 @@ arches: name: pcre evr: 8.44-4.el9 sourcerpm: pcre-8.44-4.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-6.el9.x86_64.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-10.40-7.el9_8.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 241900 - checksum: sha256:75db1e5a50e7b1794d7ba18212d95cd2684559da9e7c52eee46490302c7f24dd + size: 244785 + checksum: sha256:ef2c65a463e1ef1bfbf3d9ee66986e14cca0fbae88343f099dd5b16619276f1f name: pcre2 - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm - - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-6.el9.noarch.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm + - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/pcre2-syntax-10.40-7.el9_8.noarch.rpm repoid: ubi-9-for-x86_64-baseos-rpms - size: 147926 - checksum: sha256:d386b5e9b3a4b077b2ba143882e605750855dd3354f13c55fa12ed26908cb442 + size: 150069 + checksum: sha256:e7466a43ef2bfda2b1dc07e855b0db743d48b11c0bdc1402e1d2ea6fb23d33a3 name: pcre2-syntax - evr: 10.40-6.el9 - sourcerpm: pcre2-10.40-6.el9.src.rpm + evr: 10.40-7.el9_8 + sourcerpm: pcre2-10.40-7.el9_8.src.rpm - url: https://cdn-ubi.redhat.com/content/public/ubi/dist/ubi9/9/x86_64/baseos/os/Packages/p/python3-3.9.25-7.el9_8.3.x86_64.rpm repoid: ubi-9-for-x86_64-baseos-rpms size: 33200 diff --git a/deploy/rhoai/README.md b/deploy/rhoai/README.md new file mode 100644 index 0000000000..c5e488d1b0 --- /dev/null +++ b/deploy/rhoai/README.md @@ -0,0 +1,353 @@ +# OpenShell and Dashboard installer for OpenShift + +This ODH-fork installer installs two independent Helm releases, `openshell` and +`openshell-dashboard`, into the administrator-selected namespace. It reuses an +existing namespace or creates it if missing. Each namespace has one gateway and +its dashboard; reruns upgrade the same releases. The gateway chart supplies +the gateway and the configuration for launching supervisor and sandbox workloads. +The UI chart remains in the [dashboard repository](https://github.com/opendatahub-io/openshell-dashboard/tree/main/deploy/helm/openshell-dashboard). + +This is an initial installer, not a qualified RHOAI deployment. Gateway chart +`0.1.3` and dashboard chart `0.1.0`, using their default images, passed the live +smoke checks below on OpenShift `4.21.8`. Matching `0.1` release lines alone do +not prove API compatibility. Choose exact published versions and validate them +together. The script intentionally has no default release versions. Exact SemVer +prereleases such as `0.0.0-dev.` are accepted for downstream builds; +accepting a version does not establish chart or image compatibility. + +## Live validation + +On October 13, 2026, the installer was exercised on an OpenShift `4.21.8` cluster +with RHOAI installed, OVN Kubernetes, `gp3-csi` storage, Agent Sandbox `v1.0.6`, and +an existing Red Hat build of Keycloak `26.4.10`. A dedicated test realm supplied +OIDC; existing customer realms were not changed. + +The following checks passed with gateway chart `0.1.3` and dashboard chart `0.1.0`: + +- First installation into a new namespace and a second run with the same inputs. +- Installation and rerun in a separately pre-created namespace selected with + `--namespace`, with both releases reaching revision 2 and the namespace's labels, + annotations, and OpenShift SCC allocations preserved. +- Gateway readiness, both dashboard containers ready, and persistent storage bound. +- Public HTTPS routing and HTTP 401 for unauthenticated dashboard API requests. +- Browser OIDC redirect/login/callback flow and authenticated workspace and gateway API calls. +- Manual browser confirmation of the workspaces page, platform-admin identity, + healthy gateway indicator, and active default workspace. +- Dashboard compatibility response reporting the gateway release line as supported. +- Sandbox creation through the dashboard API, workload/supervisor readiness, + authenticated terminal WebSocket execution with exit code 0, and completed deletion. +- Upgrade using YAML resource overrides and explicit OIDC scopes, with legacy + installer state migrated to record the gateway identity. +- Read-only authorization verification: admin access to both test workspaces, + ordinary-user access to its assigned workspace and HTTP 403 for the other. + A separate live check confirmed access was denied after membership revocation. + Browser login and sandbox/terminal smoke checks also passed after the upgrade. + +These checks do not establish FIPS qualification, production certificate renewal, +isolation beyond the exercised workspace checks, or a full UI interaction test. The live +configuration and credentials are kept outside the repository. + +## Prerequisites + +- `uv`, Helm 3.17.3 or later, and `kubectl` on the administrator's workstation. +- An OpenShift cluster, an explicit kubeconfig context, and permission to manage + the namespaced/cluster resources required by the charts. Permission to create + namespaces is needed only when the selected namespace does not exist. Preflight + also reads StorageClasses, nodes, ClusterVersion, and installer-labeled ConfigMaps + across namespaces. System namespaces such as `default`, `kube-*`, and + `openshift-*` are rejected. +- Agent Sandbox CRDs and a running controller installed by the cluster + administrator. The installer checks the supported served Sandbox API and waits + for the controller Deployment. Override `agent_sandbox_namespace` and + `agent_sandbox_deployment` in the JSON if your installation uses different names. +- A CNI enforcing NetworkPolicy, a working default StorageClass, and cluster SCC + configuration appropriate for the pinned gateway's sandbox runtime. +- Customer-managed HTTPS OIDC, a confidential browser client with PKCE S256, + and access tokens whose audience and role claims match the gateway settings. +- Distinct dashboard and gateway DNS names pointing at the OpenShift router. + The dashboard name must be covered by the router's default certificate. +- Registry access from the workstation for chart pulls and from cluster nodes + for all images. Private image pull-secret provisioning is outside this first + installer; use cluster-managed registry credentials if needed. + +The installer does not install an IdP, Agent Sandbox, RHOAI, a modular operator, +or console integration. Administrators retain responsibility for these cluster +prerequisites. It does not change your current kubeconfig context. + +## Configure and install + +Copy the configuration outside the checkout so local credentials cannot be +accidentally committed: + +```shell +mkdir -p "$HOME/.config/openshell-rhoai" +chmod 700 "$HOME/.config/openshell-rhoai" +cp deploy/rhoai/config.example.json "$HOME/.config/openshell-rhoai/config.json" +``` + +Edit the copy. Replace both version placeholders with exact published SemVer +chart versions, and set the namespace, kubeconfig context, DNS names, and OIDC +claims. The two versions need not have the same patch number. The dashboard OCI +reference currently points to its upstream publisher, not an ODH chart registry. +Both chart references are configurable for mirrors. + +For ODH images, select the development chart matching the upstream commit of +the downstream build, then set `gateway.images` to the downstream registry, +repositories, and image tags/digests. The installer does not infer that mapping +or silently select a newer chart. Source version `0.0.0`, floating tags, and +version ranges are rejected for installation. + +Save the browser client's secret to `dashboard-client-secret.txt` beside the JSON, +using a trusted editor or your secret manager. Restrict its permissions to `0600`. +Do not put the secret in the JSON, Helm values, or command arguments. File paths +in `oidc.client_secret_file` and optional `oidc.ca_file` are relative to the JSON. +The latter supplies a PEM CA bundle for a private OIDC issuer to both releases. + +Register `https:///oauth2/callback` with your IdP. The browser +client ID is separate from the gateway API audience. Generic OIDC uses +`allowed_groups`; Keycloak can use `provider: "keycloak-oidc"` and `allowed_roles` +such as `openshell-api:openshell-user`. These are alternatives, not assumptions +about customer claim names. Without either UI filter, any authenticated IdP user +can reach the UI, but the gateway still enforces its roles and workspace access. +Optional `oidc.scope` supplies space-separated OAuth scopes including `openid` +(for example `openid email profile offline_access`). Request only scopes your +IdP supports; include `groups` if your provider requires it for group claims. +When omitted, oauth2-proxy keeps its provider defaults. + +Validate both charts without connecting to the cluster: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --check +``` + +Install: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" +``` + +Set `namespace` in the JSON to your team/project namespace, or override it on the +command line (the override also works with `--check`): + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --namespace research-team +``` + +`--namespace` takes precedence; you may omit the JSON `namespace` when passing +this option. Both charts, Secrets, and sandbox resources target this namespace. +Gateway Service DNS and certificate SANs are derived from the selected name. +Existing namespace labels, annotations, and OpenShift SCC allocations are +preserved. Use a separate configuration with distinct public hostnames for each +gateway namespace, and register each dashboard callback with the IdP. Specifying +a different namespace creates another deployment; it does not move an existing one. + +Provision a distinct API audience and browser client per gateway, even when +gateways share an issuer. The example uses `openshell-api-openshell-team` and +`openshell-dashboard-openshell-team`; change these registrations and the JSON +when selecting another namespace. `--namespace` does not rewrite OIDC identifiers +or create IdP clients. Configure per-gateway role/group mappings at your IdP; +a namespace alone is not an authentication boundary. + +Preflight rejects an issuer/audience or issuer/browser-client pair already +recorded by another installer-managed namespace. It cannot discover registrations +outside this installer, tokens with multiple audiences, or old installation +state that has no identity metadata. Verify those cases with your IdP admin. + +`--check` still downloads the pinned charts. It discards rendered manifests and +does not validate connectivity, SCC admission, token claims, or runtime behavior. +It reads the client secret file's existence/size, not its contents. Temporary +values files have owner-only permissions and contain no client or cookie secrets. + +Run cluster preflight without installing: + +```shell +uv run --script --python 3.12 deploy/rhoai/install.py \ + --config "$HOME/.config/openshell-rhoai/config.json" --preflight +``` + +This renders both charts, checks the default and selected workspace storage +classes, checks create/patch permissions for installation resources, verifies +Agent Sandbox readiness, and detects release/identity conflicts. It reports the +OpenShift version and node kernels and flags older kernels for runtime validation. +It does not encode a Red Hat product support matrix or prove storage health, +RBAC bind/escalate permissions, quota, SCC admission, or application compatibility. +Preflight reads supplied credential files to check for unintended rotation; +`--check` does not read their contents. `uv` installs the pinned PyYAML dependency +for parsing values files. + +## Chart values + +Set `gateway.values_files` and/or `dashboard.values_files` to lists of YAML or JSON +files, relative to the configuration file. Files merge in order: maps merge, +later scalars and lists replace earlier ones. Keep the same files for reruns; +the installer passes the merged result on every upgrade. Start with +[gateway-values.example.yaml](gateway-values.example.yaml) and +[dashboard-values.example.yaml](dashboard-values.example.yaml). + +Supported operational overrides are: + +| Chart | Values | +|---|---| +| Both | `resources`, `nodeSelector`, `tolerations`, `affinity`, `imagePullSecrets`, `openshiftRoute.annotations` | +| Gateway | `server.workspaceStorageClass`, `server.workspaceDefaultStorageSize`, `server.defaultRuntimeClassName`, `server.sandboxImagePullSecrets` | +| Dashboard | `authProxy.resources`, `authProxy.cookie.refresh`, `authProxy.upstreamTimeout`, `networkPolicy.ingressNamespace`, `networkPolicy.ingressFrom` | + +For example, add `"values_files": ["gateway-values.yaml"]` to the `gateway` +section. Reference existing image-pull Secrets by name; do not put credentials +in values files. Authentication, TLS, release names, service ports, and namespace +wiring remain owned by the installer. Unsupported overrides fail before cluster +access. Extending this set requires checking the pinned charts' contracts. +The gateway chart's SQLite PVC still requires a default block-backed StorageClass; +`workspaceStorageClass` affects sandbox workspace PVCs only. + +The installer creates the namespace only when missing, then creates an installer +state ConfigMap, a client/cookie Secret, and an optional OIDC CA ConfigMap. +It installs the gateway first, checks the gateway CA Secret, +and installs the dashboard with that CA and the internal gateway Service URL. +Both releases use OpenShift-assigned UID/GID settings. It waits for Helm readiness +and checks Route admission. Set `--timeout 15m` for a slower cluster. + +## TLS and onboarding + +The gateway uses the chart's PKI bootstrap hook and a passthrough Route. Generated +server SANs include the gateway public hostname and namespace-specific Service +names. This initial mode uses a private gateway CA: export and distribute its +`ca.crt` from `openshell-client-tls` through your trusted administrator process. +Register the gateway using the installed CLI's CA options from +`openshell gateway add --help`; do not disable certificate verification. +TLS client-certificate authentication is disabled for this OIDC-only deployment. +Supervisors retain gateway-minted sandbox JWT authentication. + +The dashboard uses an edge Route and trusts the gateway CA for internal calls. +Its authentication proxy is the only exposed dashboard port. The OpenShift +router defaults to the `openshift-ingress` namespace network identity. For a +custom or host-network router, provide reviewed `networkPolicy.ingressFrom` +peers in dashboard values instead; retain ingress restricted to your router. + +Configure a CLI OIDC registration separately with your IdP. After installation, +verify administrator and standard-user login. Administrators add workspace +memberships using the token's subject ID. Verify denied cross-workspace access, +sandbox creation/deletion, and terminal WebSockets before sharing the deployment. +Readiness and Route admission alone do not prove this flow works. + +Use the read-only verifier after provisioning two distinct identities and two +existing workspaces. The admin must access both workspaces; the ordinary user +must belong only to the allowed workspace. Save fresh access tokens in files +with mode `0600` outside the checkout, using your IdP's supported login flow: + +```shell +uv run --script --python 3.12 deploy/rhoai/verify.py \ + --context YOUR_CONTEXT --namespace research-team \ + --dashboard-url https://dashboard.apps.example.com \ + --admin-role openshell-admin \ + --admin-token-file /secure/admin-token.txt \ + --user-token-file /secure/user-token.txt \ + --allowed-workspace team-a --denied-workspace team-b +``` + +The verifier checks public unauthenticated rejection, gateway identity/roles, +admin access to both workspaces, ordinary-user access to one and denial in the +other. It opens a temporary loopback-only port-forward to the BFF to test gateway +authorization with bearer tokens; this requires `pods/portforward` permission. +It performs GET requests only and does not change users, memberships, or workloads. +It closes its port-forward on success or failure and never follows redirects +with credentials or prints tokens/response bodies. + +Optional `--other-gateway-token-file` checks rejection of a token for another +gateway. First confirm that token works on its intended gateway: an expired or +otherwise invalid token cannot prove audience isolation. Optional +`--session-cookie-file` accepts the Cookie header value from an already logged-in +browser and checks the public session endpoint. `--ca-file` adds a private Route +CA to system trust. These checks do not automate browser login, grant/revoke, +sandbox creation/deletion, terminal execution, or network-policy enforcement. + +## Reruns, upgrades, and failures + +Rerun the same command after correcting a failure. The administrator selects the +namespace; it does not need an installer ownership label. The fixed release names +provide one gateway/dashboard pair per namespace. An existing gateway chart under +a different release name, or either fixed release without installer state, is +rejected and needs explicit migration. Unmanaged gateway TLS/JWT Secrets are also +rejected so the bootstrap hook cannot silently reuse another gateway's identity. +Unrelated Helm releases in the selected namespace are left alone. +Preflight inspects stored Helm workload manifests for the `openshell-gateway` +container or executable, so renaming the chart or overriding its image does not +bypass gateway detection. Failed releases and scaled-to-zero gateways count too. +Custom packaging must retain one of these markers; workloads outside Helm or +with both markers removed require administrator inspection. +Uninstalled releases retained under another name do not count as active gateways; +conflicting remaining TLS/JWT Secrets and fixed release names still require +explicit migration. + +The original gateway hostname is recorded in the `openshell-installer` ConfigMap. +Issuer, API audience, and browser client ID are recorded too; changing them +requires an explicit identity migration. Legacy state is populated on a successful +preflight followed by an install/rerun without rewriting existing identity values. +Earlier installations recorded it on the namespace; reruns migrate that state +without changing namespace metadata. Changing the recorded hostname is +rejected because the certificate hook preserves existing certificates; migrate +the certificate and endpoint explicitly rather than editing installer state. + +The installer preserves existing client and cookie secrets. You may omit +`client_secret_file` only when installer state exists. A first installation requires +the file even if `openshell-dashboard-oidc` already exists; the supplied client +secret must match before that Secret is reused. Preflight validates local +credential files without reading namespaced objects when the namespace is absent. +A provided secret or private OIDC CA that differs from the existing object causes +an error; rotate credentials explicitly and restart the consuming pods. +Removing `ca_file` removes the mounts +on the next upgrade, but does not delete the old ConfigMap. + +Change the pinned versions to upgrade. Both Helm releases receive the generated +values with `--reset-values`, including the configured values files, so out-of-band +Helm value edits are not preserved. +There is no cross-release transaction or automatic rollback: gateway success +followed by dashboard failure leaves the upgraded gateway in place. Failed +releases, namespaces, Secrets, and PVCs are preserved for diagnosis. + +Inspect the two Helm releases, namespace pods/events, and the `openshell` and +`openshell-dashboard` Routes with the same explicit kubeconfig context. For +sandbox failures, inspect Agent Sandbox, SCC admission, storage, supervisor logs, +and NetworkPolicy enforcement. Namespace deletion is never part of this script. + +## Image overrides + +Optional `gateway.images` accepts `gateway`, `supervisor`, `sandboxRuntime`, and +`sandbox` objects. Optional `dashboard.images` accepts `dashboard` and `authProxy`. +Each object accepts `registry`, `repository`, `tag`, and `digest`, using the chart's +image semantics. For example: + +```json +"images": { + "gateway": { + "registry": "quay.io", + "repository": "your-org/openshell-gateway", + "digest": "sha256:" + } +} +``` + +Pin compatible runtime and UI images as well as chart versions. The default +workload image uses a complete registry/repository path in `sandbox.repository` +(within `gateway.images.sandbox`); its chart does not use a separate registry key. +Image overrides do not establish FIPS qualification. + +## Contributor validation + +The `RHOAI Installer` GitHub Actions workflow runs both suites for changes to +this directory. It needs no cluster credentials or IdP. Run them locally with: + +```shell +uv run --script --python 3.12 deploy/rhoai/test_install.py +uv run --no-project --python 3.12 deploy/rhoai/test_verify.py +``` + +For source-chart rendering, use `--check --gateway-chart-directory +--dashboard-chart-directory `. These overrides are rejected for installation. +Source charts carry version `0.0.0`, so this check renders images with the versions +from your configuration. The gateway source originally selected for this work is +`348a1fc62566892ba5530ac9f35acc983e4d9324`; a source commit is not a published OCI +chart version. Resolve it to the intended release before installing. diff --git a/deploy/rhoai/config.example.json b/deploy/rhoai/config.example.json new file mode 100644 index 0000000000..c8659ed66b --- /dev/null +++ b/deploy/rhoai/config.example.json @@ -0,0 +1,27 @@ +{ + "namespace": "openshell-team", + "context": "REPLACE_WITH_KUBECONFIG_CONTEXT", + "gateway": { + "chart": "oci://ghcr.io/nvidia/openshell/helm-chart", + "version": "REPLACE_WITH_EXACT_GATEWAY_VERSION", + "values_files": [], + "host": "gateway.apps.example.com" + }, + "dashboard": { + "chart": "oci://ghcr.io/gkrumbach07/openshell-dashboard/helm-chart/openshell-dashboard", + "version": "REPLACE_WITH_EXACT_DASHBOARD_VERSION", + "values_files": [], + "host": "dashboard.apps.example.com" + }, + "oidc": { + "provider": "oidc", + "issuer": "https://identity.example.com/realms/customer", + "client_id": "openshell-dashboard-openshell-team", + "client_secret_file": "dashboard-client-secret.txt", + "audience": "openshell-api-openshell-team", + "roles_claim": "roles", + "admin_role": "openshell-admin", + "user_role": "openshell-user", + "allowed_groups": [] + } +} diff --git a/deploy/rhoai/dashboard-values.example.yaml b/deploy/rhoai/dashboard-values.example.yaml new file mode 100644 index 0000000000..fe3a3c59ee --- /dev/null +++ b/deploy/rhoai/dashboard-values.example.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# Optional operational overrides, referenced by dashboard.values_files. +resources: + requests: + cpu: 100m + memory: 128Mi +authProxy: + resources: + requests: + cpu: 50m + memory: 64Mi +# For custom/host-network routers, set networkPolicy.ingressFrom to peers +# reviewed for your ingress controller. Keep the proxy restricted to the router. diff --git a/deploy/rhoai/gateway-values.example.yaml b/deploy/rhoai/gateway-values.example.yaml new file mode 100644 index 0000000000..769087478f --- /dev/null +++ b/deploy/rhoai/gateway-values.example.yaml @@ -0,0 +1,14 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# Optional operational overrides. Copy beside your private config.json and add +# its filename to gateway.values_files. Use sizes appropriate for your workload. +resources: + requests: + cpu: 100m + memory: 256Mi +# imagePullSecrets: +# - name: registry-credentials +# server: +# workspaceStorageClass: block-storage +# sandboxImagePullSecrets: +# - name: registry-credentials diff --git a/deploy/rhoai/install.py b/deploy/rhoai/install.py new file mode 100644 index 0000000000..4290f4ac55 --- /dev/null +++ b/deploy/rhoai/install.py @@ -0,0 +1,986 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = ["PyYAML==6.0.3"] +# /// +"""Install the gateway and dashboard as separate Helm releases on OpenShift.""" + +import argparse +import base64 +import copy +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.parse import urlsplit + +import yaml + +OWNER = "openshell.ai/installer" +OWNER_VALUE = "odh-dashboard" +GATEWAY_HOST = "openshell.ai/installer-gateway-host" +INSTALL_STATE = "openshell-installer" +OIDC_SECRET = "openshell-dashboard-oidc" +OIDC_CA = "openshell-oidc-ca" +VERSION = re.compile( + r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)" + r"(?:-(?:0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*)" + r"(?:\.(?:0|[1-9][0-9]*|[0-9A-Za-z-]*[A-Za-z-][0-9A-Za-z-]*))*)?" + r"(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?" +) +DNS = re.compile(r"[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?") + +# Expose operational chart settings without allowing overlays to replace the +# installer's identity, TLS, service naming, or namespace contract. +COMMON_VALUES = dict.fromkeys( + ("resources", "nodeSelector", "tolerations", "affinity", "imagePullSecrets") +) +EXTRA_VALUES = { + "gateway": { + **COMMON_VALUES, + "server": dict.fromkeys( + ( + "workspaceStorageClass", + "workspaceDefaultStorageSize", + "defaultRuntimeClassName", + "sandboxImagePullSecrets", + ) + ), + "openshiftRoute": {"annotations": None}, + }, + "dashboard": { + **COMMON_VALUES, + "authProxy": { + "resources": None, + "cookie": {"refresh": None}, + "upstreamTimeout": None, + }, + "networkPolicy": {"ingressNamespace": None, "ingressFrom": None}, + "openshiftRoute": {"annotations": None}, + }, +} + + +def merge_values(target, overlay, allowed, prefix=""): + require(isinstance(overlay, dict), f"{prefix or 'Values'} must be a mapping") + for key, value in overlay.items(): + require( + key in allowed, f"Unsupported or installer-owned Helm value: {prefix}{key}" + ) + if allowed[key] is not None: + merge_values( + target.setdefault(key, {}), value, allowed[key], f"{prefix}{key}." + ) + elif isinstance(value, dict) and isinstance(target.get(key), dict): + # Helm's map merge semantics; arrays/scalars are replaced. + merge_unrestricted(target[key], value) + else: + target[key] = copy.deepcopy(value) + + +def merge_unrestricted(target, overlay): + for key, value in overlay.items(): + if isinstance(value, dict) and isinstance(target.get(key), dict): + merge_unrestricted(target[key], value) + else: + target[key] = copy.deepcopy(value) + + +def require(condition, message): + if not condition: + raise ValueError(message) + + +def fields(obj, required, optional=()): + require(isinstance(obj, dict), "Configuration sections must be JSON objects") + require( + not (set(obj) - set(required) - set(optional)), + f"Unknown configuration fields: {set(obj) - set(required) - set(optional)}", + ) + for key in required: + require(key in obj and obj[key] != "", f"Missing configuration field: {key}") + + +def load_config(path, namespace=None): + config = json.loads(path.read_text()) + if namespace is not None: + require(isinstance(config, dict), "Configuration must be a JSON object") + config["namespace"] = namespace + fields( + config, + ("namespace", "context", "gateway", "dashboard", "oidc"), + ("agent_sandbox_namespace", "agent_sandbox_deployment"), + ) + ns = config["namespace"] + require( + isinstance(ns, str) + and len(ns) <= 63 + and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]*[a-z0-9])?", ns), + "Invalid namespace", + ) + require( + ns not in ("default", "openshift", "kube-system", "kube-public") + and not ns.startswith(("kube-", "openshift-")), + "Choose an application namespace, not a Kubernetes/OpenShift system namespace", + ) + for key in ("agent_sandbox_namespace", "agent_sandbox_deployment"): + if key in config: + require( + isinstance(config[key], str) and DNS.fullmatch(config[key]), + f"Invalid {key}", + ) + require( + isinstance(config["context"], str) and config["context"].strip(), + "context is required", + ) + for name in ("gateway", "dashboard"): + item = config[name] + fields(item, ("chart", "version", "host"), ("images", "values_files")) + require( + isinstance(item["chart"], str) and item["chart"].startswith("oci://"), + f"{name}.chart must be an OCI chart reference", + ) + require( + isinstance(item["version"], str) and VERSION.fullmatch(item["version"]), + f"{name}.version must be exact SemVer (no latest or version ranges)", + ) + require( + item["version"] != "0.0.0", + "Use a published release or pinned development version", + ) + sources = item.get("values_files", []) + require( + isinstance(sources, list) + and all(isinstance(p, str) and p for p in sources), + f"{name}.values_files must be a list of file paths", + ) + overrides = {} + for source in sources: + # SafeLoader does not construct Python objects. Do not include file + # contents in diagnostics: user-supplied files may contain secrets. + try: + overlay = yaml.safe_load((path.parent / source).read_text()) + except yaml.YAMLError: + raise ValueError(f"Invalid YAML in {name} values file") from None + merge_values(overrides, overlay, EXTRA_VALUES[name]) + item["values"] = overrides + host = item["host"] + require( + isinstance(host, str) + and len(host) <= 253 + and DNS.fullmatch(host) + and all( + 0 < len(label) <= 63 + and not label.startswith("-") + and not label.endswith("-") + for label in host.split(".") + ), + f"Invalid {name}.host", + ) + components = ( + ("gateway", "supervisor", "sandboxRuntime", "sandbox") + if name == "gateway" + else ("dashboard", "authProxy") + ) + fields(item.get("images", {}), (), components) + for component, image in item.get("images", {}).items(): + keys = ( + ("repository", "tag", "digest") + if component == "sandbox" + else ("registry", "repository", "tag", "digest") + ) + fields(image, (), keys) + require( + all(isinstance(value, str) for value in image.values()), + "Image values must be strings", + ) + require( + config["gateway"]["host"] != config["dashboard"]["host"], + "Gateway and dashboard need distinct hosts", + ) + oidc = config["oidc"] + fields( + oidc, + ("issuer", "client_id", "audience", "roles_claim", "admin_role", "user_role"), + ( + "provider", + "allowed_groups", + "allowed_roles", + "client_secret_file", + "ca_file", + "scope", + ), + ) + for key in ( + "issuer", + "client_id", + "audience", + "roles_claim", + "admin_role", + "user_role", + ): + require( + isinstance(oidc[key], str) and oidc[key].strip(), + f"oidc.{key} must be a nonempty string", + ) + issuer = urlsplit(oidc["issuer"]) + require( + issuer.scheme == "https" + and issuer.hostname + and not issuer.username + and not issuer.password + and not issuer.query + and not issuer.fragment, + "OIDC issuer must be an HTTPS URL", + ) + require( + oidc["client_id"] != oidc["audience"], + "Browser client ID and gateway audience must be distinct", + ) + require( + oidc["admin_role"] != oidc["user_role"], "Admin and user roles must be distinct" + ) + require( + oidc.get("provider", "oidc") in ("oidc", "keycloak-oidc"), + "Unsupported OIDC provider", + ) + for key in ("allowed_groups", "allowed_roles"): + require( + isinstance(oidc.get(key, []), list) + and all(isinstance(v, str) and v for v in oidc.get(key, [])), + f"oidc.{key} must be a list of strings", + ) + require( + not oidc.get("allowed_roles") or oidc.get("provider") == "keycloak-oidc", + "allowed_roles requires keycloak-oidc; use allowed_groups for generic OIDC", + ) + if "scope" in oidc: + require( + isinstance(oidc["scope"], str) + and re.fullmatch( + r"[\x21\x23-\x5B\x5D-\x7E]+(?: [\x21\x23-\x5B\x5D-\x7E]+)*", + oidc["scope"], + ) + and "openid" in oidc["scope"].split(), + "oidc.scope must be space-separated OAuth scopes including openid", + ) + for key in ("client_secret_file", "ca_file"): + if oidc.get(key): + source = (path.parent / oidc[key]).resolve() + require( + source.is_file() and source.stat().st_size > 0, + f"Missing or empty oidc.{key}", + ) + oidc[key] = str(source) + return config + + +def chart_values(config): + ns, oidc = config["namespace"], config["oidc"] + ca = OIDC_CA if oidc.get("ca_file") else "" + security = {"runAsUser": None, "seccompProfile": {"type": "RuntimeDefault"}} + gateway = { + "fullnameOverride": "openshell", + "podSecurityContext": None, + "securityContext": security, + "replicaCount": 1, + "server": { + "disableTls": False, + "sandboxNamespace": ns, + "auth": {"allowUnauthenticatedUsers": False}, + "tls": { + "enableMtls": False, + "clientCaSecretName": "", + "certSecretName": "openshell-server-tls", + "clientTlsSecretName": "openshell-client-tls", + }, + "oidc": { + "issuer": oidc["issuer"], + "audience": oidc["audience"], + "rolesClaim": oidc["roles_claim"], + "adminRole": oidc["admin_role"], + "userRole": oidc["user_role"], + "caConfigMapName": ca, + }, + }, + "pkiInitJob": { + "enabled": True, + "serverDnsNames": [ + config["gateway"]["host"], + f"openshell.{ns}.svc", + f"openshell.{ns}.svc.cluster.local", + ], + }, + "openshiftRoute": {"enabled": True, "host": config["gateway"]["host"]}, + } + dashboard = { + "fullnameOverride": "openshell-dashboard", + "podSecurityContext": None, + "securityContext": security, + "gateway": { + "url": f"grpcs://openshell.{ns}.svc.cluster.local:8080", + "caSecretName": "openshell-client-tls", + }, + "oidc": { + "provider": oidc.get("provider", "oidc"), + "issuer": oidc["issuer"], + "clientId": oidc["client_id"], + "existingSecret": OIDC_SECRET, + "audience": oidc["audience"], + "rolesClaim": oidc["roles_claim"], + "adminRole": oidc["admin_role"], + "codeChallengeMethod": "S256", + "allowedGroups": oidc.get("allowed_groups", []), + "allowedRoles": oidc.get("allowed_roles", []), + "caConfigMapName": ca, + "scope": oidc.get("scope", ""), + }, + "openshiftRoute": { + "enabled": True, + "host": config["dashboard"]["host"], + "termination": "edge", + }, + "networkPolicy": {"ingressNamespace": "openshift-ingress"}, + } + for name, values in (("gateway", gateway), ("dashboard", dashboard)): + merge_values(values, config[name].get("values", {}), EXTRA_VALUES[name]) + for component, image in config[name].get("images", {}).items(): + values.setdefault(component, {})["image"] = image + return gateway, dashboard + + +class Installer: + def __init__(self, config, timeout): + self.config = config + self.timeout = timeout + self.kube = ["kubectl", "--context", config["context"], "--request-timeout=30s"] + self.ns = config["namespace"] + + def run(self, args, payload=None): + # Never log stdin: it can contain Secret data. Do not echo commands. + result = subprocess.run( + args, input=payload, text=True, capture_output=True, check=False + ) + if result.returncode: + raise RuntimeError(f"{args[0]} failed: {result.stderr.strip()}") + return result.stdout + + def get(self, kind, name, namespaced=True): + args = self.kube + (["-n", self.ns] if namespaced else []) + output = self.run( + [*args, "get", kind, name, "--ignore-not-found", "-o", "json"] + ) + return json.loads(output) if output.strip() else None + + def create(self, resource): + self.run([*self.kube, "create", "-f", "-"], json.dumps(resource)) + + def release_has_gateway(self, release): + # Chart names and images can change in downstream packages. Inspect the + # stored workload instead, including failed releases with no running pod. + manifest = self.run( + [ + "helm", + "get", + "manifest", + release, + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + ] + ) + try: + for resource in yaml.safe_load_all(manifest): + if not isinstance(resource, dict): + continue + if resource.get("kind") == "Pod": + pod = resource + elif resource.get("kind") in ("Deployment", "StatefulSet", "DaemonSet"): + pod = resource.get("spec", {}).get("template", {}) + else: + continue + for container in pod.get("spec", {}).get("containers", []): + if container.get("name") == "openshell-gateway" or any( + Path(arg).name == "openshell-gateway" + for arg in container.get("command", []) + ): + return True + except yaml.YAMLError: + # Helm manifests can contain Secrets; never include parser excerpts. + raise ValueError(f"Cannot inspect manifest for release {release}") from None + return False + + def check_releases(self, managed): + # Explicit status flags work with both Helm 3 and Helm 4. Paginate so a + # busy administrator-managed namespace cannot hide a conflicting release. + offset = 0 + while True: + releases = json.loads( + self.run( + [ + "helm", + "list", + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + "--output", + "json", + "--max", + "256", + "--offset", + str(offset), + "--deployed", + "--failed", + "--pending", + "--uninstalling", + "--uninstalled", + "--superseded", + ] + ) + ) + for release in releases: + name = release["name"] + # --keep-history leaves an uninstalled record, not a gateway. + # Keep fixed-name ownership checks: Helm still owns that history. + if release.get("status") == "uninstalled" and name not in ( + "openshell", + "openshell-dashboard", + ): + continue + require( + managed or name not in ("openshell", "openshell-dashboard"), + f"Release {name} already exists outside this installer; migrate it explicitly", + ) + if name != "openshell": + gateway = release.get("chart", "").startswith( + "helm-chart-" + ) or self.release_has_gateway(name) + require( + not gateway, + f"Namespace {self.ns} already has gateway release {name}; use one gateway per namespace", + ) + if len(releases) < 256: + break + offset += len(releases) + + def cluster_checks(self, namespace): + version = self.get("clusterversion.config.openshift.io", "version", False) + print( + f"OpenShift: {(version or {}).get('status', {}).get('desired', {}).get('version', 'unknown')}" + ) + storage = json.loads( + self.run([*self.kube, "get", "storageclasses", "-o", "json"]) + ) + defaults = [ + item["metadata"]["name"] + for item in storage["items"] + if any( + item["metadata"].get("annotations", {}).get(key) == "true" + for key in ( + "storageclass.kubernetes.io/is-default-class", + "storageclass.beta.kubernetes.io/is-default-class", + ) + ) + ] + require( + defaults, + "No default StorageClass; the gateway chart's SQLite PVC requires one", + ) + print( + f"Default storage: {', '.join(defaults)} (verify block-backed SQLite storage)" + ) + custom = ( + self.config["gateway"] + .get("values", {}) + .get("server", {}) + .get("workspaceStorageClass") + ) + if custom: + require( + any(item["metadata"]["name"] == custom for item in storage["items"]), + "Configured workspaceStorageClass does not exist", + ) + nodes = json.loads(self.run([*self.kube, "get", "nodes", "-o", "json"])) + kernels = sorted( + { + item.get("status", {}) + .get("nodeInfo", {}) + .get("kernelVersion", "unknown") + for item in nodes["items"] + } + ) + print(f"Node kernels: {', '.join(kernels) or 'none reported'}") + if any( + (match := re.match(r"(\d+)\.(\d+)", kernel)) + and tuple(map(int, match.groups())) < (5, 19) + for kernel in kernels + ): + print( + "Note: kernels before 5.19 need an OpenShell build with the legacy-kernel fixes; verify sandbox networking." + ) + # Check actual installation operations instead of requiring wildcard + # cluster-admin. These checks cannot prove RBAC bind/escalate or SCC admission. + scoped = [ + "secrets", + "configmaps", + "services", + "serviceaccounts", + "deployments.apps", + "statefulsets.apps", + "jobs.batch", + "roles.rbac.authorization.k8s.io", + "rolebindings.rbac.authorization.k8s.io", + "routes.route.openshift.io", + "networkpolicies.networking.k8s.io", + ] + operations = [ + (verb, resource, True) + for resource in scoped + for verb in ("create", "patch") + ] + operations += [ + (verb, resource, False) + for resource in ( + "clusterroles.rbac.authorization.k8s.io", + "clusterrolebindings.rbac.authorization.k8s.io", + ) + for verb in ("create", "patch") + ] + if not namespace: + operations.append(("create", "namespaces", False)) + for verb, resource, namespaced in operations: + try: + answer = self.run( + [ + *self.kube, + "auth", + "can-i", + verb, + resource, + *(["-n", self.ns] if namespaced else []), + ] + ) + require(answer.strip() == "yes", "denied") + except (RuntimeError, ValueError): + raise ValueError( + f"Preflight permission check failed: {verb} {resource}" + + (f" in {self.ns}" if namespaced else " cluster-wide") + ) from None + + def identity_state(self): + return { + "gateway-host": self.config["gateway"]["host"], + **{ + f"oidc-{key.replace('_', '-')}": self.config["oidc"][key] + for key in ("issuer", "audience", "client_id") + }, + } + + def check_identity(self, state): + desired = self.identity_state() + for key, value in (state or {}).get("data", {}).items(): + if key in desired: + require( + value == desired[key], + f"Installer identity {key} changed; explicit identity/certificate migration is required", + ) + states = json.loads( + self.run( + [ + *self.kube, + "get", + "configmaps", + "--all-namespaces", + "-l", + f"{OWNER}={OWNER_VALUE}", + "-o", + "json", + ] + ) + ) + for other in states["items"]: + if ( + other["metadata"]["name"] != INSTALL_STATE + or other["metadata"].get("namespace") == self.ns + ): + continue + data = other.get("data", {}) + if "oidc-issuer" not in data: + print( + f"Note: legacy installer state in {other['metadata']['namespace']} has no identity metadata; verify audience isolation manually." + ) + continue + if data["oidc-issuer"].rstrip("/") == desired["oidc-issuer"].rstrip("/"): + require( + data.get("oidc-audience") != desired["oidc-audience"] + and data.get("oidc-client-id") != desired["oidc-client-id"], + f"OIDC audience/browser client already used by namespace {other['metadata']['namespace']}; provision distinct per-gateway registrations", + ) + + def preflight(self): + self.run([*self.kube, "get", "--raw", "/apis/route.openshift.io/v1"]) + crd = self.get("crd", "sandboxes.agents.x-k8s.io", False) + require( + crd + and any( + v["served"] and v["name"] in ("v1alpha1", "v1beta1") + for v in crd["spec"]["versions"] + ), + "Install the Agent Sandbox CRDs and controller first", + ) + self.run( + [ + *self.kube, + "-n", + self.config.get("agent_sandbox_namespace", "agent-sandbox-system"), + "rollout", + "status", + "deployment/" + + self.config.get( + "agent_sandbox_deployment", "agent-sandbox-controller" + ), + "--timeout", + self.timeout, + ] + ) + namespace = self.get("namespace", self.ns, False) + require( + not namespace or namespace.get("status", {}).get("phase") != "Terminating", + "Namespace is terminating", + ) + self.cluster_checks(namespace) + state = self.get("configmap", INSTALL_STATE) if namespace else None + require( + not state or state["metadata"].get("labels", {}).get(OWNER) == OWNER_VALUE, + f"ConfigMap {INSTALL_STATE} already exists outside this installer", + ) + # Preserve compatibility with installations made before state moved off + # the Namespace. Never modify administrator-owned Namespace metadata. + legacy_host = ( + (namespace or {}) + .get("metadata", {}) + .get("annotations", {}) + .get(GATEWAY_HOST) + ) + original_host = ( + state.get("data", {}).get("gateway-host") if state else legacy_host + ) + require( + not (state or legacy_host) + or original_host == self.config["gateway"]["host"], + "Gateway hostname differs from the original install; certificate migration is required", + ) + self.check_identity(state) + if namespace: + self.check_releases(managed=bool(state or legacy_host)) + # The chart's bootstrap hook reuses existing TLS material. Do not + # accidentally use certificates belonging to another gateway. + if not (state or legacy_host): + for secret in ( + "openshell-server-tls", + "openshell-client-tls", + "openshell-jwt-keys", + ): + require( + not self.get("secret", secret), + f"Existing gateway Secret {secret} requires explicit migration", + ) + require( + state or legacy_host or self.config["oidc"].get("client_secret_file"), + "oidc.client_secret_file is required for the first install", + ) + self.credentials(check_only=True, namespace_exists=bool(namespace)) + return namespace + + def credentials(self, check_only=False, *, namespace_exists=True): + require( + namespace_exists or check_only, "Create the namespace before credentials" + ) + oidc = self.config["oidc"] + existing = self.get("secret", OIDC_SECRET) if namespace_exists else None + supplied = ( + Path(oidc["client_secret_file"]).read_bytes().rstrip(b"\r\n") + if oidc.get("client_secret_file") + else None + ) + if supplied is not None: + require(supplied, "Client secret file contains no secret") + if existing: + data = existing.get("data", {}) + require( + data.get("client-secret") and data.get("cookie-secret"), + "Existing OIDC Secret is missing required keys", + ) + require( + supplied is None or base64.b64decode(data["client-secret"]) == supplied, + "OIDC Secret differs from file; rotate it explicitly before rerunning", + ) + else: + require(supplied, "Client secret file is required") + if not check_only: + self.create( + { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": {"name": OIDC_SECRET, "namespace": self.ns}, + "data": { + "client-secret": base64.b64encode(supplied).decode(), + "cookie-secret": base64.b64encode( + secrets.token_urlsafe(32).encode() + ).decode(), + }, + } + ) + if oidc.get("ca_file"): + ca = Path(oidc["ca_file"]).read_text() + existing_ca = self.get("configmap", OIDC_CA) if namespace_exists else None + require( + not existing_ca or existing_ca.get("data", {}).get("ca.crt") == ca, + "OIDC CA differs; rotate the ConfigMap and restart both releases explicitly", + ) + if not existing_ca and not check_only: + self.create( + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": OIDC_CA, "namespace": self.ns}, + "data": {"ca.crt": ca}, + } + ) + + def wait_for_route(self, release): + seconds = ( + int(self.timeout[:-1]) * {"s": 1, "m": 60, "h": 3600}[self.timeout[-1]] + ) + deadline = time.monotonic() + seconds + while True: + route = self.get("route", release) + if route and any( + condition.get("type") == "Admitted" + and condition.get("status") == "True" + for ingress in route.get("status", {}).get("ingress", []) + for condition in ingress.get("conditions", []) + ): + return + require( + time.monotonic() < deadline, + f"Route {release} is not admitted; inspect it and rerun", + ) + time.sleep(min(2, max(0, deadline - time.monotonic()))) + + def install(self, charts, values): + namespace = self.preflight() + if not namespace: + self.create( + { + "apiVersion": "v1", + "kind": "Namespace", + "metadata": { + "name": self.ns, + "labels": {OWNER: OWNER_VALUE}, + }, + } + ) + if not self.get("configmap", INSTALL_STATE): + self.create( + { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": INSTALL_STATE, + "namespace": self.ns, + "labels": {OWNER: OWNER_VALUE}, + }, + "data": self.identity_state(), + } + ) + else: + # Add identity metadata to legacy state only after preflight passed. + self.run( + [ + *self.kube, + "-n", + self.ns, + "patch", + "configmap", + INSTALL_STATE, + "--type=merge", + "-p", + json.dumps({"data": self.identity_state()}), + ] + ) + self.credentials() + for index, release in enumerate(("openshell", "openshell-dashboard")): + print(f"Installing {release} in {self.ns}...", flush=True) + self.run( + [ + "helm", + "upgrade", + "--install", + release, + str(charts[index]), + "--kube-context", + self.config["context"], + "--namespace", + self.ns, + "--values", + str(values[index]), + "--reset-values", + "--wait", + "--wait-for-jobs", + "--timeout", + self.timeout, + "--history-max", + "5", + ] + ) + if index == 0: + ca = self.get("secret", "openshell-client-tls") + require( + ca and ca.get("data", {}).get("ca.crt"), + "Gateway did not create its CA Secret", + ) + self.wait_for_route(release) + print(f"Dashboard: https://{self.config['dashboard']['host']}") + print(f"Gateway: https://{self.config['gateway']['host']}") + print( + "Register the gateway with the CLI using its CA, then log in through customer OIDC." + ) + print( + "An administrator must add users to workspaces. Verify login, denied access, sandbox creation and terminals." + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument( + "--namespace", + help="Administrator-selected namespace; overrides config.namespace (created if missing)", + ) + mode = parser.add_mutually_exclusive_group() + mode.add_argument( + "--check", + action="store_true", + help="Validate and render charts without accessing the cluster", + ) + mode.add_argument( + "--preflight", + action="store_true", + help="Render charts and check cluster prerequisites without installing", + ) + parser.add_argument( + "--gateway-chart-directory", + type=Path, + help="Local source chart for --check only", + ) + parser.add_argument( + "--dashboard-chart-directory", + type=Path, + help="Local source chart for --check only", + ) + parser.add_argument( + "--timeout", default="10m", help="Helm/controller wait timeout (default: 10m)" + ) + args = parser.parse_args() + require( + args.check + or not (args.gateway_chart_directory or args.dashboard_chart_directory), + "Local source charts are only supported with --check; install versioned OCI artifacts", + ) + require( + re.fullmatch(r"[1-9][0-9]*[smh]", args.timeout), + "Timeout must be a positive duration such as 10m", + ) + for tool in ["helm"] if args.check else ["helm", "kubectl"]: + require(shutil.which(tool), f"Required tool not found: {tool}") + config = load_config(args.config.resolve(), args.namespace) + installer = Installer(config, args.timeout) + os.umask(0o077) + with tempfile.TemporaryDirectory(prefix="openshell-install-") as temporary: + root = Path(temporary) + charts, files = [], [] + for index, (name, values, local) in enumerate( + zip( + ("gateway", "dashboard"), + chart_values(config), + (args.gateway_chart_directory, args.dashboard_chart_directory), + strict=True, + ) + ): + if local: + chart = local.resolve() + require((chart / "Chart.yaml").is_file(), f"Missing chart: {chart}") + # Source charts carry 0.0.0; render with the requested release image tag. + values.setdefault("global", {})["image"] = { + "tag": config[name]["version"] + } + else: + destination = root / name + destination.mkdir() + installer.run( + [ + "helm", + "pull", + config[name]["chart"], + "--version", + config[name]["version"], + "--destination", + str(destination), + ] + ) + packages = list(destination.glob("*.tgz")) + require(len(packages) == 1, f"Expected one downloaded {name} chart") + chart = packages[0] + value_file = root / f"{name}.json" + value_file.write_text(json.dumps(values)) + release = ("openshell", "openshell-dashboard")[index] + installer.run( + [ + "helm", + "template", + release, + str(chart), + "--namespace", + config["namespace"], + "--values", + str(value_file), + "--api-versions", + "route.openshift.io/v1", + "--api-versions", + "agents.x-k8s.io/v1beta1", + ] + ) + charts.append(chart) + files.append(value_file) + print( + "Both charts rendered successfully. This does not verify cluster or API compatibility.", + flush=True, + ) + if args.preflight: + installer.preflight() + print("Cluster preflight passed; no installation resources were changed.") + elif not args.check: + installer.install(charts, files) + + +if __name__ == "__main__": + try: + main() + except (ValueError, RuntimeError, OSError, KeyError, TypeError) as error: + print( + f"Installation stopped: {error}\nCluster resources were not rolled back or deleted.", + file=sys.stderr, + ) + sys.exit(1) diff --git a/deploy/rhoai/test_install.py b/deploy/rhoai/test_install.py new file mode 100644 index 0000000000..ad1b5c66cf --- /dev/null +++ b/deploy/rhoai/test_install.py @@ -0,0 +1,769 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = ["PyYAML==6.0.3"] +# /// +"""Behavioral tests with a simulated cluster; no network or cluster access.""" + +import base64 +import copy +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location( + "installer", Path(__file__).with_name("install.py") +) +installer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(installer) + + +def configuration(): + config = json.loads(Path(__file__).with_name("config.example.json").read_text()) + config["context"] = "test-context" + config["gateway"]["version"] = "0.1.3" + config["dashboard"]["version"] = "0.1.7" + config["oidc"].pop("client_secret_file") + return config + + +class FakeCluster(installer.Installer): + def __init__(self, config): + super().__init__(config, "10m") + self.objects = { + ("crd", "sandboxes.agents.x-k8s.io"): { + "spec": {"versions": [{"name": "v1beta1", "served": True}]} + } + } + self.calls = [] + self.releases = [] + self.manifests = {} + self.fail_dashboard = False + self.storage = [ + { + "metadata": { + "name": "block", + "annotations": { + "storageclass.kubernetes.io/is-default-class": "true" + }, + } + } + ] + self.other_states = [] + self.denied_permission = None + + def run(self, args, payload=None): + self.calls.append((args, payload)) + if ( + args[0] == "kubectl" + and "-n" in args + and "get" in args + and ("namespace", args[args.index("-n") + 1]) not in self.objects + ): + raise RuntimeError("namespace not found") + if "can-i" in args: + return ( + "no" + if self.denied_permission and self.denied_permission in args + else "yes" + ) + if "storageclasses" in args: + return json.dumps({"items": self.storage}) + if "nodes" in args: + return json.dumps({"items": []}) + if "--all-namespaces" in args: + return json.dumps({"items": self.other_states}) + if "patch" in args: + self.objects[("configmap", installer.INSTALL_STATE)]["data"].update( + json.loads(args[args.index("-p") + 1])["data"] + ) + return "" + if args[:2] == ["helm", "list"]: + offset = int(args[args.index("--offset") + 1]) + return json.dumps(self.releases[offset : offset + 256]) + if args[:3] == ["helm", "get", "manifest"]: + return self.manifests.get(args[3], "") + if args[:2] == ["helm", "upgrade"]: + if self.fail_dashboard and "openshell-dashboard" in args: + raise RuntimeError("dashboard failed") + self.objects[("secret", "openshell-client-tls")] = { + "data": {"ca.crt": "Y2E="} + } + self.objects[("route", args[3])] = { + "status": { + "ingress": [ + {"conditions": [{"type": "Admitted", "status": "True"}]} + ] + } + } + if not any(r["name"] == args[3] for r in self.releases): + self.releases.append( + { + "name": args[3], + "chart": "helm-chart-0.1.3" + if args[3] == "openshell" + else "openshell-dashboard-0.1.0", + } + ) + elif "get" in args and "--raw" not in args: + start = args.index("get") + obj = self.objects.get(tuple(args[start + 1 : start + 3])) + return json.dumps(obj) if obj is not None else "" + elif "create" in args: + obj = json.loads(payload) + key = (obj["kind"].lower(), obj["metadata"]["name"]) + if key in self.objects: + raise RuntimeError("already exists") + self.objects[key] = obj + return "" + + +class InstallTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.config = configuration() + + def load(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + return installer.load_config(path) + + def cluster(self): + secret = self.root / "client.txt" + secret.write_text("a-secret-that-must-not-appear-in-argv\n") + self.config["oidc"]["client_secret_file"] = str(secret) + return FakeCluster(self.load()) + + def test_rejects_unpinned_versions_and_unknown_keys(self): + for version in ( + "latest", + "0.0.0", + "01.2.3", + "1.2.3-01", + ">=0.1.0", + "REPLACE_ME", + ): + with self.subTest(version=version): + self.config["gateway"]["version"] = version + with self.assertRaises(ValueError): + self.load() + self.config = configuration() + self.config["oidc"]["client_secret"] = "never-inline" + with self.assertRaisesRegex(ValueError, "Unknown"): + self.load() + + def test_pinned_downstream_and_new_release_versions(self): + for version in ("0.0.0-dev.e7fdd6b", "0.2.1", "1.2.3+rhaiv.5"): + self.config["gateway"]["version"] = version + self.assertEqual(self.load()["gateway"]["version"], version) + + def test_yaml_overlays_merge_in_order_and_preserve_managed_values(self): + (self.root / "base.yaml").write_text( + "resources:\n requests:\n cpu: 100m\n memory: 256Mi\n" + ) + (self.root / "site.yaml").write_text( + "resources:\n requests:\n cpu: 200m\nimagePullSecrets:\n - name: registry-creds\n" + ) + self.config["gateway"]["values_files"] = ["base.yaml", "site.yaml"] + self.config["oidc"]["scope"] = "openid email profile offline_access" + gateway, dashboard = installer.chart_values(self.load()) + self.assertEqual( + gateway["resources"]["requests"], {"cpu": "200m", "memory": "256Mi"} + ) + self.assertEqual(gateway["imagePullSecrets"], [{"name": "registry-creds"}]) + self.assertEqual( + gateway["server"]["sandboxNamespace"], self.config["namespace"] + ) + self.assertEqual( + dashboard["oidc"]["scope"], "openid email profile offline_access" + ) + + def test_overlays_cannot_replace_authentication_or_namespace_contract(self): + for name, overlay in ( + ("gateway", "server:\n auth:\n allowUnauthenticatedUsers: true"), + ("gateway", "server: null"), + ("gateway", "fullnameOverride: foreign"), + ("dashboard", "authProxy:\n extraArgs: [--skip-auth-regex=.*]"), + ("dashboard", "networkPolicy:\n enabled: false"), + ("dashboard", "oidc:\n existingSecret: foreign"), + ): + with self.subTest(overlay=overlay): + self.config = configuration() + (self.root / "site.yaml").write_text(overlay) + self.config[name]["values_files"] = ["site.yaml"] + with self.assertRaises(ValueError): + self.load() + + def test_scope_and_system_namespace_validation(self): + for scope in (None, [], "profile email", "openid\nemail"): + self.config["oidc"]["scope"] = scope + with self.assertRaises(ValueError): + self.load() + self.config = configuration() + for ns in ("default", "openshift", "openshift-ingress", "kube-system"): + self.config["namespace"] = ns + with self.assertRaisesRegex(ValueError, "system namespace"): + self.load() + + def test_preflight_storage_and_permissions_fail_before_mutation(self): + for issue in ("storage", "permission", "workspace-storage"): + cluster = self.cluster() + if issue == "storage": + cluster.storage = [] + elif issue == "permission": + cluster.denied_permission = "deployments.apps" + else: + cluster.config["gateway"]["values"] = { + "server": {"workspaceStorageClass": "missing"} + } + with self.subTest(issue=issue), self.assertRaises(ValueError): + cluster.install([], []) + self.assertFalse( + any( + "create" in args and "can-i" not in args + for args, _ in cluster.calls + ) + ) + self.assertFalse( + any(args[:2] == ["helm", "upgrade"] for args, _ in cluster.calls) + ) + + def test_cross_namespace_identity_reuse_is_rejected(self): + for field in ("oidc-audience", "oidc-client-id"): + cluster = self.cluster() + other = { + **cluster.identity_state(), + "oidc-audience": "other-api", + "oidc-client-id": "other-browser", + } + other[field] = cluster.identity_state()[field] + cluster.other_states = [ + { + "metadata": {"name": installer.INSTALL_STATE, "namespace": "other"}, + "data": other, + } + ] + with ( + self.subTest(field=field), + self.assertRaisesRegex(ValueError, "distinct per-gateway"), + ): + cluster.preflight() + cluster.other_states[0]["data"]["oidc-issuer"] = ( + "https://another-issuer.example" + ) + cluster.preflight() + + def test_identity_change_requires_migration(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + cluster.config["oidc"]["audience"] = "different-api" + with self.assertRaisesRegex(ValueError, "identity.*changed"): + cluster.preflight() + + def test_uninstalled_other_release_history_does_not_block_install(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [ + { + "name": "retired-gateway", + "chart": "helm-chart-0.1.3", + "status": "uninstalled", + } + ] + cluster.manifests["retired-gateway"] = self.gateway_manifest() + cluster.preflight() + self.assertFalse( + any(args[:3] == ["helm", "get", "manifest"] for args, _ in cluster.calls) + ) + cluster.objects[("secret", "openshell-server-tls")] = {"data": {}} + with self.assertRaisesRegex(ValueError, "explicit migration"): + cluster.preflight() + + def test_preflight_rejects_credential_drift_without_changing_state(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + before = copy.deepcopy(cluster.objects) + Path(cluster.config["oidc"]["client_secret_file"]).write_text("changed") + cluster.calls.clear() + with self.assertRaisesRegex(ValueError, "rotate"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any("patch" in args and "can-i" not in args for args, _ in cluster.calls) + ) + + def test_preflight_mode_renders_and_checks_without_installing(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + + def run(_instance, args, payload=None): + self.assertIsNone(payload) + if args[:2] == ["helm", "pull"]: + (Path(args[args.index("--destination") + 1]) / "chart.tgz").write_bytes( + b"fake" + ) + return "" + + with ( + patch("sys.argv", ["install.py", "--config", str(path), "--preflight"]), + patch.object(installer.shutil, "which", return_value="tool"), + patch.object(installer.Installer, "run", run), + patch.object(installer.Installer, "preflight") as preflight, + patch.object(installer.Installer, "install") as install, + ): + installer.main() + preflight.assert_called_once() + install.assert_not_called() + + def test_rejects_invalid_oidc_and_namespace(self): + for section, key, value in ( + ("oidc", "issuer", "http://issuer.example"), + ("oidc", "issuer", "https://user:password@issuer.example"), + ("oidc", "audience", configuration()["oidc"]["client_id"]), + ("oidc", "allowed_roles", ["admin"]), + ("gateway", "host", "gateway..example"), + ): + self.config = configuration() + self.config[section][key] = value + with self.subTest(key=key), self.assertRaises(ValueError): + self.load() + self.config = configuration() + self.config["namespace"] = "openshell/another" + with self.assertRaises(ValueError): + self.load() + + def test_generated_connection_and_identity_contract(self): + gateway, dashboard = installer.chart_values(self.load()) + self.assertIn( + "openshell.openshell-team.svc.cluster.local", + gateway["pkiInitJob"]["serverDnsNames"], + ) + self.assertEqual( + dashboard["gateway"]["url"], + "grpcs://openshell.openshell-team.svc.cluster.local:8080", + ) + self.assertEqual( + gateway["server"]["oidc"]["issuer"], dashboard["oidc"]["issuer"] + ) + self.assertFalse(gateway["server"]["tls"]["enableMtls"]) + self.assertIsNone(gateway["securityContext"]["runAsUser"]) + + def test_existing_admin_namespace_preserves_metadata_and_targets_releases(self): + self.config["namespace"] = "research-team" + cluster = self.cluster() + namespace = { + "metadata": { + "name": cluster.ns, + "labels": {"team": "research"}, + "annotations": {"openshift.io/sa.scc.uid-range": "1000100000/10000"}, + }, + "status": {"phase": "Active"}, + } + cluster.objects[("namespace", cluster.ns)] = copy.deepcopy(namespace) + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("namespace", cluster.ns)], namespace) + self.assertEqual( + {r["name"] for r in cluster.releases}, {"openshell", "openshell-dashboard"} + ) + for args, payload in cluster.calls: + if args[:2] == ["helm", "upgrade"]: + self.assertEqual(args[args.index("--namespace") + 1], "research-team") + if "create" in args and "can-i" not in args: + resource = json.loads(payload) + self.assertNotEqual(resource["kind"], "Namespace") + self.assertEqual(resource["metadata"]["namespace"], "research-team") + gateway, dashboard = installer.chart_values(cluster.config) + self.assertEqual(gateway["server"]["sandboxNamespace"], "research-team") + self.assertIn( + "openshell.research-team.svc", gateway["pkiInitJob"]["serverDnsNames"] + ) + self.assertEqual( + dashboard["gateway"]["url"], + "grpcs://openshell.research-team.svc.cluster.local:8080", + ) + + def test_namespace_override_validated_and_config_namespace_optional_with_override( + self, + ): + self.config.pop("namespace") + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + self.assertEqual( + installer.load_config(path, "admin-project")["namespace"], "admin-project" + ) + with self.assertRaisesRegex(ValueError, "Invalid namespace"): + installer.load_config(path, "invalid/namespace") + with self.assertRaisesRegex(ValueError, "namespace"): + installer.load_config(path) + + def test_existing_gateway_or_foreign_release_rejected_before_mutation(self): + for release in ( + {"name": "another-gateway", "chart": "helm-chart-0.1.3"}, + {"name": "openshell", "chart": "helm-chart-0.1.3"}, + {"name": "openshell-dashboard", "chart": "other-chart-1.0.0"}, + ): + with self.subTest(release=release): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [release] + with self.assertRaises(ValueError): + cluster.install([], []) + self.assertFalse( + any( + ("create" in args and "can-i" not in args) + or args[:2] == ["helm", "upgrade"] + for args, _ in cluster.calls + ) + ) + + @staticmethod + def gateway_manifest(kind="StatefulSet", container=None): + return json.dumps( + { + "apiVersion": "apps/v1", + "kind": kind, + "metadata": {"name": "renamed-gateway"}, + "spec": { + "replicas": 0, + "template": { + "spec": { + "containers": [ + container + or { + "name": "openshell-gateway", + "image": "registry.example/custom/runtime:1.0", + } + ] + } + }, + }, + } + ) + + def test_repackaged_gateway_rejected_before_mutation(self): + for kind, status in (("StatefulSet", "deployed"), ("Deployment", "failed")): + with self.subTest(kind=kind, status=status): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [ + {"name": "team-gateway", "chart": "custom-1.0.0", "status": status} + ] + cluster.manifests["team-gateway"] = self.gateway_manifest(kind) + before = copy.deepcopy(cluster.objects) + with self.assertRaisesRegex(ValueError, "one gateway per namespace"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any(args[:2] == ["helm", "upgrade"] for args, _ in cluster.calls) + ) + + def test_gateway_executable_detected_with_renamed_container(self): + cluster = self.cluster() + cluster.manifests["custom"] = self.gateway_manifest( + container={ + "name": "server", + "command": ["/usr/local/bin/openshell-gateway"], + } + ) + self.assertTrue(cluster.release_has_gateway("custom")) + + def test_unrelated_release_with_workload_is_preserved(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.releases = [{"name": "other-app", "chart": "custom-1.0.0"}] + cluster.manifests["other-app"] = self.gateway_manifest( + "Deployment", {"name": "web", "image": "registry.example/web:1.0"} + ) + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn("other-app", [release["name"] for release in cluster.releases]) + + def test_invalid_release_manifest_stops_without_echoing_content(self): + cluster = self.cluster() + cluster.manifests["custom"] = "secret: [do-not-print-this" + with self.assertRaisesRegex(ValueError, "Cannot inspect manifest") as caught: + cluster.release_has_gateway("custom") + self.assertNotIn("do-not-print-this", str(caught.exception)) + + def test_conflict_on_later_release_page_is_detected(self): + cluster = self.cluster() + cluster.releases = [ + {"name": f"other-{i}", "chart": "unrelated-1.0.0"} for i in range(256) + ] + cluster.releases.append( + {"name": "another-gateway", "chart": "helm-chart-0.1.3"} + ) + with self.assertRaisesRegex(ValueError, "one gateway"): + cluster.check_releases(managed=True) + + def test_terminating_namespace_is_rejected(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = { + "metadata": {}, + "status": {"phase": "Terminating"}, + } + with self.assertRaisesRegex(ValueError, "terminating"): + cluster.install([], []) + + def test_preexisting_gateway_certificate_is_not_reused(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.objects[("secret", "openshell-server-tls")] = { + "data": {"tls.crt": "old"} + } + with self.assertRaisesRegex(ValueError, "explicit migration"): + cluster.install([], []) + + def test_legacy_install_migrates_without_modifying_namespace(self): + cluster = self.cluster() + namespace = { + "metadata": { + "labels": {installer.OWNER: installer.OWNER_VALUE}, + "annotations": { + installer.GATEWAY_HOST: cluster.config["gateway"]["host"] + }, + } + } + cluster.objects[("namespace", cluster.ns)] = copy.deepcopy(namespace) + cluster.releases = [{"name": "openshell", "chart": "helm-chart-0.1.3"}] + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("namespace", cluster.ns)], namespace) + self.assertEqual( + cluster.objects[("configmap", installer.INSTALL_STATE)]["data"][ + "gateway-host" + ], + cluster.config["gateway"]["host"], + ) + + def test_missing_prerequisite_stops_before_mutation(self): + cluster = self.cluster() + cluster.objects.clear() + with self.assertRaisesRegex(ValueError, "Agent Sandbox"): + cluster.install([], []) + self.assertFalse( + any("create" in args and "can-i" not in args for args, _ in cluster.calls) + ) + + def test_install_order_context_and_secret_transport(self): + cluster = self.cluster() + cluster.install(["gateway.tgz", "dashboard.tgz"], ["g.json", "d.json"]) + helm = [args for args, _ in cluster.calls if args[0] == "helm"] + self.assertEqual( + [args[3] for args in helm], ["openshell", "openshell-dashboard"] + ) + for args, _ in cluster.calls: + self.assertNotIn("a-secret-that", " ".join(args)) + self.assertIn("test-context", args) + self.assertNotIn("--atomic", args) + self.assertNotIn("delete", args) + data = cluster.objects[("secret", installer.OIDC_SECRET)]["data"] + self.assertEqual( + base64.b64decode(data["client-secret"]), + b"a-secret-that-must-not-appear-in-argv", + ) + cookie = base64.b64decode(data["cookie-secret"]).decode() + self.assertEqual(len(base64.urlsafe_b64decode(cookie + "=")), 32) + + def test_rerun_preserves_secrets_without_input_file(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + before = copy.deepcopy(cluster.objects[("secret", installer.OIDC_SECRET)]) + cluster.config["oidc"].pop("client_secret_file") + cluster.calls.clear() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects[("secret", installer.OIDC_SECRET)], before) + self.assertFalse( + any("create" in args and "can-i" not in args for args, _ in cluster.calls) + ) + + def test_first_install_requires_supplied_secret_even_with_existing_secret(self): + for supplied in (None, "different", "matching"): + with self.subTest(supplied=supplied): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + existing = { + "data": { + "client-secret": base64.b64encode(b"matching").decode(), + "cookie-secret": base64.b64encode(b"existing-cookie").decode(), + } + } + cluster.objects[("secret", installer.OIDC_SECRET)] = copy.deepcopy( + existing + ) + path = Path(cluster.config["oidc"]["client_secret_file"]) + if supplied is None: + cluster.config["oidc"].pop("client_secret_file") + else: + path.write_text(supplied) + before = copy.deepcopy(cluster.objects) + if supplied == "matching": + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual( + cluster.objects[("secret", installer.OIDC_SECRET)], existing + ) + else: + expected = ( + "required for the first install" + if supplied is None + else "rotate" + ) + with self.assertRaisesRegex(ValueError, expected): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertEqual(cluster.objects, before) + + def test_new_namespace_preflight_checks_files_without_namespaced_reads(self): + (self.root / "issuer.pem").write_text("test CA") + self.config["oidc"]["ca_file"] = "issuer.pem" + cluster = self.cluster() + before = copy.deepcopy(cluster.objects) + self.assertIsNone(cluster.preflight()) + self.assertEqual(cluster.objects, before) + self.assertFalse( + any("-n" in args and "get" in args for args, _ in cluster.calls) + ) + path = Path(cluster.config["oidc"]["client_secret_file"]) + secret = path.read_text() + path.write_text("\n") + with self.assertRaisesRegex(ValueError, "contains no secret"): + cluster.preflight() + self.assertEqual(cluster.objects, before) + path.write_text(secret) + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn(("namespace", cluster.ns), cluster.objects) + self.assertIn(("secret", installer.OIDC_SECRET), cluster.objects) + self.assertIn(("configmap", installer.OIDC_CA), cluster.objects) + + def test_changed_gateway_host_requires_certificate_migration(self): + cluster = self.cluster() + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + cluster.config["gateway"]["host"] = "other.apps.example.com" + cluster.calls.clear() + with self.assertRaisesRegex(ValueError, "certificate migration"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertFalse( + any( + args[0] == "helm" or ("create" in args and "can-i" not in args) + for args, _ in cluster.calls + ) + ) + + def test_different_secret_is_not_silently_rotated(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.credentials() + before = copy.deepcopy(cluster.objects) + Path(cluster.config["oidc"]["client_secret_file"]).write_text("changed") + with self.assertRaisesRegex(ValueError, "rotate"): + cluster.credentials() + self.assertEqual(cluster.objects, before) + + def test_private_ca_and_relative_path(self): + (self.root / "issuer.pem").write_text("test CA") + self.config["oidc"]["ca_file"] = "issuer.pem" + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + cluster.credentials() + gateway, dashboard = installer.chart_values(cluster.config) + self.assertEqual( + gateway["server"]["oidc"]["caConfigMapName"], installer.OIDC_CA + ) + self.assertEqual(dashboard["oidc"]["caConfigMapName"], installer.OIDC_CA) + self.assertEqual( + cluster.objects[("configmap", installer.OIDC_CA)]["data"]["ca.crt"], + "test CA", + ) + + def test_dashboard_failure_keeps_gateway_and_credentials(self): + cluster = self.cluster() + cluster.fail_dashboard = True + with self.assertRaisesRegex(RuntimeError, "dashboard failed"): + cluster.install(["g.tgz", "d.tgz"], ["g.json", "d.json"]) + self.assertIn(("namespace", cluster.ns), cluster.objects) + self.assertIn(("secret", installer.OIDC_SECRET), cluster.objects) + self.assertIn(("secret", "openshell-client-tls"), cluster.objects) + self.assertFalse( + any("delete" in args or "uninstall" in args for args, _ in cluster.calls) + ) + + def test_route_admission_waits_and_times_out(self): + cluster = self.cluster() + cluster.objects[("namespace", cluster.ns)] = {"metadata": {}} + admitted = { + "status": { + "ingress": [{"conditions": [{"type": "Admitted", "status": "True"}]}] + } + } + with ( + patch.object(cluster, "get", side_effect=[{}, admitted]), + patch.object(installer.time, "sleep") as sleep, + ): + cluster.wait_for_route("openshell") + sleep.assert_called_once() + with ( + patch.object(installer.time, "monotonic", side_effect=[0, 601]), + self.assertRaisesRegex(ValueError, "not admitted"), + ): + cluster.wait_for_route("openshell") + + def test_render_failure_prevents_cluster_access(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + commands = [] + + def run(_instance, args, payload=None): + self.assertIsNone(payload) + commands.append(args) + if args[:2] == ["helm", "pull"]: + destination = Path(args[args.index("--destination") + 1]) + (destination / "chart.tgz").write_bytes(b"fake") + return "" + raise RuntimeError("invalid chart values") + + with ( + patch("sys.argv", ["install.py", "--config", str(path)]), + patch.object(installer.shutil, "which", return_value="tool"), + patch.object(installer.Installer, "run", run), + self.assertRaisesRegex(RuntimeError, "invalid chart values"), + ): + installer.main() + self.assertTrue(all(args[0] == "helm" for args in commands)) + + def test_check_only_never_calls_kubectl(self): + path = self.root / "config.json" + path.write_text(json.dumps(self.config)) + for chart in ("gateway", "dashboard"): + (self.root / chart).mkdir() + (self.root / chart / "Chart.yaml").write_text("version: 0.0.0\n") + argv = [ + "install.py", + "--config", + str(path), + "--check", + "--namespace", + "render-project", + "--gateway-chart-directory", + str(self.root / "gateway"), + "--dashboard-chart-directory", + str(self.root / "dashboard"), + ] + with ( + patch("sys.argv", argv), + patch.object(installer.shutil, "which", return_value="helm"), + patch.object(installer.Installer, "run", return_value="") as run, + ): + installer.main() + self.assertEqual(run.call_count, 2) + for call in run.call_args_list: + self.assertEqual(call.args[0][:2], ["helm", "template"]) + args = call.args[0] + self.assertEqual(args[args.index("--namespace") + 1], "render-project") + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/rhoai/test_verify.py b/deploy/rhoai/test_verify.py new file mode 100644 index 0000000000..f562169b90 --- /dev/null +++ b/deploy/rhoai/test_verify.py @@ -0,0 +1,131 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +"""Authorization verifier regression tests; no cluster or identity provider.""" + +import importlib.util +import io +import threading +import unittest +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from unittest.mock import Mock, patch + +spec = importlib.util.spec_from_file_location( + "verifier", Path(__file__).with_name("verify.py") +) +verifier = importlib.util.module_from_spec(spec) +spec.loader.exec_module(verifier) + + +class VerifyTests(unittest.TestCase): + def response(self, url, token=None, **_kwargs): + if url.startswith("https:") or not token: + return 401, None + if url.endswith("whoami"): + return 200, { + "subject": token, + "roles": ["admin"] if token == "admin-token" else ["user"], + } + if token == "other-token": + return 401, None + if token == "user-token" and "/private/" in url: + return 403, None + return 200, [] + + def run_verify(self, http): + with redirect_stdout(io.StringIO()) as output: + verifier.verify( + "http://127.0.0.1:1234", + "https://dashboard.example", + "admin-token", + "user-token", + "team", + "private", + "admin", + other="other-token", + http=http, + ) + self.assertNotIn("admin-token", output.getvalue()) + self.assertNotIn("user-token", output.getvalue()) + + def test_allowed_and_denied_access(self): + self.run_verify(self.response) + + def test_false_positive_authorization_results_fail(self): + for failure in ( + "admin", + "user", + "missing-workspace", + "public-redirect", + "missing-subject", + "wrong-audience", + ): + + def response(url, token=None, failure=failure, **kwargs): + if failure == "admin" and token == "admin-token": + return 403, None + if failure == "user" and token == "user-token" and "/private/" in url: + return 200, [] + if failure == "missing-workspace" and "/private/" in url: + return 404, None + if failure == "public-redirect" and url.startswith("https:"): + return 302, None + if failure == "missing-subject" and url.endswith("whoami"): + return 200, {"roles": ["admin"]} + if failure == "wrong-audience" and token == "other-token": + return 200, [] + return self.response(url, token, **kwargs) + + with self.subTest(failure=failure), self.assertRaises(ValueError): + self.run_verify(response) + + def test_redirect_does_not_forward_credentials(self): + received = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + received.append((self.path, self.headers.get("Authorization"))) + self.send_response(302) + self.send_header("Location", "/leak") + self.end_headers() + + def log_message(self, *_args): + pass + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + status, _ = verifier.request( + f"http://127.0.0.1:{server.server_port}/start", + token="private-token", + ) + self.assertEqual(status, 302) + self.assertEqual(received, [("/start", "Bearer private-token")]) + finally: + server.shutdown() + thread.join() + + def test_port_forward_is_terminated_when_verification_fails(self): + process = Mock() + process.poll.return_value = None + + def start(_args, stdout, **_kwargs): + stdout.write("Forwarding from 127.0.0.1:4567 -> 8080\n") + stdout.flush() + return process + + with ( + patch.object(verifier.subprocess, "Popen", side_effect=start), + self.assertRaisesRegex(ValueError, "test failure"), + verifier.forward("context", "team") as url, + ): + self.assertEqual(url, "http://127.0.0.1:4567") + raise ValueError("test failure") + process.terminate.assert_called_once() + process.wait.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/deploy/rhoai/verify.py b/deploy/rhoai/verify.py new file mode 100644 index 0000000000..34df477a90 --- /dev/null +++ b/deploy/rhoai/verify.py @@ -0,0 +1,264 @@ +#!/usr/bin/env -S uv run --script +# SPDX-FileCopyrightText: Copyright (c) 2026 OpenShell contributors +# SPDX-License-Identifier: Apache-2.0 +# /// script +# requires-python = ">=3.11" +# dependencies = [] +# /// +"""Read-only gateway authorization checks through the dashboard BFF and public Route.""" + +import argparse +import contextlib +import json +import re +import shutil +import ssl +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote, urlsplit +from urllib.request import ( + HTTPRedirectHandler, + HTTPSHandler, + ProxyHandler, + Request, + build_opener, +) + + +class NoRedirect(HTTPRedirectHandler): + def redirect_request(self, _req, _fp, _code, _msg, _headers, _newurl): + # Never forward an access token or session cookie to a redirect target. + return None + + +def secret_file(path): + value = path.read_text().strip() + if not value or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise ValueError("Credential file must contain one nonempty header value") + return value + + +def request(url, token=None, cookie=None, ca_file=None): + headers = {} + if token: + headers["Authorization"] = f"Bearer {token}" + if cookie: + headers["Cookie"] = cookie + context = ssl.create_default_context() + if ca_file: + context.load_verify_locations(cafile=str(ca_file)) + # Loopback traffic must never pass through an environment-configured proxy. + handlers = [NoRedirect(), HTTPSHandler(context=context)] + if urlsplit(url).hostname == "127.0.0.1": + handlers.append(ProxyHandler({})) + try: + with build_opener(*handlers).open( + Request(url, headers=headers), timeout=30 + ) as response: + body = response.read(1024 * 1024) + try: + data = json.loads(body) + except ValueError: + data = None + return response.status, data + except HTTPError as error: + error.close() + return error.code, None + except (URLError, OSError): + raise ValueError( + "HTTP connection or TLS verification failed; check endpoint reachability and CA trust" + ) from None + + +@contextlib.contextmanager +def forward(context, namespace): + # kubectl allocates the port. Keep its diagnostics private and never write + # tokens into argv, environment variables, or a subprocess's stdin. + with tempfile.TemporaryFile(mode="w+") as output: + process = subprocess.Popen( + [ + "kubectl", + "--context", + context, + "-n", + namespace, + "port-forward", + "--address=127.0.0.1", + "deployment/openshell-dashboard", + ":8080", + ], + stdout=output, + stderr=subprocess.STDOUT, + ) + try: + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + output.seek(0) + match = re.search( + r"Forwarding from 127\.0\.0\.1:(\d+) -> 8080", output.read() + ) + if match: + yield f"http://127.0.0.1:{match[1]}" + return + if process.poll() is not None: + break + time.sleep(0.2) + raise ValueError( + "Could not port-forward the dashboard; check context, namespace, pod readiness and pods/portforward permissions" + ) + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +def verify( + base, + public, + admin, + user, + allowed, + denied, + admin_role, + other=None, + cookie=None, + ca_file=None, + http=request, +): + failures = [] + + def check(label, url, expected, **kwargs): + status, data = http(url, **kwargs) + passed = status in expected + print(f"{'PASS' if passed else 'FAIL'}: {label} (HTTP {status})") + if not passed: + failures.append(label) + return data + + def identity(label, token, is_admin): + data = check(label, f"{base}/api/v1/auth/whoami", (200,), token=token) + valid = ( + isinstance(data, dict) + and bool(data.get("subject")) + and isinstance(data.get("roles"), list) + and (admin_role in data["roles"]) == is_admin + ) + if not valid: + failures.append(f"{label} identity/role") + print(f"FAIL: {label} identity/role") + return data.get("subject") if isinstance(data, dict) else None + + check( + "public API rejects missing session", + f"{public}/api/v1/workspaces", + (401,), + ca_file=ca_file, + ) + check("gateway rejects missing token", f"{base}/api/v1/workspaces", (401,)) + admin_subject = identity("admin token", admin, True) + user_subject = identity("ordinary-user token", user, False) + if admin_subject == user_subject: + failures.append("admin and user must be distinct identities") + for name in (allowed, denied): + path = f"/api/v1/workspaces/{quote(name, safe='')}/sandboxes" + # Establish that BOTH workspaces exist and are accessible to the admin, + # so a missing workspace cannot count as isolation success. + check(f"admin accesses {name}", base + path, (200,), token=admin) + check( + f"ordinary user {'accesses' if name == allowed else 'is denied in'} {name}", + base + path, + (200,) if name == allowed else (403,), + token=user, + ) + if other: + check( + "other-gateway token is rejected", + f"{base}/api/v1/workspaces", + (401, 403), + token=other, + ) + if cookie: + data = check( + "browser session reaches public API", + f"{public}/api/v1/auth/whoami", + (200,), + cookie=cookie, + ca_file=ca_file, + ) + if not isinstance(data, dict) or not data.get("subject"): + failures.append("browser session identity") + if failures: + raise ValueError(f"{len(failures)} authorization check(s) failed") + print( + "Authorization checks passed. Browser login, grant/revoke, sandbox lifecycle and terminal checks require separate validation." + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ( + "context", + "namespace", + "dashboard-url", + "allowed-workspace", + "denied-workspace", + "admin-role", + ): + parser.add_argument(f"--{name}", required=True) + for name in ("admin-token-file", "user-token-file"): + parser.add_argument(f"--{name}", type=Path, required=True) + for name in ("other-gateway-token-file", "session-cookie-file", "ca-file"): + parser.add_argument(f"--{name}", type=Path) + args = parser.parse_args() + url = urlsplit(args.dashboard_url) + if ( + url.scheme != "https" + or not url.hostname + or url.username + or url.password + or url.path not in ("", "/") + or url.query + or url.fragment + ): + parser.error("dashboard-url must be an HTTPS origin without credentials") + if args.allowed_workspace == args.denied_workspace: + parser.error("allowed and denied workspaces must be different") + if not shutil.which("kubectl"): + parser.error("kubectl is required") + admin = secret_file(args.admin_token_file) + user = secret_file(args.user_token_file) + other = ( + secret_file(args.other_gateway_token_file) + if args.other_gateway_token_file + else None + ) + cookie = secret_file(args.session_cookie_file) if args.session_cookie_file else None + with forward(args.context, args.namespace) as base: + verify( + base, + args.dashboard_url.rstrip("/"), + admin, + user, + args.allowed_workspace, + args.denied_workspace, + args.admin_role, + other, + cookie, + args.ca_file, + ) + + +if __name__ == "__main__": + try: + main() + except (ValueError, OSError) as error: + print(f"Verification stopped: {error}", file=sys.stderr) + sys.exit(1) diff --git a/docs/kubernetes/setup.mdx b/docs/kubernetes/setup.mdx index 53ecd932c5..ba7033f838 100644 --- a/docs/kubernetes/setup.mdx +++ b/docs/kubernetes/setup.mdx @@ -88,6 +88,22 @@ OpenShell detects the served Agent Sandbox `Sandbox` API when the Kubernetes gat ## Install OpenShell +For an administrator-managed OpenShift deployment with a browser UI, the ODH +fork provides a [gateway and dashboard installer](https://github.com/opendatahub-io/openshell/tree/main/deploy/rhoai). +It installs separately versioned gateway and dashboard charts using customer OIDC +in the administrator-selected namespace (`namespace` in the configuration or +`--namespace` on the command line), creating it only if missing. Existing namespace +metadata is preserved. Each namespace has one gateway/dashboard pair, which reruns +upgrade in place. Agent Sandbox remains a prerequisite. +The installer includes a render-only check, preserves resources on failure, and +requires explicit chart versions. Validate login, workspace authorization, +sandbox lifecycle, and terminal access on your target cluster before sharing the +deployment; chart rendering does not establish RHOAI qualification. +The installer accepts operational Helm values files and custom OIDC scopes, and +provides a read-only cluster preflight and token-based authorization verifier. +Provision distinct OIDC audiences and browser clients for separate gateways; +namespace separation alone does not establish identity isolation. + ## Create the namespace diff --git a/skills/debug-openshell-cluster/SKILL.md b/skills/debug-openshell-cluster/SKILL.md index 418b67ec51..ae91455fe3 100644 --- a/skills/debug-openshell-cluster/SKILL.md +++ b/skills/debug-openshell-cluster/SKILL.md @@ -92,6 +92,37 @@ Use gateway metadata, deployment values, or the user's setup notes to identify t ### Step 3: Check Gateway Startup Dependencies +For an ODH gateway-and-dashboard installation, inspect both Helm releases +(`openshell` and `openshell-dashboard`) in the administrator-selected namespace +(`namespace` in the configuration or the `--namespace` override). Existing +namespaces do not need an installer ownership label. The installer +preserves partial installations and credentials on failure; gateway readiness +does not establish dashboard login or sandbox readiness. Check the +`openshell-installer` state ConfigMap, the dashboard OIDC Secret, the shared +gateway CA Secret, and admission of both Routes. A gateway chart under another +release name or an existing fixed release without installer state requires +explicit migration; do not change state to bypass these checks. Detection also +inspects stored Helm workloads for the `openshell-gateway` container or executable, +including failed and scaled-to-zero releases with renamed charts. Custom packaging +that removes both markers and workloads outside Helm require manual inspection. +A first installation requires a client-secret file matching any existing dashboard +OIDC Secret; only an installation with recorded state can omit the file. +For a new namespace, credential preflight checks local files without querying +namespaced objects before namespace creation. Earlier +installations used namespace annotations for state, which reruns migrate while +preserving namespace metadata. Use the published +[Kubernetes setup guide](https://docs.nvidia.com/openshell/latest/kubernetes/setup.md) +for the installation entry point. Diagnose Agent Sandbox availability separately; +the installer does not provision that controller or RHOAI itself. +Its read-only preflight checks storage, installation permissions, and recorded +OIDC identities across installer-managed namespaces. Distinct gateways need +distinct audiences and browser clients. Legacy state without OIDC metadata needs +manual review; retained history for another uninstalled release is not an active +gateway. Operational values files are reapplied on every upgrade; authentication, +TLS, naming, and namespace overrides are rejected. The deployment guide also +provides a read-only verifier for admin/ordinary-user workspace authorization; +it does not establish browser login, sandbox lifecycle, or network-policy enforcement. + Before debugging the compute platform, inspect gateway logs for failures in dependencies initialized before the listener becomes ready. The gateway container uses a Distroless Debian runtime. For OS-library