From cc92b17feb782c85bbc9f87acb37f7a3ae694c93 Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 4 Sep 2026 16:33:36 +0100 Subject: [PATCH] feat(data-app): let `secrets-set` write plain (unencrypted) env vars (#738) `parameters.dataApp.secrets` has always held two kinds of entry, and the Keboola UI offers exactly that choice ("optional encryption"): a `#`-prefixed key is encrypted under the project KMS, a bare key is a plain env var whose value stays readable. `secrets-list` / `secrets-get` / `secrets-remove` have handled both since 0.43.9 -- only WRITING a plain key was impossible, because `secrets-set` demanded the `#` and encrypted unconditionally. Storing a non-secret (a Storage Files tag, a feature flag, a log level) then had two bad options: encrypt it -- write-only, since the Encryption API has no decrypt endpoint, so nobody can ever read back what it was set to -- or `sync pull`, hand-edit `_config.yml`, `sync push`. - The `#` now DECIDES encryption instead of being mandatory: `#KEY=VALUE` is encrypted (unchanged), `KEY=VALUE` is written verbatim. Same rule for `--secrets-file`. - A payload with no `#` key never calls the Encryption API at all, so an unreachable API cannot fail a write that needs no encryption. - A `KBC::` value is still rejected under EITHER form: under `#` it would double-encrypt, under a bare key it would park a foreign project's ciphertext where nothing can decrypt it. - `--allow-plaintext-on-encrypt-failure` is unrelated and unchanged -- it covers a `#` key whose encryption FAILED, and `plaintext_written` stays distinct from the new `plaintext_keys`. - Response gains `encrypted_keys` / `plaintext_keys`; `secrets_set` still lists both, so existing parsers are unaffected. `secrets-list` entries gain `encrypted` and `value` (`null` for every encrypted entry) and human output prints plain values inline. The validation loop moved to `partition_secret_entries` in `services/_data_app_bodies.py` -- `data_app_service.py` is grandfathered by the file-size gate and may only shrink. Tested: 9 new unit tests (plain-only write skips encryption, mixed payload encrypts only the `#` half, existing entries of both kinds survive, `KBC::` refusal on a bare key, dry-run masking, list tagging, CLI plumbing) plus a live E2E case asserting the plain value lands verbatim in the config while its sibling lands as `KBC::` ciphertext. `make check` green. --- CLAUDE.md | 18 +- plugins/kbagent/skills/kbagent/SKILL.md | 2 +- .../kbagent/references/commands-reference.md | 4 +- .../skills/kbagent/references/gotchas.md | 52 +++++ .../commands/_data_app_runtime.py | 58 +++-- src/keboola_agent_cli/commands/context.py | 32 ++- .../services/_data_app_bodies.py | 55 +++++ .../services/data_app_service.py | 121 ++++++----- tests/test_data_app_plain_env_keys.py | 199 +++++++++++++++++- tests/test_data_app_secrets_service.py | 5 +- tests/test_e2e.py | 127 +++++++++++ 11 files changed, 590 insertions(+), 83 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 376610b79..f3bb3c90c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -807,7 +807,23 @@ kbagent data-app delete --project NAME --app-id ID [--yes] kbagent data-app password --project NAME --app-id ID kbagent data-app logs --project NAME --app-id ID [--lines N] [--since ISO8601] kbagent data-app runs --project NAME --app-id ID [--limit N] -kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] +kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE'|'KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] +# secrets-set plaintext entries (since vNEXT, #738): the '#' prefix now DECIDES encryption +# instead of being mandatory, mirroring the UI's "optional encryption" checkbox. +# '#KEY=VALUE' is encrypted under the project KMS (unchanged); 'KEY=VALUE' (no '#') is +# written VERBATIM as a plain env var, so `secrets-list` / `config detail` / the UI show +# the real value instead of . Use it for non-secret runtime settings (a Storage +# Files tag, a feature flag, a log level) -- before vNEXT the only ways were to encrypt a +# non-secret (invisible to everyone) or to `sync pull` + hand-edit _config.yml + `sync push`. +# Both halves of `parameters.dataApp.secrets` were always readable/removable; only WRITING a +# plain key was impossible. A payload with NO '#' key never calls the Encryption API at all, +# so an unreachable Encryption API cannot fail a write that needs no encryption. A `KBC::` +# value is still rejected under EITHER form (double-encryption / a foreign project's +# ciphertext parked where nothing can decrypt it). --allow-plaintext-on-encrypt-failure is +# unrelated and unchanged: it covers a '#' key whose encryption FAILED, never a deliberate +# plain key. New response keys `encrypted_keys` / `plaintext_keys` (env-var names); +# `secrets_set` still lists both. `secrets-list` gains per-entry `encrypted` + `value` +# (`value` is null for encrypted entries -- the CLI never decrypts). kbagent data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint] kbagent data-app secrets-get --project ALIAS --app-id ID --key 'KEY' [--branch ID] # '#' optional; plain values return their value, encrypted return metadata only kbagent data-app secrets-remove --project ALIAS --app-id ID --key 'KEY' [--key ...] [--branch ID] [--yes] [--dry-run] # '#' optional diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 5b70f8ad0..0529878ff 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -149,7 +149,7 @@ When working inside a git repository or project directory, run `kbagent init` (o | Show the clone URLs of a data app's configured git repository | `kbagent data-app git-repo --project PROJECT --app-id APP-ID` | | List the credentials of a data app's MANAGED git repository | `kbagent data-app git-credentials --project PROJECT --app-id APP-ID` | | Create a git credential (SSH key or HTTP token) for a MANAGED repo | `kbagent data-app git-credentials-create --project PROJECT --app-id APP-ID --type CRED-TYPE --permissions PERMISSIONS` | -| Encrypt and write app-runtime secrets to the linked Storage config | `kbagent data-app secrets-set --project PROJECT --app-id APP-ID` | +| Write app-runtime env vars to the linked Storage config | `kbagent data-app secrets-set --project PROJECT --app-id APP-ID` | | List the keys in parameters.dataApp.secrets, with derived runtime env-var names | `kbagent data-app secrets-list --project PROJECT --app-id APP-ID` | | Show ONE key from parameters.dataApp.secrets | `kbagent data-app secrets-get --project PROJECT --app-id APP-ID --key KEY` | | Remove one or more app-runtime secrets. | `kbagent data-app secrets-remove --project PROJECT --app-id APP-ID --key KEY` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 83281d11d..00a126751 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -266,8 +266,8 @@ Lifecycle for `keboola.data-apps`. Combines Storage API (config body, git block, - `data-app password --project NAME --app-id ID` -- read the simpleAuth password. Manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+. Auto-generated, not rotatable -- delete + recreate to mint a new one. - `data-app logs --project NAME --app-id ID [--lines N] [--since ISO8601]` -- tail container logs (Data Science `/apps/{id}/logs/tail`). Plain-text body covering the full spin-up trace ([TIMING] git_clone, Cloning into /app, uv install, supervisord, runtime stack traces). Default `--lines 500`; pass `--lines 0` for the full current buffer (no server-side cap). `--lines` and `--since` are mutually exclusive on the server; `--since` requires a timezone (Z or +00:00). App must be running or recently-stopped — never-started apps return 400 "App X is not running" (recover with `data-app start` or `data-app deploy`). Closes the upstream `keboola-mcp-server` gap where `get_data_apps` hardcodes a 20-line cap; this CLI surface is unconstrained. The log buffer can echo runtime secrets the app printed to stdout/stderr — consider hygiene before piping `--json` output into AI agent context. - `data-app runs --project NAME --app-id ID [--limit N]` -- list deployment attempts newest-first (Data Science `/apps/{id}/runs`), each with `failure_reason` + `startup_logs`. Captures setup-phase failures (e.g. git-clone errors) that produce NO container logs, so unlike `data-app logs` it works on never-started / failed apps where `data-app logs` returns HTTP 400. This is the way to find WHY a deploy reverted to stopped. Auth: ordinary project storage token only. -- `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- encrypt and write `#`-prefixed secrets to `parameters.dataApp.secrets`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. -- `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. +- `data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE'|'KEY=VALUE' [--secret ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next]` -- write env-var entries to `parameters.dataApp.secrets`. **Since vNEXT the `#` prefix DECIDES encryption instead of being mandatory**, mirroring the UI's "optional encryption" checkbox: `'#KEY=VALUE'` is encrypted under the project KMS, `'KEY=VALUE'` is written verbatim as a plain env var whose value stays readable in `secrets-list` / `config detail` / the UI. Use the plain form for non-secret runtime settings (a Storage Files tag, a feature flag, a log level) -- before vNEXT the only options were to encrypt a non-secret (invisible to everyone) or to `sync pull` + hand-edit `_config.yml` + `sync push`; reading and removing plain keys always worked, only writing them did not. A payload with NO `#` key never calls the Encryption API at all. A `KBC::` value is rejected under EITHER form. `--allow-plaintext-on-encrypt-failure` is unrelated and unchanged: it covers a `#` key whose encryption FAILED, never a deliberate plain key. Response adds `encrypted_keys` / `plaintext_keys` beside the combined `secrets_set`. Per-project KMS encryption, fail-closed. Read-modify-write at the service layer (NOT Storage `merge=True` -- shallow). Runtime exposes each key as an env var with `#` stripped, `-` -> `_`, uppercased. Adding bumps the Storage version; the running container keeps the OLD config until the next `data-app deploy`. +- `data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint]` -- list secret keys + derived runtime env-var names. Never echoes encrypted ciphertext in full. `--show-fingerprint` opt-in for a short ciphertext fingerprint. Since vNEXT each entry carries `encrypted` and `value`: a PLAIN entry shows its literal value (already stored in clear, visible via `config detail`), an ENCRYPTED entry always has `value: null` -- the CLI never decrypts. - `data-app secrets-get --project ALIAS --app-id ID --key 'KEY' [--branch ID]` -- show ONE key from `parameters.dataApp.secrets`. The leading `#` is OPTIONAL (since v0.43.9); the block holds both encrypted secrets (`#`) and plain unencrypted env-var values, and `secrets-list` enumerates both. For an ENCRYPTED secret it stays metadata-only (`encrypted: true`, `value: null`, fingerprint/prefix) -- the decrypted plaintext is NEVER echoed (Encryption API is one-way). For a PLAIN value it returns the literal value (`encrypted: false`), which is already visible via `config detail`. NOT_FOUND on absent key (exact match, no `#KEY`<->`KEY` fuzzing); never enumerates siblings. - `data-app secrets-remove --project ALIAS --app-id ID --key 'KEY' [--key ...] [--branch ID] [--yes] [--dry-run]` -- destructive (can break a running app at next deploy). Leading `#` OPTIONAL: removes both encrypted secrets and plain env-var keys. Idempotent: missing keys exit 0 with `removed: 0`. - `data-app validate-repo --git-repo URL [--git-branch BRANCH] [--git-public/--no-git-public] [--git-pat-env VAR | --git-pat-file PATH] [--type python-js] [--strict]` -- pre-flight Golden-Rule check for a data-app git repo (https://help.keboola.com/data-apps/python-js/). GitHub-only; ≤5 API calls (1 tree + ≤4 contents) regardless of repo size. `--type` restricted to `python-js` in 0.28.0; streamlit / pure-Python / R / Node-only follow-up. `--strict` treats WARNs as failures. Since 0.88.0 (#636) the two `setup.sh` rules (`golden-rule.setup-sh-no-pip`, `golden-rule.setup-sh-uv-sync`) match against **comment-stripped code**: a comment reading `# never pip install` no longer BLOCKS, and a comment merely mentioning `uv sync` no longer satisfies the uv-sync rule diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 750daf78d..e049776f1 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -4501,6 +4501,58 @@ the web UI. loudly when it is off. The REST surface mirrors the flag as `"workspace": true` in the `POST /data-apps/{project}` body. +## `data-app secrets-set` writes plain env vars too -- the `#` decides encryption + +*(since vNEXT, #738)* + +`parameters.dataApp.secrets` has always held **two** kinds of entry, and the +Keboola UI offers exactly that choice ("optional encryption"): a `#`-prefixed +key is encrypted under the project KMS, a bare key is a plain env var whose +value stays readable. `secrets-list`, `secrets-get` and `secrets-remove` +handled both since 0.43.9. **Only writing a plain key was impossible** -- +`secrets-set` demanded the `#` and encrypted unconditionally: + +```bash +# before vNEXT +kbagent data-app secrets-set ... --secret 'SCORING_BACKUP_TAG=ppl-assessment-db' +# rejected: entries must be '#KEY=VALUE' +kbagent data-app secrets-set ... --secret '#SCORING_BACKUP_TAG=ppl-assessment-db' +# accepted -- and the tag is now to everyone, including the operator +``` + +The workaround was `sync pull`, hand-editing `_config.yml`, `sync push`. Now: + +```bash +kbagent data-app secrets-set --project P --app-id 74021026 \ + --secret '#API_KEY=s3cr3t' \ + --secret 'SCORING_BACKUP_TAG=ppl-assessment-db' +``` + +- **The `#` is the switch, not a syntax requirement.** `#KEY=VALUE` encrypts + (behaviour unchanged); `KEY=VALUE` writes the value verbatim. Put non-secret + runtime configuration -- a Storage Files tag, a feature flag, a log level -- + in the plain form so `secrets-list`, `config detail` and the UI show the real + value instead of ``. +- **Encrypting a non-secret is not a safe default.** It is write-only: the + Encryption API has no decrypt endpoint, so once a tag is encrypted **nobody** + -- operator, UI or CLI -- can read back what it was set to. +- **A payload with no `#` key never calls the Encryption API.** An unreachable + Encryption API must not fail a write that needs no encryption. +- **A `KBC::` value is still rejected under EITHER form.** Under `#` it would + double-encrypt; under a bare key it would park a foreign project's ciphertext + where nothing can ever decrypt it. +- **`--allow-plaintext-on-encrypt-failure` is unrelated and unchanged.** It + covers a `#` key whose *encryption failed*, and still warns loudly. It has + nothing to do with a deliberate plain key, and the response's + `plaintext_written` (encryption failures) stays distinct from the new + `plaintext_keys` (deliberate plain entries). +- **New response keys:** `encrypted_keys` and `plaintext_keys` (derived + env-var names); `secrets_set` still lists both, so existing parsers are + unaffected. `secrets-list` entries gain `encrypted` and `value` -- `value` is + `null` for every encrypted entry. +- **`--secrets-file` follows the same rule**: a JSON object whose `#`-prefixed + keys are encrypted and whose bare keys are written in clear. + ## `token list --with-last-used`: `never` and `unknown` are not the same answer (since v0.88.0) `--with-last-used` derives each token's most recent activity so you can tell diff --git a/src/keboola_agent_cli/commands/_data_app_runtime.py b/src/keboola_agent_cli/commands/_data_app_runtime.py index c35dfb1d1..d48135be4 100644 --- a/src/keboola_agent_cli/commands/_data_app_runtime.py +++ b/src/keboola_agent_cli/commands/_data_app_runtime.py @@ -3,7 +3,8 @@ Split out of ``data_app.py`` to keep that module under the CONTRIBUTING.md file-size budget (see "File-size budgets"). These commands manage the ``parameters.dataApp.secrets`` block of a data app's Storage config (the -app-runtime env-var secrets); the business logic stays on ``DataAppService``. +app-runtime env vars -- ``#``-prefixed keys encrypted, bare keys plain); the +business logic stays on ``DataAppService``. They attach to the existing ``data-app`` Typer sub-app via :func:`register_secrets_commands`, called at the bottom of ``data_app.py``, so they still surface as ``kbagent data-app secrets-*`` with identical names, @@ -28,19 +29,22 @@ def _parse_secret_arg(arg: str) -> tuple[str, str]: - """Split ``#KEY=VALUE`` into ``(key, value)``. + """Split ``#KEY=VALUE`` or ``KEY=VALUE`` into ``(key, value)``. - The value may contain ``=``; only the FIRST ``=`` is the separator. + The ``#`` is kept on the key -- it is what tells the service to encrypt + the value rather than write it in clear. The value may contain ``=``; + only the FIRST ``=`` is the separator. """ if "=" not in arg: raise typer.BadParameter( - f"Expected '#KEY=VALUE'; got {arg!r} (no '=' separator).", + f"Expected '#KEY=VALUE' (encrypted) or 'KEY=VALUE' (plain); " + f"got {arg!r} (no '=' separator).", param_hint="--secret", ) key, _, value = arg.partition("=") if not key: raise typer.BadParameter( - f"Empty secret key in {arg!r}; expected '#KEY=VALUE'.", + f"Empty key in {arg!r}; expected '#KEY=VALUE' or 'KEY=VALUE'.", param_hint="--secret", ) return key, value @@ -63,7 +67,8 @@ def _read_secrets_file(path: Path) -> dict[str, str]: ) from exc if not isinstance(parsed, dict): raise typer.BadParameter( - f"Secrets file {path} must be a JSON object mapping #KEY -> value.", + f"Secrets file {path} must be a JSON object mapping key -> value " + '("#KEY" is encrypted, a bare "KEY" is written in clear).', param_hint="--secrets-file", ) out: dict[str, str] = {} @@ -90,14 +95,19 @@ def data_app_secrets_set( None, "--secret", help=( - "One or more '#KEY=VALUE' plaintext entries. Repeatable. " + "One or more entries, repeatable. '#KEY=VALUE' is ENCRYPTED under the " + "project KMS; 'KEY=VALUE' (no '#') is written as a PLAIN env var whose " + "value stays readable in the config -- use it for non-secret settings. " "Mutually exclusive with --secrets-file." ), ), secrets_file: Path | None = typer.Option( None, "--secrets-file", - help="Path to a JSON file mapping '#KEY' -> 'plaintext value'.", + help=( + "Path to a JSON file mapping key -> value. A '#KEY' is encrypted; " + "a bare 'KEY' is written as a plain env var." + ), exists=True, readable=True, dir_okay=False, @@ -126,10 +136,15 @@ def data_app_secrets_set( help="Suppress the 'now run kbagent data-app deploy' hint in the output.", ), ) -> None: - """Encrypt and write app-runtime secrets to the linked Storage config. + """Write app-runtime env vars to the linked Storage config. - The '#'-prefix is required on every key (Keboola encryption convention). - The runtime exposes each secret as an env var with '#' stripped, '-' + The '#'-prefix decides encryption, mirroring the UI's "optional + encryption" checkbox: '#KEY=VALUE' is encrypted under the project KMS, + 'KEY=VALUE' is written in clear so operators (and `secrets-list`, + `config detail`, the UI) can read it. Put non-secret settings -- a + Storage Files tag, a feature flag, a log level -- in the plain form. + + The runtime exposes each entry as an env var with '#' stripped, '-' replaced with '_', and uppercased ('#my-api-key' -> 'MY_API_KEY'). The command never auto-deploys; the running container keeps the old @@ -150,7 +165,10 @@ def data_app_secrets_set( if not secret and not secrets_file: formatter.error( - message=("Provide at least one --secret '#KEY=VALUE' or --secrets-file PATH."), + message=( + "Provide at least one --secret '#KEY=VALUE' (encrypted) / " + "'KEY=VALUE' (plain), or --secrets-file PATH." + ), error_code=ErrorCode.MISSING_PARAMETER, ) raise typer.Exit(code=2) from None @@ -250,7 +268,10 @@ def data_app_secrets_list( ) -> None: """List the keys in parameters.dataApp.secrets, with derived runtime env-var names. - Never echoes the encrypted ciphertext in full and never decrypts. + Each entry is tagged `encrypted`. A PLAIN (unencrypted) entry shows its + literal value -- it is already stored in clear and visible via + `config detail`. An ENCRYPTED entry shows ``: the ciphertext is + never echoed in full and the CLI never decrypts (the API is one-way). Reference: https://help.keboola.com/data-apps/python-js/ """ @@ -281,10 +302,10 @@ def data_app_secrets_list( return if not result["secrets"]: - formatter.console.print("[dim]No secrets set on this data app.[/dim]") + formatter.console.print("[dim]No env-var entries set on this data app.[/dim]") return formatter.console.print( - f"\n[bold]{result['count']} secret(s)[/bold] on data app " + f"\n[bold]{result['count']} entr(ies)[/bold] on data app " f"[cyan]{result['app_id']}[/cyan] in [magenta]{result['project_alias']}[/magenta]:" ) for entry in result["secrets"]: @@ -292,6 +313,13 @@ def data_app_secrets_list( " [yellow](shadowed by runtime)[/yellow]" if entry.get("shadowed_by_runtime") else "" ) line = f" [bold]{entry['key']}[/bold] -> env [cyan]{entry['env_var']}[/cyan]{marker}" + # A plain value is stored in clear and readable via `config detail`, so + # printing it leaks nothing; an encrypted one is never decryptable here. + line += ( + " = [dim][/dim]" + if entry.get("encrypted", True) + else f" = {entry.get('value', '')}" + ) if "fingerprint" in entry: line += f" [dim]fingerprint={entry['fingerprint']} prefix={entry.get('encryption_prefix', '')}[/dim]" formatter.console.print(line) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 012f64668..73098a38a 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1360,21 +1360,39 @@ WHY a deploy reverted to stopped. Project storage token only. kbagent data-app secrets-set --project ALIAS --app-id ID --secret '#KEY=VALUE' - [--secret '#KEY2=VALUE2' ...] [--secrets-file PATH] [--branch ID] + [--secret 'KEY2=VALUE2' ...] [--secrets-file PATH] [--branch ID] [--allow-plaintext-on-encrypt-failure] [--dry-run] [--no-hint-next] - Encrypt and write '#'-prefixed secrets into parameters.dataApp.secrets. + Write env-var entries into parameters.dataApp.secrets. Since vNEXT the + '#' prefix DECIDES encryption instead of being mandatory, mirroring the + UI's "optional encryption" checkbox: + '#KEY=VALUE' -> encrypted under the project KMS (unchanged behaviour). + 'KEY=VALUE' -> written VERBATIM as a plain env var, so secrets-list / + config detail / the UI show the real value instead of + . Use it for non-secret settings (a Storage + Files tag, a feature flag, a log level). + Before vNEXT a bare key was REJECTED, so a non-secret had to be either + encrypted (invisible to everyone) or written via sync pull + hand-edited + _config.yml + sync push. Reading and removing both kinds always worked. Per-project KMS via the Encryption API; ciphertext does not cross - projects (writeup §8). Read-modify-write at the service layer to - preserve sibling keys; never use Storage merge=True for nested edits. + projects (writeup §8). A payload with NO '#' key never calls the + Encryption API at all. A 'KBC::' value is rejected under EITHER form. + --allow-plaintext-on-encrypt-failure is unrelated: it covers a '#' key + whose encryption FAILED, never a deliberate plain key. + Read-modify-write at the service layer to preserve sibling keys; never + use Storage merge=True for nested edits. The runtime exposes each key as an env var with '#' stripped, '-' replaced with '_', uppercased ('#my-api-key' -> 'MY_API_KEY'). - Adding a secret bumps the Storage version; the running container - keeps the OLD config until the next 'kbagent data-app deploy'. + Response carries encrypted_keys / plaintext_keys (env-var names) beside + the combined secrets_set. Writing bumps the Storage version; the running + container keeps the OLD config until the next 'kbagent data-app deploy'. kbagent data-app secrets-list --project ALIAS --app-id ID [--branch ID] [--show-fingerprint] List the keys in parameters.dataApp.secrets with derived runtime - env-var names. Never echoes encrypted ciphertext in full and never + env-var names. Each entry carries `encrypted` and `value` (since vNEXT): + a PLAIN entry shows its literal value -- already stored in clear and + visible via config detail -- while an ENCRYPTED entry always has + value: null. Never echoes encrypted ciphertext in full and never decrypts. --show-fingerprint includes a short fingerprint per key. kbagent data-app secrets-get --project ALIAS --app-id ID --key 'KEY' diff --git a/src/keboola_agent_cli/services/_data_app_bodies.py b/src/keboola_agent_cli/services/_data_app_bodies.py index aeb30b53b..3db1dbc07 100644 --- a/src/keboola_agent_cli/services/_data_app_bodies.py +++ b/src/keboola_agent_cli/services/_data_app_bodies.py @@ -10,8 +10,11 @@ from __future__ import annotations import json +from collections.abc import Callable from typing import Any +from ..errors import ErrorCode, KeboolaApiError + # Encrypted-secret prefixes produced by the Encryption API for PROJECT-scoped # ciphertext -- one variant per cloud, and exactly these three exist: # ``KBC::ProjectSecure::`` (AWS KMS), ``KBC::ProjectSecureGKMS::`` (Google KMS) @@ -272,3 +275,55 @@ def _redact_storage_config(storage_config: dict[str, Any]) -> dict[str, Any]: configuration["parameters"] = parameters redacted["configuration"] = configuration return redacted + + +def partition_secret_entries( + secrets: dict[str, str], + *, + validate_key: Callable[[str], None], +) -> tuple[dict[str, str], dict[str, str]]: + """Split a ``secrets-set`` payload into the encrypted and plaintext halves. + + ``parameters.dataApp.secrets`` holds BOTH kinds and the Keboola UI offers + exactly that choice ("optional encryption"): a ``#``-prefixed key is + encrypted under the project KMS, a bare key is a plain env var whose value + stays readable in the config. Non-secret configuration -- a Storage Files + tag, a feature flag, a log level -- belongs in the second group so + ``secrets-list`` / ``config detail`` and the UI show the real value instead + of ````. + + ``validate_key`` is the service's own key validator (invoked with the + ``#`` made optional); it raises on a malformed key. Values are checked + here: a non-string is rejected, and so is a ``KBC::`` value in EITHER + group -- for a ``#`` key it would be double-encrypted, and for a plain key + it would park a foreign project's ciphertext in the config where nothing + can ever decrypt it. + + Returns ``(to_encrypt, plaintext)`` keyed exactly as the caller passed + them, so the merge back into the config block preserves the ``#`` prefix. + """ + to_encrypt: dict[str, str] = {} + plaintext: dict[str, str] = {} + for key, value in secrets.items(): + validate_key(key) + if not isinstance(value, str): + raise KeboolaApiError( + message=f"Entry '{key}' value must be a string; got {type(value).__name__}.", + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + if value.startswith("KBC::"): + raise KeboolaApiError( + message=( + f"Entry '{key}' value starts with 'KBC::', which suggests an " + "already-encrypted ciphertext. --secret expects plaintext; " + "re-encrypt under THIS project's KMS via " + "`kbagent encrypt values --component-id keboola.data-apps`." + ), + status_code=0, + error_code=ErrorCode.DATA_APP_INVALID_SECRET, + retryable=False, + ) + (to_encrypt if key.startswith("#") else plaintext)[key] = value + return to_encrypt, plaintext diff --git a/src/keboola_agent_cli/services/data_app_service.py b/src/keboola_agent_cli/services/data_app_service.py index 33eabfa79..8f09f56b4 100644 --- a/src/keboola_agent_cli/services/data_app_service.py +++ b/src/keboola_agent_cli/services/data_app_service.py @@ -43,6 +43,7 @@ _redact_git_block, _redact_storage_config, _secret_fingerprint, + partition_secret_entries, ) from .base import BaseService, ClientFactory, project_error_entry from .encrypt_service import EncryptService @@ -1004,7 +1005,17 @@ def set_data_app_secrets( allow_plaintext_on_encrypt_failure: bool = False, dry_run: bool = False, ) -> dict[str, Any]: - """Encrypt and write ``#``-prefixed secrets to the linked Storage config. + """Write entries to ``parameters.dataApp.secrets`` on the linked Storage config. + + The block holds two kinds of entry and this method writes both, keyed + on the ``#`` prefix exactly as the Keboola UI's "optional encryption" + checkbox does: + + - ``#KEY`` -- encrypted under THIS project's KMS before it is written. + - ``KEY`` -- a plain env var, written verbatim. Non-secret runtime + configuration (a Storage Files tag, a feature flag, a log level) + belongs here so ``secrets-list`` / ``config detail`` and the UI show + the real value instead of ````. Read-modify-write at the service layer. The Storage API's ``configuration`` field is a full-document overwrite; relying on @@ -1016,11 +1027,12 @@ def set_data_app_secrets( Fail-closed: any encryption failure aborts before Storage is touched. ``allow_plaintext_on_encrypt_failure`` is bootstrap/debug - only and emits a stderr warning when used. + only and emits a stderr warning when used; it never applies to a + deliberate plaintext key, which was never going to be encrypted. """ if not secrets: raise KeboolaApiError( - message="At least one --secret '#KEY=VALUE' is required.", + message="At least one --secret '#KEY=VALUE' or 'KEY=VALUE' is required.", status_code=0, error_code=ErrorCode.DATA_APP_INVALID_SECRET, retryable=False, @@ -1028,30 +1040,11 @@ def set_data_app_secrets( # Validate every key + value at the service boundary; we don't # trust the command layer to have caught everything. - validated: dict[str, str] = {} - for key, value in secrets.items(): - self._validate_secret_key(key) - if not isinstance(value, str): - raise KeboolaApiError( - message=(f"Secret '{key}' value must be a string; got {type(value).__name__}."), - status_code=0, - error_code=ErrorCode.DATA_APP_INVALID_SECRET, - retryable=False, - ) - if value.startswith("KBC::"): - raise KeboolaApiError( - message=( - f"Secret '{key}' value starts with 'KBC::', which suggests an " - "already-encrypted ciphertext. The --secret flag expects " - "plaintext; pass pre-encrypted values via --secrets-file or " - "re-encrypt under THIS project's KMS via " - "`kbagent encrypt values --component-id keboola.data-apps`." - ), - status_code=0, - error_code=ErrorCode.DATA_APP_INVALID_SECRET, - retryable=False, - ) - validated[key] = value + to_encrypt, plain = partition_secret_entries( + secrets, + validate_key=lambda key: self._validate_secret_key(key, require_hash=False), + ) + validated: dict[str, str] = {**to_encrypt, **plain} # Reserved-name warnings -- WARN (not BLOCKING). The platform # silently shadows colliding env vars at runtime. We still write @@ -1083,8 +1076,9 @@ def set_data_app_secrets( if dry_run: preview_secrets = dict(existing_secrets) - for key in validated: + for key in to_encrypt: preview_secrets[key] = "" + preview_secrets.update(plain) preview_body = self._merge_secrets_into_body(current_body, preview_secrets) return { "dry_run": True, @@ -1094,7 +1088,9 @@ def set_data_app_secrets( "secrets_set": sorted(_derive_runtime_env_var_name(k) for k in validated), "secrets_unchanged": unchanged, "shadowed_by_runtime": shadowed, - "encryption_request_keys": sorted(validated.keys()), + "encryption_request_keys": sorted(to_encrypt.keys()), + "encrypted_keys": sorted(_derive_runtime_env_var_name(k) for k in to_encrypt), + "plaintext_keys": sorted(_derive_runtime_env_var_name(k) for k in plain), "put_storage_config_preview": _redact_storage_config( {"configuration": preview_body} ), @@ -1104,20 +1100,25 @@ def set_data_app_secrets( ), } - # Encrypt every plaintext value under THIS project's KMS. - try: - encrypted = self._encrypt_service.encrypt( - alias=alias, - component_id=DATA_APP_COMPONENT_ID, - input_data=validated, - ) - except ConfigError as exc: - raise KeboolaApiError( - message=f"Failed to prepare secrets for encryption: {exc.message}", - status_code=0, - error_code=ErrorCode.ENCRYPTION_FAILED, - retryable=False, - ) from exc + # Encrypt the '#' half under THIS project's KMS. A payload of only + # plain keys must not call the Encryption API at all -- an + # unreachable API would otherwise fail a write that needs no + # encryption. + encrypted: dict[str, str] = {} + if to_encrypt: + try: + encrypted = self._encrypt_service.encrypt( + alias=alias, + component_id=DATA_APP_COMPONENT_ID, + input_data=to_encrypt, + ) + except ConfigError as exc: + raise KeboolaApiError( + message=f"Failed to prepare secrets for encryption: {exc.message}", + status_code=0, + error_code=ErrorCode.ENCRYPTION_FAILED, + retryable=False, + ) from exc # Validate every returned ciphertext starts with a project-scoped # prefix. Mirror the fail-closed check from _build_git_block at @@ -1155,6 +1156,7 @@ def set_data_app_secrets( # replaced; everything else is preserved bit-identical. updated_secrets = dict(existing_secrets) updated_secrets.update(encrypted) + updated_secrets.update(plain) new_body = self._merge_secrets_into_body(current_body, updated_secrets) put_response = storage_client.update_config( @@ -1162,7 +1164,7 @@ def set_data_app_secrets( config_id=config_id, configuration=new_body, change_description=( - f"Set {len(validated)} secret(s) via kbagent data-app secrets set" + f"Set {len(validated)} env-var entr(ies) via kbagent data-app secrets-set" ), branch_id=branch_id, ) @@ -1175,6 +1177,8 @@ def set_data_app_secrets( "app_id": str(app_id), "config_id": config_id, "secrets_set": secrets_set, + "encrypted_keys": sorted(_derive_runtime_env_var_name(k) for k in to_encrypt), + "plaintext_keys": sorted(_derive_runtime_env_var_name(k) for k in plain), "secrets_unchanged": unchanged, "shadowed_by_runtime": shadowed, # Keys whose ciphertext check failed but were written anyway under @@ -1189,8 +1193,9 @@ def set_data_app_secrets( f"kbagent data-app deploy --project {alias} --app-id {app_id} --wait" ), "message": ( - f"{len(secrets_set)} secret(s) encrypted and written. " - "The running container keeps the old config until you redeploy." + f"{len(to_encrypt)} secret(s) encrypted, {len(plain)} plain value(s) " + "written as-is. The running container keeps the old config until " + "you redeploy." ), } finally: @@ -1208,10 +1213,14 @@ def list_data_app_secrets( """Return metadata for every key in ``parameters.dataApp.secrets``. The block holds both ``#``-prefixed encrypted secrets and plain - (unencrypted) env-var values; both are enumerated. Never returns an - encrypted ciphertext in full and never attempts to decrypt -- the - Encryption API is one-way; decryption from the CLI is impossible by - design. + (unencrypted) env-var values; both are enumerated, tagged with + ``encrypted``. A plain entry carries its literal ``value`` -- it is + already stored in clear and visible through ``config detail``, so + showing it leaks nothing the caller could not already read, and it is + the whole point of storing non-secret configuration unencrypted. An + encrypted entry always has ``value: None``: the Encryption API is + one-way, so the CLI never returns a ciphertext in full and cannot + decrypt even if it wanted to. """ projects = self.resolve_projects([alias]) project = projects[alias] @@ -1229,15 +1238,21 @@ def list_data_app_secrets( entries: list[dict[str, Any]] = [] for key in sorted(raw_secrets.keys()): env_var = _derive_runtime_env_var_name(key) + stored = raw_secrets[key] + is_encrypted = isinstance(stored, str) and stored.startswith("KBC::") entry: dict[str, Any] = { "key": key, "env_var": env_var, "shadowed_by_runtime": env_var in RESERVED_RUNTIME_ENV_VARS, + "encrypted": is_encrypted, + # A plain entry is already stored in clear and readable via + # `config detail`, so echoing it leaks nothing new; an + # encrypted one is never decrypted (the API is one-way). + "value": None if is_encrypted else stored, } if show_fingerprint: - ciphertext = raw_secrets[key] - entry["fingerprint"] = _secret_fingerprint(ciphertext) - entry["encryption_prefix"] = self._derive_encryption_prefix(ciphertext) + entry["fingerprint"] = _secret_fingerprint(stored) + entry["encryption_prefix"] = self._derive_encryption_prefix(stored) entries.append(entry) return { diff --git a/tests/test_data_app_plain_env_keys.py b/tests/test_data_app_plain_env_keys.py index 43409f4f8..cdf34be6f 100644 --- a/tests/test_data_app_plain_env_keys.py +++ b/tests/test_data_app_plain_env_keys.py @@ -67,6 +67,7 @@ def _make_service( *, secrets: dict[str, str] | None = None, apps: list[dict] | None = None, + encrypt_service: MagicMock | None = None, ) -> tuple[DataAppService, MagicMock, MagicMock]: ds_mock = MagicMock() storage_mock = MagicMock() @@ -83,7 +84,7 @@ def _make_service( config_store=store, client_factory=lambda url, token: storage_mock, ds_client_factory=lambda url, token: ds_mock, - encrypt_service=MagicMock(), + encrypt_service=encrypt_service or MagicMock(), ) return service, ds_mock, storage_mock @@ -185,10 +186,15 @@ def test_remove_malformed_key_rejected(self, tmp_path: Path, bad_key: str) -> No class TestValidateSecretKeyRequireHash: - def test_set_path_still_requires_hash(self, tmp_path: Path) -> None: + def test_require_hash_true_still_rejects_a_plain_key(self, tmp_path: Path) -> None: + """The strict gate still exists; since #738 no caller passes it. + + `secrets-set` moved to ``require_hash=False`` so a bare ``KEY`` is a + valid plaintext env var, but the strict mode is kept for callers that + genuinely need the encryption convention enforced. + """ store = _make_store(tmp_path) service, *_ = _make_service(store) - # require_hash=True (the secrets-set path) rejects a plain key. with pytest.raises(KeboolaApiError) as excinfo: service._validate_secret_key("ADMIN_EMAILS", require_hash=True) assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET @@ -347,3 +353,190 @@ def test_plain_json_envelope(self, tmp_path: Path) -> None: body = json.loads(result.output) assert body["data"]["encrypted"] is False assert body["data"]["value"] == "a@x.io" + + +# --------------------------------------------------------------------------- +# set_data_app_secrets -- plaintext entries (issue #738) +# --------------------------------------------------------------------------- + + +class TestSetPlaintextEntries: + """A bare ``KEY`` is written in clear; a ``#KEY`` is still encrypted.""" + + def test_plain_key_is_written_verbatim_without_encrypting(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + encrypt = MagicMock() + service, _ds, storage = _make_service(store, encrypt_service=encrypt) + + result = service.set_data_app_secrets( + alias="prod", + app_id="42", + secrets={"SCORING_BACKUP_TAG": "ppl-assessment-db"}, + ) + + written = storage.update_config.call_args.kwargs["configuration"] + assert written["parameters"]["dataApp"]["secrets"] == { + "SCORING_BACKUP_TAG": "ppl-assessment-db" + } + # A payload with no '#' key must not reach the Encryption API at all -- + # an unreachable API would otherwise fail a write needing no encryption. + encrypt.encrypt.assert_not_called() + assert result["plaintext_keys"] == ["SCORING_BACKUP_TAG"] + assert result["encrypted_keys"] == [] + assert result["secrets_set"] == ["SCORING_BACKUP_TAG"] + + def test_mixed_payload_encrypts_only_the_hashed_half(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + encrypt = MagicMock() + encrypt.encrypt.return_value = {"#API_KEY": ENCRYPTED_VALUE} + service, _ds, storage = _make_service(store, encrypt_service=encrypt) + + result = service.set_data_app_secrets( + alias="prod", + app_id="42", + secrets={"#API_KEY": "s3cr3t", "LOG_LEVEL": "debug"}, + ) + + assert encrypt.encrypt.call_args.kwargs["input_data"] == {"#API_KEY": "s3cr3t"} + written = storage.update_config.call_args.kwargs["configuration"] + assert written["parameters"]["dataApp"]["secrets"] == { + "#API_KEY": ENCRYPTED_VALUE, + "LOG_LEVEL": "debug", + } + assert result["encrypted_keys"] == ["API_KEY"] + assert result["plaintext_keys"] == ["LOG_LEVEL"] + + def test_existing_entries_of_both_kinds_survive(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _ds, storage = _make_service( + store, secrets={"#OLD_SECRET": ENCRYPTED_VALUE, "OLD_PLAIN": "keep-me"} + ) + service.set_data_app_secrets(alias="prod", app_id="42", secrets={"NEW_PLAIN": "v"}) + + written = storage.update_config.call_args.kwargs["configuration"] + assert written["parameters"]["dataApp"]["secrets"] == { + "#OLD_SECRET": ENCRYPTED_VALUE, + "OLD_PLAIN": "keep-me", + "NEW_PLAIN": "v", + } + + def test_ciphertext_value_rejected_on_a_plain_key(self, tmp_path: Path) -> None: + """A `KBC::` value under a bare key would be undecryptable dead weight.""" + store = _make_store(tmp_path) + service, _ds, storage = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.set_data_app_secrets( + alias="prod", app_id="42", secrets={"TAG": ENCRYPTED_VALUE} + ) + assert excinfo.value.error_code == ErrorCode.DATA_APP_INVALID_SECRET + storage.update_config.assert_not_called() + + def test_dry_run_shows_the_plain_value_and_masks_the_encrypted_one( + self, tmp_path: Path + ) -> None: + store = _make_store(tmp_path) + service, _ds, storage = _make_service(store) + result = service.set_data_app_secrets( + alias="prod", + app_id="42", + secrets={"#API_KEY": "s3cr3t", "TAG": "public-tag"}, + dry_run=True, + ) + + preview = result["put_storage_config_preview"]["configuration"] + block = preview["parameters"]["dataApp"]["secrets"] + assert block["TAG"] == "public-tag" + assert block["#API_KEY"] == "" + assert result["encryption_request_keys"] == ["#API_KEY"] + storage.update_config.assert_not_called() + + def test_removing_a_plain_key_still_works(self, tmp_path: Path) -> None: + """`secrets-remove` already accepted bare keys; keep it wired to set.""" + store = _make_store(tmp_path) + service, _ds, storage = _make_service( + store, secrets={"TAG": "v", "#API_KEY": ENCRYPTED_VALUE} + ) + result = service.remove_data_app_secrets(alias="prod", app_id="42", keys=["TAG"]) + + assert result["removed"] == ["TAG"] + written = storage.update_config.call_args.kwargs["configuration"] + assert written["parameters"]["dataApp"]["secrets"] == {"#API_KEY": ENCRYPTED_VALUE} + + +class TestListReportsPlaintextValues: + def test_list_tags_each_entry_and_shows_plain_values(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, *_ = _make_service( + store, secrets={"#API_KEY": ENCRYPTED_VALUE, "TAG": "ppl-assessment-db"} + ) + entries = service.list_data_app_secrets(alias="prod", app_id="42")["secrets"] + by_key = {entry["key"]: entry for entry in entries} + + assert by_key["TAG"]["encrypted"] is False + assert by_key["TAG"]["value"] == "ppl-assessment-db" + # The security boundary: an encrypted entry never carries its value. + assert by_key["#API_KEY"]["encrypted"] is True + assert by_key["#API_KEY"]["value"] is None + + +class TestSecretsSetCliAcceptsPlainEntries: + def test_cli_forwards_a_bare_key(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock = MagicMock() + mock.set_data_app_secrets.return_value = { + "message": "1 plain value(s) written.", + "secrets_set": ["TAG"], + } + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.DataAppService") as MockService, + ): + MockStore.return_value = store + MockService.return_value = mock + result = runner.invoke( + app, + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "42", + "--secret", + "SCORING_BACKUP_TAG=ppl-assessment-db", + ], + ) + + assert result.exit_code == 0, result.output + assert mock.set_data_app_secrets.call_args.kwargs["secrets"] == { + "SCORING_BACKUP_TAG": "ppl-assessment-db" + } + + def test_cli_still_rejects_an_entry_without_an_equals_sign(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + mock = MagicMock() + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.DataAppService") as MockService, + ): + MockStore.return_value = store + MockService.return_value = mock + result = runner.invoke( + app, + [ + "--json", + "data-app", + "secrets-set", + "--project", + "prod", + "--app-id", + "42", + "--secret", + "NO_EQUALS", + ], + ) + + assert result.exit_code == 2 + assert json.loads(result.output)["error"]["code"] == "DATA_APP_INVALID_SECRET" + mock.set_data_app_secrets.assert_not_called() diff --git a/tests/test_data_app_secrets_service.py b/tests/test_data_app_secrets_service.py index 157b783b7..ab7ad5366 100644 --- a/tests/test_data_app_secrets_service.py +++ b/tests/test_data_app_secrets_service.py @@ -152,7 +152,10 @@ def test_empty_secrets_rejected(self, tmp_path: Path) -> None: @pytest.mark.parametrize( "bad_key", [ - "BAD", # no # + # "BAD" (no '#') is NOT here: since #738 a bare key is a valid + # PLAIN env var. See test_data_app_plain_env_keys.py. + "1bad", # starts with digit + "bad key", # space (plain form) "#1bad", # starts with digit "#bad key", # space "#bad\x00null", # NUL diff --git a/tests/test_e2e.py b/tests/test_e2e.py index a9d7be8a8..5b9b183f9 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -10002,6 +10002,133 @@ def test_data_app_secrets_round_trip(self) -> None: "second remove of the same key must be idempotent (removed=[], exit 0)" ) + @skip_without_data_app_public + def test_data_app_secrets_set_writes_plain_env_vars(self) -> None: + """`secrets-set` writes a bare KEY in clear and a #KEY encrypted (#738). + + Before this, a bare key was rejected outright; the only way to store a + non-secret was to encrypt it (invisible to everyone afterwards) or to + hand-edit the config through `sync`. Verified against a real project + KMS so the mixed payload's two halves are provably different on the + wire. + """ + repo = os.environ[ENV_DATA_APP_GIT_REPO_PUBLIC] + plain_value = "ppl-assessment-db" + secret_value = "supersecret-do-not-leak" + + _step(1, "Create shell app") + create = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "create", + "--project", + self.alias, + "--name", + f"E2E PlainEnv {RUN_ID}", + "--slug", + f"e2e-plainenv-{RUN_ID}"[:60], + "--git-repo", + repo, + "--git-public", + "--auth", + "public", + "--no-deploy", + ], + ) + )["data"] + app_id = create["app_id"] + self._created_app_ids.append(app_id) + + _step(2, "secrets-set: one plain entry + one encrypted entry") + set_raw = _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-set", + "--project", + self.alias, + "--app-id", + app_id, + "--secret", + f"E2E_PLAIN_TAG={plain_value}", + "--secret", + f"#E2E_ENC_KEY={secret_value}", + "--no-hint-next", + ], + ) + body = _json_ok(set_raw)["data"] + assert body["plaintext_keys"] == ["E2E_PLAIN_TAG"], body + assert body["encrypted_keys"] == ["E2E_ENC_KEY"], body + assert secret_value not in set_raw.output, ( + "the encrypted half's plaintext MUST NEVER appear in secrets-set output" + ) + + _step(3, "secrets-list reports the plain value and masks the encrypted one") + listed = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-list", + "--project", + self.alias, + "--app-id", + app_id, + ], + ) + )["data"]["secrets"] + by_key = {entry["key"]: entry for entry in listed} + assert by_key["E2E_PLAIN_TAG"]["encrypted"] is False + assert by_key["E2E_PLAIN_TAG"]["value"] == plain_value + assert by_key["#E2E_ENC_KEY"]["encrypted"] is True + assert by_key["#E2E_ENC_KEY"]["value"] is None + + _step(4, "the plain value is stored VERBATIM in the config, not as ciphertext") + stored = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "config", + "detail", + "--project", + self.alias, + "--component-id", + "keboola.data-apps", + "--config-id", + create["config_id"], + ], + ) + )["data"] + block = stored["configuration"]["parameters"]["dataApp"]["secrets"] + assert block["E2E_PLAIN_TAG"] == plain_value, block + assert block["#E2E_ENC_KEY"].startswith("KBC::"), block + + _step(5, "secrets-remove drops the plain entry") + removed = _json_ok( + _invoke( + self.config_dir, + [ + "--json", + "data-app", + "secrets-remove", + "--project", + self.alias, + "--app-id", + app_id, + "--key", + "E2E_PLAIN_TAG", + "--yes", + ], + ) + )["data"] + assert removed["removed"] == ["E2E_PLAIN_TAG"], removed + @skip_without_data_app_public def test_data_app_logs_validation_and_not_running(self) -> None: """E2E coverage for ``data-app logs`` (since v0.43.8).