From 013881ecbab36068c7b01db0693918a0a8ff822d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 28 Aug 2026 14:51:18 +0200 Subject: [PATCH 1/6] feat(AI-3790): metastore scope/ACL support (targeted + org-wide) for semantic layer Adds kbagent CLI support for PSGO-140's metastore scope model: `--scope project|organization|targeted` and `--target-project` on `model create` / `add `, plus a new `semantic-layer scope` command group (status/grant/request-elevation/withdraw-elevation/elevate/pending) for managing target-project grants and organization-wide elevation on existing items. Bumps the metastore envelope schema version from 1.0.0 to 1.1.0 -- every semantic-* schema only supports scope="project" at 1.0.0; 1.1.0 is what adds organization/targeted support (purely additive ACL block, verified against go-monorepo). Without this the whole feature would 400 server-side. The DELETE+POST `edit` path now reads and re-applies an item's original scope/target-project grants, so editing an organization/targeted-scope item no longer silently resets it to project scope. Widening visibility always requires an explicit --target-project (or an interactive picker on a real terminal; hard fail in --json/non-interactive) -- never a silent default. keboola-expert.md gets an explicit rule to always ask the user before passing --scope organization|targeted. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 20 +- plugins/kbagent/agents/keboola-expert.md | 17 +- plugins/kbagent/skills/kbagent/SKILL.md | 12 + .../kbagent/references/commands-reference.md | 13 + .../references/metastore-scope-workflow.md | 125 +++++ .../references/semantic-layer-workflow.md | 5 + .../commands/_semantic_layer_crud.py | 58 +- .../commands/_semantic_layer_helpers.py | 79 ++- .../commands/_semantic_layer_scope.py | 222 ++++++++ src/keboola_agent_cli/commands/context.py | 73 ++- .../commands/semantic_layer.py | 26 +- src/keboola_agent_cli/metastore_client.py | 249 ++++++-- src/keboola_agent_cli/permissions.py | 12 + .../services/_semantic_layer_crud.py | 48 +- .../services/_semantic_layer_scope.py | 128 +++++ .../services/semantic_layer_service.py | 161 +++++- tests/test_metastore_client.py | 183 +++++- tests/test_semantic_layer_scope.py | 338 +++++++++++ tests/test_semantic_layer_scope_cli.py | 531 ++++++++++++++++++ 19 files changed, 2230 insertions(+), 70 deletions(-) create mode 100644 plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md create mode 100644 src/keboola_agent_cli/commands/_semantic_layer_scope.py create mode 100644 src/keboola_agent_cli/services/_semantic_layer_scope.py create mode 100644 tests/test_semantic_layer_scope.py create mode 100644 tests/test_semantic_layer_scope_cli.py diff --git a/CLAUDE.md b/CLAUDE.md index bcc70fb54..23ce65ec0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1054,7 +1054,7 @@ kbagent dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run] kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] kbagent semantic-layer model list --project P -kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] +kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS ...] kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] @@ -1092,6 +1092,24 @@ kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,. kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes] kbagent semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH] kbagent semantic-layer token --encrypt --project P --component-id C +# scope (PSGO-140, new): visibility scope for a semantic-layer item -- "project" +# (default, owner only), "organization" (every project in the org), or "targeted" (owner + +# explicit --target-project grants). --scope/--target-project are also accepted by `model +# create` and every `add ` above (default "project", unchanged behavior). With --scope +# targeted and no --target-project: a real terminal launches an interactive picker over the +# other registered projects; --json/non-interactive fails fast (exit 2) instead of guessing. +# Creating directly at --scope organization requires the organization-admin ROLE (403 +# otherwise) -- an ordinary token instead uses request-elevation + an org-admin's `scope +# elevate`. Elevation is ONE-WAY (no downgrade endpoint). `scope grant` is a REPLACE on the +# wire (`--replace`/`--clear`); the default add/remove mode is a client-side read-modify-write +# merge, not atomic against a concurrent grant change. `edit` preserves an item's scope/grants +# across its DELETE+POST cycle automatically -- it is never silently reset to "project". +kbagent semantic-layer scope status --project P --type model|dataset|metric|relationship|constraint|glossary --id ID +kbagent semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear] +kbagent semantic-layer scope request-elevation --project P --type T --id ID +kbagent semantic-layer scope withdraw-elevation --project P --type T --id ID +kbagent semantic-layer scope elevate --project P --type T --id ID [--yes] +kbagent semantic-layer scope pending --project P --type T [--limit N] [--offset N] kbagent semantic-layer reference-data list --project P [--model M] kbagent semantic-layer reference-data get --project P (--id ID | --dimension D) kbagent semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X] diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index f5ffbb8fc..83f37f38a 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -151,8 +151,9 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` -- uses the `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` the scheduler injects; `http get /openapi.json` discovers endpoints | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated) | `curl $KBAGENT_SERVE_URL/...` by hand (loses the auth header, error mapping and JSON formatting `kbagent http` adds) | | Launch the web UI for an end-user | `kbagent serve --ui [--port PORT]` -- one FastAPI process mounts the bundled React SPA and sets an HttpOnly `kbagent_session` cookie, so SSE works with no token in the URL or JS heap | the legacy Vite + Node BFF dev flow (`web/README.md`) | inventing a `--token-in-url` flag; running uvicorn directly against the dist (the cookie bootstrap only fires from `serve --ui`) | | Schedule / manage Agent Tasks | `kbagent agent ` -- CRUD, `run [--stream]`, history, `test`/`cron-preview`/`prompt-improve`. Local-only; cron firing needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` inside scheduled subprocesses | hand-editing `agents.json` | -| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. The WHOLE `semantic-layer` family needs a MASTER token: a valid non-master token gets `MISSING_MASTER_TOKEN` (0.92.0+, #711; the Metastore's opaque 401 "Failed to create project scope") -- register a master token, do not escalate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | -| ANY semantic-layer write (add / edit / remove / import / promote / build) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.93.0+, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | +| ANY semantic-layer write (add / edit / remove / import / promote / build / scope grant / scope elevate) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope elevate`/`scope request-elevation` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS` grants specific projects; `scope grant --project P --type T --id ID --target-project ALIAS ...` adjusts an existing item's grants (merge by default, `--replace`/`--clear` for the API's native replace semantics); org-wide needs `scope request-elevation` then an org-admin's `scope elevate --yes` -- **irreversible, no downgrade endpoint**. `scope status --type T --id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope pending --type T` to find items an org-admin should act on | calling `scope elevate` or passing `--scope organization\|targeted` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | @@ -250,11 +251,13 @@ its absence is NOT a promise the entry is version-independent (see §1 Rule 6). **A 401 is not always `INVALID_TOKEN` (0.92.0+)** - A 401 whose body names something other than a bad credential now raises `AUTH_REJECTED`; the semantic-layer family's opaque metastore 401 - reclassifies to `MISSING_MASTER_TOKEN` instead (see its matrix row). - Exit code is unchanged (3) for all three -- code checking - `error.code == "INVALID_TOKEN"` alone now misses two real cases; accept - all three before concluding the credential itself is bad. 401/403/404 - now also carry `[exceptionId: ...]` -- quote it when escalating. gotchas.md. + reclassified to `MISSING_MASTER_TOKEN` on a write against a non-admin + token (see its matrix row) -- since PSGO-282 (0.93.0+) a plain read never + hits this at all, any valid token passes. Exit code is unchanged (3) for + all three -- code checking `error.code == "INVALID_TOKEN"` alone still + misses two real cases; accept all three before concluding the credential + itself is bad. 401/403/404 now also carry `[exceptionId: ...]` -- quote it + when escalating. gotchas.md. **Finding an existing Storage token** - `kbagent token list -p P` (0.86.0+) -- the only source of the `--token-id` diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 2d0fe1a19..c052b27f5 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -321,6 +321,12 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent semantic-layer reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent semantic-layer reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent semantic-layer reference-data delete --project PROJECT --id ID-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent semantic-layer scope status --project PROJECT --type TYPE- --id ITEM-ID` | +| Grant, revoke, or replace the target-project list of a targeted-scope item | `kbagent semantic-layer scope grant --project PROJECT --type TYPE- --id ITEM-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent semantic-layer scope request-elevation --project PROJECT --type TYPE- --id ITEM-ID` | +| Withdraw a pending scope-elevation request. | `kbagent semantic-layer scope withdraw-elevation --project PROJECT --type TYPE- --id ITEM-ID` | +| Step an item up to organization scope. | `kbagent semantic-layer scope elevate --project PROJECT --type TYPE- --id ITEM-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent semantic-layer scope pending --project PROJECT --type TYPE-` | | Encrypt the project's storage token for transformation `user_properties` | `kbagent sl token --project PROJECT --component-id COMPONENT-ID` | | Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent sl build --project PROJECT` | | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | @@ -354,6 +360,12 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent sl reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent sl reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent sl reference-data delete --project PROJECT --id ID-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent sl scope status --project PROJECT --type TYPE- --id ITEM-ID` | +| Grant, revoke, or replace the target-project list of a targeted-scope item | `kbagent sl scope grant --project PROJECT --type TYPE- --id ITEM-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent sl scope request-elevation --project PROJECT --type TYPE- --id ITEM-ID` | +| Withdraw a pending scope-elevation request. | `kbagent sl scope withdraw-elevation --project PROJECT --type TYPE- --id ITEM-ID` | +| Step an item up to organization scope. | `kbagent sl scope elevate --project PROJECT --type TYPE- --id ITEM-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent sl scope pending --project PROJECT --type TYPE-` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 2b53faa1b..3f5274c92 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -438,6 +438,19 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN = the table's Storage location, *(since 0.95.0)* no longer a hardcoded `"KEBOOLA"` database; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). **Rollback on push failure**: every successfully-POSTed child is DELETEd in reverse PUSH_ORDER, and the model itself is DELETEd if we created it during this call. The wrapped `KeboolaApiError` carries `details.rollback={attempted, posted_children, deleted, failed_deletes, model_created_here, model_deleted, model_uuid}` so operators get full diagnostics. Pass `--keep-on-failure` to preserve the partial state for forensic inspection (mirrors `data-app create --keep-on-failure`); the wrapped error then carries `details.rollback.attempted=False, reason='keep_on_failure'`. - `semantic-layer token --encrypt --project P --component-id C` -- encrypt the project's storage token for a transformation's `user_properties`. Builds `{"#metastore_token": }` from the project's already-stored Storage API token and delegates to the existing EncryptService. `--encrypt` is currently required; other modes are refused with `USAGE_ERROR` (exit 2). Output (human): the raw envelope ready to paste. JSON: full `{encrypted, component_id, project}`. +### Scope / target-project grants / elevation (`scope` sub-app, PSGO-140) + +*(since vNEXT)* Every semantic-layer item carries a visibility `scope`: `project` (default, owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above (default `project`, so existing behavior is unchanged unless the flag is passed). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other registered projects; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2) instead of guessing which project(s) should see the object. **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule. `edit` (DELETE+POST) reads the item's scope/grants before deleting and re-applies them on the POST half, so editing an organization/targeted-scope item is never silently downgraded back to `project`. + +- `semantic-layer scope status --project P --type model|dataset|metric|relationship|constraint|glossary --id ID` -- show an item's current `scope`, `target_project_ids`, and any pending `scope_elevation_requested_at`. +- `semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear]` -- update the target-project grant list of a `--scope targeted` item. Default mode is an additive/subtractive **merge** (read the current grants, apply `--target-project`/`--remove-target-project`, PUT the result) -- **not atomic** against a concurrent grant change on the same item, last write wins. `--replace` sends exactly `--target-project` as the whole set, matching the metastore's native `PUT .../target-projects` semantics (a REPLACE, not a merge) in one round trip. `--clear` is `--replace` with an empty set (revokes every grant; the item becomes owner-only). Only the owning project or an organization-admin may call this; the item must have `scope="targeted"` already (400 otherwise -- including against an organization-scoped item, where targeting is meaningless). +- `semantic-layer scope request-elevation --project P --type T --id ID` -- owner-only. Flags a project-scoped item as awaiting an organization admin's step-up decision. Idempotent: calling again just refreshes the timestamp. +- `semantic-layer scope withdraw-elevation --project P --type T --id ID` -- owner-only. Clears a pending elevation request. Idempotent no-op if none is pending. +- `semantic-layer scope elevate --project P --type T --id ID [--yes]` -- steps an item up to organization scope. Requires the **organization-admin role** (a normal project token gets `ACCESS_DENIED`, 403) -- same role required to create directly with `--scope organization`, so most callers reach organization scope via `request-elevation` + this command run by an org-admin-capable token. **ONE-WAY: there is no downgrade endpoint.** Prompts for confirmation unless `--yes` or `--json`. Widens the item's visibility (and its full revision history) to every project in the organization, irreversibly -- never call it without the user having explicitly named the item. +- `semantic-layer scope pending --project P --type T [--limit N] [--offset N]` -- lists items of `--type` awaiting an elevation decision, across the whole organization (`GET .../organization?scope_elevation_requested_at[not][null]=true`). This is the org-admin's discovery queue for `scope elevate`. + +There is no bulk-elevate endpoint: promoting every semantic-layer object in an existing project to organization/targeted scope ("elevate existing projects") means one `scope request-elevation` + `scope elevate` call per item, deliberately, only for objects the user has named -- never loop this speculatively over an entire project. + ### Reference data (dimension members, e.g. a Chart of Accounts) `semantic-reference-data` is a per-dimension member store: ONE record per dimension holding the full member list in a `members[]` array. The driving use case is a Chart of Accounts (the account list + all attributes) held in the metastore instead of a hardcoded Storage table. It is deliberately kept **outside** `build` / `export` / `diff` / cascade / `PUSH_ORDER` — its members come from `DIM_COA`, not from AI generation — so it has its own self-contained CRUD surface. Member field names mirror the `DIM_COA` columns 1:1 (snake_case: `account_code`, `account_name`, `parent_code`, `is_leaf`, `level_1_code`, `cf_category`, …). diff --git a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md new file mode 100644 index 000000000..7955f4ff4 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md @@ -0,0 +1,125 @@ +# Metastore Scope Workflow -- Sharing & Organization-Wide Objects (PSGO-140) + +Every metastore item (semantic-layer model, dataset, metric, relationship, +constraint, glossary term) carries a visibility **scope**: + +- `project` (default) -- visible only to the owning project. Unchanged + behavior; every command that doesn't mention `--scope` still creates + project-scoped items exactly as before this feature. +- `targeted` -- owner project + an explicit **target-project grant list** + (repeatable project alias). Any normal project token can create at this + scope; no elevated role needed. +- `organization` -- visible to **every** project in the organization. + Creating directly at this scope, or *elevating* an existing item to it, + requires the caller to hold the **organization-admin** role -- a normal + project token gets `ACCESS_DENIED` (403). + +For one-line command reference, see +[commands-reference.md](commands-reference.md#scope--target-project-grants--elevation-scope-sub-app-since-vnext-psgo-140). +For the schema-version / replace-vs-merge / 403-vs-404 surprises, see +[gotchas.md](gotchas.md). + +**Hard rule for AI agents**: never pass `--scope organization|targeted`, and +never run `scope elevate`/`scope request-elevation`, without the user having +explicitly named which project(s) should gain visibility. Widening an +object's visibility is a security-relevant decision, and organization-scope +elevation has **no downgrade endpoint** -- it cannot be undone via the API. +When in doubt, create/leave the item at the default `project` scope and ask. + +## When to use what + +| Goal | Command | +|------|---------| +| Check an item's current scope/grants/pending elevation | `semantic-layer scope status --type T --id ID` | +| Create a new item visible to a couple of named projects | `model create` / `add ` with `--scope targeted --target-project ALIAS ...` | +| Add/remove target projects on an existing targeted item | `scope grant --target-project ALIAS ...` / `--remove-target-project ALIAS ...` | +| Replace the whole target-project list in one call | `scope grant --replace --target-project ALIAS ...` | +| Revoke every grant (item becomes owner-only again) | `scope grant --clear` | +| Ask an org-admin to make an item org-wide | `scope request-elevation` (owner-only) | +| Cancel a pending elevation request | `scope withdraw-elevation` (owner-only) | +| Actually make an item org-wide (org-admin token) | `scope elevate --yes` | +| Find items awaiting an elevation decision | `scope pending --type T` (org-admin token) | + +--- + +## Workflow 1 -- Create an item shared with specific projects + +```bash +# Ask the user which project(s) should see this metric BEFORE running this. +kbagent --json semantic-layer add metric \ + --project prod --model core_model \ + --name gross_margin --sql "SUM(revenue) - SUM(cogs)" --dataset out.c-fin.fact_pnl \ + --scope targeted --target-project analytics-prod --target-project finance-prod +``` + +`--target-project` is repeatable and takes project **aliases** (resolved to +numeric Storage project IDs internally). Omitting `--target-project` with +`--scope targeted`: + +- On a real terminal: launches an interactive checkbox picker over every + *other* registered project. +- In `--json` / non-interactive mode: fails fast with `INVALID_ARGUMENT` + (exit 2) instead of guessing. There is always an explicit target list -- + never a silent default. + +## Workflow 2 -- Adjust grants on an existing targeted item + +```bash +# See what's granted today +kbagent --json semantic-layer scope status --project prod --type metric --id +# -> {"scope": "targeted", "target_project_ids": [1234, 5678], ...} + +# Add one more project (merge -- read-modify-write, NOT atomic against a +# concurrent grant change on the same item) +kbagent semantic-layer scope grant --project prod --type metric --id \ + --target-project new-team-prod + +# Replace the whole set in one round trip (matches the API's native semantics) +kbagent semantic-layer scope grant --project prod --type metric --id \ + --replace --target-project analytics-prod + +# Revoke every grant -- item becomes owner-only again +kbagent semantic-layer scope grant --project prod --type metric --id --clear +``` + +`scope grant` only works on an item already created with `scope="targeted"` +-- it 400s against a project-scoped or organization-scoped item (targeting a +project-scoped item makes no sense; an organization-scoped item is already +visible everywhere). + +## Workflow 3 -- Make an item organization-wide + +Two-step, deliberately: the owner requests, an organization-admin decides. + +```bash +# 1. Owner project flags the item (idempotent -- re-running just refreshes +# the timestamp) +kbagent semantic-layer scope request-elevation --project prod --type dataset --id + +# 2. An organization-admin discovers the queue... +kbagent --json semantic-layer scope pending --project prod --type dataset +# -> [{"id": "", "name": "fact_pnl", "scope_elevation_requested_at": "..."}] + +# 3. ...and elevates. IRREVERSIBLE -- confirmation prompt unless --yes/--json. +kbagent semantic-layer scope elevate --project prod --type dataset --id --yes +``` + +A non-admin token can also call `scope elevate` directly (skipping step 1) if +the caller already holds the organization-admin role -- `request-elevation` +exists for the common case where the owner and the admin are different +people/tokens. `scope withdraw-elevation` cancels a pending request before +an admin acts on it. + +There is **no bulk-elevate endpoint**. "Elevate an existing project's +objects" as a migration means one `request-elevation` + `elevate` call per +item, run deliberately for objects the user has named -- never loop this +over every object in a project speculatively. + +## Workflow 4 -- Editing a scoped item + +`semantic-layer edit ` is DELETE+POST (the metastore has no PATCH for +data changes). The edit path automatically reads the item's current +scope/target-project grants before deleting and re-applies them on the POST +half -- editing an `organization`/`targeted`-scope item never silently +resets it back to `project` scope. Nothing to do here; this is just so you +don't have to re-grant after every rename. diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index fd165e76f..713900ef5 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -18,6 +18,10 @@ For one-line command reference, see For the live-validated metastore contract surprises (constraint rule shape, name regex, CODE_METRIC cascade), see [gotchas.md](gotchas.md#semantic-layer-constraint-rule-is-a-string-not-an-object-since-v0340). +To share an item with specific other projects, or make it organization-wide, +see [metastore-scope-workflow.md](metastore-scope-workflow.md) -- **always +ask the user which project(s) first**, this is a security-relevant, mostly +irreversible decision. ## When to use what @@ -34,6 +38,7 @@ name regex, CODE_METRIC cascade), see | Promote dev -> prod | `semantic-layer promote --from-project dev --to-project prod` | | Bootstrap a model from storage tables | `semantic-layer build --tables ...` (heuristic) | | Encrypt the storage token for a Python container | `semantic-layer token --encrypt` | +| Share an item with named projects, or make it org-wide | see [metastore-scope-workflow.md](metastore-scope-workflow.md) | --- diff --git a/src/keboola_agent_cli/commands/_semantic_layer_crud.py b/src/keboola_agent_cli/commands/_semantic_layer_crud.py index 18e1d84fc..d85207cc1 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_crud.py @@ -18,7 +18,13 @@ get_formatter, get_service, ) -from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty +from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty, resolve_scope_targets + +_SCOPE_HELP = ( + "Visibility: 'project' (default, owner only), 'organization' (every " + "project in the org), or 'targeted' (owner + explicit --target-project grants)." +) +_TARGET_PROJECT_HELP = "Project alias to grant visibility to (repeatable; --scope targeted only)." # --------------------------------------------------------------------------- # semantic-layer add -- one sub-subcommand per entity type @@ -68,10 +74,18 @@ def add_metric( dataset: str = typer.Option(..., "--dataset", help="Dataset tableId this metric belongs to"), description: str = typer.Option("", "--description", help="Optional description"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the dataset-mismatch warning"), + scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a metric to a semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.add_metric, @@ -84,6 +98,8 @@ def add_metric( assume_yes=yes, is_tty=_is_stdin_tty(), confirm_cb=typer.confirm, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output(result, _print_item_added("metric")) @@ -115,10 +131,18 @@ def add_dataset( 'location from Storage, e.g. \'"DB"."out.c-bucket"."table"\'.' ), ), + scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a dataset (FQN read from the table's Storage location).""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.add_dataset, @@ -131,6 +155,8 @@ def add_dataset( primary_key=primary_key, deep_fields=deep_fields, fqn=fqn, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output(result, _print_item_added("dataset")) @@ -145,10 +171,18 @@ def add_relationship( to: str = typer.Option(..., "--to", help="Target dataset tableId"), on: str = typer.Option(..., "--on", help="Join condition"), type_: str = typer.Option("left", "--type", help="Join type: 'left' or 'inner'."), + scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a relationship between two datasets.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.add_relationship, @@ -159,6 +193,8 @@ def add_relationship( to=to, on=on, type_=type_, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output(result, _print_item_added("relationship")) @@ -194,6 +230,8 @@ def add_constraint( severity: str = typer.Option( "warning", "--severity", help="One of: error|warning|info (the 3-level API enum)." ), + scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a constraint.""" formatter = get_formatter(ctx) @@ -205,6 +243,12 @@ def add_constraint( error_code=ErrorCode.VALIDATION_ERROR, ) raise typer.Exit(code=2) + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.add_constraint, @@ -215,6 +259,8 @@ def add_constraint( rule=rule, metrics=metrics_list, severity=severity, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output(result, _print_item_added("constraint")) @@ -226,10 +272,18 @@ def add_glossary( model: str | None = typer.Option(None, "--model", help="Model name or UUID"), term: str = typer.Option(..., "--term", help="Glossary term"), definition: str = typer.Option("", "--definition", help="Optional definition"), + scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a glossary term.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.add_glossary, @@ -237,6 +291,8 @@ def add_glossary( model_name_or_uuid=model, term=term, definition=definition, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output(result, _print_item_added("glossary")) diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py index b48bb42ce..5d21071df 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -13,7 +13,8 @@ import typer from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ._helpers import get_formatter, map_error_to_exit_code +from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select +from ._helpers import get_formatter, get_service, map_error_to_exit_code def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ignore[no-untyped-def] @@ -42,3 +43,79 @@ def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ig def _is_stdin_tty() -> bool: """Return ``True`` when stdin is attached to a TTY (interactive shell).""" return hasattr(sys.stdin, "isatty") and sys.stdin.isatty() + + +def resolve_scope_targets( + ctx: typer.Context, + *, + scope: str, + target_project: list[str] | None, + owner_alias: str, +) -> list[str] | None: + """Resolve ``--target-project`` aliases for ``--scope targeted``. + + Returns ``None`` when ``scope`` isn't ``"targeted"`` (nothing to + resolve) or the raw alias list when the caller already gave explicit + ``--target-project`` values. When ``--scope targeted`` is chosen with + none given, this is the "ask when uncertain" mechanism: on a real + terminal it launches the checkbox picker over every OTHER registered + project; in a non-TTY or ``--json`` context it hard-fails instead of + silently defaulting -- widening an object's visibility across projects + is not a guess this CLI makes on the caller's behalf. + """ + if scope != "targeted": + return None + if target_project: + return list(target_project) + + formatter = get_formatter(ctx) + project_service = get_service(ctx, "project_service") + candidates = [p for p in project_service.list_projects() if p["alias"] != owner_alias] + if not candidates: + formatter.error( + message=( + "--scope targeted needs at least one other registered project to " + "target. Register one with `kbagent project add`, pass " + "--target-project, or use --scope project." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + if formatter.json_mode: + formatter.error( + message=( + "--scope targeted requires --target-project (repeatable) in " + "--json mode -- there is no terminal to pick from." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + items = [ + CheckboxItem( + label=p["alias"], + hint=f"{p.get('project_name', '')} ({p.get('project_id', '?')})", + ) + for p in candidates + ] + try: + selected = checkbox_select( + items, title="Select project(s) to grant visibility to (targeted scope):" + ) + except CheckboxUnavailable: + formatter.error( + message=( + "--scope targeted requires --target-project (repeatable) -- no " + "interactive terminal available to pick from." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + if not selected: + formatter.error( + message="No target project selected. Pass --target-project or use --scope project.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + return [candidates[i]["alias"] for i in selected] diff --git a/src/keboola_agent_cli/commands/_semantic_layer_scope.py b/src/keboola_agent_cli/commands/_semantic_layer_scope.py new file mode 100644 index 000000000..5ecdf99ca --- /dev/null +++ b/src/keboola_agent_cli/commands/_semantic_layer_scope.py @@ -0,0 +1,222 @@ +"""Typer sub-app for ``kbagent semantic-layer scope`` (PSGO-140). + +Extracted from :mod:`commands.semantic_layer` for the same LOC-ceiling reason +as ``_semantic_layer_crud.py``. Thin per the 3-layer architecture: all scope +resolution and metastore calls live in +:meth:`SemanticLayerService.scope_*` / :mod:`services._semantic_layer_scope`. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.table import Table + +from ..errors import ErrorCode +from ._helpers import check_cli_permission, get_formatter, get_service +from ._semantic_layer_helpers import _handle_service_call + +scope_app = typer.Typer( + name="scope", + help=( + "Manage an item's visibility scope (project/organization/targeted), " + "target-project grants, and organization-scope elevation requests." + ), + no_args_is_help=True, +) + + +@scope_app.callback(invoke_without_command=True) +def _scope_permission_check(ctx: typer.Context) -> None: + """Permission check for the ``scope`` sub-app. + + Composes ``semantic-layer.scope.{subcommand}`` -- ``status``/``pending`` + are ``read``, ``grant``/``request-elevation``/``withdraw-elevation`` are + ``write``, ``elevate`` is ``destructive`` (irreversible, widens + visibility org-wide). See ``permissions.OPERATION_REGISTRY``. + """ + check_cli_permission(ctx, "semantic-layer.scope") + + +def _print_scope_status(console: Console, data: dict) -> None: + console.print(f"[bold]scope:[/bold] {data.get('scope', 'project')}") + targets = data.get("target_project_ids") + if targets: + console.print(f"[bold]target_project_ids:[/bold] {targets}") + pending = data.get("scope_elevation_requested_at") + if pending: + console.print(f"[bold]scope_elevation_requested_at:[/bold] {pending}") + + +def _print_pending_table(console: Console, data: dict) -> None: + items = data if isinstance(data, list) else data.get("items", []) + if not items: + console.print("[dim]No items awaiting scope elevation.[/dim]") + return + table = Table(title="Pending scope-elevation requests") + table.add_column("Name", style="bold cyan") + table.add_column("ID", style="dim") + table.add_column("Requested at") + for item in items: + table.add_row( + item.get("name") or "", + item.get("id") or "", + str(item.get("scope_elevation_requested_at") or ""), + ) + console.print(table) + + +@scope_app.command("status") +def scope_status( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + item_id: str = typer.Option(..., "--id", help="Item UUID"), +) -> None: + """Show an item's current scope, target-project grants, and pending elevation.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_status, alias=project, kind=type_, item_id=item_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("grant") +def scope_grant( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Owning project alias"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + item_id: str = typer.Option(..., "--id", help="Item UUID (must have scope='targeted')"), + target_project: list[str] = typer.Option( + [], "--target-project", help="Project alias to add to the grant list (repeatable)." + ), + remove_target_project: list[str] = typer.Option( + [], "--remove-target-project", help="Project alias to remove from the grant list." + ), + replace: bool = typer.Option( + False, + "--replace", + help="Replace the whole grant list with exactly --target-project (matches the API's " + "native semantics) instead of merging with the current set.", + ), + clear: bool = typer.Option( + False, "--clear", help="Clear every grant (same as --replace with no --target-project)." + ), +) -> None: + """Grant, revoke, or replace the target-project list of a targeted-scope item. + + Default is an additive/subtractive merge against the current grants. + Pass --replace to send exactly --target-project (the server's native + replace-only semantics, one round trip); --clear is shorthand for + --replace with no --target-project. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + if clear and (target_project or remove_target_project): + formatter.error( + message="--clear cannot be combined with --target-project/--remove-target-project.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + result = _handle_service_call( + ctx, + service.scope_grant, + alias=project, + kind=type_, + item_id=item_id, + add=target_project or None, + remove=remove_target_project or None, + replace=([] if clear else (target_project if replace else None)), + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("request-elevation") +def scope_request_elevation( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Owning project alias"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + item_id: str = typer.Option(..., "--id", help="Item UUID"), +) -> None: + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_request_elevation, alias=project, kind=type_, item_id=item_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("withdraw-elevation") +def scope_withdraw_elevation( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Owning project alias"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + item_id: str = typer.Option(..., "--id", help="Item UUID"), +) -> None: + """Withdraw a pending scope-elevation request. Idempotent no-op if none is pending.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_withdraw_elevation, alias=project, kind=type_, item_id=item_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("elevate") +def scope_elevate( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Owning project alias"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + item_id: str = typer.Option(..., "--id", help="Item UUID"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."), +) -> None: + """Step an item up to organization scope. Requires org-admin; ONE-WAY, no downgrade. + + Every project in the organization gains read access to this item and its + full revision history once elevated -- there is no API to reverse it. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + if not yes and not formatter.json_mode: + confirmed = typer.confirm( + f"Elevate {type_} {item_id!r} to organization scope? This is IRREVERSIBLE " + "and makes it visible to every project in the organization. Continue?" + ) + if not confirmed: + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + result = _handle_service_call( + ctx, service.scope_elevate, alias=project, kind=type_, item_id=item_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("pending") +def scope_pending( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias (org-admin token)"), + type_: str = typer.Option( + ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + ), + limit: int | None = typer.Option(None, "--limit", help="Max results"), + offset: int | None = typer.Option(None, "--offset", help="Skip this many results"), +) -> None: + """List items of --type awaiting an org-admin's elevation decision, across the org.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_pending, alias=project, kind=type_, limit=limit, offset=offset + ) + formatter.output(result, _print_pending_table) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 2b7593fbf..152a5eb38 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1761,8 +1761,21 @@ kbagent semantic-layer model list --project P List all semantic-layer models in a project. - kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] - Create a new model (default sql-dialect: Snowflake). + kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS ...] + Create a new model (default sql-dialect: Snowflake). --scope defaults to + "project" (owner-only, unchanged). "targeted" = owner + explicit + --target-project grants (repeatable alias); any normal project token can + create at this scope. "organization" = visible to every project in the + org from creation -- the schema's ACL requires the organization-admin + ROLE to create directly at this scope (a normal project token gets 403 + ACCESS_DENIED); an ordinary caller instead creates at project/targeted + scope and uses `scope request-elevation` + an org-admin's + `scope elevate` (below) to get there. ASK THE USER which project(s) + before ever passing --scope organization|targeted -- never guess. With + --scope targeted and no --target-project: on a real terminal this + launches an interactive picker over the other registered projects; in + --json/non-interactive it fails fast (exit 2) rather than silently + defaulting. kbagent semantic-layer model delete --project P --model M [--yes] Delete a model. Fails if the model still has child entities. @@ -1829,7 +1842,9 @@ else dimension). Constraint name regex `^[a-z][a-z0-9_]*$`, severity is error|warning|info (the 4-band health convention lives in the NAME suffix `_critical/_warning/_healthy/_review`, not the API severity). `--rule` is - a STRING expression (e.g. "value >= 0"), NEVER an object. + a STRING expression (e.g. "value >= 0"), NEVER an object. --scope / + --target-project: same semantics as `model create --scope` above -- + ASK THE USER before ever passing --scope organization|targeted. kbagent semantic-layer edit metric|dataset|constraint|relationship|glossary ... DELETE+POST (no PATCH on metastore). Metric rename cascades through every @@ -1847,6 +1862,10 @@ --new-metrics ...`. Human-mode CLI prints a red `PARTIAL STATE` banner above the per-entry list. `edit_simple` (no-cascade variants) carries `partial_state: false, recovery_hint: null` for envelope uniformity. + The item's scope/target-project grants (`scope status` below) are + read from the pre-edit item and re-applied on the POST half of every + DELETE+POST -- editing an organization/targeted-scope item never + silently downgrades it back to project scope. kbagent semantic-layer remove metric|dataset|constraint|relationship|glossary ... Destructive. `remove metric` pre-scans constraints whose metrics[] includes @@ -1883,6 +1902,54 @@ Builds {{"#metastore_token": }} and delegates to EncryptService. --encrypt is currently required; other modes refused with USAGE_ERROR. + kbagent semantic-layer scope status --project P --type T --id ID + Show an item's current scope ("project"|"organization"|"targeted"), + target_project_ids, and any pending scope_elevation_requested_at. + + kbagent semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear] + Grant/revoke/replace the target-project grant list of a --scope targeted + item. Default is an additive/subtractive merge (read current grants, + apply the delta, PUT the result) -- NOT atomic against a concurrent + grant change, last write wins. --replace sends exactly --target-project + as the whole set (the server's native replace-only semantics, one round + trip); --clear is --replace with an empty set (revokes every grant, + object becomes owner-only). Only the owning project or an + organization-admin may call this; the item must have been created with + scope="targeted" (400 otherwise, including against an + organization-scoped item -- targeting is meaningless there). + + kbagent semantic-layer scope request-elevation --project P --type T --id ID + Owner-only. Flags a project-scoped item as awaiting an organization + admin's step-up decision (`scope elevate` below). Idempotent -- calling + again just refreshes the timestamp. + + kbagent semantic-layer scope withdraw-elevation --project P --type T --id ID + Owner-only. Clears a pending elevation request. Idempotent no-op if + none is pending. + + kbagent semantic-layer scope elevate --project P --type T --id ID [--yes] + Steps an item up to organization scope. REQUIRES the organization-admin + ROLE (a normal project token gets 403 ACCESS_DENIED) -- creating + directly with `--scope organization` needs the same role, so most + callers reach organization scope via request-elevation + this command, + run by whoever holds an org-admin-capable token. ONE-WAY: there is no + downgrade endpoint. Prompts for confirmation unless --yes or --json. + NEVER call this without the user explicitly naming the item to elevate + -- it makes the item (and its full revision history) visible to every + project in the organization, irreversibly. + + kbagent semantic-layer scope pending --project P --type T [--limit N] [--offset N] + Lists items of --type awaiting an elevation decision, across the whole + organization (`GET .../organization?scope_elevation_requested_at[not][null]=true`). + This is the org-admin's discovery queue for `scope elevate`. + + Elevating an EXISTING project's semantic-layer objects in bulk ("elevate + existing projects" as a migration): there is no bulk-elevate endpoint -- + each object needs its own `scope request-elevation` + an org-admin's + `scope elevate`, one call per item. Do this deliberately, one object at a + time, only when the user has named which objects should become org-wide; + never loop this over every object in a project speculatively. + ### Self-call HTTP (inside `kbagent serve` subprocesses) diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 1a3748ab7..12ffd1837 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -24,8 +24,9 @@ get_service, ) from ._semantic_layer_crud import add_app, edit_app, remove_app -from ._semantic_layer_helpers import _handle_service_call +from ._semantic_layer_helpers import _handle_service_call, resolve_scope_targets from ._semantic_layer_reference_data import reference_data_app +from ._semantic_layer_scope import scope_app semantic_layer_app = typer.Typer( name="semantic-layer", @@ -112,10 +113,30 @@ def model_create( sql_dialect: str = typer.Option( "Snowflake", "--sql-dialect", help="SQL dialect (default: Snowflake)" ), + scope: str = typer.Option( + "project", + "--scope", + help=( + "Visibility: 'project' (default, owner only), 'organization' " + "(every project in the org), or 'targeted' (owner + explicit " + "--target-project grants)." + ), + ), + target_project: list[str] = typer.Option( + [], + "--target-project", + help="Project alias to grant visibility to (repeatable; --scope targeted only).", + ), ) -> None: """Create a new semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_aliases = resolve_scope_targets( + ctx, scope=scope, target_project=target_project, owner_alias=project + ) + target_project_ids = ( + service.resolve_target_project_ids(target_aliases) if target_aliases else None + ) result = _handle_service_call( ctx, service.create_model, @@ -123,6 +144,8 @@ def model_create( name=name, description=description, sql_dialect=sql_dialect, + scope=scope, + target_project_ids=target_project_ids, ) formatter.output( result, @@ -143,6 +166,7 @@ def model_create( semantic_layer_app.add_typer(edit_app, name="edit") semantic_layer_app.add_typer(remove_app, name="remove") semantic_layer_app.add_typer(reference_data_app, name="reference-data") +semantic_layer_app.add_typer(scope_app, name="scope") @model_app.command("delete") diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 7e92a3624..210f91a25 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -20,6 +20,31 @@ :data:`ErrorCode.ALREADY_EXISTS`. - Error envelope has top-level ``error``, ``code``, ``exception``, ``status``, ``context.path``, and an ``errors[]`` list for 422 validation failures. + +Scope/target-project contract (grounded in go-monorepo commit ``e4f62941``, +services/metastore -- Linear PSGO-140; read from source, **not yet probed +live**): + +- Every item's ``scope`` (``"project"`` | ``"organization"`` | ``"targeted"``), + ``targetProjectIds``, and ``scopeElevationRequestedAt`` live under the + response's ``meta`` block, not ``attributes`` (server: + ``MetaObjectResponse.JSONAPIMeta``). +- ``POST`` accepts ``scope``/``targetProjectIds`` in the create envelope. + ``PUT /{id}`` (plain update) does **not** -- its server-side request struct + (``MetaObjectUpdatePutRequest``) has no such fields, so scope/grants can + only be set at creation or through the dedicated endpoints below. +- Elevating to organization scope is ``PATCH /{id}`` with body + ``{"scope": "organization"}`` **only** -- it cannot be combined with a + name/data change in the same request, is one-way (no downgrade endpoint + exists), and requires the organization-admin role. +- ``PUT /{id}/target-projects`` **replaces the whole grant set** (not + additive); only valid for an object created with ``scope="targeted"``. + 204 on success, no body. +- ``PUT``/``DELETE /{id}/scope-elevation-request`` self-service "request a + step-up" flag: empty body, 200 with the updated item. Idempotent. +- ``GET /{type}/organization`` lists organization-visible items across + projects; supports the same generic ``field[op]=value`` filter query + language as every other list endpoint, plus bare ``limit=N``/``offset=N``. """ import logging @@ -34,15 +59,24 @@ logger = logging.getLogger(__name__) -# The metastore's auth middleware collapses EVERY project-scope resolution -# failure into this one opaque 401 string (go-monorepo +# Pre-PSGO-282, the metastore's auth middleware collapsed EVERY project-scope +# resolution failure into this one opaque 401 string (go-monorepo # ``services/metastore/internal/middleware/auth.go``, ``resolveProjectScope`` -# -- the underlying error is logged server-side and discarded). The dominant -# cause is its master-token gate: ``NewProjectDeps`` is called without -# ``WithoutMasterToken()``, so unlike the Storage API the metastore accepts -# ONLY a master (project admin) Storage token, and every valid non-master -# token lands here (issue #711; A/B-verified live on us-east4.gcp: non-master -# token -> this 401 on every call, master token on the same stack -> 200). +# -- the underlying error was logged server-side and discarded), because +# ``NewProjectDeps`` was called without ``WithoutMasterToken()``: unlike the +# Storage API, the metastore accepted ONLY a master (project admin) Storage +# token and every valid non-master token landed here (issue #711; +# A/B-verified live on us-east4.gcp: non-master token -> this 401 on every +# call, master token on the same stack -> 200). +# +# Fixed server-side in go-monorepo#596 (PSGO-282, merged 2026-08-31): reads +# now work with any valid, non-disabled, non-expired Storage token, writes +# still require the token to belong to a project admin, and a 401 now relays +# the Storage API's real error message instead of this generic string. A +# fixed metastore will not emit this exact phrase again -- this +# reclassification is kept only as a safety net for a deployment that has +# not rolled the fix out yet (or an older regional stack); the message below +# no longer claims every call needs a master token. _PROJECT_SCOPE_401_EXCEPTION = "Failed to create project scope" # Re-extracts the ``[exceptionId: ...]`` suffix `_raise_api_error` appended, @@ -72,10 +106,20 @@ ) +ObjectScope = Literal["project", "organization", "targeted"] + # Envelope fields kept constant across every POST (per metastore contract). _ENVELOPE_BRANCH = "main" -_ENVELOPE_SCHEMA_VERSION = "1.0.0" -_ENVELOPE_SCOPE = "project" +# 1.1.0, not 1.0.0: every semantic-* schema's x-metastore.scope.supported is +# ["project"] ONLY at 1.0.0 -- 1.1.0 is what adds "organization"/"targeted" +# (go-monorepo migrations/schema/semantic-*_schema_1.1.0.json, diffed against +# 1.0.0 at commit e4f62941: purely additive x-metastore.acl/scope blocks, no +# `data` schema change, so this is safe for every existing caller). Sending +# scope="organization"/"targeted" against 1.0.0 gets a clean 400 +# ErrScopeNotSupported from prepareCreateSchema -- the server resolves the +# EXACT version string sent, never silently upgrades it. +_ENVELOPE_SCHEMA_VERSION = "1.1.0" +_DEFAULT_SCOPE: ObjectScope = "project" class MetastoreClient(BaseHttpClient): @@ -87,11 +131,12 @@ class MetastoreClient(BaseHttpClient): no in-memory caching. All such concerns live in the service layer. """ - # The metastore answers a VALID non-master token with 401 (its master-token - # gate, #711), so a bearer 401 here is not an expired token. Opt out of the - # refresh-and-retry: otherwise every non-master session call burns a - # refresh-token rotation and can report SESSION_EXPIRED instead of the real - # MISSING_MASTER_TOKEN. See ``_do_request`` below and ``BearerAuth``. + # The metastore answers a VALID non-admin token with 401 on a WRITE (its + # project-admin gate, #711 / PSGO-282), so a bearer 401 here is not an + # expired token. Opt out of the refresh-and-retry: otherwise every + # non-admin session write burns a refresh-token rotation and can report + # SESSION_EXPIRED instead of the real MISSING_MASTER_TOKEN. See + # ``_do_request`` below and ``BearerAuth``. BEARER_REFRESH_ON_401 = False def __init__(self, stack_url: str, token: str, *, http_auth: httpx.Auth | None = None) -> None: @@ -111,14 +156,18 @@ def __exit__(self, *args: object) -> None: def _do_request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: """Funnel every metastore call through the project-scope-401 diagnosis. - A 401 carrying ``"Failed to create project scope"`` is the metastore's - master-token gate rejecting a valid non-master token (see - :data:`_PROJECT_SCOPE_401_EXCEPTION`), not a bad credential and not a - server fault to escalate -- so it is reclassified from the generic - 401 mapping into :data:`ErrorCode.MISSING_MASTER_TOKEN` with the - actual remedy, mirroring the ``token create`` / ``config oauth-url`` - pre-flight guards (#599). Kept at the request funnel (rather than one - try/except per verb method) so no endpoint can miss it. + A 401 carrying ``"Failed to create project scope"`` was the + metastore's pre-PSGO-282 master-token gate rejecting a valid + non-master token (see :data:`_PROJECT_SCOPE_401_EXCEPTION`), not a + bad credential and not a server fault to escalate -- so it is + reclassified from the generic 401 mapping into + :data:`ErrorCode.MISSING_MASTER_TOKEN`, mirroring the + ``token create`` / ``config oauth-url`` pre-flight guards (#599). + Kept at the request funnel (rather than one try/except per verb + method) so no endpoint can miss it, but since PSGO-282 a fixed + metastore only 401s this way on a WRITE against a non-admin token + (reads succeed for any valid token) -- the remedy below reflects + that, not a blanket master-token requirement. """ try: return super()._do_request(method, path, **kwargs) @@ -131,25 +180,27 @@ def _do_request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: # Session (browser-login) project: the token is a bearer, so the # masked-token note and `project edit --token` remedy do not apply. remedy = ( - "This browser-login session is not a project master. Register " - "the project with a static MASTER Storage token " + "This browser-login session is not a project admin. Register " + "the project with a static project-admin Storage token " "(`kbagent project add`), or sign in as a project admin, to " - "use semantic-layer commands." + "use semantic-layer write commands." ) else: remedy = ( - f"The token ({self._masked_token}) is not a master token. Check " + f"The token ({self._masked_token}) is not a project admin. Check " f"`kbagent project info` -> is_master_token, and register a " - f"master token (`kbagent project edit --token ...`) to use " - f"semantic-layer commands." + f"project-admin token (`kbagent project edit --token ...`) to " + f"use semantic-layer write commands." ) raise KeboolaApiError( message=( f"The Metastore API (semantic layer) rejected the request with " - f"HTTP 401 {_PROJECT_SCOPE_401_EXCEPTION!r}. Unlike the Storage " - f"API, the metastore accepts only a MASTER (project admin) " - f"Storage token, and this is how it answers a valid non-master " - f"token. {remedy}{id_suffix}" + f"HTTP 401 {_PROJECT_SCOPE_401_EXCEPTION!r}. This is the " + f"metastore's project-admin gate: writes need a token whose " + f"Storage Admin role is `admin` on this project (a master " + f"token qualifies, but so does any other project-admin " + f"user's token); reads work with any valid token. " + f"{remedy}{id_suffix}" ), status_code=exc.status_code, error_code=ErrorCode.MISSING_MASTER_TOKEN, @@ -223,24 +274,50 @@ def post_item( item_type: SemanticType, name: str, data: dict[str, Any], + *, + scope: ObjectScope = _DEFAULT_SCOPE, + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create an item. Returns the server's stored representation. ``data`` is the inner ``attributes`` payload (including ``modelUUID`` for non-model types). The outer envelope is added here. + ``scope`` defaults to ``"project"`` (today's only behavior, unchanged + for every existing caller). ``target_project_ids`` is only meaningful + with ``scope="targeted"``; the server rejects the combination + otherwise (``meta_object_repository.go`` ``Create``: not a typed + sentinel error there, so it may surface as a 500 rather than a clean + 400 -- checked client-side below instead of relying on that). Neither + can be changed later via :meth:`put_item` -- use + :meth:`elevate_to_organization` / :meth:`put_target_projects`. + Normalizes the duplicate-name conflict into a clean :data:`ErrorCode.ALREADY_EXISTS`, accepting both shapes the metastore has used: HTTP 409 (post go-monorepo PR #513) and HTTP 500 with ``"Failed to create meta object"`` (legacy / pre-fix deployments). """ - envelope = { + if scope not in ("project", "organization", "targeted"): + raise KeboolaApiError( + message=f"scope must be one of 'project'|'organization'|'targeted', got {scope!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if target_project_ids and scope != "targeted": + raise KeboolaApiError( + message=( + f"target_project_ids is only valid with scope='targeted', got scope={scope!r}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + envelope: dict[str, Any] = { "name": name, "data": data, "branch": _ENVELOPE_BRANCH, "schemaVersion": _ENVELOPE_SCHEMA_VERSION, - "scope": _ENVELOPE_SCOPE, + "scope": scope, } + if target_project_ids is not None: + envelope["targetProjectIds"] = target_project_ids try: response = self._do_request( "POST", @@ -288,7 +365,12 @@ def put_item( use, ``PUT`` updates the record in place and increments ``meta.revision`` server-side, preserving the metastore's revision history. ``data`` is the inner ``attributes`` payload; the outer - envelope is added here (identical shape to :meth:`post_item`). + envelope is added here. + + Deliberately carries no ``scope``/``targetProjectIds`` -- the + server's request struct for this endpoint (``MetaObjectUpdatePutRequest``) + has no such fields, so scope/grants are untouched by a plain PUT. + Use :meth:`elevate_to_organization` / :meth:`put_target_projects`. Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. """ @@ -297,7 +379,6 @@ def put_item( "data": data, "branch": _ENVELOPE_BRANCH, "schemaVersion": _ENVELOPE_SCHEMA_VERSION, - "scope": _ENVELOPE_SCOPE, } response = self._do_request( "PUT", @@ -313,3 +394,97 @@ def delete_item(self, item_type: SemanticType, item_id: str) -> None: Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. """ self._do_request("DELETE", f"/api/v1/repository/{item_type}/{item_id}") + + # ------------------------------------------------------------------ + # Scope / target-project primitives (PSGO-140) + # ------------------------------------------------------------------ + + def elevate_to_organization(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Step an item up from project/targeted scope to organization scope. + + ``PATCH /{id}`` with ``{"scope": "organization"}`` only -- the server + rejects combining a scope change with name/data in the same request, + so this method sends nothing else. One-way: there is no downgrade + endpoint. Requires the caller to hold the organization-admin role; + a caller without it gets ``ACCESS_DENIED`` (403). + """ + response = self._do_request( + "PATCH", + f"/api/v1/repository/{item_type}/{item_id}", + json={"scope": "organization"}, + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def put_target_projects( + self, + item_type: SemanticType, + item_id: str, + target_project_ids: list[int], + ) -> None: + """Replace the full set of projects granted access to a targeted-scope item. + + This is a **replace**, not a merge -- an empty list clears every + grant. Only the owning project or an organization admin may call + this; only valid for an item created with ``scope="targeted"`` + (the server 400s otherwise, including for organization-scoped + items, where targeting is meaningless). 204 on success, no body. + """ + self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}/target-projects", + json={"targetProjectIds": target_project_ids}, + ) + + def request_scope_elevation(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Flag a project-scoped item as awaiting an org-admin's step-up decision. + + Owner-only, idempotent (repeating just refreshes the timestamp). No + request body. Returns the updated item. + """ + response = self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}/scope-elevation-request", + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def withdraw_scope_elevation(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" + response = self._do_request( + "DELETE", + f"/api/v1/repository/{item_type}/{item_id}/scope-elevation-request", + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def list_organization_items( + self, + item_type: SemanticType, + *, + pending_elevation_only: bool = False, + limit: int | None = None, + offset: int | None = None, + ) -> list[dict[str, Any]]: + """List organization-visible items of ``item_type`` across projects. + + ``GET /{type}/organization``. ``pending_elevation_only`` applies the + server's documented discovery filter for an org-admin's queue + (``scope_elevation_requested_at[not][null]=true``); the generic + filter/limit/offset query language is shared with every other list + endpoint in this API. + """ + params: dict[str, str] = {} + if pending_elevation_only: + params["scope_elevation_requested_at[not][null]"] = "true" + if limit is not None: + params["limit"] = str(limit) + if offset is not None: + params["offset"] = str(offset) + response = self._do_request( + "GET", + f"/api/v1/repository/{item_type}/organization", + params=params or None, + ) + body = response.json() + return body.get("data", []) if isinstance(body, dict) else [] diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 489685269..cec08293f 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -357,6 +357,18 @@ "semantic-layer.reference-data.get": "read", "semantic-layer.reference-data.set": "write", "semantic-layer.reference-data.delete": "destructive", + # `scope` sub-app: visibility scope / target-project grants / elevation + # requests (PSGO-140). Parent key at the least-privileged level (read), + # same pattern as `reference-data` above. `elevate` is `destructive`, + # not `write` -- it is one-way (no downgrade endpoint exists) and + # widens an item's visibility to every project in the organization. + "semantic-layer.scope": "read", + "semantic-layer.scope.status": "read", + "semantic-layer.scope.pending": "read", + "semantic-layer.scope.grant": "write", + "semantic-layer.scope.request-elevation": "write", + "semantic-layer.scope.withdraw-elevation": "write", + "semantic-layer.scope.elevate": "destructive", # Raw HTTP client against `kbagent serve` (used by AI subprocesses). # Categorised by the underlying HTTP method under the taxonomy at the top # of this registry: GET = read, POST/PATCH = write (they create/modify), diff --git a/src/keboola_agent_cli/services/_semantic_layer_crud.py b/src/keboola_agent_cli/services/_semantic_layer_crud.py index 20b0ab310..24a851218 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/services/_semantic_layer_crud.py @@ -23,7 +23,19 @@ from ..errors import ErrorCode, KeboolaApiError if TYPE_CHECKING: - from ..metastore_client import MetastoreClient, SemanticType + from ..metastore_client import MetastoreClient, ObjectScope, SemanticType + + +def _item_scope(item: dict[str, Any]) -> tuple[ObjectScope, list[int] | None]: + """Extract ``(scope, target_project_ids)`` from a raw item's ``meta`` block. + + Defaults to ``("project", None)`` for an item predating PSGO-140 (no + ``meta.scope`` key) so the edit path is a no-op change for anything + already project-scoped. + """ + meta = item.get("meta") or {} + return meta.get("scope", "project"), meta.get("targetProjectIds") + # Kinds accepted by `remove_item` / `preview_remove`. Relationship and # glossary were added in iter-4 (NB-5) -- neither is referenced by @@ -54,9 +66,18 @@ def delete_then_post( original_attrs: dict[str, Any], new_name: str, new_attrs: dict[str, Any], + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> tuple[dict[str, Any], dict[str, Any] | None]: """Run a safe DELETE+POST, rolling back to ``original_attrs`` on POST failure. + ``scope``/``target_project_ids`` are the ORIGINAL item's values (read + from its ``meta`` block by the caller) and are re-applied on both the + primary POST and the rollback POST -- ``post_item`` defaults to + ``scope="project"``, and without passing these through explicitly, every + edit of an organization/targeted-scope object would silently downgrade + it back to project-only visibility on every save. + Returns ``(new_item, rollback)`` where ``rollback`` is None on success. On POST failure we re-POST the original payload; the rollback dict records whether that re-POST succeeded. The original @@ -65,7 +86,13 @@ def delete_then_post( """ client.delete_item(item_type, old_id) try: - new_item = client.post_item(item_type, name=new_name, data=new_attrs) + new_item = client.post_item( + item_type, + name=new_name, + data=new_attrs, + scope=scope, + target_project_ids=target_project_ids, + ) except KeboolaApiError as exc: rollback_status: dict[str, Any] = { "attempted": True, @@ -74,7 +101,13 @@ def delete_then_post( } try: old_name = original_attrs.get("name") or original_attrs.get("term", "") - restored = client.post_item(item_type, name=old_name, data=original_attrs) + restored = client.post_item( + item_type, + name=old_name, + data=original_attrs, + scope=scope, + target_project_ids=target_project_ids, + ) rollback_status["status"] = "succeeded" rollback_status["restored_id"] = restored.get("id", "") except KeboolaApiError as rollback_exc: @@ -124,6 +157,7 @@ def edit_metric_with_cascade( ) original_attrs = dict(target.get("attributes") or {}) old_id = target["id"] + target_scope, target_project_ids = _item_scope(target) effective_new_name = new_name if new_name is not None else current_name cascade_required: list[dict[str, Any]] = [] @@ -174,6 +208,8 @@ def edit_metric_with_cascade( original_attrs=original_attrs, new_name=effective_new_name, new_attrs=new_attrs, + scope=target_scope, + target_project_ids=target_project_ids, ) # Cascade constraints individually (each is independent -- @@ -185,6 +221,7 @@ def edit_metric_with_cascade( cmetrics = [effective_new_name if x == current_name else x for x in cmetrics] cattrs["metrics"] = cmetrics cname = cattrs.get("name", "") + c_scope, c_target_project_ids = _item_scope(c) try: cascaded_item, _ = delete_then_post( client, @@ -193,6 +230,8 @@ def edit_metric_with_cascade( original_attrs=c.get("attributes") or {}, new_name=cname, new_attrs=cattrs, + scope=c_scope, + target_project_ids=c_target_project_ids, ) cascaded.append({"constraint": cname, "status": "updated", "id": cascaded_item["id"]}) except KeboolaApiError as exc: @@ -340,6 +379,7 @@ def edit_simple( if v is not None: new_attrs[k] = v effective_new = new_attrs.get(id_key) or current_key + target_scope, target_project_ids = _item_scope(target) new_item, rollback = delete_then_post( client, item_type, @@ -347,6 +387,8 @@ def edit_simple( original_attrs=original_attrs, new_name=effective_new, new_attrs=new_attrs, + scope=target_scope, + target_project_ids=target_project_ids, ) return { "updated": new_item, diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py new file mode 100644 index 000000000..ca59a38bc --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -0,0 +1,128 @@ +"""Scope / target-project / elevation logic for ``semantic-layer scope`` (PSGO-140). + +Composes the :class:`MetastoreClient` scope primitives +(``elevate_to_organization``, ``put_target_projects``, +``request_scope_elevation``, ``withdraw_scope_elevation``, +``list_organization_items``) into the operations +:class:`SemanticLayerService` exposes as ``scope_*``. Alias<->numeric +``project_id`` resolution happens here (service layer), not in the client, +following this project's client/service split -- see CLAUDE.md +"Architecture: 3-Layer Design". + +Split out (like the other ``_semantic_layer_*.py`` helpers) so the +orchestrator class stays under the CONTRIBUTING.md services LOC ceiling. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ..errors import ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from ..config_store import ConfigStore + from ..metastore_client import MetastoreClient, SemanticType + + +def resolve_target_project_ids(config_store: ConfigStore, target_projects: list[str]) -> list[int]: + """Resolve project aliases to numeric Storage project IDs. + + Raises ``NOT_FOUND`` for an unregistered alias, ``CONFIG_ERROR`` for a + registered project with no numeric ``project_id`` on record (would need + a ``project refresh``/re-add to populate it -- see + ``services/org_service.py``, which relies on the same invariant). + """ + ids: list[int] = [] + for alias in target_projects: + project = config_store.get_project(alias) + if project is None: + raise KeboolaApiError( + message=f"Unknown project alias {alias!r}. See `kbagent project list`.", + error_code=ErrorCode.NOT_FOUND, + ) + if project.project_id is None: + raise KeboolaApiError( + message=( + f"Project {alias!r} has no numeric project_id on record " + "(re-run `kbagent project refresh` or `project add`)." + ), + error_code=ErrorCode.CONFIG_ERROR, + ) + ids.append(project.project_id) + return ids + + +def item_status(item: dict[str, Any]) -> dict[str, Any]: + """Extract the scope/grant/elevation-request fields from a raw item for display.""" + meta = item.get("meta") or {} + attrs = item.get("attributes") or {} + return { + "id": item.get("id"), + "type": item.get("type"), + "name": attrs.get("name") or attrs.get("term"), + "scope": meta.get("scope", "project"), + "target_project_ids": meta.get("targetProjectIds"), + "scope_elevation_requested_at": meta.get("scopeElevationRequestedAt"), + "project_id": meta.get("projectId"), + } + + +def grant_target_projects( + client: MetastoreClient, + item_type: SemanticType, + item_id: str, + *, + add: list[int] | None = None, + remove: list[int] | None = None, + replace: list[int] | None = None, +) -> dict[str, Any]: + """Update the target-project grant list for a targeted-scope item. + + ``replace`` sends exactly that set, matching the server's native + replace-only semantics with no extra round trip. ``add``/``remove`` are + a convenience merge: read the item's current grants, apply the delta, + then PUT the result. This merge is **not atomic** against a concurrent + grant change on the same item -- last write wins, same as every other + read-modify-write in this CLI. + """ + if replace is not None: + new_ids = sorted(set(replace)) + else: + current = client.get_item(item_type, item_id) + current_ids = set((current.get("meta") or {}).get("targetProjectIds") or []) + current_ids |= set(add or []) + current_ids -= set(remove or []) + new_ids = sorted(current_ids) + client.put_target_projects(item_type, item_id, new_ids) + return item_status(client.get_item(item_type, item_id)) + + +def request_elevation( + client: MetastoreClient, item_type: SemanticType, item_id: str +) -> dict[str, Any]: + return item_status(client.request_scope_elevation(item_type, item_id)) + + +def withdraw_elevation( + client: MetastoreClient, item_type: SemanticType, item_id: str +) -> dict[str, Any]: + return item_status(client.withdraw_scope_elevation(item_type, item_id)) + + +def elevate_to_organization( + client: MetastoreClient, item_type: SemanticType, item_id: str +) -> dict[str, Any]: + return item_status(client.elevate_to_organization(item_type, item_id)) + + +def list_pending_elevations( + client: MetastoreClient, + item_type: SemanticType, + *, + limit: int | None = None, + offset: int | None = None, +) -> list[dict[str, Any]]: + items = client.list_organization_items( + item_type, pending_elevation_only=True, limit=limit, offset=offset + ) + return [item_status(i) for i in items] diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 67061b55e..e90b3d1fa 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -24,9 +24,10 @@ from ..auth.sentinel import require_static_token from ..config_store import ConfigStore from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..metastore_client import MetastoreClient, SemanticType +from ..metastore_client import MetastoreClient, ObjectScope, SemanticType from ..models import ProjectConfig from . import _semantic_layer_reference_data as _refdata +from . import _semantic_layer_scope as _scope from ._semantic_layer_cascade import cascade_delete_model as _cascade_delete_model_impl from ._semantic_layer_crud import REMOVE_KINDS as _REMOVE_KINDS_HELPER from ._semantic_layer_crud import code_metric as _code_metric_helper @@ -684,6 +685,9 @@ def create_model( name: str, description: str = "", sql_dialect: str = "Snowflake", + *, + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a semantic-layer model and return the server-stored item.""" project = self._resolve_one_project(alias) @@ -691,7 +695,13 @@ def create_model( data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} if description: data["description"] = description - created = client.post_item("semantic-model", name=name, data=data) + created = client.post_item( + "semantic-model", + name=name, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) return {"project": alias, "model": created} def delete_model( @@ -735,6 +745,8 @@ def add_metric( assume_yes: bool = False, is_tty: bool = False, confirm_cb: Callable[[str], bool] | None = None, + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a metric. The ``dataset`` argument is a tableId. @@ -774,7 +786,13 @@ def add_metric( } if description: data["description"] = description - return client.post_item("semantic-metric", name=name, data=data) + return client.post_item( + "semantic-metric", + name=name, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) def add_dataset( self, @@ -788,6 +806,8 @@ def add_dataset( primary_key: list[str] | None = None, deep_fields: bool = False, fqn: str | None = None, + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a dataset. @@ -829,7 +849,13 @@ def add_dataset( fields = _synthesize_role_classified_fields(detail, _classify_field_role) if fields: data["fields"] = fields - return client.post_item("semantic-dataset", name=name, data=data) + return client.post_item( + "semantic-dataset", + name=name, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) def add_relationship( self, @@ -841,6 +867,8 @@ def add_relationship( to: str, on: str, type_: str, + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a relationship. ``from``/``to`` are tableIds; ``type``='left'|'inner'.""" if type_ not in ("left", "inner"): @@ -859,7 +887,13 @@ def add_relationship( "type": type_, "modelUUID": model_uuid, } - return client.post_item("semantic-relationship", name=name, data=data) + return client.post_item( + "semantic-relationship", + name=name, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) def add_constraint( self, @@ -871,6 +905,8 @@ def add_constraint( rule: str, metrics: list[str], severity: str = "warning", + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a constraint after validating every field locally.""" _validate_constraint_attrs( @@ -905,7 +941,13 @@ def add_constraint( "severity": severity, "modelUUID": model_uuid, } - return client.post_item("semantic-constraint", name=name, data=data) + return client.post_item( + "semantic-constraint", + name=name, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) def add_glossary( self, @@ -914,6 +956,8 @@ def add_glossary( *, term: str, definition: str = "", + scope: ObjectScope = "project", + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" project = self._resolve_one_project(alias) @@ -922,7 +966,110 @@ def add_glossary( data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} if definition: data["definition"] = definition - return client.post_item("semantic-glossary", name=term, data=data) + return client.post_item( + "semantic-glossary", + name=term, + data=data, + scope=scope, + target_project_ids=target_project_ids, + ) + + # ------------------------------------------------------------------ + # Phase 5 — Scope / target-project / elevation (PSGO-140) + # ------------------------------------------------------------------ + + def resolve_target_project_ids(self, target_projects: list[str]) -> list[int]: + """Resolve project aliases to numeric project IDs (for ``--target-project``).""" + return _scope.resolve_target_project_ids(self._config_store, target_projects) + + def _resolve_scope_type(self, kind: str) -> SemanticType: + item_type = SCHEMA_TYPE_ALIAS.get(kind) + if item_type is None: + raise KeboolaApiError( + message=f"--type must be one of {sorted(SCHEMA_TYPE_ALIAS)}, got {kind!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + return item_type + + def scope_status(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: + """Show the current scope/grants/pending-elevation state of one item.""" + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.item_status(client.get_item(item_type, item_id)) + + def scope_grant( + self, + alias: str, + kind: str, + item_id: str, + *, + add: list[str] | None = None, + remove: list[str] | None = None, + replace: list[str] | None = None, + ) -> dict[str, Any]: + """Grant/revoke/replace the target-project list of a targeted-scope item. + + ``add``/``remove``/``replace`` are project ALIASES, resolved to + numeric project IDs here before calling the metastore. ``replace`` + (including ``[]`` to clear every grant) is a direct pass-through of + the server's native replace-only semantics; ``add``/``remove`` do a + read-modify-write merge (see :func:`_semantic_layer_scope.grant_target_projects`). + """ + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.grant_target_projects( + client, + item_type, + item_id, + add=_scope.resolve_target_project_ids(self._config_store, add) if add else None, + remove=( + _scope.resolve_target_project_ids(self._config_store, remove) + if remove + else None + ), + replace=( + _scope.resolve_target_project_ids(self._config_store, replace) + if replace is not None + else None + ), + ) + + def scope_request_elevation(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.request_elevation(client, item_type, item_id) + + def scope_withdraw_elevation(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: + """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.withdraw_elevation(client, item_type, item_id) + + def scope_elevate(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: + """Step an item up to organization scope. Requires org-admin; one-way, no downgrade.""" + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.elevate_to_organization(client, item_type, item_id) + + def scope_pending( + self, + alias: str, + kind: str, + *, + limit: int | None = None, + offset: int | None = None, + ) -> list[dict[str, Any]]: + """List items of ``kind`` awaiting an org-admin's elevation decision.""" + item_type = self._resolve_scope_type(kind) + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + return _scope.list_pending_elevations(client, item_type, limit=limit, offset=offset) # ------------------------------------------------------------------ # Reference data — dimension-member records (e.g. a Chart of Accounts). diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index b6d99141a..3503e4343 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -169,10 +169,11 @@ def test_post_envelope(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "rev" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" + assert body["schemaVersion"] == "1.1.0" assert body["scope"] == "project" assert body["data"]["sql"] == "SUM(x)" assert body["data"]["modelUUID"] == "u" + assert "targetProjectIds" not in body @pytest.fixture @@ -303,8 +304,8 @@ def test_put_envelope_and_url(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "chart_of_accounts" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" - assert body["scope"] == "project" + assert body["schemaVersion"] == "1.1.0" + assert "scope" not in body assert body["data"]["dimensionName"] == "chart_of_accounts" assert body["data"]["members"] == [] @@ -342,12 +343,15 @@ def test_semantic_types_complete(self) -> None: class TestProjectScope401Reclassification: - """The metastore's master-token gate 401 becomes MISSING_MASTER_TOKEN. - - The metastore auth middleware collapses every project-scope resolution - failure into a 401 ``"Failed to create project scope"``; for a valid - non-master token that IS the master-token gate firing (issue #711). - The client funnels every verb through the reclassification. + """The metastore's project-admin gate 401 becomes MISSING_MASTER_TOKEN. + + Pre-PSGO-282, the metastore auth middleware collapsed every project-scope + resolution failure into a 401 ``"Failed to create project scope"``, + including plain reads from a valid non-master token (issue #711). Since + PSGO-282 (go-monorepo#596) a fixed metastore only emits this 401 for a + WRITE against a token that is not a project admin -- reads succeed for + any valid token. The client still funnels every verb through the + reclassification as a safety net for a not-yet-upgraded deployment. """ _SCOPE_401_BODY: ClassVar[dict] = { @@ -520,3 +524,164 @@ def test_default_bearer_would_refresh_reproducing_the_bug(self, httpx_mock) -> N client.close() assert spy.refresh_calls == 1 assert len(httpx_mock.get_requests()) == 2 +class TestPostItemScopeValidation: + """post_item validates scope/target_project_ids client-side (PSGO-140).""" + + def test_rejects_unknown_scope(self, metastore_client) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item("semantic-metric", name="x", data={}, scope="bogus") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_rejects_target_project_ids_without_targeted_scope(self, metastore_client) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item( + "semantic-metric", name="x", data={}, scope="project", target_project_ids=[1] + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_targeted_scope_sends_target_project_ids(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "semantic-metric", + name="x", + data={}, + scope="targeted", + target_project_ids=[123, 456], + ) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "targeted" + assert body["targetProjectIds"] == [123, 456] + + def test_organization_scope_omits_target_project_ids_key( + self, httpx_mock, metastore_client + ) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item("semantic-metric", name="x", data={}, scope="organization") + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "organization" + assert "targetProjectIds" not in body + + +class TestElevateToOrganization: + """PATCH /{type}/{id} with {"scope": "organization"} only.""" + + def test_sends_scope_only_patch(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PATCH", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc", + json={ + "data": { + "type": "semantic-dataset", + "id": "abc", + "attributes": {}, + "meta": {"scope": "organization"}, + } + }, + status_code=200, + ) + result = metastore_client.elevate_to_organization("semantic-dataset", "abc") + assert result["meta"]["scope"] == "organization" + request = httpx_mock.get_requests()[0] + assert request.method == "PATCH" + assert json.loads(request.content) == {"scope": "organization"} + + def test_403_maps_to_access_denied(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PATCH", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc", + status_code=403, + json={"error": "Insufficient permissions"}, + ) + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.elevate_to_organization("semantic-dataset", "abc") + assert excinfo.value.error_code == ErrorCode.ACCESS_DENIED + + +class TestPutTargetProjects: + """PUT /{type}/{id}/target-projects replaces the whole grant set; 204 no body.""" + + def test_replaces_target_projects(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/target-projects", + status_code=204, + ) + assert metastore_client.put_target_projects("semantic-dataset", "abc", [1, 2]) is None + request = httpx_mock.get_requests()[0] + assert json.loads(request.content) == {"targetProjectIds": [1, 2]} + + def test_clears_with_empty_list(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/target-projects", + status_code=204, + ) + metastore_client.put_target_projects("semantic-dataset", "abc", []) + request = httpx_mock.get_requests()[0] + assert json.loads(request.content) == {"targetProjectIds": []} + + +class TestScopeElevationRequest: + """PUT/DELETE .../scope-elevation-request: empty body, 200 with the updated item.""" + + def test_request_elevation(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/scope-elevation-request", + json={ + "data": { + "type": "semantic-dataset", + "id": "abc", + "attributes": {}, + "meta": {"scopeElevationRequestedAt": "2026-08-28T00:00:00Z"}, + } + }, + status_code=200, + ) + result = metastore_client.request_scope_elevation("semantic-dataset", "abc") + assert result["meta"]["scopeElevationRequestedAt"] + + def test_withdraw_elevation(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="DELETE", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/scope-elevation-request", + json={"data": {"type": "semantic-dataset", "id": "abc", "attributes": {}, "meta": {}}}, + status_code=200, + ) + result = metastore_client.withdraw_scope_elevation("semantic-dataset", "abc") + assert result["id"] == "abc" + + +class TestListOrganizationItems: + """GET /{type}/organization with the generic filter/limit/offset query language.""" + + def test_pending_elevation_filter_and_pagination(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=( + f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/organization" + "?scope_elevation_requested_at%5Bnot%5D%5Bnull%5D=true&limit=5&offset=10" + ), + json={"data": [{"type": "semantic-dataset", "id": "a", "attributes": {}}]}, + status_code=200, + ) + result = metastore_client.list_organization_items( + "semantic-dataset", pending_elevation_only=True, limit=5, offset=10 + ) + assert len(result) == 1 + assert result[0]["id"] == "a" + + def test_no_filters_sends_bare_request(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-model/organization", + json={"data": []}, + status_code=200, + ) + assert metastore_client.list_organization_items("semantic-model") == [] diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py new file mode 100644 index 000000000..eb4e52f15 --- /dev/null +++ b/tests/test_semantic_layer_scope.py @@ -0,0 +1,338 @@ +"""Tests for metastore scope/target-project/elevation support (PSGO-140). + +Covers ``services._semantic_layer_scope`` (pure alias-resolution + merge +logic) and ``SemanticLayerService.scope_*`` (orchestration), plus the +edit-preserves-scope regression: a DELETE+POST edit of an +organization/targeted-scope item must never silently reset it to +``project`` scope. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services import _semantic_layer_scope as scope_helpers +from keboola_agent_cli.services.semantic_layer_service import SemanticLayerService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + store.add_project( + "analytics", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="analytics", + project_id=1234, + ), + ) + return store + + +def _make_service(store: ConfigStore, *, metastore_mock: MagicMock | None = None): + mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + service = SemanticLayerService( + config_store=store, + metastore_client_factory=lambda url, token: mock, + ) + return service, mock + + +def _model_item(uuid: str = "u-model", name: str = "default") -> dict[str, Any]: + return {"type": "semantic-model", "id": uuid, "attributes": {"name": name}} + + +def _child_item( + item_type: str, item_id: str, attrs: dict[str, Any], meta: dict[str, Any] | None = None +) -> dict[str, Any]: + item: dict[str, Any] = {"type": item_type, "id": item_id, "attributes": dict(attrs)} + if meta is not None: + item["meta"] = meta + return item + + +# --------------------------------------------------------------------------- +# services._semantic_layer_scope -- pure helpers +# --------------------------------------------------------------------------- + + +class TestResolveTargetProjectIds: + def test_resolves_known_aliases(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + assert scope_helpers.resolve_target_project_ids(store, ["prod", "analytics"]) == [ + 5725, + 1234, + ] + + def test_unknown_alias_raises_not_found(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.resolve_target_project_ids(store, ["ghost"]) + assert excinfo.value.error_code == ErrorCode.NOT_FOUND + + def test_missing_project_id_raises_config_error(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + store.add_project( + "no-id", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="no-id", + project_id=None, + ), + ) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.resolve_target_project_ids(store, ["no-id"]) + assert excinfo.value.error_code == ErrorCode.CONFIG_ERROR + + +class TestItemStatus: + def test_extracts_scope_fields_from_meta(self) -> None: + item = _child_item( + "semantic-dataset", + "d1", + {"name": "fact_pnl"}, + meta={ + "scope": "targeted", + "targetProjectIds": [1, 2], + "scopeElevationRequestedAt": "2026-08-28T00:00:00Z", + "projectId": 5725, + }, + ) + status = scope_helpers.item_status(item) + assert status == { + "id": "d1", + "type": "semantic-dataset", + "name": "fact_pnl", + "scope": "targeted", + "target_project_ids": [1, 2], + "scope_elevation_requested_at": "2026-08-28T00:00:00Z", + "project_id": 5725, + } + + def test_defaults_to_project_scope_without_meta(self) -> None: + item = _child_item("semantic-metric", "m1", {"name": "rev"}) + status = scope_helpers.item_status(item) + assert status["scope"] == "project" + assert status["target_project_ids"] is None + + def test_glossary_falls_back_to_term(self) -> None: + item = _child_item("semantic-glossary", "g1", {"term": "GMV"}) + assert scope_helpers.item_status(item)["name"] == "GMV" + + +class TestGrantTargetProjects: + def test_replace_sends_exact_set_without_reading_current(self) -> None: + client = MagicMock() + client.get_item.return_value = _child_item( + "semantic-dataset", "d1", {"name": "x"}, meta={"scope": "targeted"} + ) + scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", replace=[2, 1, 2]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 2]) + # replace mode never reads the current grant set before writing. + assert client.get_item.call_count == 1 # only the post-write status re-fetch + + def test_add_merges_with_current_grants(self) -> None: + client = MagicMock() + client.get_item.return_value = _child_item( + "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": [1, 2]} + ) + scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", add=[3]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 2, 3]) + + def test_remove_merges_with_current_grants(self) -> None: + client = MagicMock() + client.get_item.return_value = _child_item( + "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": [1, 2, 3]} + ) + scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", remove=[2]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 3]) + + def test_clear_via_empty_replace(self) -> None: + client = MagicMock() + client.get_item.return_value = _child_item("semantic-dataset", "d1", {"name": "x"}) + scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", replace=[]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", []) + + +# --------------------------------------------------------------------------- +# SemanticLayerService.scope_* -- orchestration +# --------------------------------------------------------------------------- + + +class TestServiceScopeMethods: + def test_resolve_target_project_ids_public_wrapper(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _mock = _make_service(store) + assert service.resolve_target_project_ids(["analytics"]) == [1234] + + def test_resolve_scope_type_rejects_unknown_kind(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, _mock = _make_service(store) + with pytest.raises(KeboolaApiError) as excinfo: + service.scope_status("prod", "bogus-kind", "id-1") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_scope_status_delegates_to_client_get_item(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _child_item( + "semantic-dataset", "d1", {"name": "x"}, meta={"scope": "organization"} + ) + result = service.scope_status("prod", "dataset", "d1") + mock.get_item.assert_called_once_with("semantic-dataset", "d1") + assert result["scope"] == "organization" + + def test_scope_request_elevation_delegates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.request_scope_elevation.return_value = _child_item( + "semantic-metric", "m1", {"name": "x"}, meta={"scopeElevationRequestedAt": "t"} + ) + result = service.scope_request_elevation("prod", "metric", "m1") + mock.request_scope_elevation.assert_called_once_with("semantic-metric", "m1") + assert result["scope_elevation_requested_at"] == "t" + + def test_scope_withdraw_elevation_delegates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.withdraw_scope_elevation.return_value = _child_item( + "semantic-metric", "m1", {"name": "x"} + ) + service.scope_withdraw_elevation("prod", "metric", "m1") + mock.withdraw_scope_elevation.assert_called_once_with("semantic-metric", "m1") + + def test_scope_elevate_delegates(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.elevate_to_organization.return_value = _child_item( + "semantic-metric", "m1", {"name": "x"}, meta={"scope": "organization"} + ) + result = service.scope_elevate("prod", "metric", "m1") + mock.elevate_to_organization.assert_called_once_with("semantic-metric", "m1") + assert result["scope"] == "organization" + + def test_scope_pending_delegates_with_pagination(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_organization_items.return_value = [ + _child_item("semantic-dataset", "d1", {"name": "x"}, meta={}) + ] + result = service.scope_pending("prod", "dataset", limit=5, offset=1) + mock.list_organization_items.assert_called_once_with( + "semantic-dataset", pending_elevation_only=True, limit=5, offset=1 + ) + assert len(result) == 1 + + def test_scope_grant_resolves_aliases_before_calling_client(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _child_item( + "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": []} + ) + service.scope_grant("prod", "dataset", "d1", add=["analytics"]) + mock.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1234]) + + +# --------------------------------------------------------------------------- +# Regression: edit (DELETE+POST) must preserve the original item's scope +# --------------------------------------------------------------------------- + + +class TestEditPreservesScope: + def test_edit_metric_preserves_targeted_scope_and_grants(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item( + "semantic-metric", + "m1", + {"name": "rev", "sql": "1"}, + meta={"scope": "targeted", "targetProjectIds": [1234]}, + ) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item()] + if item_type == "semantic-metric": + return [original] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} + + service.edit_metric("prod", None, current_name="rev", new_description="updated") + + mock.post_item.assert_called_once() + _args, kwargs = mock.post_item.call_args + assert kwargs["scope"] == "targeted" + assert kwargs["target_project_ids"] == [1234] + + def test_edit_dataset_preserves_organization_scope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item( + "semantic-dataset", + "d1", + {"name": "fact_pnl", "tableId": "out.c-fin.fact_pnl"}, + meta={"scope": "organization"}, + ) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item()] + if item_type == "semantic-dataset": + return [original] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "d_new", "attributes": {"name": "fact_pnl"}} + + service.edit_dataset("prod", None, current_name="fact_pnl", new_grain="daily") + + _args, kwargs = mock.post_item.call_args + assert kwargs["scope"] == "organization" + assert kwargs["target_project_ids"] is None + + def test_edit_metric_defaults_to_project_scope_when_no_meta(self, tmp_path: Path) -> None: + """Pre-PSGO-140 items (no meta.scope) edit as plain project scope, unchanged.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item()] + if item_type == "semantic-metric": + return [original] + return [] + + mock.list_items.side_effect = _list + mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} + + service.edit_metric("prod", None, current_name="rev", new_sql="2") + + _args, kwargs = mock.post_item.call_args + assert kwargs["scope"] == "project" + assert kwargs["target_project_ids"] is None diff --git a/tests/test_semantic_layer_scope_cli.py b/tests/test_semantic_layer_scope_cli.py new file mode 100644 index 000000000..8e584e978 --- /dev/null +++ b/tests/test_semantic_layer_scope_cli.py @@ -0,0 +1,531 @@ +"""CLI-layer tests for ``semantic-layer scope`` and the ``--scope``/ +``--target-project`` flags on ``model create`` / ``add `` (PSGO-140). + +Mirrors the test_semantic_layer_cli.py pattern: patch cli.py's service +factory so the runner sees a MagicMock SemanticLayerService, plus a REAL +ProjectService (built from the test ConfigStore) since ``resolve_scope_targets`` +reads project aliases through it for the interactive-picker fallback. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.constants import EXIT_PERMISSION_DENIED +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + + +def _setup_config(config_dir: Path, projects: dict[str, dict]) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _invoke(args: list[str], *, store: ConfigStore, sl_mock: MagicMock): + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.SemanticLayerService") as MockSL, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + MockSL.return_value = sl_mock + return runner.invoke(app, args) + + +@pytest.fixture +def cfg_dir(tmp_path: Path) -> Path: + d = tmp_path / "config" + d.mkdir() + return d + + +@pytest.fixture +def store(cfg_dir: Path) -> ConfigStore: + return _setup_config( + cfg_dir, + { + "prod": {"token": TEST_TOKEN, "project_id": 5725}, + "analytics": {"token": TEST_TOKEN, "project_id": 1234}, + }, + ) + + +# --------------------------------------------------------------------------- +# --scope / --target-project on model create / add +# --------------------------------------------------------------------------- + + +class TestModelCreateScope: + def test_default_scope_is_project_and_unrequested(self, store: ConfigStore) -> None: + """No --scope passed: create_model still gets scope='project', no picker touched.""" + mock = MagicMock() + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + result = _invoke( + ["--json", "semantic-layer", "model", "create", "--project", "prod", "--name", "n"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _args, kwargs = mock.create_model.call_args + assert kwargs["scope"] == "project" + assert kwargs["target_project_ids"] is None + + def test_targeted_scope_with_explicit_target_project(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + mock.resolve_target_project_ids.return_value = [1234] + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "n", + "--scope", + "targeted", + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.resolve_target_project_ids.assert_called_once_with(["analytics"]) + _args, kwargs = mock.create_model.call_args + assert kwargs["scope"] == "targeted" + assert kwargs["target_project_ids"] == [1234] + + def test_targeted_scope_without_target_project_fails_fast_non_tty( + self, store: ConfigStore + ) -> None: + """No --target-project + CliRunner's non-TTY stdin/stdout -> hard fail, never a hang.""" + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "n", + "--scope", + "targeted", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.create_model.assert_not_called() + body = json.loads(result.output) + assert body["error"]["code"] == "INVALID_ARGUMENT" + + def test_organization_scope_ignores_absent_target_project(self, store: ConfigStore) -> None: + """--scope organization never needs --target-project; no picker, no failure.""" + mock = MagicMock() + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + result = _invoke( + [ + "--json", + "semantic-layer", + "model", + "create", + "--project", + "prod", + "--name", + "n", + "--scope", + "organization", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _args, kwargs = mock.create_model.call_args + assert kwargs["scope"] == "organization" + assert kwargs["target_project_ids"] is None + mock.resolve_target_project_ids.assert_not_called() + + +class TestAddMetricScope: + def test_add_metric_targeted_scope(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_metric.return_value = {"id": "m1", "attributes": {"name": "rev"}} + mock.resolve_target_project_ids.return_value = [1234] + result = _invoke( + [ + "--json", + "semantic-layer", + "add", + "metric", + "--project", + "prod", + "--name", + "rev", + "--sql", + "SUM(x)", + "--dataset", + "out.c-foo.bar", + "--scope", + "targeted", + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _args, kwargs = mock.add_metric.call_args + assert kwargs["scope"] == "targeted" + assert kwargs["target_project_ids"] == [1234] + + +# --------------------------------------------------------------------------- +# semantic-layer scope +# --------------------------------------------------------------------------- + + +class TestScopeStatus: + def test_status_success(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_status.return_value = { + "id": "d1", + "scope": "targeted", + "target_project_ids": [1234], + "scope_elevation_requested_at": None, + } + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "status", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_status.assert_called_once_with(alias="prod", kind="dataset", item_id="d1") + assert json.loads(result.output)["data"]["scope"] == "targeted" + + +class TestScopeGrant: + def test_grant_add_default_merge(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": [1234]} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "grant", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_grant.assert_called_once_with( + alias="prod", + kind="dataset", + item_id="d1", + add=["analytics"], + remove=None, + replace=None, + ) + + def test_grant_replace(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": [1234]} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "grant", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + "--target-project", + "analytics", + "--replace", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _args, kwargs = mock.scope_grant.call_args + assert kwargs["replace"] == ["analytics"] + + def test_grant_clear(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": []} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "grant", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + "--clear", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + _args, kwargs = mock.scope_grant.call_args + assert kwargs["replace"] == [] + + def test_clear_combined_with_target_project_is_usage_error(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "grant", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + "--clear", + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.scope_grant.assert_not_called() + + +class TestScopeElevation: + def test_request_elevation(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_request_elevation.return_value = {"scope_elevation_requested_at": "t"} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "request-elevation", + "--project", + "prod", + "--type", + "metric", + "--id", + "m1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_request_elevation.assert_called_once_with( + alias="prod", kind="metric", item_id="m1" + ) + + def test_withdraw_elevation(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_withdraw_elevation.return_value = {"scope_elevation_requested_at": None} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "withdraw-elevation", + "--project", + "prod", + "--type", + "metric", + "--id", + "m1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_withdraw_elevation.assert_called_once_with( + alias="prod", kind="metric", item_id="m1" + ) + + def test_elevate_with_yes_skips_prompt(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_elevate.return_value = {"scope": "organization"} + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "elevate", + "--project", + "prod", + "--type", + "metric", + "--id", + "m1", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_elevate.assert_called_once_with(alias="prod", kind="metric", item_id="m1") + + def test_pending(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_pending.return_value = [ + {"id": "d1", "name": "x", "scope_elevation_requested_at": "t"} + ] + result = _invoke( + [ + "--json", + "semantic-layer", + "scope", + "pending", + "--project", + "prod", + "--type", + "dataset", + "--limit", + "5", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_pending.assert_called_once_with( + alias="prod", kind="dataset", limit=5, offset=None + ) + + +# --------------------------------------------------------------------------- +# Permission gating +# --------------------------------------------------------------------------- + + +class TestScopePermissions: + def test_deny_writes_blocks_grant(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "scope", + "grant", + "--project", + "prod", + "--type", + "dataset", + "--id", + "d1", + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.scope_grant.assert_not_called() + + def test_deny_destructive_blocks_elevate(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + [ + "--deny-destructive", + "--json", + "semantic-layer", + "scope", + "elevate", + "--project", + "prod", + "--type", + "metric", + "--id", + "m1", + "--yes", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == EXIT_PERMISSION_DENIED + mock.scope_elevate.assert_not_called() + + def test_deny_writes_does_not_block_status(self, store: ConfigStore) -> None: + """--deny-writes must not block the read-only `scope status` leaf.""" + mock = MagicMock() + mock.scope_status.return_value = {"scope": "project"} + result = _invoke( + [ + "--deny-writes", + "--json", + "semantic-layer", + "scope", + "status", + "--project", + "prod", + "--type", + "metric", + "--id", + "m1", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + mock.scope_status.assert_called_once() From 015dd44099f5c93ef278d4b2ec91b27d775979d2 Mon Sep 17 00:00:00 2001 From: jordanrburger Date: Fri, 28 Aug 2026 12:37:33 -0400 Subject: [PATCH 2/6] fix(semantic-layer): report scope grants from a re-read, not the endpoint response `scope request-elevation` / `withdraw-elevation` / `elevate` rendered the mutating endpoint's own response body. That body omits `meta.targetProjectIds`, so the commands printed `target_project_ids: null` for an item whose grants were fully intact. Verified live against metastore.us-east4 (project keboola-ai, model with scope=targeted, targets=[5024]): scope status -> targets=[5024] request-elevation -> targets=None <-- wrong, grants untouched scope status -> targets=[5024] An operator reading that output would reasonably conclude that requesting elevation had just wiped every grant on the object. `grant_target_projects` already re-read the item after its PUT for exactly this reason; the three elevation helpers now do the same. Re-verified live: command output matches `scope status` at every step. Co-Authored-By: Claude Opus 5 --- .../services/_semantic_layer_scope.py | 22 +++++++-- tests/test_semantic_layer_scope.py | 45 ++++++++++++++++--- 2 files changed, 59 insertions(+), 8 deletions(-) diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py index ca59a38bc..e0dcc8c5a 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -100,19 +100,35 @@ def grant_target_projects( def request_elevation( client: MetastoreClient, item_type: SemanticType, item_id: str ) -> dict[str, Any]: - return item_status(client.request_scope_elevation(item_type, item_id)) + """Flag an item as awaiting an org-admin's step-up decision. + + Re-reads the item instead of rendering the endpoint's own response: + the ``scope-elevation-request`` (and ``PATCH``) response bodies omit + ``meta.targetProjectIds``, so reporting them directly prints + ``target_project_ids: null`` for an item whose grants are actually + intact -- verified live: `scope status` before and after shows + ``[5024]`` while the endpoint response shows ``None``. That reads as + "requesting elevation just wiped my grants", which is not what + happened. Same re-read pattern as :func:`grant_target_projects`. + """ + client.request_scope_elevation(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) def withdraw_elevation( client: MetastoreClient, item_type: SemanticType, item_id: str ) -> dict[str, Any]: - return item_status(client.withdraw_scope_elevation(item_type, item_id)) + """Clear a pending elevation request. Re-reads for the reason in :func:`request_elevation`.""" + client.withdraw_scope_elevation(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) def elevate_to_organization( client: MetastoreClient, item_type: SemanticType, item_id: str ) -> dict[str, Any]: - return item_status(client.elevate_to_organization(item_type, item_id)) + """Step an item up to organization scope. Re-reads for the reason in :func:`request_elevation`.""" + client.elevate_to_organization(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) def list_pending_elevations( diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index eb4e52f15..3a09fb103 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -208,7 +208,7 @@ def test_scope_status_delegates_to_client_get_item(self, tmp_path: Path) -> None def test_scope_request_elevation_delegates(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) - mock.request_scope_elevation.return_value = _child_item( + mock.get_item.return_value = _child_item( "semantic-metric", "m1", {"name": "x"}, meta={"scopeElevationRequestedAt": "t"} ) result = service.scope_request_elevation("prod", "metric", "m1") @@ -218,22 +218,57 @@ def test_scope_request_elevation_delegates(self, tmp_path: Path) -> None: def test_scope_withdraw_elevation_delegates(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) - mock.withdraw_scope_elevation.return_value = _child_item( - "semantic-metric", "m1", {"name": "x"} - ) + mock.get_item.return_value = _child_item("semantic-metric", "m1", {"name": "x"}) service.scope_withdraw_elevation("prod", "metric", "m1") mock.withdraw_scope_elevation.assert_called_once_with("semantic-metric", "m1") def test_scope_elevate_delegates(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) - mock.elevate_to_organization.return_value = _child_item( + mock.get_item.return_value = _child_item( "semantic-metric", "m1", {"name": "x"}, meta={"scope": "organization"} ) result = service.scope_elevate("prod", "metric", "m1") mock.elevate_to_organization.assert_called_once_with("semantic-metric", "m1") assert result["scope"] == "organization" + @pytest.mark.parametrize( + ("method", "client_call"), + [ + ("scope_request_elevation", "request_scope_elevation"), + ("scope_withdraw_elevation", "withdraw_scope_elevation"), + ("scope_elevate", "elevate_to_organization"), + ], + ) + def test_elevation_ops_report_grants_from_a_reread_not_the_endpoint_response( + self, tmp_path: Path, method: str, client_call: str + ) -> None: + """Grants must survive in the REPORTED status, not just on the server. + + Verified live against metastore.us-east4: the + ``scope-elevation-request`` / ``PATCH`` responses omit + ``meta.targetProjectIds`` even though the item keeps its grants, so + rendering that response printed ``target_project_ids: null`` right + after a `scope status` had shown ``[5024]`` -- i.e. it looked like + requesting elevation had silently wiped every grant. + """ + store = _make_store(tmp_path) + service, mock = _make_service(store) + # What the mutating endpoint returns: grants missing from meta. + getattr(mock, client_call).return_value = _child_item( + "semantic-metric", "m1", {"name": "x"}, meta={"scope": "targeted"} + ) + # What the item actually looks like when re-read. + mock.get_item.return_value = _child_item( + "semantic-metric", + "m1", + {"name": "x"}, + meta={"scope": "targeted", "targetProjectIds": [5024]}, + ) + result = getattr(service, method)("prod", "metric", "m1") + assert result["target_project_ids"] == [5024] + mock.get_item.assert_called_with("semantic-metric", "m1") + def test_scope_pending_delegates_with_pagination(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) From 4140a98520d8e3af5233af349b5b670000d6997a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 31 Aug 2026 14:57:24 +0200 Subject: [PATCH 3/6] fix(semantic-layer): stop telling agents metastore needs a master token for every call (PSGO-282) go-monorepo#596 (PSGO-282) fixed the metastore to accept any valid, non-disabled, non-expired Storage token for reads -- writes still need a project-admin token. Every doc this repo carries about the old behavior (added in #711/#717) still claimed the *whole* semantic-layer family needed a master token, which is now false and would make agents refuse a plain read or hunt for a master token they don't need. Updates CLAUDE.md, keboola-expert.md, commands-reference.md, gotchas.md, semantic-layer-workflow.md and docs/error-codes.md to state the real split (reads: any valid token: writes: project-admin token), and softens metastore_client.py's 401-reclassification message so it no longer overclaims a blanket master-token requirement -- kept as a safety net for a deployment that predates the fix. New gate entries use the vNEXT placeholder per this repo's release convention (check_version_gates.py). Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 20 +++++++++++-------- docs/error-codes.md | 2 +- plugins/kbagent/agents/keboola-expert.md | 4 ++-- .../kbagent/references/commands-reference.md | 2 +- .../references/semantic-layer-workflow.md | 17 ++++++++++------ 5 files changed, 27 insertions(+), 18 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 23ce65ec0..1a6949073 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1115,14 +1115,18 @@ kbagent semantic-layer reference-data get --project P (--id ID | --dimension D) kbagent semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X] kbagent semantic-layer reference-data delete --project P --id ID [--yes] # Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. -# semantic-layer REQUIRES A MASTER (project admin) Storage token (#711): the Metastore's -# auth gate -- unlike the Storage API -- rejects every valid non-master token with an opaque -# 401 "Failed to create project scope" (the underlying MasterTokenRequiredError is swallowed -# server-side). kbagent reclassifies exactly that 401 to MISSING_MASTER_TOKEN (exit 3) with -# the remedy; other unexplained 401s anywhere map to AUTH_REJECTED instead of the false -# "Invalid or expired token" (INVALID_TOKEN stays for 401s that DO blame the credential). -# Version gate for this entry lives in gotchas.md -- `(since vNEXT)` cannot be written on -# these `# ` comment lines (check_version_gates.py parses them as ATX headings). +# semantic-layer READS work with any valid, non-disabled, non-expired Storage token +# (vNEXT, PSGO-282): the Metastore no longer requires a master token for GET/List. WRITES +# (add/edit/remove/import/promote/build/scope grant/elevate) still need a project-admin token +# (master token or any admin-role user token) -- a non-admin token 403s on the write. Before +# PSGO-282 the Metastore's auth gate rejected EVERY valid non-master token, including reads, +# with an opaque 401 "Failed to create project scope" (#711, MasterTokenRequiredError swallowed +# server-side); kbagent still reclassifies exactly that 401 to MISSING_MASTER_TOKEN (exit 3) as +# a safety net for a deployment that predates the fix. Other unexplained 401s anywhere map to +# AUTH_REJECTED instead of the false "Invalid or expired token" (INVALID_TOKEN stays for 401s +# that DO blame the credential). Version gate for this entry lives in gotchas.md -- `(since +# vNEXT)` cannot be written on these `# ` comment lines (check_version_gates.py parses them as +# ATX headings). kbagent http get PATH [--timeout SECONDS] kbagent http post PATH [--body JSON|@file|-] [--timeout SECONDS] diff --git a/docs/error-codes.md b/docs/error-codes.md index 6977cd7a9..f37d951b6 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -19,7 +19,7 @@ of `ErrorCode` in `src/keboola_agent_cli/errors.py`. | `INVALID_TOKEN` | Storage API token is invalid or expired | | `ACCESS_DENIED` | Token lacks the required permission for this API call | | `PERMISSION_DENIED` | Operation blocked by the active kbagent permission policy | -| `MISSING_MASTER_TOKEN` | Operation requires a master (admin) Storage token (e.g. `token create`, `config oauth-url` pre-flight, and every `semantic-layer` command -- the Metastore's opaque 401 `Failed to create project scope` is its master-token gate and is reclassified to this code); maps to exit 3 | +| `MISSING_MASTER_TOKEN` | Operation requires a master (admin) Storage token (e.g. `token create`, `config oauth-url` pre-flight, and a `semantic-layer` WRITE against a non-admin token -- `semantic-layer` reads work with any valid token since PSGO-282; the Metastore's opaque 401 `Failed to create project scope` was its master-token gate pre-PSGO-282 and is reclassified to this code as a safety net); maps to exit 3 | | `UNAUTHORIZED` | `kbagent serve` rejected the request's Bearer token | | `AUTH_REJECTED` | Upstream answered HTTP 401 but its own error text did not blame the credential. Same exit 3 as `INVALID_TOKEN`; rotating the token is not the fix. (The Metastore's `Failed to create project scope` 401 has a known cause and lands on `MISSING_MASTER_TOKEN` instead) | diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 83f37f38a..7017de331 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -151,7 +151,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` -- uses the `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` the scheduler injects; `http get /openapi.json` discovers endpoints | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated) | `curl $KBAGENT_SERVE_URL/...` by hand (loses the auth header, error mapping and JSON formatting `kbagent http` adds) | | Launch the web UI for an end-user | `kbagent serve --ui [--port PORT]` -- one FastAPI process mounts the bundled React SPA and sets an HttpOnly `kbagent_session` cookie, so SSE works with no token in the URL or JS heap | the legacy Vite + Node BFF dev flow (`web/README.md`) | inventing a `--token-in-url` flag; running uvicorn directly against the dist (the cookie bootstrap only fires from `serve --ui`) | | Schedule / manage Agent Tasks | `kbagent agent ` -- CRUD, `run [--stream]`, history, `test`/`cron-preview`/`prompt-improve`. Local-only; cron firing needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` inside scheduled subprocesses | hand-editing `agents.json` | -| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.93.0+, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | +| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (vNEXT, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope grant / scope elevate) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope elevate`/`scope request-elevation` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS` grants specific projects; `scope grant --project P --type T --id ID --target-project ALIAS ...` adjusts an existing item's grants (merge by default, `--replace`/`--clear` for the API's native replace semantics); org-wide needs `scope request-elevation` then an org-admin's `scope elevate --yes` -- **irreversible, no downgrade endpoint**. `scope status --type T --id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope pending --type T` to find items an org-admin should act on | calling `scope elevate` or passing `--scope organization\|targeted` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | @@ -252,7 +252,7 @@ its absence is NOT a promise the entry is version-independent (see §1 Rule 6). - A 401 whose body names something other than a bad credential now raises `AUTH_REJECTED`; the semantic-layer family's opaque metastore 401 reclassified to `MISSING_MASTER_TOKEN` on a write against a non-admin - token (see its matrix row) -- since PSGO-282 (0.93.0+) a plain read never + token (see its matrix row) -- since PSGO-282 (vNEXT) a plain read never hits this at all, any valid token passes. Exit code is unchanged (3) for all three -- code checking `error.code == "INVALID_TOKEN"` alone still misses two real cases; accept all three before concluding the credential diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 3f5274c92..22ea43d4d 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -406,7 +406,7 @@ MFA login sends `challenge: SOFTWARE_TOKEN_MFA` explicitly to fix a 404 on perso ## Semantic Layer (Metastore) -Reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars. The scheduler auto-injects these (plus `KBAGENT_CONFIG_DIR`) into every AI-agent / `cli_command` subprocess. Outside a serve subprocess context the command refuses to run with exit code 2. **Inside a scheduled-agent task, prefer `kbagent http get /openapi.json` then a typed call over forking another `kbagent` CLI -- the HTTP path always sees the operator's live config (not the global `~/.config/keboola-agent-cli/` one).** -Manage Keboola metastore models -- datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from the stack URL by replacing `connection.` with `metastore.` (cloud/region-agnostic). Auth: same `X-StorageApi-Token` as Storage, **but the metastore accepts only a MASTER (project admin) token** -- a valid non-master token gets its opaque 401 `Failed to create project scope` on every call, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711; see [gotchas.md](gotchas.md)). Hidden alias: `kbagent sl ...` is equivalent to `kbagent semantic-layer ...`. See [semantic-layer-workflow.md](semantic-layer-workflow.md) for full recipes. +Manage Keboola metastore models -- datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from the stack URL by replacing `connection.` with `metastore.` (cloud/region-agnostic). Auth: same `X-StorageApi-Token` as Storage -- **reads work with any valid, non-disabled, non-expired token** (vNEXT, PSGO-282); **writes** still need a project-admin token (master, or any other admin-role user token) -- a non-admin token gets the opaque 401 `Failed to create project scope` on a write, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711; see [gotchas.md](gotchas.md)). Hidden alias: `kbagent sl ...` is equivalent to `kbagent semantic-layer ...`. See [semantic-layer-workflow.md](semantic-layer-workflow.md) for full recipes. - `semantic-layer model list --project P` -- list all models in a project. Output: `{models: [{id, name, sql_dialect, description}, ...]}`. Use to disambiguate when `--model` is required and the project has more than one model. - `semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake]` -- create a new model. `--sql-dialect` defaults to `Snowflake`. Returns the new model UUID; subsequent commands accept either name or UUID via `--model`. diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index 713900ef5..07953384e 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -4,12 +4,17 @@ The Keboola semantic layer (aka **metastore**) is a project-scoped catalogue of datasets, metrics, relationships, constraints, and glossary terms. It is served from a separate API at `metastore.` (derived from `connection.` by string-substitution; cloud/region-agnostic). Auth is -the same `X-StorageApi-Token` as Storage, **with one extra requirement: it -must be a MASTER (project admin) token**. The metastore's auth gate rejects -every valid non-master token with an opaque 401 `Failed to create project -scope`, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy -(since 0.92.0, #711). Pre-flight: `kbagent --json project info --project P` --> `is_master_token`. +the same `X-StorageApi-Token` as Storage: **reads work with any valid, +non-disabled, non-expired token** (vNEXT, PSGO-282); **writes** (add / +edit / remove / import / promote / build / scope grant / elevate) still need +a project-admin token -- a master token qualifies, but so does any other +project-admin user's token. Before PSGO-282, the metastore's auth gate +rejected every valid non-master token -- including reads -- with an opaque +401 `Failed to create project scope`, which kbagent reclassifies to +`MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711); that +reclassification now only fires on a write against a non-admin token, or +against a metastore deployment that predates the fix. Pre-flight for a +write: `kbagent --json project info --project P` -> `is_master_token`. `kbagent semantic-layer ...` (alias `kbagent sl ...`, hidden) wraps the metastore so AI agents and CI scripts don't roll their own `urllib` loops. From ffb8f04ea2868b7e3f0c5f1a0b3eae5a172a158d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 2 Oct 2026 12:46:40 +0200 Subject: [PATCH 4/6] fix(AI-3790): address review of scope support (in-place edits, scope CLI redesign, serve routes) Correctness (review by Martin Struzsky): - edit, import --overwrite and promote update in place with put_item instead of DELETE+POST: the item keeps its id, scope, grants and pending elevation request, and a failed update changes nothing. Removes delete_then_post. - scope add/remove refuse from a project that does not own the item (the server hides the grants from a non-owner, so a merge would wipe them). - --target-project accepts an alias or numeric project ID (repeatable or comma-separated); an alias must be on the owner's stack. - add without --scope inherits the model's scope and targets. - schemaVersion 1.1.0 is sent only for non-project scopes. - post_item rejects target_project_ids=[] without targeted scope. CLI spec alignment (#791): - scope verbs: get/add/remove/set/request-create/request-delete/request-list, --id -> --context-id, scope set takes exactly one of --scope organization, --target-project or --clear, with --dry-run/--yes. - --scope/--type are fixed choices; bad values, unknown aliases and --target-project without --scope targeted exit 2. - --scope organization is destructive-class via FLAG_ESCALATIONS. - request-list returns limit/offset/has_more. Serve: 7 scope routes plus scope/target_projects on POST /models and /items/{kind}, with command-map entries and regenerated endpoint docs. Docs: CLAUDE.md, context.py, commands-reference, gotchas, scope workflow, expert prompt, SKILL.md (regenerated + workflow row); PSGO-282 wording. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 47 +- docs/web-server-endpoints.md | 11 +- plugins/kbagent/agents/keboola-expert.md | 4 +- plugins/kbagent/skills/kbagent/SKILL.md | 35 +- .../kbagent/references/commands-reference.md | 28 +- .../skills/kbagent/references/gotchas.md | 153 ++++- .../references/metastore-scope-workflow.md | 142 +++-- .../references/semantic-layer-workflow.md | 38 +- .../commands/_semantic_layer_crud.py | 99 +-- .../commands/_semantic_layer_helpers.py | 74 ++- .../commands/_semantic_layer_scope.py | 245 ++++---- src/keboola_agent_cli/commands/context.py | 144 +++-- .../commands/semantic_layer.py | 28 +- src/keboola_agent_cli/metastore_client.py | 25 +- src/keboola_agent_cli/permissions.py | 29 +- .../server/_serve_command_map.py | 13 + .../server/routers/semantic_layer.py | 160 ++++- .../services/_semantic_layer_crud.py | 132 +--- .../services/_semantic_layer_internals.py | 13 +- .../services/_semantic_layer_scope.py | 147 +++-- .../services/semantic_layer_service.py | 219 ++++--- tests/test_metastore_client.py | 50 +- tests/test_project_ref.py | 6 +- tests/test_semantic_layer_scope.py | 507 ++++++++-------- tests/test_semantic_layer_scope_cli.py | 572 +++++++----------- tests/test_semantic_layer_service.py | 105 ++-- tests/test_server_router_calls.py | 146 +++++ 27 files changed, 1879 insertions(+), 1293 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 1a6949073..cebc8c4e6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1054,7 +1054,7 @@ kbagent dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run] kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] kbagent semantic-layer model list --project P -kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS ...] +kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS|ID ...] kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] @@ -1093,23 +1093,36 @@ kbagent semantic-layer promote --from-project A --to-project B [--from-model M] kbagent semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH] kbagent semantic-layer token --encrypt --project P --component-id C # scope (PSGO-140, new): visibility scope for a semantic-layer item -- "project" -# (default, owner only), "organization" (every project in the org), or "targeted" (owner + +# (owner only), "organization" (every project in the org), or "targeted" (owner + # explicit --target-project grants). --scope/--target-project are also accepted by `model -# create` and every `add ` above (default "project", unchanged behavior). With --scope -# targeted and no --target-project: a real terminal launches an interactive picker over the -# other registered projects; --json/non-interactive fails fast (exit 2) instead of guessing. +# create` and every `add ` above. --scope omitted: `model create` makes a "project" +# item, `add ` INHERITS its model's scope (an org-level model gets org-level children; +# a targeted model, its target projects). --target-project takes a registered alias OR a +# numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack); +# without --scope targeted it exits 2. With --scope targeted and no --target-project: a real +# terminal launches an interactive picker over the other projects on the stack; --json fails +# fast (exit 2). A bad alias/ID/--scope/--type is a usage error (exit 2), never a traceback. # Creating directly at --scope organization requires the organization-admin ROLE (403 -# otherwise) -- an ordinary token instead uses request-elevation + an org-admin's `scope -# elevate`. Elevation is ONE-WAY (no downgrade endpoint). `scope grant` is a REPLACE on the -# wire (`--replace`/`--clear`); the default add/remove mode is a client-side read-modify-write -# merge, not atomic against a concurrent grant change. `edit` preserves an item's scope/grants -# across its DELETE+POST cycle automatically -- it is never silently reset to "project". -kbagent semantic-layer scope status --project P --type model|dataset|metric|relationship|constraint|glossary --id ID -kbagent semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear] -kbagent semantic-layer scope request-elevation --project P --type T --id ID -kbagent semantic-layer scope withdraw-elevation --project P --type T --id ID -kbagent semantic-layer scope elevate --project P --type T --id ID [--yes] -kbagent semantic-layer scope pending --project P --type T [--limit N] [--offset N] +# otherwise) -- an ordinary token instead uses `scope request-create` + an org-admin's +# `scope set --scope organization`. Elevation is ONE-WAY (no downgrade endpoint), which is why +# every `--scope organization` (here and on `model create` / `add `) is destructive-class +# (FLAG_ESCALATIONS). `edit`, `import --overwrite` and `promote` update in place (PUT): an item +# keeps its id, scope, grants and pending elevation request. `scope add|remove` merge into the +# grants client-side (not atomic; refused from a project that does not own the item, because +# the server hides the grants from a non-owner -- use `scope set --target-project` there). +# Verbs follow the CLI spec (#791): get / add / remove / set / request-create / request-delete / +# request-list; `scope set` takes exactly ONE of --scope organization, --target-project (replace +# the whole list) or --clear. +kbagent semantic-layer scope get --project P --type model|dataset|metric|relationship|constraint|glossary --context-id ID +kbagent semantic-layer scope add --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] +kbagent semantic-layer scope remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] +kbagent semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID [...] | --clear) [--dry-run] [--yes] +kbagent semantic-layer scope request-create --project P --type T --context-id ID +kbagent semantic-layer scope request-delete --project P --type T --context-id ID +kbagent semantic-layer scope request-list --project P --type T [--limit N] [--offset N] +# scope over `kbagent serve`: GET/PUT /semantic-layer/scope/{context_id}, POST/DELETE +# .../target-projects, PUT/DELETE .../elevation-request, GET /semantic-layer/scope/elevation-requests; +# POST /semantic-layer/models and /items/{kind} take `scope` + `target_projects`. kbagent semantic-layer reference-data list --project P [--model M] kbagent semantic-layer reference-data get --project P (--id ID | --dimension D) kbagent semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X] @@ -1117,7 +1130,7 @@ kbagent semantic-layer reference-data delete --project P --id ID [--yes] # Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. # semantic-layer READS work with any valid, non-disabled, non-expired Storage token # (vNEXT, PSGO-282): the Metastore no longer requires a master token for GET/List. WRITES -# (add/edit/remove/import/promote/build/scope grant/elevate) still need a project-admin token +# (add/edit/remove/import/promote/build/scope add|remove|set) still need a project-admin token # (master token or any admin-role user token) -- a non-admin token 403s on the write. Before # PSGO-282 the Metastore's auth gate rejected EVERY valid non-master token, including reads, # with an opaque 401 "Failed to create project scope" (#711, MasterTokenRequiredError swallowed diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 95710a58e..774f65de0 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -9,7 +9,7 @@ auth, and the concepts behind these routes live in [`web-server.md`](web-server.md); a running server serves the same spec interactively at `/docs` (Swagger) and `/openapi.json`. -**248 operations** across **216 paths** and **31 routers**. +**255 operations** across **220 paths** and **31 routers**. Paths are shown as the server registers them. Reaching them through the Node BFF (or single-process `--ui` mode) prefixes every path with `/api`. @@ -388,7 +388,7 @@ Build and query cross-project data lineage (table-level and column-level). Mirro | `GET` | `/lineage/walk` | Walk lineage graph from a node | | `GET` | `/lineage/mermaid` | Render lineage as Mermaid | -### `semantic-layer` (21 operations) +### `semantic-layer` (28 operations) Model, validate, import/export, diff, promote, and build semantic layer artifacts (datasets, metrics, relationships, constraints, glossary). Mirrors `kbagent semantic-layer *`. @@ -415,6 +415,13 @@ Model, validate, import/export, diff, promote, and build semantic layer artifact | `PUT` | `/semantic-layer/reference-data` | Create or replace a reference-data record | | `GET` | `/semantic-layer/reference-data/{record_id}` | Get one reference-data record | | `DELETE` | `/semantic-layer/reference-data/{record_id}` | Delete a reference-data record | +| `GET` | `/semantic-layer/scope/elevation-requests` | List scope-elevation requests | +| `GET` | `/semantic-layer/scope/{context_id}` | Get an item's scope | +| `PUT` | `/semantic-layer/scope/{context_id}` | Set an item's scope or target projects | +| `POST` | `/semantic-layer/scope/{context_id}/target-projects` | Add target projects | +| `DELETE` | `/semantic-layer/scope/{context_id}/target-projects` | Remove target projects | +| `PUT` | `/semantic-layer/scope/{context_id}/elevation-request` | Request scope elevation | +| `DELETE` | `/semantic-layer/scope/{context_id}/elevation-request` | Withdraw scope elevation request | ## AI & Tools diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 7017de331..24e510bf8 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -152,8 +152,8 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Launch the web UI for an end-user | `kbagent serve --ui [--port PORT]` -- one FastAPI process mounts the bundled React SPA and sets an HttpOnly `kbagent_session` cookie, so SSE works with no token in the URL or JS heap | the legacy Vite + Node BFF dev flow (`web/README.md`) | inventing a `--token-in-url` flag; running uvicorn directly against the dist (the cookie bootstrap only fires from `serve --ui`) | | Schedule / manage Agent Tasks | `kbagent agent ` -- CRUD, `run [--stream]`, history, `test`/`cron-preview`/`prompt-improve`. Local-only; cron firing needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` inside scheduled subprocesses | hand-editing `agents.json` | | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (vNEXT, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | -| ANY semantic-layer write (add / edit / remove / import / promote / build / scope grant / scope elevate) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | -| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope elevate`/`scope request-elevation` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS` grants specific projects; `scope grant --project P --type T --id ID --target-project ALIAS ...` adjusts an existing item's grants (merge by default, `--replace`/`--clear` for the API's native replace semantics); org-wide needs `scope request-elevation` then an org-admin's `scope elevate --yes` -- **irreversible, no downgrade endpoint**. `scope status --type T --id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope pending --type T` to find items an org-admin should act on | calling `scope elevate` or passing `--scope organization\|targeted` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating) | +| ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index c052b27f5..233e4cfbe 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -309,8 +309,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Add a glossary term | `kbagent semantic-layer add glossary --project PROJECT --term TERM` | | Edit a metric. | `kbagent semantic-layer edit metric --project PROJECT --name NAME` | | Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent semantic-layer edit dataset --project PROJECT --name NAME` | -| Edit a constraint (DELETE+POST, with local validators) | `kbagent semantic-layer edit constraint --project PROJECT --name NAME` | -| Edit a relationship (DELETE+POST). | `kbagent semantic-layer edit relationship --project PROJECT --name NAME` | +| Edit a constraint (in-place update, with local validators) | `kbagent semantic-layer edit constraint --project PROJECT --name NAME` | +| Edit a relationship (in-place update). | `kbagent semantic-layer edit relationship --project PROJECT --name NAME` | | Edit a glossary term. | `kbagent semantic-layer edit glossary --project PROJECT --term TERM` | | Remove a metric. | `kbagent semantic-layer remove metric --project PROJECT --name NAME` | | Remove a dataset | `kbagent semantic-layer remove dataset --project PROJECT --name NAME` | @@ -321,12 +321,13 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent semantic-layer reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent semantic-layer reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent semantic-layer reference-data delete --project PROJECT --id ID-` | -| Show an item's current scope, target-project grants, and pending elevation | `kbagent semantic-layer scope status --project PROJECT --type TYPE- --id ITEM-ID` | -| Grant, revoke, or replace the target-project list of a targeted-scope item | `kbagent semantic-layer scope grant --project PROJECT --type TYPE- --id ITEM-ID` | -| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent semantic-layer scope request-elevation --project PROJECT --type TYPE- --id ITEM-ID` | -| Withdraw a pending scope-elevation request. | `kbagent semantic-layer scope withdraw-elevation --project PROJECT --type TYPE- --id ITEM-ID` | -| Step an item up to organization scope. | `kbagent semantic-layer scope elevate --project PROJECT --type TYPE- --id ITEM-ID` | -| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent semantic-layer scope pending --project PROJECT --type TYPE-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent semantic-layer scope get --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Add target projects to a targeted-scope item (merges with the current grants) | `kbagent semantic-layer scope add --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Remove target projects from a targeted-scope item (merges with the current grants) | `kbagent semantic-layer scope remove --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Write an item's scope: elevate to organization, or replace/clear its target projects | `kbagent semantic-layer scope set --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent semantic-layer scope request-create --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Withdraw a pending scope-elevation request. | `kbagent semantic-layer scope request-delete --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent semantic-layer scope request-list --project PROJECT --type TYPE-` | | Encrypt the project's storage token for transformation `user_properties` | `kbagent sl token --project PROJECT --component-id COMPONENT-ID` | | Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent sl build --project PROJECT` | | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | @@ -348,8 +349,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Add a glossary term | `kbagent sl add glossary --project PROJECT --term TERM` | | Edit a metric. | `kbagent sl edit metric --project PROJECT --name NAME` | | Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent sl edit dataset --project PROJECT --name NAME` | -| Edit a constraint (DELETE+POST, with local validators) | `kbagent sl edit constraint --project PROJECT --name NAME` | -| Edit a relationship (DELETE+POST). | `kbagent sl edit relationship --project PROJECT --name NAME` | +| Edit a constraint (in-place update, with local validators) | `kbagent sl edit constraint --project PROJECT --name NAME` | +| Edit a relationship (in-place update). | `kbagent sl edit relationship --project PROJECT --name NAME` | | Edit a glossary term. | `kbagent sl edit glossary --project PROJECT --term TERM` | | Remove a metric. | `kbagent sl remove metric --project PROJECT --name NAME` | | Remove a dataset | `kbagent sl remove dataset --project PROJECT --name NAME` | @@ -360,12 +361,13 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent sl reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent sl reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent sl reference-data delete --project PROJECT --id ID-` | -| Show an item's current scope, target-project grants, and pending elevation | `kbagent sl scope status --project PROJECT --type TYPE- --id ITEM-ID` | -| Grant, revoke, or replace the target-project list of a targeted-scope item | `kbagent sl scope grant --project PROJECT --type TYPE- --id ITEM-ID` | -| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent sl scope request-elevation --project PROJECT --type TYPE- --id ITEM-ID` | -| Withdraw a pending scope-elevation request. | `kbagent sl scope withdraw-elevation --project PROJECT --type TYPE- --id ITEM-ID` | -| Step an item up to organization scope. | `kbagent sl scope elevate --project PROJECT --type TYPE- --id ITEM-ID` | -| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent sl scope pending --project PROJECT --type TYPE-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent sl scope get --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Add target projects to a targeted-scope item (merges with the current grants) | `kbagent sl scope add --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Remove target projects from a targeted-scope item (merges with the current grants) | `kbagent sl scope remove --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Write an item's scope: elevate to organization, or replace/clear its target projects | `kbagent sl scope set --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent sl scope request-create --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Withdraw a pending scope-elevation request. | `kbagent sl scope request-delete --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent sl scope request-list --project PROJECT --type TYPE-` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | @@ -459,6 +461,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | Reading synced data | [reading-synced-data](references/reading-synced-data.md) | | SQL migration (input mapping removal) | [sql-migration-workflow](references/sql-migration-workflow.md) | | **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | +| **Share a semantic-layer item across projects / make it org-wide** -- `--scope`, `--target-project`, `scope get\|add\|remove\|set\|request-*` | [metastore-scope-workflow](references/metastore-scope-workflow.md) | | **Developer Portal** (identity CRUD, list/get apps, create/patch/upload-icon/publish/deprecate; TTY-confirm on writes) | [dev-portal-workflow](references/dev-portal-workflow.md) | | **Config metadata** (list/get/set/delete arbitrary key-value metadata on a configuration) | [config-metadata-workflow](references/config-metadata-workflow.md) | | **Storage descriptions** (describe bucket / table / column, batch from YAML) | [storage-describe-workflow](references/storage-describe-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 22ea43d4d..cebb8caa7 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -423,33 +423,33 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer add relationship --project P [--model M] --name N --from TABLE_ID --to TABLE_ID --on EXPR [--type left|inner]` -- add a join relationship. `--type` defaults to `left`. - `semantic-layer add constraint --project P [--model M] --name N --constraint-type T --rule "EXPR" --metrics M1,M2 [--severity error|warning|info]` -- add a constraint. `--constraint-type` is the closed enum `inequality|equality|range|composition|exclusion|temporal|conditional`. `--rule` is a **STRING expression** (e.g. `"value >= 0"`), NEVER a `{bounds: {min, max}}` object (sl-builder docs are wrong -- see [gotchas.md](gotchas.md)). `--metrics` is a comma-separated list of metric names that must already exist in the model. `--severity` defaults to `warning`. Name regex `^[a-z][a-z0-9_]*$`; the 4-band health convention lives in the name suffix `_critical / _warning / _healthy / _review`, distinct from the 3-value API `severity`. - `semantic-layer add glossary --project P [--model M] --term TERM [--definition D]` -- add a glossary term. -- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. The metastore has NO PATCH endpoint, so this is DELETE+POST. Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. +- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. Updates the metric in place (`PUT`: same id, scope and grants kept). Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. - `semantic-layer edit dataset --project P [--model M] --name N [--new-name N2] [--new-description D] [--new-grain G]` -- edit a dataset. No cascade -- metrics reference the dataset's tableId, not its name. -- `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (DELETE+POST). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. -- `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (DELETE+POST). No constraint cascade. Rollback on POST failure. -- `semantic-layer edit glossary --project P [--model M] --term TERM [--new-term T2] [--new-definition D]` -- edit a glossary entry (DELETE+POST). `--new-term` is a destructive cascade through any downstream consumer that joins on the term; warns but allows. +- `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (in-place `PUT`). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. +- `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (in-place `PUT`). No constraint cascade. Rollback on POST failure. +- `semantic-layer edit glossary --project P [--model M] --term TERM [--new-term T2] [--new-definition D]` -- edit a glossary entry (in-place `PUT`). `--new-term` is a destructive cascade through any downstream consumer that joins on the term; warns but allows. - `semantic-layer remove metric --project P [--model M] --name N [--yes]` -- destructive. Pre-deletion scan lists every constraint whose `metrics[]` includes the target name; warning is ALWAYS printed (even with `--yes`) about the resulting orphan + dangling `DIM_METRIC_THRESHOLD` reference. Non-TTY without `--yes` refuses with exit 2. - `semantic-layer remove dataset --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. - `semantic-layer remove constraint --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. - `semantic-layer remove relationship --project P [--model M] --name N [--yes]` -- destructive. Relationships aren't referenced by other entities; no orphan-check. - `semantic-layer remove glossary --project P [--model M] --term TERM [--yes]` -- destructive. Glossary entries aren't referenced by other entities; no orphan-check. -- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into DELETE+POST for conflicting items. `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. +- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into an in-place update of conflicting items (they keep their scope). `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. - `semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes]` -- cross-project model copy with `modelUUID` rewrite to the target model's UUID. Classifies items NEW / IDENTICAL / CHANGED (deep-equality after stripping `modelUUID` + timestamps). Additive + overwrite only: NEVER deletes target items absent from source. Holds two MetastoreClients in try/finally. Response: per-type counts + `changes[]` with `diff_keys` and `failed[]`. - `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN = the table's Storage location, *(since 0.95.0)* no longer a hardcoded `"KEBOOLA"` database; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). **Rollback on push failure**: every successfully-POSTed child is DELETEd in reverse PUSH_ORDER, and the model itself is DELETEd if we created it during this call. The wrapped `KeboolaApiError` carries `details.rollback={attempted, posted_children, deleted, failed_deletes, model_created_here, model_deleted, model_uuid}` so operators get full diagnostics. Pass `--keep-on-failure` to preserve the partial state for forensic inspection (mirrors `data-app create --keep-on-failure`); the wrapped error then carries `details.rollback.attempted=False, reason='keep_on_failure'`. - `semantic-layer token --encrypt --project P --component-id C` -- encrypt the project's storage token for a transformation's `user_properties`. Builds `{"#metastore_token": }` from the project's already-stored Storage API token and delegates to the existing EncryptService. `--encrypt` is currently required; other modes are refused with `USAGE_ERROR` (exit 2). Output (human): the raw envelope ready to paste. JSON: full `{encrypted, component_id, project}`. ### Scope / target-project grants / elevation (`scope` sub-app, PSGO-140) -*(since vNEXT)* Every semantic-layer item carries a visibility `scope`: `project` (default, owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above (default `project`, so existing behavior is unchanged unless the flag is passed). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other registered projects; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2) instead of guessing which project(s) should see the object. **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule. `edit` (DELETE+POST) reads the item's scope/grants before deleting and re-applies them on the POST half, so editing an organization/targeted-scope item is never silently downgraded back to `project`. +*(since vNEXT)* Every semantic-layer item carries a visibility `scope`: `project` (owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above. With `--scope` omitted, `model create` makes a `project` model and `add ` **inherits its model's scope and target projects**. `--target-project` takes a registered alias or a numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack) and needs `--scope targeted` (else exit 2). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other projects on the stack; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2). **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule; `--scope organization` is irreversible and gated as destructive. `edit`, `import --overwrite` and `promote` update the item in place (PUT), so an organization/targeted item keeps its scope, grants, id and pending elevation request. -- `semantic-layer scope status --project P --type model|dataset|metric|relationship|constraint|glossary --id ID` -- show an item's current `scope`, `target_project_ids`, and any pending `scope_elevation_requested_at`. -- `semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear]` -- update the target-project grant list of a `--scope targeted` item. Default mode is an additive/subtractive **merge** (read the current grants, apply `--target-project`/`--remove-target-project`, PUT the result) -- **not atomic** against a concurrent grant change on the same item, last write wins. `--replace` sends exactly `--target-project` as the whole set, matching the metastore's native `PUT .../target-projects` semantics (a REPLACE, not a merge) in one round trip. `--clear` is `--replace` with an empty set (revokes every grant; the item becomes owner-only). Only the owning project or an organization-admin may call this; the item must have `scope="targeted"` already (400 otherwise -- including against an organization-scoped item, where targeting is meaningless). -- `semantic-layer scope request-elevation --project P --type T --id ID` -- owner-only. Flags a project-scoped item as awaiting an organization admin's step-up decision. Idempotent: calling again just refreshes the timestamp. -- `semantic-layer scope withdraw-elevation --project P --type T --id ID` -- owner-only. Clears a pending elevation request. Idempotent no-op if none is pending. -- `semantic-layer scope elevate --project P --type T --id ID [--yes]` -- steps an item up to organization scope. Requires the **organization-admin role** (a normal project token gets `ACCESS_DENIED`, 403) -- same role required to create directly with `--scope organization`, so most callers reach organization scope via `request-elevation` + this command run by an org-admin-capable token. **ONE-WAY: there is no downgrade endpoint.** Prompts for confirmation unless `--yes` or `--json`. Widens the item's visibility (and its full revision history) to every project in the organization, irreversibly -- never call it without the user having explicitly named the item. -- `semantic-layer scope pending --project P --type T [--limit N] [--offset N]` -- lists items of `--type` awaiting an elevation decision, across the whole organization (`GET .../organization?scope_elevation_requested_at[not][null]=true`). This is the org-admin's discovery queue for `scope elevate`. +- `semantic-layer scope get --project P --type model|dataset|metric|relationship|constraint|glossary --context-id ID` -- show an item's current `scope`, `target_project_ids`, and any pending `scope_elevation_requested_at`. +- `semantic-layer scope add|remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...]` -- attach / detach target projects of a `--scope targeted` item (alias or numeric ID; repeatable or comma-separated). A client-side **merge** (read the grants, apply the delta, PUT) -- **not atomic** against a concurrent grant change. Refused (exit 2) from a project that does not own the item -- the server hides the grants from a non-owner, so use `scope set --target-project` there. +- `semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID ... | --clear) [--dry-run] [--yes]` -- exactly ONE of: `--target-project` (REPLACES the whole list), `--clear` (revokes every grant), or `--scope organization` (the only accepted value; elevation). A mixed request exits 2. Elevation needs the **organization-admin role** (403 otherwise), is **ONE-WAY** (no downgrade), is gated as destructive, and prompts unless `--yes`/`--json`; `--dry-run` only shows the change. +- `semantic-layer scope request-create --project P --type T --context-id ID` -- owner-only. Flags a project-scoped item as awaiting an organization admin's step-up decision. Idempotent: calling again just refreshes the timestamp. +- `semantic-layer scope request-delete --project P --type T --context-id ID` -- owner-only. Clears a pending elevation request. Idempotent no-op if none is pending. +- `semantic-layer scope request-list --project P --type T [--limit N] [--offset N]` -- one page (default `--limit 50`) of items of `--type` awaiting an elevation decision, across the whole organization; returns `{items, limit, offset, has_more}`. The org-admin's discovery queue. -There is no bulk-elevate endpoint: promoting every semantic-layer object in an existing project to organization/targeted scope ("elevate existing projects") means one `scope request-elevation` + `scope elevate` call per item, deliberately, only for objects the user has named -- never loop this speculatively over an entire project. +There is no bulk-elevate endpoint: promoting every semantic-layer object in an existing project to organization/targeted scope ("elevate existing projects") means one `scope request-create` + `scope set --scope organization` call per item, deliberately, only for objects the user has named -- never loop this speculatively over an entire project. ### Reference data (dimension members, e.g. a Chart of Accounts) @@ -457,7 +457,7 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e - `semantic-layer reference-data list --project P [--model M]` -- list dimension records (summaries: `id`, `dimension_name`, `model_uuid`, `dataset_id`, `member_count`). `--model` filters to one model. Members are omitted from the summary; use `get` for them. - `semantic-layer reference-data get --project P (--id ID | --dimension D)` -- fetch one record with all members. Resolve by record UUID (`--id`) or by `--dimension`. The dimension is **unique per project**, so the lookup is project-wide and needs no model. Passing both `--id` and `--dimension` (or neither) is a usage error (exit 2). Returns `{id, dimension_name, model_uuid, dataset_id, member_count, revision, members[]}`. -- `semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X]` -- create-or-replace. `--members-file` is a JSON array of member objects (`-` reads stdin). **Idempotent** on `dimension`: the lookup is project-wide (the envelope `name` = dimension is unique per project per type), so an existing record is replaced in place via `PUT` (the metastore increments `meta.revision`, preserving history) — distinct from the DELETE+POST used by `edit` — regardless of which `--model` is passed (the resolved model is stored on the record); otherwise a new record is `POST`-ed. Response: `{id, dimension_name, member_count, action: "created"|"updated"}`. +- `semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X]` -- create-or-replace. `--members-file` is a JSON array of member objects (`-` reads stdin). **Idempotent** on `dimension`: the lookup is project-wide (the envelope `name` = dimension is unique per project per type), so an existing record is replaced in place via `PUT` (the metastore increments `meta.revision`, preserving history) — like `edit` — regardless of which `--model` is passed (the resolved model is stored on the record); otherwise a new record is `POST`-ed. Response: `{id, dimension_name, member_count, action: "created"|"updated"}`. - `semantic-layer reference-data delete --project P --id ID [--yes]` -- delete by UUID (server-side soft-delete; the record stays in revision history). Non-TTY without `--yes` refuses with exit 2. ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index ecc76fd95..ef481485e 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -180,18 +180,31 @@ Versioning convention: verbatim instead of asserting the token is invalid. **Rotating the token does not fix an `AUTH_REJECTED`**; verify the token against another endpoint on the same stack, then escalate with the exceptionId. -- **The reported trigger case has its own, more specific mapping: every - `semantic-layer` / `sl` command needs a MASTER token.** The Metastore's - 401 `{"exception": "Failed to create project scope"}` is its master-token - gate: unlike the Storage API, the metastore accepts only a master (project - admin) Storage token, and this opaque 401 is how it answers a valid - non-master token -- which blocked every `semantic-layer` command while - `project status` reported the token healthy (issue #711; the underlying - `MasterTokenRequiredError` is swallowed server-side). kbagent reclassifies - exactly that 401 to `ErrorCode.MISSING_MASTER_TOKEN` with the remedy in - the message. Fix: check `kbagent project info` -> `is_master_token`, and - register a master token (`kbagent project edit --token ...`). Do NOT - escalate this one to support -- it is by design. +- **The reported trigger case has its own, more specific mapping: a + `semantic-layer` / `sl` WRITE needs a project-admin token.** The + Metastore's 401 `{"exception": "Failed to create project scope"}` is its + admin-token gate: writes (add/edit/remove/import/promote/build/scope + add|remove|set|request-*) require the token's Storage Admin role to be + `admin` -- a master token qualifies, so does any other project-admin + user's token -- and this opaque 401 was how it answered a valid non-admin + token, on every call, while `project status` reported the token healthy + (issue #711; the underlying `MasterTokenRequiredError` was swallowed + server-side). kbagent reclassifies exactly that 401 to + `ErrorCode.MISSING_MASTER_TOKEN` with the remedy in the message. Fix: + check `kbagent project info` -> `is_master_token`, and register a + project-admin token (`kbagent project edit --token ...`). Do NOT escalate + this one to support -- it is by design. + - **(updated vNEXT -- PSGO-282, go-monorepo#596):** the "every call" + part above is now history. The Metastore no longer requires a master + token for a plain read (`show`, `model list`, `search-context`, + `get-context`, `validate`, `diff`, `scope get`, `scope request-list`) -- + any valid, non-disabled, non-expired Storage token works. Only writes + still hit this gate. `MISSING_MASTER_TOKEN` on a read now means either + an outdated kbagent replaying the old client-side reclassification, or + a metastore deployment that predates the fix -- not a real requirement. + See [Metastore no longer requires a master token for reads + (PSGO-282)](#metastore-no-longer-requires-a-master-token-for-reads-psgo-282) + below for the full writeup. - **Exit code is unchanged: 3, same as `INVALID_TOKEN`.** All three are authentication-class failures; only the diagnosis differs. A script branching on `$?` sees nothing new -- one branching on @@ -1168,7 +1181,7 @@ record per member. Consequences an agent must internalize: and `set` the whole thing back. There is no per-member endpoint. - **It uses the metastore's real `PUT`** (revisioned update, `meta.revision` increments, history preserved) when a record for that dimension already - exists — distinct from the DELETE+POST that `edit metric|…` uses. + exists — as `edit metric|…` does since vNEXT (before, those used DELETE+POST). A brand-new dimension is `POST`-ed. - **The envelope `name` is the dimension, unique per project per type, so the `set`/`get` lookup is project-wide.** Because the dimension name is the @@ -1594,11 +1607,12 @@ events and emits a final `done` SSE frame mirroring the same record. ## Metric rename auto-cascades through `CODE_METRIC` -- `kbagent semantic-layer edit metric --new-name NEW` does DELETE+POST - on the metric and ALSO DELETE+POST on every constraint whose - `metrics[]` referenced the old name (POST new with `metrics[]` - updated to the new name). The metastore has no PATCH endpoint, so - every "edit" is a delete-then-create. +- `kbagent semantic-layer edit metric --new-name NEW` updates the metric + and ALSO every constraint whose `metrics[]` referenced the old name + (`metrics[]` rewritten to the new name). *(updated vNEXT)* Each is an + in-place `PUT` -- same id, scope, grants and revision history, and a failed + update changes nothing; before, every "edit" was a delete-then-create with + a rollback re-POST. - The `CODE_METRIC` derived value (used in downstream SQL joins on `DIM_METRIC_THRESHOLD` / `FACT_METRIC_*` lookups) is computed via ```python @@ -1620,8 +1634,8 @@ events and emits a final `done` SSE frame mirroring the same record. is left in a partial state -- surface that to the operator and recommend running `semantic-layer validate` immediately. - **Partial-state envelope signal (updated v0.41.10 -- closes #294)**: - the cascade has per-item rollback only (each constraint DELETE+POST - rolls back individually), NOT whole-operation atomicity. If the + the cascade is per-item only (each constraint is updated on its own; + *since vNEXT* a PUT, so there is nothing to roll back), NOT whole-operation atomicity. If the metric rename succeeds but M of N dependent constraints fail to repoint, the response envelope sets `partial_state: true` and `recovery_hint: ""` @@ -1629,8 +1643,8 @@ events and emits a final `done` SSE frame mirroring the same record. buried inside `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Atomic two-phase commit was intentionally NOT - implemented: the metastore has no PATCH endpoint, so every - cascade 'stage' is itself a DELETE+POST that can fail; true + implemented: every + cascade 'stage' is itself a write that can fail; true atomicity would require side-staging every cascade item, which is disproportionate for a rename. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then @@ -5730,3 +5744,98 @@ workspaces". (also `--force`) keeps it and reports it as `skipped`; only `--theirs` deletes it. A `kbc` manifest save removes the key too (`kbc` writes back only the keys it knows). + +## Metastore no longer requires a master token for reads (PSGO-282) + +*(since vNEXT)* Every `semantic-layer` read (`show`, `model list`, +`search-context`, `get-context`, `validate`, `diff`, `scope get`, `scope +request-list`) works with any valid, non-disabled, non-expired Storage token -- +programmatic (`kbc_at_*`/`kbc_pat_*`) included. Before this, the metastore +rejected every non-master token with an opaque 401 `"Failed to create +project scope"` on every single endpoint, including plain reads (#711); a +valid non-master token used to get reclassified client-side to +`MISSING_MASTER_TOKEN` (0.92.0+, #717) because there was no other way to +succeed. That reclassification still fires, but now only on a genuine +authorization failure: + +- **Writes still need a project-admin token.** `add`/`edit`/`remove`/ + `import`/`promote`/`build`/`scope add|remove|set`/`scope request-create` + all require the token's Storage `Admin.Role` to be `admin` -- a master + token qualifies, and so does any other project-admin user's token. A + non-admin token gets 403 (or `MISSING_MASTER_TOKEN` if the install is still + on the old reclassification path) on the write, never on a preceding read. +- **Do not pre-flight `isMasterToken` before a read.** That workaround + predates the fix and now rejects tokens the server happily accepts -- + drop it if you see it in older automation. +- **401 body is honest now.** The metastore relays the Storage API's real + error message instead of the generic "Failed to create project scope", + so an actually-invalid/expired token surfaces as such rather than looking + like a master-token gate. + +## Metastore `scope` / `--target-project` / `scope set --scope organization` (PSGO-140) + +*(since vNEXT)* `model create` / `add ` gained `--scope +project|organization|targeted` + `--target-project ALIAS|ID ...`, and a new +`semantic-layer scope ` +sub-app (also `GET/PUT /semantic-layer/scope/...` over `kbagent serve`). See +[metastore-scope-workflow.md](metastore-scope-workflow.md) for worked +examples. Surprises worth knowing before you touch this: + +- **A write that changes scope sends metastore schema `1.1.0`.** Every + `semantic-*` schema's `x-metastore.scope.supported` is `["project"]` ONLY at + `1.0.0` -- `1.1.0` adds `organization`/`targeted` (purely additive ACL/scope + blocks, no `data` shape change). The server resolves the EXACT version + string sent and never upgrades it, so kbagent sends `1.1.0` only when the + scope is not `project` (create) or the item being edited is not `project` + scoped (edit); a plain project-scope write keeps sending `1.0.0`, so a stack + without the 1.1.0 schemas is unaffected until someone uses a scope. +- **`--scope` omitted means "inherit" for child items.** `add + metric|dataset|relationship|constraint|glossary` take their model's scope + (and, for `targeted`, its target projects) when `--scope` is not given; + `model create` defaults to `project`. Pass `--scope` to override. Without + this an org-level model would show up in consumer projects with no + datasets or metrics. +- **`--target-project` takes an alias or a numeric project ID** (repeatable or + comma-separated), so the target need not be registered in kbagent. An alias + must be on the owner project's stack (a project ID only means the same + project on its own stack; a bare ID cannot be checked and is trusted). + `--target-project` without `--scope targeted`, an unknown alias, or a bad + `--scope`/`--type` value exits 2 -- never a traceback or a silent ignore. +- **`scope set --target-project` is a REPLACE on the wire; `scope add|remove` + are a client-side merge.** The underlying `PUT .../target-projects` replaces + the *whole* list (an empty array clears every grant); add/remove read the + current grants, apply the delta and PUT the result -- **not atomic** against + a concurrent grant change (last write wins). The server returns + `meta.targetProjectIds` only to the OWNING project, so add/remove from any + other project (an org admin elsewhere) is refused with exit 2 instead of + silently replacing every grant -- use `scope set --target-project` there. +- **Creating directly at `--scope organization` needs the organization-admin + ROLE**, the same role `scope set --scope organization` needs -- a normal + project token gets `ACCESS_DENIED` (403) on both. `--scope targeted` needs a + project-admin token (the same gate as every write). +- **Elevation is one-way, so `--scope organization` is destructive-class.** + There is no downgrade endpoint: the only way "back" is deleting and + re-creating the item as project-scoped (losing its revision history). The + permission engine escalates `--scope organization` on `scope set`, `model + create` and every `add ` (`FLAG_ESCALATIONS`), so `--deny-destructive` + blocks them; a `scope set --scope organization --dry-run` is not blocked. +- **Elevating items created before this release can fail.** The server checks + `PATCH {"scope":"organization"}` and the elevation request against the + item's STORED `schemaVersion`; an item created by an older kbagent has + `1.0.0`, which supports only `project`. *Not yet verified on a live stack.* +- **A 403 vs a 404 on a scope call encodes something deliberate.** The + metastore returns 404 (not 403) when the caller cannot see the object at + all -- not its owner, not a granted project, not viewing an + organization-scoped object -- so a 403 never leaks "an object owned by + some other project exists." Do not collapse the two when surfacing errors. +- **`edit`, `import --overwrite` and `promote` update in place (PUT).** The + item keeps its id, scope, grants and pending elevation request. They used to + DELETE+POST, which reset an organization/targeted item to `project` scope + (`import --overwrite`, `promote`) or could lose the item when the re-create + was refused (`edit`). A failed PUT changes nothing. Note that `promote`'s + diff compares `attributes` only, so a scope difference between source and + target is neither shown nor copied. +- **No bulk-elevate.** There is no "promote every object in this project" + endpoint -- each item needs its own `request-create` + `set --scope + organization` call. Never loop this over a whole project's objects without + the user having named which ones should become org-wide. diff --git a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md index 7955f4ff4..5737aa44d 100644 --- a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md @@ -3,16 +3,19 @@ Every metastore item (semantic-layer model, dataset, metric, relationship, constraint, glossary term) carries a visibility **scope**: -- `project` (default) -- visible only to the owning project. Unchanged - behavior; every command that doesn't mention `--scope` still creates - project-scoped items exactly as before this feature. +- `project` -- visible only to the owning project. A `model create` without + `--scope` creates a project-scoped model. - `targeted` -- owner project + an explicit **target-project grant list** - (repeatable project alias). Any normal project token can create at this - scope; no elevated role needed. + (repeatable `--target-project`, alias or numeric project ID). Creating at + this scope needs a **project-admin** token, like every metastore write. - `organization` -- visible to **every** project in the organization. Creating directly at this scope, or *elevating* an existing item to it, - requires the caller to hold the **organization-admin** role -- a normal - project token gets `ACCESS_DENIED` (403). + requires the **organization-admin** role -- a normal project token gets + `ACCESS_DENIED` (403). + +`add ` without `--scope` **inherits its model's scope** (and target +projects), so an org-level model gets org-level children. Pass `--scope` to +override. For one-line command reference, see [commands-reference.md](commands-reference.md#scope--target-project-grants--elevation-scope-sub-app-since-vnext-psgo-140). @@ -20,25 +23,26 @@ For the schema-version / replace-vs-merge / 403-vs-404 surprises, see [gotchas.md](gotchas.md). **Hard rule for AI agents**: never pass `--scope organization|targeted`, and -never run `scope elevate`/`scope request-elevation`, without the user having -explicitly named which project(s) should gain visibility. Widening an -object's visibility is a security-relevant decision, and organization-scope -elevation has **no downgrade endpoint** -- it cannot be undone via the API. -When in doubt, create/leave the item at the default `project` scope and ask. +never run `scope set --scope organization` / `scope request-create`, without +the user having explicitly named which project(s) should gain visibility. +Widening an object's visibility is a security-relevant decision, and +organization-scope elevation has **no downgrade endpoint** -- it cannot be +undone via the API. When in doubt, create/leave the item at `project` scope +and ask. ## When to use what | Goal | Command | |------|---------| -| Check an item's current scope/grants/pending elevation | `semantic-layer scope status --type T --id ID` | -| Create a new item visible to a couple of named projects | `model create` / `add ` with `--scope targeted --target-project ALIAS ...` | -| Add/remove target projects on an existing targeted item | `scope grant --target-project ALIAS ...` / `--remove-target-project ALIAS ...` | -| Replace the whole target-project list in one call | `scope grant --replace --target-project ALIAS ...` | -| Revoke every grant (item becomes owner-only again) | `scope grant --clear` | -| Ask an org-admin to make an item org-wide | `scope request-elevation` (owner-only) | -| Cancel a pending elevation request | `scope withdraw-elevation` (owner-only) | -| Actually make an item org-wide (org-admin token) | `scope elevate --yes` | -| Find items awaiting an elevation decision | `scope pending --type T` (org-admin token) | +| Check an item's current scope/grants/pending elevation | `semantic-layer scope get --type T --context-id ID` | +| Create a new item visible to a couple of named projects | `model create` / `add ` with `--scope targeted --target-project ALIAS\|ID ...` | +| Add / remove target projects on an existing targeted item | `scope add --target-project X` / `scope remove --target-project X` (from the owning project) | +| Replace the whole target-project list in one call | `scope set --target-project X --target-project Y` | +| Revoke every grant (item becomes owner-only again) | `scope set --clear` | +| Ask an org-admin to make an item org-wide | `scope request-create` (owner-only) | +| Cancel a pending elevation request | `scope request-delete` (owner-only) | +| Actually make an item org-wide (org-admin token) | `scope set --scope organization --yes` (preview with `--dry-run`) | +| Find items awaiting an elevation decision | `scope request-list --type T` (org-admin token) | --- @@ -49,43 +53,50 @@ When in doubt, create/leave the item at the default `project` scope and ask. kbagent --json semantic-layer add metric \ --project prod --model core_model \ --name gross_margin --sql "SUM(revenue) - SUM(cogs)" --dataset out.c-fin.fact_pnl \ - --scope targeted --target-project analytics-prod --target-project finance-prod + --scope targeted --target-project analytics-prod,5678 ``` -`--target-project` is repeatable and takes project **aliases** (resolved to -numeric Storage project IDs internally). Omitting `--target-project` with -`--scope targeted`: +`--target-project` is repeatable or comma-separated and takes a registered +project **alias** or a numeric project **ID** (so the target need not be +registered). An alias must be on the owner project's stack. Omitting +`--target-project` with `--scope targeted`: - On a real terminal: launches an interactive checkbox picker over every - *other* registered project. + *other* registered project on the same stack. - In `--json` / non-interactive mode: fails fast with `INVALID_ARGUMENT` (exit 2) instead of guessing. There is always an explicit target list -- never a silent default. +`--target-project` without `--scope targeted`, or an unknown alias, also +exits 2. + ## Workflow 2 -- Adjust grants on an existing targeted item ```bash # See what's granted today -kbagent --json semantic-layer scope status --project prod --type metric --id +kbagent --json semantic-layer scope get --project prod --type metric --context-id # -> {"scope": "targeted", "target_project_ids": [1234, 5678], ...} -# Add one more project (merge -- read-modify-write, NOT atomic against a -# concurrent grant change on the same item) -kbagent semantic-layer scope grant --project prod --type metric --id \ +# Attach / detach one project (client-side merge -- NOT atomic against a +# concurrent grant change; only from the OWNING project) +kbagent semantic-layer scope add --project prod --type metric --context-id \ --target-project new-team-prod +kbagent semantic-layer scope remove --project prod --type metric --context-id \ + --target-project 5678 -# Replace the whole set in one round trip (matches the API's native semantics) -kbagent semantic-layer scope grant --project prod --type metric --id \ - --replace --target-project analytics-prod +# Replace the whole set in one round trip (the API's native semantics) +kbagent semantic-layer scope set --project prod --type metric --context-id \ + --target-project analytics-prod,5678 # Revoke every grant -- item becomes owner-only again -kbagent semantic-layer scope grant --project prod --type metric --id --clear +kbagent semantic-layer scope set --project prod --type metric --context-id --clear ``` -`scope grant` only works on an item already created with `scope="targeted"` --- it 400s against a project-scoped or organization-scoped item (targeting a -project-scoped item makes no sense; an organization-scoped item is already -visible everywhere). +These only work on an item already created with `scope="targeted"` -- they +400 against a project- or organization-scoped item. `scope add|remove` from a +project that does not own the item (an org admin elsewhere) are refused +(exit 2): the server hides the current grants from a non-owner, so a merge +would overwrite them. Use `scope set --target-project` in that case. ## Workflow 3 -- Make an item organization-wide @@ -94,32 +105,47 @@ Two-step, deliberately: the owner requests, an organization-admin decides. ```bash # 1. Owner project flags the item (idempotent -- re-running just refreshes # the timestamp) -kbagent semantic-layer scope request-elevation --project prod --type dataset --id +kbagent semantic-layer scope request-create --project prod --type dataset --context-id # 2. An organization-admin discovers the queue... -kbagent --json semantic-layer scope pending --project prod --type dataset -# -> [{"id": "", "name": "fact_pnl", "scope_elevation_requested_at": "..."}] - -# 3. ...and elevates. IRREVERSIBLE -- confirmation prompt unless --yes/--json. -kbagent semantic-layer scope elevate --project prod --type dataset --id --yes +kbagent --json semantic-layer scope request-list --project prod --type dataset --limit 50 +# -> {"items": [{"id": "", "name": "fact_pnl", ...}], "limit": 50, "offset": 0, "has_more": false} + +# 3. ...previews, then elevates. IRREVERSIBLE -- confirmation prompt unless --yes/--json. +kbagent semantic-layer scope set --project prod --type dataset --context-id \ + --scope organization --dry-run +kbagent semantic-layer scope set --project prod --type dataset --context-id \ + --scope organization --yes ``` -A non-admin token can also call `scope elevate` directly (skipping step 1) if -the caller already holds the organization-admin role -- `request-elevation` -exists for the common case where the owner and the admin are different -people/tokens. `scope withdraw-elevation` cancels a pending request before -an admin acts on it. +A caller who already holds the organization-admin role can run step 3 +directly (skipping step 1) -- `request-create` exists for the common case +where the owner and the admin are different people/tokens. +`scope request-delete` cancels a pending request before an admin acts on it. + +`--scope organization` is gated as **destructive** by the permission engine +(`--deny-destructive` blocks it, here and on `model create` / `add `); +`--dry-run` is not blocked. There is **no bulk-elevate endpoint**. "Elevate an existing project's -objects" as a migration means one `request-elevation` + `elevate` call per -item, run deliberately for objects the user has named -- never loop this -over every object in a project speculatively. +objects" as a migration means one `request-create` + `set --scope +organization` call per item, run deliberately for objects the user has named +-- never loop this over every object in a project speculatively. Items +created by a kbagent older than this feature may refuse elevation (their +stored schema version only supports `project`); if so, re-create them. ## Workflow 4 -- Editing a scoped item -`semantic-layer edit ` is DELETE+POST (the metastore has no PATCH for -data changes). The edit path automatically reads the item's current -scope/target-project grants before deleting and re-applies them on the POST -half -- editing an `organization`/`targeted`-scope item never silently -resets it back to `project` scope. Nothing to do here; this is just so you -don't have to re-grant after every rename. +`semantic-layer edit `, `import --overwrite` and `promote` update the +item **in place** (PUT): it keeps its id, scope, target projects and any +pending elevation request, and a failed update changes nothing. Nothing to do +here; this is just so you don't have to re-grant after every rename. + +## Over `kbagent serve` + +`GET|PUT /semantic-layer/scope/{context_id}`, `POST|DELETE +.../target-projects`, `PUT|DELETE .../elevation-request` and `GET +/semantic-layer/scope/elevation-requests` mirror the commands above; +`POST /semantic-layer/models` and `/items/{kind}` accept `scope` + +`target_projects`. Like the other semantic-layer routes they do not consult the +`permissions` policy -- the serve bearer token is the gate. diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index 07953384e..278c7bc3e 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -6,7 +6,7 @@ served from a separate API at `metastore.` (derived from `connection.` by string-substitution; cloud/region-agnostic). Auth is the same `X-StorageApi-Token` as Storage: **reads work with any valid, non-disabled, non-expired token** (vNEXT, PSGO-282); **writes** (add / -edit / remove / import / promote / build / scope grant / elevate) still need +edit / remove / import / promote / build / scope add|remove|set) still need a project-admin token -- a master token qualifies, but so does any other project-admin user's token. Before PSGO-282, the metastore's auth gate rejected every valid non-master token -- including reads -- with an opaque @@ -227,7 +227,7 @@ kbagent semantic-layer edit metric \ --new-name revenue_growth_qoq # Will print something like: # CODE_METRIC: REVENUE_GROWTH -> REVENUE_GROWTH_QOQ -# Will DELETE+POST the following constraints (cascade): +# Will update the following constraints in place (cascade): # - revenue_growth_minimum_warning # - revenue_growth_band_review # Proceed? [y/N]: @@ -245,14 +245,13 @@ kbagent --json semantic-layer show --project prod --model core_model --type cons # These joins WILL silently start returning empty rows otherwise. ``` -If the new POST fails (e.g. the new name collides), the service -re-POSTs `original_attrs` to roll back and reports rollback -success/failure in the response envelope's `rollback` field. If the -rollback itself fails, the model is left in a partial state -- run -`semantic-layer validate` immediately. +Edits are an in-place `PUT`, so if the update fails (e.g. the new name +collides) nothing was deleted and nothing needs rolling back -- the item is +exactly as it was (the envelope's `rollback` field is always `null`). Items +keep their id, scope and revision history across an edit. -**Partial cascade state**: the cascade has per-item -rollback only -- each constraint DELETE+POST rolls back individually. +**Partial cascade state**: each cascaded constraint is +updated independently (an in-place PUT, no rollback needed). If the metric rename succeeds but M of N dependent constraints fail to repoint, the envelope sets `partial_state: true` at the top level and a `recovery_hint` string. Human-mode CLI prints a bright red @@ -267,10 +266,9 @@ kbagent semantic-layer edit constraint --project prod --model core_model \ --name revenue_growth_minimum_warning --new-metrics revenue_growth_qoq ``` -Atomic two-phase commit was rejected as disproportionate: the -metastore has no PATCH endpoint, so every cascade 'stage' is itself a -DELETE+POST that can fail; true atomicity would require side-staging -every cascade item. +Atomic two-phase commit was rejected as disproportionate: every cascade +'stage' is itself a PUT that can fail; true atomicity would require +side-staging every cascade item. --- @@ -328,7 +326,7 @@ kbagent --json semantic-layer promote \ | jq '.metrics, .constraints' # Inspect: # metrics.new -- count of items the promote will POST -# metrics.overwritten -- count of CHANGED items that will be DELETE+POSTed +# metrics.overwritten -- count of CHANGED items that will be updated in place # metrics.identical -- count of items skipped (already match) # metrics.changes[] -- per-item diff with diff_keys # metrics.failed[] -- per-item failures (e.g. dry-run validators) @@ -512,7 +510,8 @@ For the live-validated metastore contract surprises -- where the Quick reminders: - **POST envelope**: `{name, data, branch: "main", schemaVersion: - "1.0.0", scope: "project"}` -> 201 with `{data: {type, id, + "1.0.0", scope: "project"}` (`schemaVersion: "1.1.0"` when the scope is + `organization`/`targeted`) -> 201 with `{data: {type, id, attributes, meta}}`. kbagent handles this. - **Duplicate-name POST -> 409 Conflict** (post go-monorepo PR #513) with `"Object with this name already exists in this project"`, or **500** with @@ -521,10 +520,11 @@ Quick reminders: the fix-deployed path avoids the `MAX_RETRIES` round-trips the 500 path still pays. - **DELETE -> 204** empty body. -- **No PATCH endpoint** -- every "edit" is DELETE+POST. kbagent's - `edit metric / dataset / relationship / constraint / glossary` - (five sub-subcommands matching `add`) rolls back on POST failure. - Same five types are available under `remove`. +- **Data edits are a `PUT`** (revisioned, in place; `PATCH` only changes + the scope). kbagent's `edit metric / dataset / relationship / constraint + / glossary` (five sub-subcommands matching `add`) keep the item's id, + scope and grants, and a failed PUT changes nothing. Same five types are + available under `remove`. - **`X-StorageApi-Token` is the only auth** -- no separate metastore token. `kbagent semantic-layer token --encrypt` encrypts the STORAGE token for a `user_properties` slot named `#metastore_token`, diff --git a/src/keboola_agent_cli/commands/_semantic_layer_crud.py b/src/keboola_agent_cli/commands/_semantic_layer_crud.py index d85207cc1..0756fb01b 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_crud.py @@ -18,13 +18,21 @@ get_formatter, get_service, ) -from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty, resolve_scope_targets +from ._semantic_layer_helpers import ( + ScopeChoice, + _handle_service_call, + _is_stdin_tty, + resolve_scope_targets, +) _SCOPE_HELP = ( - "Visibility: 'project' (default, owner only), 'organization' (every " - "project in the org), or 'targeted' (owner + explicit --target-project grants)." + "Visibility: 'project' (owner only), 'organization' (every project in the org), or " + "'targeted' (owner + --target-project grants). Default: the model's own scope." +) +_TARGET_PROJECT_HELP = ( + "Project alias or ID to grant visibility to (repeatable or comma-separated; " + "--scope targeted only)." ) -_TARGET_PROJECT_HELP = "Project alias to grant visibility to (repeatable; --scope targeted only)." # --------------------------------------------------------------------------- # semantic-layer add -- one sub-subcommand per entity type @@ -74,17 +82,18 @@ def add_metric( dataset: str = typer.Option(..., "--dataset", help="Dataset tableId this metric belongs to"), description: str = typer.Option("", "--description", help="Optional description"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the dataset-mismatch warning"), - scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a metric to a semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.metric", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -99,7 +108,7 @@ def add_metric( is_tty=_is_stdin_tty(), confirm_cb=typer.confirm, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output(result, _print_item_added("metric")) @@ -131,17 +140,18 @@ def add_dataset( 'location from Storage, e.g. \'"DB"."out.c-bucket"."table"\'.' ), ), - scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a dataset (FQN read from the table's Storage location).""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.dataset", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -156,7 +166,7 @@ def add_dataset( deep_fields=deep_fields, fqn=fqn, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output(result, _print_item_added("dataset")) @@ -171,17 +181,18 @@ def add_relationship( to: str = typer.Option(..., "--to", help="Target dataset tableId"), on: str = typer.Option(..., "--on", help="Join condition"), type_: str = typer.Option("left", "--type", help="Join type: 'left' or 'inner'."), - scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a relationship between two datasets.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.relationship", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -194,7 +205,7 @@ def add_relationship( on=on, type_=type_, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output(result, _print_item_added("relationship")) @@ -230,7 +241,7 @@ def add_constraint( severity: str = typer.Option( "warning", "--severity", help="One of: error|warning|info (the 3-level API enum)." ), - scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a constraint.""" @@ -243,11 +254,12 @@ def add_constraint( error_code=ErrorCode.VALIDATION_ERROR, ) raise typer.Exit(code=2) - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.constraint", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -260,7 +272,7 @@ def add_constraint( metrics=metrics_list, severity=severity, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output(result, _print_item_added("constraint")) @@ -272,17 +284,18 @@ def add_glossary( model: str | None = typer.Option(None, "--model", help="Model name or UUID"), term: str = typer.Option(..., "--term", help="Glossary term"), definition: str = typer.Option("", "--definition", help="Optional definition"), - scope: str = typer.Option("project", "--scope", help=_SCOPE_HELP), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a glossary term.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.glossary", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -292,13 +305,13 @@ def add_glossary( term=term, definition=definition, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output(result, _print_item_added("glossary")) # --------------------------------------------------------------------------- -# semantic-layer edit -- DELETE+POST with rollback + rename cascade +# semantic-layer edit -- in-place PUT + rename cascade # --------------------------------------------------------------------------- @@ -306,7 +319,7 @@ def add_glossary( name="edit", help=( "Edit a metric, dataset, constraint, relationship, or glossary term " - "(DELETE+POST with rollback)." + "(in-place update; keeps scope and id)." ), no_args_is_help=True, ) @@ -439,7 +452,7 @@ def edit_constraint( None, "--new-metrics", help="Comma-separated list of metric names" ), ) -> None: - """Edit a constraint (DELETE+POST, with local validators).""" + """Edit a constraint (in-place update, with local validators).""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") metrics_list = ( @@ -476,7 +489,7 @@ def edit_relationship( None, "--new-type", help="Replace join type (left | inner)" ), ) -> None: - """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally.""" + """Edit a relationship (in-place update). Validates ``--new-type`` locally.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") result = _handle_service_call( diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py index 5d21071df..e39d5ac11 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -9,12 +9,38 @@ from __future__ import annotations import sys +from enum import StrEnum import typer from ..errors import ConfigError, ErrorCode, KeboolaApiError from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select -from ._helpers import get_formatter, get_service, map_error_to_exit_code +from ._helpers import check_cli_operation, get_formatter, get_service, map_error_to_exit_code + + +class ScopeChoice(StrEnum): + """Fixed values of ``--scope`` on the create commands (rendered as a click Choice).""" + + PROJECT = "project" + ORGANIZATION = "organization" + TARGETED = "targeted" + + +class ElevationScope(StrEnum): + """``scope set --scope`` accepts only ``organization`` (scope changes are one-way).""" + + ORGANIZATION = "organization" + + +class ItemType(StrEnum): + """Semantic item kinds addressable by ``scope`` commands (``--type``).""" + + MODEL = "model" + DATASET = "dataset" + METRIC = "metric" + RELATIONSHIP = "relationship" + CONSTRAINT = "constraint" + GLOSSARY = "glossary" def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ignore[no-untyped-def] @@ -37,7 +63,9 @@ def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ig retryable=exc.retryable, details=exc.details, ) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + # A bad option value is a usage error (exit 2), not a general failure. + is_usage_error = exc.error_code == ErrorCode.INVALID_ARGUMENT + raise typer.Exit(code=2 if is_usage_error else map_error_to_exit_code(exc)) from None def _is_stdin_tty() -> bool: @@ -48,29 +76,47 @@ def _is_stdin_tty() -> bool: def resolve_scope_targets( ctx: typer.Context, *, - scope: str, + operation: str, + scope: str | None, target_project: list[str] | None, owner_alias: str, ) -> list[str] | None: - """Resolve ``--target-project`` aliases for ``--scope targeted``. - - Returns ``None`` when ``scope`` isn't ``"targeted"`` (nothing to - resolve) or the raw alias list when the caller already gave explicit - ``--target-project`` values. When ``--scope targeted`` is chosen with - none given, this is the "ask when uncertain" mechanism: on a real - terminal it launches the checkbox picker over every OTHER registered - project; in a non-TTY or ``--json`` context it hard-fails instead of - silently defaulting -- widening an object's visibility across projects - is not a guess this CLI makes on the caller's behalf. + """Validate ``--scope`` / ``--target-project`` of a create command. + + Returns the ``--target-project`` values (aliases or IDs; the service + resolves them) for ``--scope targeted``, else ``None``. ``--scope + organization`` is gated here as destructive (``operation`` is the command's + permission key, e.g. ``semantic-layer.add.dataset``). ``--target-project`` + without ``--scope targeted`` is a usage error, never silently ignored. + + With ``--scope targeted`` and no target, this is the "ask when uncertain" + mechanism: on a real terminal it launches the checkbox picker over every + OTHER registered project on the owner's stack; in a non-TTY or ``--json`` + context it hard-fails instead of silently defaulting -- widening an + object's visibility across projects is not a guess this CLI makes. """ + if scope == "organization": + check_cli_operation(ctx, f"{operation} --scope organization") if scope != "targeted": + if target_project: + get_formatter(ctx).error( + message="--target-project requires --scope targeted.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) return None if target_project: return list(target_project) formatter = get_formatter(ctx) project_service = get_service(ctx, "project_service") - candidates = [p for p in project_service.list_projects() if p["alias"] != owner_alias] + projects = project_service.list_projects() + owner_stack = next((p["stack_url"] for p in projects if p["alias"] == owner_alias), None) + candidates = [ + p + for p in projects + if p["alias"] != owner_alias and (owner_stack is None or p["stack_url"] == owner_stack) + ] if not candidates: formatter.error( message=( diff --git a/src/keboola_agent_cli/commands/_semantic_layer_scope.py b/src/keboola_agent_cli/commands/_semantic_layer_scope.py index 5ecdf99ca..e68646b16 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_scope.py @@ -4,6 +4,11 @@ as ``_semantic_layer_crud.py``. Thin per the 3-layer architecture: all scope resolution and metastore calls live in :meth:`SemanticLayerService.scope_*` / :mod:`services._semantic_layer_scope`. + +Verbs follow the CLI spec (#791): ``get`` reads one attribute, ``add`` / +``remove`` attach / detach a target project, ``set`` writes the scope, +``request-create`` / ``request-delete`` / ``request-list`` manage the +elevation request. """ from __future__ import annotations @@ -13,8 +18,9 @@ from rich.table import Table from ..errors import ErrorCode -from ._helpers import check_cli_permission, get_formatter, get_service -from ._semantic_layer_helpers import _handle_service_call +from ..services._semantic_layer_scope import DEFAULT_REQUEST_LIST_LIMIT +from ._helpers import check_cli_operation, check_cli_permission, get_formatter, get_service +from ._semantic_layer_helpers import ElevationScope, ItemType, _handle_service_call scope_app = typer.Typer( name="scope", @@ -25,31 +31,48 @@ no_args_is_help=True, ) +_PROJECT = typer.Option(..., "--project", help="Owning project alias") +_TYPE = typer.Option(..., "--type", help="Semantic item type") +_CONTEXT_ID = typer.Option(..., "--context-id", help="Item UUID") +_TARGET_PROJECT = typer.Option( + [], + "--target-project", + help="Project alias or ID (repeatable, or comma-separated).", +) + @scope_app.callback(invoke_without_command=True) def _scope_permission_check(ctx: typer.Context) -> None: """Permission check for the ``scope`` sub-app. - Composes ``semantic-layer.scope.{subcommand}`` -- ``status``/``pending`` - are ``read``, ``grant``/``request-elevation``/``withdraw-elevation`` are - ``write``, ``elevate`` is ``destructive`` (irreversible, widens - visibility org-wide). See ``permissions.OPERATION_REGISTRY``. + Composes ``semantic-layer.scope.{subcommand}``: ``get`` / ``request-list`` + are ``read``, the rest are ``write``. ``set --scope organization`` is + escalated to ``destructive`` in the command body (irreversible, widens + visibility org-wide) -- see ``permissions.FLAG_ESCALATIONS``. """ check_cli_permission(ctx, "semantic-layer.scope") def _print_scope_status(console: Console, data: dict) -> None: + if data.get("dry_run"): + console.print("[yellow]Dry run -- nothing was changed.[/yellow]") console.print(f"[bold]scope:[/bold] {data.get('scope', 'project')}") targets = data.get("target_project_ids") - if targets: + if targets is not None: console.print(f"[bold]target_project_ids:[/bold] {targets}") + if data.get("would_set_scope"): + console.print(f"[bold]would set scope:[/bold] {data['would_set_scope']}") + if data.get("would_set_target_project_ids") is not None: + console.print( + f"[bold]would set target_project_ids:[/bold] {data['would_set_target_project_ids']}" + ) pending = data.get("scope_elevation_requested_at") if pending: console.print(f"[bold]scope_elevation_requested_at:[/bold] {pending}") -def _print_pending_table(console: Console, data: dict) -> None: - items = data if isinstance(data, list) else data.get("items", []) +def _print_request_table(console: Console, data: dict) -> None: + items = data.get("items", []) if not items: console.print("[dim]No items awaiting scope elevation.[/dim]") return @@ -64,159 +87,167 @@ def _print_pending_table(console: Console, data: dict) -> None: str(item.get("scope_elevation_requested_at") or ""), ) console.print(table) + if data.get("has_more"): + console.print( + f"[dim]More results: re-run with --offset {data['offset'] + data['limit']}.[/dim]" + ) -@scope_app.command("status") -def scope_status( +@scope_app.command("get") +def scope_get( ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Project alias"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" - ), - item_id: str = typer.Option(..., "--id", help="Item UUID"), + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, ) -> None: """Show an item's current scope, target-project grants, and pending elevation.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") result = _handle_service_call( - ctx, service.scope_status, alias=project, kind=type_, item_id=item_id + ctx, service.scope_get, alias=project, kind=type_, context_id=context_id ) formatter.output(result, _print_scope_status) -@scope_app.command("grant") -def scope_grant( - ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Owning project alias"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" - ), - item_id: str = typer.Option(..., "--id", help="Item UUID (must have scope='targeted')"), - target_project: list[str] = typer.Option( - [], "--target-project", help="Project alias to add to the grant list (repeatable)." - ), - remove_target_project: list[str] = typer.Option( - [], "--remove-target-project", help="Project alias to remove from the grant list." - ), - replace: bool = typer.Option( - False, - "--replace", - help="Replace the whole grant list with exactly --target-project (matches the API's " - "native semantics) instead of merging with the current set.", - ), - clear: bool = typer.Option( - False, "--clear", help="Clear every grant (same as --replace with no --target-project)." - ), -) -> None: - """Grant, revoke, or replace the target-project list of a targeted-scope item. - - Default is an additive/subtractive merge against the current grants. - Pass --replace to send exactly --target-project (the server's native - replace-only semantics, one round trip); --clear is shorthand for - --replace with no --target-project. - """ +def _update_targets(ctx: typer.Context, project: str, type_: str, context_id: str, **delta) -> None: # type: ignore[no-untyped-def] formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - if clear and (target_project or remove_target_project): - formatter.error( - message="--clear cannot be combined with --target-project/--remove-target-project.", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) result = _handle_service_call( ctx, - service.scope_grant, + service.scope_update_targets, alias=project, kind=type_, - item_id=item_id, - add=target_project or None, - remove=remove_target_project or None, - replace=([] if clear else (target_project if replace else None)), + context_id=context_id, + **delta, ) formatter.output(result, _print_scope_status) -@scope_app.command("request-elevation") -def scope_request_elevation( +def _require_targets(ctx: typer.Context, target_project: list[str]) -> None: + if not target_project: + get_formatter(ctx).error( + message="--target-project is required.", error_code=ErrorCode.INVALID_ARGUMENT + ) + raise typer.Exit(code=2) + + +@scope_app.command("add") +def scope_add( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + target_project: list[str] = _TARGET_PROJECT, +) -> None: + """Add target projects to a targeted-scope item (merges with the current grants).""" + _require_targets(ctx, target_project) + _update_targets(ctx, project, type_, context_id, add=target_project) + + +@scope_app.command("remove") +def scope_remove( ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Owning project alias"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + target_project: list[str] = _TARGET_PROJECT, +) -> None: + """Remove target projects from a targeted-scope item (merges with the current grants).""" + _require_targets(ctx, target_project) + _update_targets(ctx, project, type_, context_id, remove=target_project) + + +@scope_app.command("set") +def scope_set( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + scope: ElevationScope | None = typer.Option( + None, + "--scope", + help="Step the item up to organization scope. Requires org-admin; ONE-WAY, no downgrade.", ), - item_id: str = typer.Option(..., "--id", help="Item UUID"), + target_project: list[str] = _TARGET_PROJECT, + clear: bool = typer.Option(False, "--clear", help="Clear every target project."), + dry_run: bool = typer.Option(False, "--dry-run", help="Show the change without applying it."), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the elevation confirmation."), ) -> None: - """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + """Write an item's scope: elevate to organization, or replace/clear its target projects. + + Pass exactly one of --scope organization, --target-project (replaces the + whole list) or --clear. Elevating gives every project in the organization + read access to the item and its full revision history, and cannot be undone. + """ formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + if scope == "organization": + if not dry_run: + check_cli_operation(ctx, "semantic-layer.scope.set --scope organization") + if not (yes or dry_run or formatter.json_mode) and not typer.confirm( + f"Elevate {type_} {context_id!r} to organization scope? This is IRREVERSIBLE " + "and makes it visible to every project in the organization. Continue?" + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) result = _handle_service_call( - ctx, service.scope_request_elevation, alias=project, kind=type_, item_id=item_id + ctx, + service.scope_set, + alias=project, + kind=type_, + context_id=context_id, + scope=scope, + target_projects=target_project, + clear=clear, + dry_run=dry_run, ) formatter.output(result, _print_scope_status) -@scope_app.command("withdraw-elevation") -def scope_withdraw_elevation( +@scope_app.command("request-create") +def scope_request_create( ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Owning project alias"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" - ), - item_id: str = typer.Option(..., "--id", help="Item UUID"), + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, ) -> None: - """Withdraw a pending scope-elevation request. Idempotent no-op if none is pending.""" + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") result = _handle_service_call( - ctx, service.scope_withdraw_elevation, alias=project, kind=type_, item_id=item_id + ctx, service.scope_request_create, alias=project, kind=type_, context_id=context_id ) formatter.output(result, _print_scope_status) -@scope_app.command("elevate") -def scope_elevate( +@scope_app.command("request-delete") +def scope_request_delete( ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Owning project alias"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" - ), - item_id: str = typer.Option(..., "--id", help="Item UUID"), - yes: bool = typer.Option(False, "--yes", "-y", help="Skip the confirmation prompt."), + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, ) -> None: - """Step an item up to organization scope. Requires org-admin; ONE-WAY, no downgrade. - - Every project in the organization gains read access to this item and its - full revision history once elevated -- there is no API to reverse it. - """ + """Withdraw a pending scope-elevation request. Idempotent no-op if none is pending.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - if not yes and not formatter.json_mode: - confirmed = typer.confirm( - f"Elevate {type_} {item_id!r} to organization scope? This is IRREVERSIBLE " - "and makes it visible to every project in the organization. Continue?" - ) - if not confirmed: - formatter.console.print("Aborted.") - raise typer.Exit(code=0) result = _handle_service_call( - ctx, service.scope_elevate, alias=project, kind=type_, item_id=item_id + ctx, service.scope_request_delete, alias=project, kind=type_, context_id=context_id ) formatter.output(result, _print_scope_status) -@scope_app.command("pending") -def scope_pending( +@scope_app.command("request-list") +def scope_request_list( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias (org-admin token)"), - type_: str = typer.Option( - ..., "--type", help="model|dataset|metric|relationship|constraint|glossary" - ), - limit: int | None = typer.Option(None, "--limit", help="Max results"), - offset: int | None = typer.Option(None, "--offset", help="Skip this many results"), + type_: ItemType = _TYPE, + limit: int = typer.Option(DEFAULT_REQUEST_LIST_LIMIT, "--limit", min=1, help="Max results"), + offset: int = typer.Option(0, "--offset", min=0, help="Skip this many results"), ) -> None: """List items of --type awaiting an org-admin's elevation decision, across the org.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") result = _handle_service_call( - ctx, service.scope_pending, alias=project, kind=type_, limit=limit, offset=offset + ctx, service.scope_request_list, alias=project, kind=type_, limit=limit, offset=offset ) - formatter.output(result, _print_pending_table) + formatter.output(result, _print_request_table) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 152a5eb38..7a2feb631 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1751,31 +1751,34 @@ Manage Keboola metastore models: datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from stack URL by replacing `connection.` -with `metastore.`. Auth: same `X-StorageApi-Token` as Storage, but it MUST be a -MASTER (project admin) token -- the metastore rejects valid non-master tokens -with an opaque 401 "Failed to create project scope", reclassified by kbagent to -MISSING_MASTER_TOKEN (exit 3) with the remedy (#711). Pre-flight: +with `metastore.`. Auth: same `X-StorageApi-Token` as Storage. READS work with +any valid, non-disabled, non-expired token (PSGO-282); WRITES (add, edit, +remove, import, promote, build, scope add|remove|set|request-*) need a +project-admin token (master, or any admin-role user's). A non-admin write is +the metastore's opaque 401 "Failed to create project scope", reclassified by +kbagent to MISSING_MASTER_TOKEN (exit 3) with the remedy (#711). Pre-flight: `kbagent --json project info --project P` -> is_master_token. Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. kbagent semantic-layer model list --project P List all semantic-layer models in a project. - kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS ...] - Create a new model (default sql-dialect: Snowflake). --scope defaults to - "project" (owner-only, unchanged). "targeted" = owner + explicit - --target-project grants (repeatable alias); any normal project token can - create at this scope. "organization" = visible to every project in the - org from creation -- the schema's ACL requires the organization-admin - ROLE to create directly at this scope (a normal project token gets 403 - ACCESS_DENIED); an ordinary caller instead creates at project/targeted - scope and uses `scope request-elevation` + an org-admin's - `scope elevate` (below) to get there. ASK THE USER which project(s) - before ever passing --scope organization|targeted -- never guess. With - --scope targeted and no --target-project: on a real terminal this - launches an interactive picker over the other registered projects; in - --json/non-interactive it fails fast (exit 2) rather than silently - defaulting. + kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS|ID ...] + Create a new model (default sql-dialect: Snowflake). --scope omitted = + "project" (owner-only). "targeted" = owner + explicit --target-project + grants (alias or numeric project ID; repeatable or comma-separated; an + alias must be on the owner's stack; a project-admin token is needed). + "organization" = visible to every project in the org from creation -- the + schema's ACL requires the organization-admin ROLE to create directly at + this scope (a normal project token gets 403 ACCESS_DENIED); an ordinary + caller instead creates at project/targeted scope and uses + `scope request-create` + an org-admin's `scope set --scope organization` + (below). `--scope organization` is irreversible and is gated as + destructive. ASK THE USER which project(s) before ever passing --scope + organization|targeted -- never guess. --target-project without --scope + targeted exits 2. With --scope targeted and no --target-project: on a + real terminal this launches an interactive picker over the other projects + on the stack; in --json/non-interactive it fails fast (exit 2). kbagent semantic-layer model delete --project P --model M [--yes] Delete a model. Fails if the model still has child entities. @@ -1847,11 +1850,11 @@ ASK THE USER before ever passing --scope organization|targeted. kbagent semantic-layer edit metric|dataset|constraint|relationship|glossary ... - DELETE+POST (no PATCH on metastore). Metric rename cascades through every - constraint referencing the old name (DELETE old + POST new with updated + In-place PUT (same id, scope and grants). Metric rename cascades through + every constraint referencing the old name (each PUT with updated metrics[]); CODE_METRIC warning shown - (re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")). On POST failure, - rollback re-POSTs original_attrs and reports success/failure explicitly. + (re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")). A failed PUT + changes nothing (the `rollback` field is always null). --yes skips the confirm prompt. `edit relationship` accepts --new-from / --new-to / --new-on / --new-type (left|inner). `edit glossary` accepts --new-term (destructive cascade; requires --yes in non-TTY) / --new-definition. @@ -1862,10 +1865,10 @@ --new-metrics ...`. Human-mode CLI prints a red `PARTIAL STATE` banner above the per-entry list. `edit_simple` (no-cascade variants) carries `partial_state: false, recovery_hint: null` for envelope uniformity. - The item's scope/target-project grants (`scope status` below) are - read from the pre-edit item and re-applied on the POST half of every - DELETE+POST -- editing an organization/targeted-scope item never - silently downgrades it back to project scope. + Edits update the item in place (PUT): it keeps its id, scope, + target-project grants (`scope get` below), pending elevation request and + revision history -- editing an organization/targeted item never + downgrades it. A failed edit changes nothing. kbagent semantic-layer remove metric|dataset|constraint|relationship|glossary ... Destructive. `remove metric` pre-scans constraints whose metrics[] includes @@ -1877,7 +1880,7 @@ kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] Replay a snapshot. Default: skip on conflict. --overwrite opts into - DELETE+POST. Dependency-ordered push (datasets -> metrics -> relationships + an in-place update (keeps the existing item's scope). Dependency-ordered push (datasets -> metrics -> relationships -> glossary -> constraints). kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types ...] [--dry-run] [--yes] @@ -1902,53 +1905,60 @@ Builds {{"#metastore_token": }} and delegates to EncryptService. --encrypt is currently required; other modes refused with USAGE_ERROR. - kbagent semantic-layer scope status --project P --type T --id ID + kbagent semantic-layer scope get --project P --type T --context-id ID Show an item's current scope ("project"|"organization"|"targeted"), target_project_ids, and any pending scope_elevation_requested_at. + --type is one of model|dataset|metric|relationship|constraint|glossary. + + kbagent semantic-layer scope add --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] + kbagent semantic-layer scope remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] + Attach / detach target projects of a --scope targeted item (alias or + numeric ID; repeatable or comma-separated). A client-side merge (read the + grants, apply the delta, PUT) -- NOT atomic against a concurrent grant + change. Refused (exit 2) from a project that does not own the item: the + server hides the grants from a non-owner, so a merge would overwrite them + -- use `scope set --target-project` there. The item must have been created + with scope="targeted" (400 otherwise). + + kbagent semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID ... | --clear) [--dry-run] [--yes] + Write the scope. Exactly ONE of: --target-project (REPLACES the whole + list), --clear (revokes every grant; owner-only again), or --scope + organization (elevate; the only value --scope accepts). A mixed + request exits 2. Elevating REQUIRES the organization-admin ROLE (403 + otherwise), is ONE-WAY (no downgrade), and is gated as destructive; + it prompts unless --yes/--json, and --dry-run shows the change without + applying it. NEVER elevate without the user explicitly naming the item + -- it makes the item (and its full revision history) visible to every + project in the organization, irreversibly. - kbagent semantic-layer scope grant --project P --type T --id ID [--target-project ALIAS ...] [--remove-target-project ALIAS ...] [--replace] [--clear] - Grant/revoke/replace the target-project grant list of a --scope targeted - item. Default is an additive/subtractive merge (read current grants, - apply the delta, PUT the result) -- NOT atomic against a concurrent - grant change, last write wins. --replace sends exactly --target-project - as the whole set (the server's native replace-only semantics, one round - trip); --clear is --replace with an empty set (revokes every grant, - object becomes owner-only). Only the owning project or an - organization-admin may call this; the item must have been created with - scope="targeted" (400 otherwise, including against an - organization-scoped item -- targeting is meaningless there). - - kbagent semantic-layer scope request-elevation --project P --type T --id ID + kbagent semantic-layer scope request-create --project P --type T --context-id ID Owner-only. Flags a project-scoped item as awaiting an organization - admin's step-up decision (`scope elevate` below). Idempotent -- calling - again just refreshes the timestamp. + admin's step-up decision (`scope set --scope organization`). Idempotent -- + calling again just refreshes the timestamp. - kbagent semantic-layer scope withdraw-elevation --project P --type T --id ID + kbagent semantic-layer scope request-delete --project P --type T --context-id ID Owner-only. Clears a pending elevation request. Idempotent no-op if none is pending. - kbagent semantic-layer scope elevate --project P --type T --id ID [--yes] - Steps an item up to organization scope. REQUIRES the organization-admin - ROLE (a normal project token gets 403 ACCESS_DENIED) -- creating - directly with `--scope organization` needs the same role, so most - callers reach organization scope via request-elevation + this command, - run by whoever holds an org-admin-capable token. ONE-WAY: there is no - downgrade endpoint. Prompts for confirmation unless --yes or --json. - NEVER call this without the user explicitly naming the item to elevate - -- it makes the item (and its full revision history) visible to every - project in the organization, irreversibly. - - kbagent semantic-layer scope pending --project P --type T [--limit N] [--offset N] - Lists items of --type awaiting an elevation decision, across the whole - organization (`GET .../organization?scope_elevation_requested_at[not][null]=true`). - This is the org-admin's discovery queue for `scope elevate`. - - Elevating an EXISTING project's semantic-layer objects in bulk ("elevate - existing projects" as a migration): there is no bulk-elevate endpoint -- - each object needs its own `scope request-elevation` + an org-admin's - `scope elevate`, one call per item. Do this deliberately, one object at a - time, only when the user has named which objects should become org-wide; - never loop this over every object in a project speculatively. + kbagent semantic-layer scope request-list --project P --type T [--limit N] [--offset N] + One page (default --limit 50) of items of --type awaiting an elevation + decision, across the whole organization; returns {{items, limit, offset, + has_more}}. The org-admin's discovery queue. + + Child items (`add metric|dataset|...`) with --scope omitted INHERIT their + model's scope and target projects; pass --scope to override. + + Elevating an EXISTING project's semantic-layer objects in bulk: there is no + bulk-elevate endpoint -- each object needs its own `scope request-create` + + an org-admin's `scope set --scope organization`, one call per item. Do this + deliberately, one object at a time, only when the user has named which + objects should become org-wide; never loop this over every object in a + project speculatively. + + Over `kbagent serve`: GET/PUT /semantic-layer/scope/{{context_id}}, + POST/DELETE .../target-projects, PUT/DELETE .../elevation-request, + GET /semantic-layer/scope/elevation-requests; POST /semantic-layer/models + and /items/{{kind}} take `scope` + `target_projects`. ### Self-call HTTP (inside `kbagent serve` subprocesses) diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 12ffd1837..ff3197f6d 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -24,7 +24,7 @@ get_service, ) from ._semantic_layer_crud import add_app, edit_app, remove_app -from ._semantic_layer_helpers import _handle_service_call, resolve_scope_targets +from ._semantic_layer_helpers import ScopeChoice, _handle_service_call, resolve_scope_targets from ._semantic_layer_reference_data import reference_data_app from ._semantic_layer_scope import scope_app @@ -113,8 +113,8 @@ def model_create( sql_dialect: str = typer.Option( "Snowflake", "--sql-dialect", help="SQL dialect (default: Snowflake)" ), - scope: str = typer.Option( - "project", + scope: ScopeChoice | None = typer.Option( + None, "--scope", help=( "Visibility: 'project' (default, owner only), 'organization' " @@ -125,17 +125,21 @@ def model_create( target_project: list[str] = typer.Option( [], "--target-project", - help="Project alias to grant visibility to (repeatable; --scope targeted only).", + help=( + "Project alias or ID to grant visibility to (repeatable or comma-separated; " + "--scope targeted only)." + ), ), ) -> None: """Create a new semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") - target_aliases = resolve_scope_targets( - ctx, scope=scope, target_project=target_project, owner_alias=project - ) - target_project_ids = ( - service.resolve_target_project_ids(target_aliases) if target_aliases else None + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.model.create", + scope=scope, + target_project=target_project, + owner_alias=project, ) result = _handle_service_call( ctx, @@ -145,7 +149,7 @@ def model_create( description=description, sql_dialect=sql_dialect, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) formatter.output( result, @@ -555,7 +559,7 @@ def semantic_layer_promote( """Promote a model from one project to another (NEW + overwrite CHANGED; never deletes). Default behaviour: NEW items are POSTed, CHANGED items are - DELETE+POSTed, IDENTICAL items are skipped. Items only present in + updated in place (scope kept), IDENTICAL items are skipped. Items only present in the target are never touched (additive-only). """ formatter = get_formatter(ctx) @@ -608,7 +612,7 @@ def semantic_layer_import( False, "--dry-run", help="Plan the import without calling any write API" ), overwrite: bool = typer.Option( - False, "--overwrite", help="DELETE+POST conflicting items (default: skip)" + False, "--overwrite", help="Update conflicting items in place (default: skip)" ), yes: bool = typer.Option( False, "--yes", "-y", help="Skip confirmation (alias for default SKIP behavior)" diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 210f91a25..37a62ab1b 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -110,7 +110,7 @@ # Envelope fields kept constant across every POST (per metastore contract). _ENVELOPE_BRANCH = "main" -# 1.1.0, not 1.0.0: every semantic-* schema's x-metastore.scope.supported is +# Every semantic-* schema's x-metastore.scope.supported is # ["project"] ONLY at 1.0.0 -- 1.1.0 is what adds "organization"/"targeted" # (go-monorepo migrations/schema/semantic-*_schema_1.1.0.json, diffed against # 1.0.0 at commit e4f62941: purely additive x-metastore.acl/scope blocks, no @@ -118,10 +118,18 @@ # scope="organization"/"targeted" against 1.0.0 gets a clean 400 # ErrScopeNotSupported from prepareCreateSchema -- the server resolves the # EXACT version string sent, never silently upgrades it. -_ENVELOPE_SCHEMA_VERSION = "1.1.0" +# Only ``scope != "project"`` needs the 1.1.0 ACL schemas (additive; every +# ``semantic-*`` type supports ``scope=project`` at 1.0.0). Project-scope writes +# keep sending 1.0.0 so a stack without the 1.1.0 schemas is unaffected. +_ENVELOPE_SCHEMA_VERSION = "1.0.0" +_ACL_ENVELOPE_SCHEMA_VERSION = "1.1.0" _DEFAULT_SCOPE: ObjectScope = "project" +def _schema_version(scope: str) -> str: + return _ENVELOPE_SCHEMA_VERSION if scope == "project" else _ACL_ENVELOPE_SCHEMA_VERSION + + class MetastoreClient(BaseHttpClient): """HTTP client for the Keboola Metastore (semantic layer repository). @@ -302,7 +310,7 @@ def post_item( message=f"scope must be one of 'project'|'organization'|'targeted', got {scope!r}.", error_code=ErrorCode.VALIDATION_ERROR, ) - if target_project_ids and scope != "targeted": + if target_project_ids is not None and scope != "targeted": raise KeboolaApiError( message=( f"target_project_ids is only valid with scope='targeted', got scope={scope!r}." @@ -313,7 +321,7 @@ def post_item( "name": name, "data": data, "branch": _ENVELOPE_BRANCH, - "schemaVersion": _ENVELOPE_SCHEMA_VERSION, + "schemaVersion": _schema_version(scope), "scope": scope, } if target_project_ids is not None: @@ -358,6 +366,8 @@ def put_item( item_id: str, name: str, data: dict[str, Any], + *, + scope: ObjectScope = _DEFAULT_SCOPE, ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). @@ -367,7 +377,10 @@ def put_item( history. ``data`` is the inner ``attributes`` payload; the outer envelope is added here. - Deliberately carries no ``scope``/``targetProjectIds`` -- the + ``scope`` is the item's CURRENT scope and only selects the envelope + ``schemaVersion`` (an organization/targeted item needs 1.1.0); it is + never sent as a field. Deliberately carries no + ``scope``/``targetProjectIds`` -- the server's request struct for this endpoint (``MetaObjectUpdatePutRequest``) has no such fields, so scope/grants are untouched by a plain PUT. Use :meth:`elevate_to_organization` / :meth:`put_target_projects`. @@ -378,7 +391,7 @@ def put_item( "name": name, "data": data, "branch": _ENVELOPE_BRANCH, - "schemaVersion": _ENVELOPE_SCHEMA_VERSION, + "schemaVersion": _schema_version(scope), } response = self._do_request( "PUT", diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index cec08293f..a52aa8a76 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -359,16 +359,18 @@ "semantic-layer.reference-data.delete": "destructive", # `scope` sub-app: visibility scope / target-project grants / elevation # requests (PSGO-140). Parent key at the least-privileged level (read), - # same pattern as `reference-data` above. `elevate` is `destructive`, - # not `write` -- it is one-way (no downgrade endpoint exists) and - # widens an item's visibility to every project in the organization. + # same pattern as `reference-data` above. `set --scope organization` is + # escalated to `destructive` (FLAG_ESCALATIONS): it is one-way (no + # downgrade endpoint exists) and widens an item's visibility to every + # project in the organization. "semantic-layer.scope": "read", - "semantic-layer.scope.status": "read", - "semantic-layer.scope.pending": "read", - "semantic-layer.scope.grant": "write", - "semantic-layer.scope.request-elevation": "write", - "semantic-layer.scope.withdraw-elevation": "write", - "semantic-layer.scope.elevate": "destructive", + "semantic-layer.scope.get": "read", + "semantic-layer.scope.request-list": "read", + "semantic-layer.scope.add": "write", + "semantic-layer.scope.remove": "write", + "semantic-layer.scope.set": "write", + "semantic-layer.scope.request-create": "write", + "semantic-layer.scope.request-delete": "write", # Raw HTTP client against `kbagent serve` (used by AI subprocesses). # Categorised by the underlying HTTP method under the taxonomy at the top # of this registry: GET = read, POST/PATCH = write (they create/modify), @@ -452,6 +454,15 @@ FLAG_ESCALATIONS: dict[str, str] = { "auth.logout --remove-projects": "admin", "sync.push --force": "destructive", + # `--scope organization` makes an item (and its revision history) visible to + # every project in the org with no downgrade endpoint -- irreversible. + "semantic-layer.scope.set --scope organization": "destructive", + "semantic-layer.model.create --scope organization": "destructive", + "semantic-layer.add.metric --scope organization": "destructive", + "semantic-layer.add.dataset --scope organization": "destructive", + "semantic-layer.add.relationship --scope organization": "destructive", + "semantic-layer.add.constraint --scope organization": "destructive", + "semantic-layer.add.glossary --scope organization": "destructive", } # Operations that exist ONLY on the `kbagent serve` REST surface. They are real diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 42eb22bdc..e45441de4 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -29,6 +29,11 @@ ("DELETE", "/notifications/{project}/{subscription_id}"): "notification delete", ("DELETE", "/projects/{alias}"): "project remove", ("DELETE", "/semantic-layer/items/{kind}/{name}"): "", + ( + "DELETE", + "/semantic-layer/scope/{context_id}/elevation-request", + ): "semantic-layer scope request-delete", + ("DELETE", "/semantic-layer/scope/{context_id}/target-projects"): "semantic-layer scope remove", ("DELETE", "/semantic-layer/models/{model}"): "semantic-layer model delete", ( "DELETE", @@ -130,6 +135,8 @@ ("GET", "/semantic-layer/reference-data"): "semantic-layer reference-data list", ("GET", "/semantic-layer/reference-data/{record_id}"): "semantic-layer reference-data get", ("GET", "/semantic-layer/schema"): "semantic-layer schema", + ("GET", "/semantic-layer/scope/elevation-requests"): "semantic-layer scope request-list", + ("GET", "/semantic-layer/scope/{context_id}"): "semantic-layer scope get", ("GET", "/semantic-layer/search-context"): "semantic-layer search-context", ("GET", "/semantic-layer/show"): "semantic-layer show", ("GET", "/semantic-layer/validate"): "semantic-layer validate", @@ -245,6 +252,7 @@ ("POST", "/semantic-layer/items/{kind}"): "", ("POST", "/semantic-layer/models"): "semantic-layer model create", ("POST", "/semantic-layer/promote"): "semantic-layer promote", + ("POST", "/semantic-layer/scope/{context_id}/target-projects"): "semantic-layer scope add", ("POST", "/semantic-layer/token/encrypt"): "semantic-layer token", ("POST", "/sharing/{project}/link"): "sharing link", ("POST", "/sharing/{project}/share"): "sharing share", @@ -289,4 +297,9 @@ ("PUT", "/projects/{alias}/description"): "project description-set", ("PUT", "/semantic-layer/items/{kind}/{name}"): "", ("PUT", "/semantic-layer/reference-data"): "semantic-layer reference-data set", + ("PUT", "/semantic-layer/scope/{context_id}"): "semantic-layer scope set", + ( + "PUT", + "/semantic-layer/scope/{context_id}/elevation-request", + ): "semantic-layer scope request-create", } diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index ad3f4b147..4ee8cfd75 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -1,4 +1,4 @@ -"""Semantic-layer endpoints — 18 routes covering 30 CLI subcommands. +"""Semantic-layer endpoints — 25 routes covering 37 CLI subcommands. Mirrors the per-subcommand surface of :class:`keboola_agent_cli.services.semantic_layer_service.SemanticLayerService`. @@ -39,7 +39,26 @@ # ── Pydantic body models ─────────────────────────────────────────── -class ModelCreate(BaseModel): +class ScopeFields(BaseModel): + """``--scope`` / ``--target-project`` of the create commands. + + ``scope`` omitted: a model is created ``project``-scoped, a child item takes + its model's scope. ``target_projects`` (alias or numeric ID) needs ``targeted``. + """ + + scope: Literal["project", "organization", "targeted"] | None = None + target_projects: list[str] | None = None + + @model_validator(mode="after") + def _targets_need_targeted_scope(self) -> ScopeFields: + if self.target_projects and self.scope != "targeted": + raise ValueError("target_projects requires scope='targeted'.") + if self.scope == "targeted" and not self.target_projects: + raise ValueError("scope='targeted' requires target_projects.") + return self + + +class ModelCreate(ScopeFields): project: str name: str description: str = "" @@ -71,7 +90,7 @@ def _exactly_one_per_side(cls, data: Any) -> Any: return data -class AddMetric(BaseModel): +class AddMetric(ScopeFields): project: str model: str | None = None name: str @@ -80,7 +99,7 @@ class AddMetric(BaseModel): description: str = "" -class AddDataset(BaseModel): +class AddDataset(ScopeFields): project: str model: str | None = None name: str @@ -92,7 +111,7 @@ class AddDataset(BaseModel): fqn: str | None = None -class AddRelationship(BaseModel): +class AddRelationship(ScopeFields): project: str model: str | None = None name: str @@ -104,7 +123,7 @@ class AddRelationship(BaseModel): model_config = {"populate_by_name": True} -class AddConstraint(BaseModel): +class AddConstraint(ScopeFields): project: str model: str | None = None name: str @@ -114,7 +133,7 @@ class AddConstraint(BaseModel): severity: str = "warning" -class AddGlossary(BaseModel): +class AddGlossary(ScopeFields): project: str model: str | None = None term: str @@ -235,6 +254,8 @@ def create_model( name=body.name, description=body.description, sql_dialect=body.sql_dialect, + scope=body.scope, + target_projects=body.target_projects, ) @@ -426,6 +447,8 @@ def add_item( description=m.description, assume_yes=True, is_tty=False, + scope=m.scope, + target_projects=m.target_projects, ) if kind == "dataset": d = AddDataset.model_validate(body) @@ -439,6 +462,8 @@ def add_item( primary_key=d.primary_key, deep_fields=d.deep_fields, fqn=d.fqn, + scope=d.scope, + target_projects=d.target_projects, ) if kind == "relationship": r = AddRelationship.model_validate(body) @@ -450,6 +475,8 @@ def add_item( to=r.to, on=r.on, type_=r.type_, + scope=r.scope, + target_projects=r.target_projects, ) if kind == "constraint": c = AddConstraint.model_validate(body) @@ -461,6 +488,8 @@ def add_item( rule=c.rule, metrics=c.metrics, severity=c.severity, + scope=c.scope, + target_projects=c.target_projects, ) if kind == "glossary": g = AddGlossary.model_validate(body) @@ -469,6 +498,8 @@ def add_item( model_name_or_uuid=g.model, term=g.term, definition=g.definition, + scope=g.scope, + target_projects=g.target_projects, ) raise HTTPException( status_code=404, @@ -678,5 +709,120 @@ def delete_reference_data( return registry.semantic_layer.delete_reference_data(alias=project, record_id=record_id) +# ── scope (visibility, target-project grants, org-elevation requests) ── +# Mirrors `kbagent semantic-layer scope *`. A write to `organization` scope is +# irreversible; like every router outside /auth, these do not consult the +# permission engine -- the bearer token of `serve` is the gate. + +ScopeKind = Literal["model", "dataset", "metric", "relationship", "constraint", "glossary"] + + +class ScopeTargets(BaseModel): + project: str + type: ScopeKind + target_projects: list[str] = Field(min_length=1) + + +class ScopeSet(BaseModel): + """Exactly one of ``scope`` ('organization'), ``target_projects`` or ``clear``.""" + + project: str + type: ScopeKind + scope: Literal["organization"] | None = None + target_projects: list[str] | None = None + clear: bool = False + dry_run: bool = False + + +@router.get("/scope/elevation-requests", summary="List scope-elevation requests") +def scope_request_list( + project: str, + type: ScopeKind, + limit: int = Query(50, ge=1), + offset: int = Query(0, ge=0), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Items of ``type`` awaiting an org-admin's elevation decision (org-admin token).""" + return registry.semantic_layer.scope_request_list( + alias=project, kind=type, limit=limit, offset=offset + ) + + +@router.get("/scope/{context_id}", summary="Get an item's scope") +def scope_get( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_get(alias=project, kind=type, context_id=context_id) + + +@router.put("/scope/{context_id}", summary="Set an item's scope or target projects") +def scope_set( + context_id: str, + body: ScopeSet, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Elevate to organization (irreversible), or replace/clear the target projects.""" + return registry.semantic_layer.scope_set( + alias=body.project, + kind=body.type, + context_id=context_id, + scope=body.scope, + target_projects=body.target_projects, + clear=body.clear, + dry_run=body.dry_run, + ) + + +@router.post("/scope/{context_id}/target-projects", summary="Add target projects") +def scope_add( + context_id: str, + body: ScopeTargets, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_update_targets( + alias=body.project, kind=body.type, context_id=context_id, add=body.target_projects + ) + + +@router.delete("/scope/{context_id}/target-projects", summary="Remove target projects") +def scope_remove( + context_id: str, + project: str, + type: ScopeKind, + target_project: list[str] = Query(min_length=1), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_update_targets( + alias=project, kind=type, context_id=context_id, remove=target_project + ) + + +@router.put("/scope/{context_id}/elevation-request", summary="Request scope elevation") +def scope_request_create( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_request_create( + alias=project, kind=type, context_id=context_id + ) + + +@router.delete("/scope/{context_id}/elevation-request", summary="Withdraw scope elevation request") +def scope_request_delete( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_request_delete( + alias=project, kind=type, context_id=context_id + ) + + # Re-export the closed set of kinds for tests / docs. __all__ = ["ErrorCode", "ItemKind", "router"] diff --git a/src/keboola_agent_cli/services/_semantic_layer_crud.py b/src/keboola_agent_cli/services/_semantic_layer_crud.py index 24a851218..d94368881 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/services/_semantic_layer_crud.py @@ -6,9 +6,11 @@ main module are thin orchestrators that resolve credentials + the model UUID, then delegate. +Edits are an in-place ``PUT`` (the item keeps its ID, scope, grants, pending +elevation request and revision history); a failed PUT leaves the original intact. + Helpers: -- :func:`delete_then_post` -- safe DELETE+POST with rollback - :func:`edit_metric_with_cascade` -- metric rename + constraint cascade - :func:`scan_orphan_constraints` -- pre-deletion orphan scan for metric - :data:`REMOVE_KINDS` -- accepted kinds for ``remove_item`` / @@ -21,20 +23,10 @@ from typing import TYPE_CHECKING, Any from ..errors import ErrorCode, KeboolaApiError +from ._semantic_layer_scope import item_scope if TYPE_CHECKING: - from ..metastore_client import MetastoreClient, ObjectScope, SemanticType - - -def _item_scope(item: dict[str, Any]) -> tuple[ObjectScope, list[int] | None]: - """Extract ``(scope, target_project_ids)`` from a raw item's ``meta`` block. - - Defaults to ``("project", None)`` for an item predating PSGO-140 (no - ``meta.scope`` key) so the edit path is a no-op change for anything - already project-scoped. - """ - meta = item.get("meta") or {} - return meta.get("scope", "project"), meta.get("targetProjectIds") + from ..metastore_client import MetastoreClient, SemanticType # Kinds accepted by `remove_item` / `preview_remove`. Relationship and @@ -58,73 +50,6 @@ def code_metric(name: str) -> str: return re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_") -def delete_then_post( - client: MetastoreClient, - item_type: SemanticType, - *, - old_id: str, - original_attrs: dict[str, Any], - new_name: str, - new_attrs: dict[str, Any], - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, -) -> tuple[dict[str, Any], dict[str, Any] | None]: - """Run a safe DELETE+POST, rolling back to ``original_attrs`` on POST failure. - - ``scope``/``target_project_ids`` are the ORIGINAL item's values (read - from its ``meta`` block by the caller) and are re-applied on both the - primary POST and the rollback POST -- ``post_item`` defaults to - ``scope="project"``, and without passing these through explicitly, every - edit of an organization/targeted-scope object would silently downgrade - it back to project-only visibility on every save. - - Returns ``(new_item, rollback)`` where ``rollback`` is None on - success. On POST failure we re-POST the original payload; the - rollback dict records whether that re-POST succeeded. The original - exception is re-raised wrapped in a KeboolaApiError with the - rollback context. - """ - client.delete_item(item_type, old_id) - try: - new_item = client.post_item( - item_type, - name=new_name, - data=new_attrs, - scope=scope, - target_project_ids=target_project_ids, - ) - except KeboolaApiError as exc: - rollback_status: dict[str, Any] = { - "attempted": True, - "original_name": original_attrs.get("name") or original_attrs.get("term", ""), - "error": exc.message, - } - try: - old_name = original_attrs.get("name") or original_attrs.get("term", "") - restored = client.post_item( - item_type, - name=old_name, - data=original_attrs, - scope=scope, - target_project_ids=target_project_ids, - ) - rollback_status["status"] = "succeeded" - rollback_status["restored_id"] = restored.get("id", "") - except KeboolaApiError as rollback_exc: - rollback_status["status"] = "failed" - rollback_status["rollback_error"] = rollback_exc.message - raise KeboolaApiError( - message=( - f"edit failed (POST after DELETE raised): {exc.message}. " - f"Rollback: {rollback_status['status']}." - ), - error_code=exc.error_code, - status_code=exc.status_code, - details={"rollback": rollback_status}, - ) from exc - return new_item, None - - def edit_metric_with_cascade( client: MetastoreClient, *, @@ -141,8 +66,8 @@ def edit_metric_with_cascade( """Body of :meth:`SemanticLayerService.edit_metric`. Resolves the target metric, computes the constraint cascade list, - enforces TTY/--yes guards, then DELETE+POSTs the metric and - DELETE+POSTs each cascaded constraint individually so per-item + enforces TTY/--yes guards, then PUTs the metric and + PUTs each cascaded constraint individually so per-item failures don't poison the rest. """ metrics = client.list_items("semantic-metric", model_uuid) @@ -157,7 +82,7 @@ def edit_metric_with_cascade( ) original_attrs = dict(target.get("attributes") or {}) old_id = target["id"] - target_scope, target_project_ids = _item_scope(target) + target_scope, _ = item_scope(target) effective_new_name = new_name if new_name is not None else current_name cascade_required: list[dict[str, Any]] = [] @@ -201,15 +126,8 @@ def edit_metric_with_cascade( if new_description is not None: new_attrs["description"] = new_description - new_item, rollback = delete_then_post( - client, - "semantic-metric", - old_id=old_id, - original_attrs=original_attrs, - new_name=effective_new_name, - new_attrs=new_attrs, - scope=target_scope, - target_project_ids=target_project_ids, + new_item = client.put_item( + "semantic-metric", old_id, effective_new_name, new_attrs, scope=target_scope ) # Cascade constraints individually (each is independent -- @@ -221,17 +139,9 @@ def edit_metric_with_cascade( cmetrics = [effective_new_name if x == current_name else x for x in cmetrics] cattrs["metrics"] = cmetrics cname = cattrs.get("name", "") - c_scope, c_target_project_ids = _item_scope(c) try: - cascaded_item, _ = delete_then_post( - client, - "semantic-constraint", - old_id=c["id"], - original_attrs=c.get("attributes") or {}, - new_name=cname, - new_attrs=cattrs, - scope=c_scope, - target_project_ids=c_target_project_ids, + cascaded_item = client.put_item( + "semantic-constraint", c["id"], cname, cattrs, scope=item_scope(c)[0] ) cascaded.append({"constraint": cname, "status": "updated", "id": cascaded_item["id"]}) except KeboolaApiError as exc: @@ -240,7 +150,6 @@ def edit_metric_with_cascade( "constraint": cname, "status": "failed", "error": exc.message, - "rollback": (exc.details or {}).get("rollback"), } ) @@ -249,7 +158,7 @@ def edit_metric_with_cascade( return { "updated": new_item, "cascaded_constraints": cascaded, - "rollback": rollback, + "rollback": None, "partial_state": partial_state, "recovery_hint": ( ( @@ -379,21 +288,14 @@ def edit_simple( if v is not None: new_attrs[k] = v effective_new = new_attrs.get(id_key) or current_key - target_scope, target_project_ids = _item_scope(target) - new_item, rollback = delete_then_post( - client, - item_type, - old_id=target["id"], - original_attrs=original_attrs, - new_name=effective_new, - new_attrs=new_attrs, - scope=target_scope, - target_project_ids=target_project_ids, + target_scope, _ = item_scope(target) + new_item = client.put_item( + item_type, target["id"], effective_new, new_attrs, scope=target_scope ) return { "updated": new_item, "cascaded_constraints": [], - "rollback": rollback, + "rollback": None, "partial_state": False, "recovery_hint": None, } diff --git a/src/keboola_agent_cli/services/_semantic_layer_internals.py b/src/keboola_agent_cli/services/_semantic_layer_internals.py index 035eb37aa..25e07c4f1 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_internals.py +++ b/src/keboola_agent_cli/services/_semantic_layer_internals.py @@ -40,6 +40,7 @@ from ..errors import ConfigError, ErrorCode, KeboolaApiError from ._semantic_layer_fqn import append_fqn_mismatch_warnings +from ._semantic_layer_scope import item_scope logger = logging.getLogger(__name__) @@ -444,8 +445,10 @@ def run_import_loop( per_type["overwritten"] += 1 continue try: - client.delete_item(type_slug, existing_by_name[key]["id"]) - client.post_item(type_slug, name=key, data=attrs) + existing = existing_by_name[key] + client.put_item( + type_slug, existing["id"], key, attrs, scope=item_scope(existing)[0] + ) per_type["overwritten"] += 1 except KeboolaApiError as exc: per_type["failed"].append({"name": key, "reason": exc.message}) @@ -517,8 +520,10 @@ def run_promote_loop( stats["changes"].append({id_key: key, "diff_keys": diff_keys}) continue try: - target_client.delete_item(type_slug, tgt_by_key[key]["id"]) - target_client.post_item(type_slug, name=key, data=src_attrs) + existing = tgt_by_key[key] + target_client.put_item( + type_slug, existing["id"], key, src_attrs, scope=item_scope(existing)[0] + ) stats["overwritten"] += 1 stats["changes"].append({id_key: key, "diff_keys": diff_keys}) except KeboolaApiError as exc: diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py index e0dcc8c5a..c44b502a9 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -21,35 +21,79 @@ if TYPE_CHECKING: from ..config_store import ConfigStore - from ..metastore_client import MetastoreClient, SemanticType + from ..metastore_client import MetastoreClient, ObjectScope, SemanticType +DEFAULT_REQUEST_LIST_LIMIT = 50 -def resolve_target_project_ids(config_store: ConfigStore, target_projects: list[str]) -> list[int]: - """Resolve project aliases to numeric Storage project IDs. - Raises ``NOT_FOUND`` for an unregistered alias, ``CONFIG_ERROR`` for a - registered project with no numeric ``project_id`` on record (would need - a ``project refresh``/re-add to populate it -- see - ``services/org_service.py``, which relies on the same invariant). +def parse_target_projects(values: list[str] | None) -> list[str]: + """Flatten repeatable and comma-separated ``--target-project`` values, de-duplicated.""" + seen: dict[str, None] = {} + for value in values or []: + for part in value.split(","): + if part.strip(): + seen.setdefault(part.strip()) + return list(seen) + + +def resolve_target_project_ids( + config_store: ConfigStore, owner_alias: str, targets: list[str] | None +) -> list[int]: + """Resolve ``--target-project`` values (alias or numeric project ID) to project IDs. + + A registered alias wins; otherwise an all-digit value is taken as a project + ID, so a target does not have to be registered in kbagent. An alias must be + on the owner project's stack -- a project ID is only meaningful there, and + an ID cannot be checked, so it is trusted as being on that stack. Any bad + value is ``INVALID_ARGUMENT`` (a usage error) naming the option. """ + owner = config_store.get_project(owner_alias) ids: list[int] = [] - for alias in target_projects: - project = config_store.get_project(alias) + for target in parse_target_projects(targets): + project = config_store.get_project(target) if project is None: + if not target.isdigit(): + raise KeboolaApiError( + message=( + f"--target-project {target!r} is neither a registered project alias " + "nor a numeric project ID. See `kbagent project list`." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + ids.append(int(target)) + continue + if owner is not None and project.stack_url.rstrip("/") != owner.stack_url.rstrip("/"): raise KeboolaApiError( - message=f"Unknown project alias {alias!r}. See `kbagent project list`.", - error_code=ErrorCode.NOT_FOUND, + message=( + f"--target-project {target!r} is on a different stack than {owner_alias!r}; " + "a project ID only means the same project on its own stack." + ), + error_code=ErrorCode.INVALID_ARGUMENT, ) if project.project_id is None: raise KeboolaApiError( message=( - f"Project {alias!r} has no numeric project_id on record " + f"--target-project {target!r} has no numeric project_id on record " "(re-run `kbagent project refresh` or `project add`)." ), - error_code=ErrorCode.CONFIG_ERROR, + error_code=ErrorCode.INVALID_ARGUMENT, ) ids.append(project.project_id) - return ids + return list(dict.fromkeys(ids)) + + +def item_scope(item: dict[str, Any]) -> tuple[ObjectScope, list[int] | None]: + """``(scope, target_project_ids)`` from a raw item's ``meta`` block (``project`` if absent).""" + meta = item.get("meta") or {} + return meta.get("scope", "project"), meta.get("targetProjectIds") + + +def inherited_scope( + client: MetastoreClient, model_uuid: str +) -> tuple[ObjectScope, list[int] | None]: + """The scope a child item takes when ``--scope`` is omitted: its model's own.""" + scope, targets = item_scope(client.get_item("semantic-model", model_uuid)) + return scope, (targets or None) if scope == "targeted" else None def item_status(item: dict[str, Any]) -> dict[str, Any]: @@ -67,32 +111,51 @@ def item_status(item: dict[str, Any]) -> dict[str, Any]: } -def grant_target_projects( +def _merge_targets( + item: dict[str, Any], caller_project_id: int | None, add: list[int], remove: list[int] +) -> list[int]: + """Apply an add/remove delta to the item's grants, refusing when they cannot be read. + + The server returns ``meta.targetProjectIds`` only to the owning project, so + a non-owner (an org admin from elsewhere) would read ``None``, apply the + delta to an empty list, and PUT it -- silently replacing every grant. + """ + meta = item.get("meta") or {} + owner_id = meta.get("projectId") + if (owner_id is not None and owner_id != caller_project_id) or ( + meta.get("scope", "project") == "targeted" and meta.get("targetProjectIds") is None + ): + raise KeboolaApiError( + message=( + "This project does not own the item, so its current target projects cannot " + "be read and add/remove would overwrite them. Use `scope set --target-project` " + "to replace the whole list from the owning project's view." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + return sorted((set(meta.get("targetProjectIds") or []) | set(add)) - set(remove)) + + +def set_target_projects( client: MetastoreClient, item_type: SemanticType, item_id: str, *, + replace: list[int] | None = None, add: list[int] | None = None, remove: list[int] | None = None, - replace: list[int] | None = None, + caller_project_id: int | None = None, ) -> dict[str, Any]: - """Update the target-project grant list for a targeted-scope item. - - ``replace`` sends exactly that set, matching the server's native - replace-only semantics with no extra round trip. ``add``/``remove`` are - a convenience merge: read the item's current grants, apply the delta, - then PUT the result. This merge is **not atomic** against a concurrent - grant change on the same item -- last write wins, same as every other - read-modify-write in this CLI. + """Update a targeted-scope item's grants: replace the list, or merge an add/remove delta. + + ``replace`` is the server's native replace-only PUT (``[]`` clears). A merge + is read-modify-write and not atomic against a concurrent grant change. """ if replace is not None: new_ids = sorted(set(replace)) else: current = client.get_item(item_type, item_id) - current_ids = set((current.get("meta") or {}).get("targetProjectIds") or []) - current_ids |= set(add or []) - current_ids -= set(remove or []) - new_ids = sorted(current_ids) + new_ids = _merge_targets(current, caller_project_id, add or [], remove or []) client.put_target_projects(item_type, item_id, new_ids) return item_status(client.get_item(item_type, item_id)) @@ -106,10 +169,7 @@ def request_elevation( the ``scope-elevation-request`` (and ``PATCH``) response bodies omit ``meta.targetProjectIds``, so reporting them directly prints ``target_project_ids: null`` for an item whose grants are actually - intact -- verified live: `scope status` before and after shows - ``[5024]`` while the endpoint response shows ``None``. That reads as - "requesting elevation just wiped my grants", which is not what - happened. Same re-read pattern as :func:`grant_target_projects`. + intact. Same re-read pattern as :func:`set_target_projects`. """ client.request_scope_elevation(item_type, item_id) return item_status(client.get_item(item_type, item_id)) @@ -124,9 +184,12 @@ def withdraw_elevation( def elevate_to_organization( - client: MetastoreClient, item_type: SemanticType, item_id: str + client: MetastoreClient, item_type: SemanticType, item_id: str, *, dry_run: bool = False ) -> dict[str, Any]: - """Step an item up to organization scope. Re-reads for the reason in :func:`request_elevation`.""" + """Step an item up to organization scope (one-way). ``dry_run`` only reports the plan.""" + if dry_run: + status = item_status(client.get_item(item_type, item_id)) + return {**status, "dry_run": True, "would_set_scope": "organization"} client.elevate_to_organization(item_type, item_id) return item_status(client.get_item(item_type, item_id)) @@ -135,10 +198,16 @@ def list_pending_elevations( client: MetastoreClient, item_type: SemanticType, *, - limit: int | None = None, - offset: int | None = None, -) -> list[dict[str, Any]]: + limit: int = DEFAULT_REQUEST_LIST_LIMIT, + offset: int = 0, +) -> dict[str, Any]: + """One page of elevation requests; ``has_more`` comes from fetching one row past ``limit``.""" items = client.list_organization_items( - item_type, pending_elevation_only=True, limit=limit, offset=offset + item_type, pending_elevation_only=True, limit=limit + 1, offset=offset ) - return [item_status(i) for i in items] + return { + "items": [item_status(i) for i in items[:limit]], + "limit": limit, + "offset": offset, + "has_more": len(items) > limit, + } diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index e90b3d1fa..9d145dff0 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -31,7 +31,6 @@ from ._semantic_layer_cascade import cascade_delete_model as _cascade_delete_model_impl from ._semantic_layer_crud import REMOVE_KINDS as _REMOVE_KINDS_HELPER from ._semantic_layer_crud import code_metric as _code_metric_helper -from ._semantic_layer_crud import delete_then_post as _delete_then_post_helper from ._semantic_layer_crud import edit_metric_with_cascade as _edit_metric_helper from ._semantic_layer_crud import edit_simple as _edit_simple_helper from ._semantic_layer_crud import find_target_for_remove as _find_target_for_remove @@ -679,6 +678,34 @@ def _collect_side( # Phase 4 — Model lifecycle (create / delete) # ------------------------------------------------------------------ + def _target_ids( + self, alias: str, scope: ObjectScope, target_projects: list[str] | None + ) -> list[int] | None: + """Resolve ``--target-project`` values for a ``targeted`` write (``None`` otherwise).""" + if scope != "targeted": + return None + return _scope.resolve_target_project_ids(self._config_store, alias, target_projects) or None + + def _post_scoped( + self, + client: MetastoreClient, + alias: str, + item_type: SemanticType, + *, + name: str, + data: dict[str, Any], + scope: ObjectScope | None, + target_projects: list[str] | None, + ) -> dict[str, Any]: + """POST a child item; with ``scope`` omitted it takes the scope of its model.""" + if scope is None: + scope, target_ids = _scope.inherited_scope(client, data["modelUUID"]) + else: + target_ids = self._target_ids(alias, scope, target_projects) + return client.post_item( + item_type, name=name, data=data, scope=scope, target_project_ids=target_ids + ) + def create_model( self, alias: str, @@ -686,8 +713,8 @@ def create_model( description: str = "", sql_dialect: str = "Snowflake", *, - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a semantic-layer model and return the server-stored item.""" project = self._resolve_one_project(alias) @@ -695,12 +722,13 @@ def create_model( data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} if description: data["description"] = description + scope = scope or "project" created = client.post_item( "semantic-model", name=name, data=data, scope=scope, - target_project_ids=target_project_ids, + target_project_ids=self._target_ids(alias, scope, target_projects), ) return {"project": alias, "model": created} @@ -745,8 +773,8 @@ def add_metric( assume_yes: bool = False, is_tty: bool = False, confirm_cb: Callable[[str], bool] | None = None, - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a metric. The ``dataset`` argument is a tableId. @@ -786,12 +814,14 @@ def add_metric( } if description: data["description"] = description - return client.post_item( + return self._post_scoped( + client, + alias, "semantic-metric", name=name, data=data, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) def add_dataset( @@ -806,8 +836,8 @@ def add_dataset( primary_key: list[str] | None = None, deep_fields: bool = False, fqn: str | None = None, - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a dataset. @@ -849,12 +879,14 @@ def add_dataset( fields = _synthesize_role_classified_fields(detail, _classify_field_role) if fields: data["fields"] = fields - return client.post_item( + return self._post_scoped( + client, + alias, "semantic-dataset", name=name, data=data, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) def add_relationship( @@ -867,8 +899,8 @@ def add_relationship( to: str, on: str, type_: str, - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a relationship. ``from``/``to`` are tableIds; ``type``='left'|'inner'.""" if type_ not in ("left", "inner"): @@ -887,12 +919,14 @@ def add_relationship( "type": type_, "modelUUID": model_uuid, } - return client.post_item( + return self._post_scoped( + client, + alias, "semantic-relationship", name=name, data=data, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) def add_constraint( @@ -905,8 +939,8 @@ def add_constraint( rule: str, metrics: list[str], severity: str = "warning", - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a constraint after validating every field locally.""" _validate_constraint_attrs( @@ -941,12 +975,14 @@ def add_constraint( "severity": severity, "modelUUID": model_uuid, } - return client.post_item( + return self._post_scoped( + client, + alias, "semantic-constraint", name=name, data=data, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) def add_glossary( @@ -956,8 +992,8 @@ def add_glossary( *, term: str, definition: str = "", - scope: ObjectScope = "project", - target_project_ids: list[int] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" project = self._resolve_one_project(alias) @@ -966,109 +1002,123 @@ def add_glossary( data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} if definition: data["definition"] = definition - return client.post_item( + return self._post_scoped( + client, + alias, "semantic-glossary", name=term, data=data, scope=scope, - target_project_ids=target_project_ids, + target_projects=target_projects, ) # ------------------------------------------------------------------ # Phase 5 — Scope / target-project / elevation (PSGO-140) # ------------------------------------------------------------------ - def resolve_target_project_ids(self, target_projects: list[str]) -> list[int]: - """Resolve project aliases to numeric project IDs (for ``--target-project``).""" - return _scope.resolve_target_project_ids(self._config_store, target_projects) - - def _resolve_scope_type(self, kind: str) -> SemanticType: + def _scope_item_type(self, kind: str) -> SemanticType: item_type = SCHEMA_TYPE_ALIAS.get(kind) if item_type is None: raise KeboolaApiError( message=f"--type must be one of {sorted(SCHEMA_TYPE_ALIAS)}, got {kind!r}.", - error_code=ErrorCode.VALIDATION_ERROR, + error_code=ErrorCode.INVALID_ARGUMENT, ) return item_type - def scope_status(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: + def scope_get(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: """Show the current scope/grants/pending-elevation state of one item.""" - item_type = self._resolve_scope_type(kind) - project = self._resolve_one_project(alias) - with self._new_metastore_client(project) as client: - return _scope.item_status(client.get_item(item_type, item_id)) + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.item_status(client.get_item(item_type, context_id)) - def scope_grant( + def scope_update_targets( self, alias: str, kind: str, - item_id: str, + context_id: str, *, add: list[str] | None = None, remove: list[str] | None = None, - replace: list[str] | None = None, ) -> dict[str, Any]: - """Grant/revoke/replace the target-project list of a targeted-scope item. - - ``add``/``remove``/``replace`` are project ALIASES, resolved to - numeric project IDs here before calling the metastore. ``replace`` - (including ``[]`` to clear every grant) is a direct pass-through of - the server's native replace-only semantics; ``add``/``remove`` do a - read-modify-write merge (see :func:`_semantic_layer_scope.grant_target_projects`). - """ - item_type = self._resolve_scope_type(kind) + """Merge ``add``/``remove`` target projects (alias or ID) into a targeted item's grants.""" + item_type = self._scope_item_type(kind) project = self._resolve_one_project(alias) + resolve = _scope.resolve_target_project_ids with self._new_metastore_client(project) as client: - return _scope.grant_target_projects( + return _scope.set_target_projects( client, item_type, - item_id, - add=_scope.resolve_target_project_ids(self._config_store, add) if add else None, - remove=( - _scope.resolve_target_project_ids(self._config_store, remove) - if remove - else None - ), - replace=( - _scope.resolve_target_project_ids(self._config_store, replace) - if replace is not None - else None - ), + context_id, + add=resolve(self._config_store, alias, add), + remove=resolve(self._config_store, alias, remove), + caller_project_id=project.project_id, ) - def scope_request_elevation(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: - """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" - item_type = self._resolve_scope_type(kind) - project = self._resolve_one_project(alias) - with self._new_metastore_client(project) as client: - return _scope.request_elevation(client, item_type, item_id) + def scope_set( + self, + alias: str, + kind: str, + context_id: str, + *, + scope: str | None = None, + target_projects: list[str] | None = None, + clear: bool = False, + dry_run: bool = False, + ) -> dict[str, Any]: + """Write the scope: elevate to ``organization``, or replace/clear the target projects. - def scope_withdraw_elevation(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: - """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" - item_type = self._resolve_scope_type(kind) + Exactly one of ``scope`` / ``target_projects`` / ``clear`` is allowed -- the + metastore changes scope only to ``organization`` and accepts a target list + only for a ``targeted`` item, so a mixed request is refused before any call. + """ + modes = [scope is not None, bool(target_projects), clear] + if sum(modes) != 1: + raise KeboolaApiError( + message="scope set takes exactly one of --scope organization, " + "--target-project, or --clear.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + if scope is not None and scope != "organization": + raise KeboolaApiError( + message="--scope can only be 'organization' (scope is one-way).", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + item_type = self._scope_item_type(kind) project = self._resolve_one_project(alias) + replace = _scope.resolve_target_project_ids(self._config_store, alias, target_projects) with self._new_metastore_client(project) as client: - return _scope.withdraw_elevation(client, item_type, item_id) + if scope is not None: + return _scope.elevate_to_organization( + client, item_type, context_id, dry_run=dry_run + ) + if dry_run: + status = _scope.item_status(client.get_item(item_type, context_id)) + return {**status, "dry_run": True, "would_set_target_project_ids": replace} + return _scope.set_target_projects(client, item_type, context_id, replace=replace) - def scope_elevate(self, alias: str, kind: str, item_id: str) -> dict[str, Any]: - """Step an item up to organization scope. Requires org-admin; one-way, no downgrade.""" - item_type = self._resolve_scope_type(kind) - project = self._resolve_one_project(alias) - with self._new_metastore_client(project) as client: - return _scope.elevate_to_organization(client, item_type, item_id) + def scope_request_create(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.request_elevation(client, item_type, context_id) - def scope_pending( + def scope_request_delete(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: + """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.withdraw_elevation(client, item_type, context_id) + + def scope_request_list( self, alias: str, kind: str, *, - limit: int | None = None, - offset: int | None = None, - ) -> list[dict[str, Any]]: - """List items of ``kind`` awaiting an org-admin's elevation decision.""" - item_type = self._resolve_scope_type(kind) - project = self._resolve_one_project(alias) - with self._new_metastore_client(project) as client: + limit: int = _scope.DEFAULT_REQUEST_LIST_LIMIT, + offset: int = 0, + ) -> dict[str, Any]: + """One page of ``kind`` items awaiting an org-admin's elevation decision.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: return _scope.list_pending_elevations(client, item_type, limit=limit, offset=offset) # ------------------------------------------------------------------ @@ -1142,7 +1192,6 @@ def delete_reference_data(self, alias: str, record_id: str) -> dict[str, Any]: # Thin delegates to ._semantic_layer_crud helpers -- bodies live # there so this orchestrator stays under the services budget. _code_metric = staticmethod(_code_metric_helper) - _delete_then_post = staticmethod(_delete_then_post_helper) def edit_metric( self, diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 3503e4343..5db3272b1 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -169,7 +169,7 @@ def test_post_envelope(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "rev" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.1.0" + assert body["schemaVersion"] == "1.0.0" assert body["scope"] == "project" assert body["data"]["sql"] == "SUM(x)" assert body["data"]["modelUUID"] == "u" @@ -304,7 +304,7 @@ def test_put_envelope_and_url(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "chart_of_accounts" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.1.0" + assert body["schemaVersion"] == "1.0.0" assert "scope" not in body assert body["data"]["dimensionName"] == "chart_of_accounts" assert body["data"]["members"] == [] @@ -524,6 +524,8 @@ def test_default_bearer_would_refresh_reproducing_the_bug(self, httpx_mock) -> N client.close() assert spy.refresh_calls == 1 assert len(httpx_mock.get_requests()) == 2 + + class TestPostItemScopeValidation: """post_item validates scope/target_project_ids client-side (PSGO-140).""" @@ -532,6 +534,50 @@ def test_rejects_unknown_scope(self, metastore_client) -> None: metastore_client.post_item("semantic-metric", name="x", data={}, scope="bogus") assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + def test_empty_target_project_ids_still_rejected_without_targeted_scope( + self, metastore_client + ) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item( + "semantic-metric", name="x", data={}, scope="organization", target_project_ids=[] + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + @pytest.mark.parametrize( + ("scope", "expected"), + [("project", "1.0.0"), ("organization", "1.1.0"), ("targeted", "1.1.0")], + ) + def test_schema_version_is_1_1_0_only_for_acl_scopes( + self, httpx_mock, metastore_client, scope, expected + ) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "semantic-metric", + name="x", + data={}, + scope=scope, + target_project_ids=[1] if scope == "targeted" else None, + ) + assert json.loads(httpx_mock.get_requests()[0].content)["schemaVersion"] == expected + + def test_put_item_schema_version_follows_the_items_scope( + self, httpx_mock, metastore_client + ) -> None: + for _ in range(2): + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric/abc", + json={"data": {"type": "semantic-metric", "id": "abc", "attributes": {}}}, + ) + metastore_client.put_item("semantic-metric", "abc", "x", {}) + metastore_client.put_item("semantic-metric", "abc", "x", {}, scope="organization") + versions = [json.loads(r.content)["schemaVersion"] for r in httpx_mock.get_requests()] + assert versions == ["1.0.0", "1.1.0"] + def test_rejects_target_project_ids_without_targeted_scope(self, metastore_client) -> None: with pytest.raises(KeboolaApiError) as excinfo: metastore_client.post_item( diff --git a/tests/test_project_ref.py b/tests/test_project_ref.py index 5dd5e60a6..299dc8771 100644 --- a/tests/test_project_ref.py +++ b/tests/test_project_ref.py @@ -148,6 +148,10 @@ def test_no_shadow_notice_for_aliases_of_the_same_project(self) -> None: "--project-ids": "already project IDs (org setup)", "--source-project-id": "already a project ID (sharing link)", "--target-project-ids": "already project IDs (sharing share)", + "--target-project": ( + "semantic-layer: alias OR numeric ID resolved by the service itself (a target need not " + "be registered); config clone's --target-project is in ALIAS_OPTIONS" + ), "--all-projects": "a boolean flag", "--register-projects": "a boolean flag", "--remove-projects": "a boolean flag", @@ -183,7 +187,7 @@ def _leaf_commands(group: Any, path: tuple[str, ...] = ()) -> dict[tuple[str, .. def _dummy_value(param: Any, existing_file: Path) -> str: """A value Click accepts for ``param`` (the command body never runs).""" type_name = type(param.type).__name__ - if type_name == "TyperChoice": + if type_name in ("TyperChoice", "Choice"): return str(param.type.choices[0]) if type_name == "TyperPath": return str(existing_file) diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index 3a09fb103..cbce2c8e9 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -1,10 +1,9 @@ """Tests for metastore scope/target-project/elevation support (PSGO-140). -Covers ``services._semantic_layer_scope`` (pure alias-resolution + merge -logic) and ``SemanticLayerService.scope_*`` (orchestration), plus the -edit-preserves-scope regression: a DELETE+POST edit of an -organization/targeted-scope item must never silently reset it to -``project`` scope. +Covers ``services._semantic_layer_scope`` (target resolution, grant merge, +elevation) and the service orchestration on top of it: ``scope_*``, scope +inheritance on child creates, and the overwrite paths (``import --overwrite``, +``promote``) keeping an item's scope. """ from __future__ import annotations @@ -46,6 +45,15 @@ def _make_store(tmp_path: Path) -> ConfigStore: project_id=1234, ), ) + store.add_project( + "eu-project", + ProjectConfig( + stack_url="https://connection.europe-west3.gcp.keboola.com", + token=TEST_TOKEN, + project_name="eu", + project_id=77, + ), + ) return store @@ -74,300 +82,315 @@ def _child_item( # --------------------------------------------------------------------------- -# services._semantic_layer_scope -- pure helpers +# services._semantic_layer_scope -- helpers # --------------------------------------------------------------------------- class TestResolveTargetProjectIds: - def test_resolves_known_aliases(self, tmp_path: Path) -> None: + def test_aliases_ids_and_comma_lists_resolve_and_dedupe(self, tmp_path: Path) -> None: store = _make_store(tmp_path) - assert scope_helpers.resolve_target_project_ids(store, ["prod", "analytics"]) == [ - 5725, - 1234, - ] + resolved = scope_helpers.resolve_target_project_ids( + store, "prod", ["analytics,5678", "analytics", "9"] + ) + assert resolved == [1234, 5678, 9] - def test_unknown_alias_raises_not_found(self, tmp_path: Path) -> None: + @pytest.mark.parametrize( + ("target", "needle"), + [ + ("ghost", "neither a registered project alias nor a numeric project ID"), + ("eu-project", "different stack"), + ], + ) + def test_bad_target_is_invalid_argument_naming_the_option( + self, tmp_path: Path, target: str, needle: str + ) -> None: store = _make_store(tmp_path) with pytest.raises(KeboolaApiError) as excinfo: - scope_helpers.resolve_target_project_ids(store, ["ghost"]) - assert excinfo.value.error_code == ErrorCode.NOT_FOUND + scope_helpers.resolve_target_project_ids(store, "prod", [target]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert "--target-project" in excinfo.value.message + assert needle in excinfo.value.message - def test_missing_project_id_raises_config_error(self, tmp_path: Path) -> None: + def test_registered_project_without_id_is_invalid_argument(self, tmp_path: Path) -> None: store = _make_store(tmp_path) store.add_project( "no-id", ProjectConfig( - stack_url="https://connection.keboola.com", - token=TEST_TOKEN, - project_name="no-id", - project_id=None, + stack_url="https://connection.keboola.com", token=TEST_TOKEN, project_id=None ), ) with pytest.raises(KeboolaApiError) as excinfo: - scope_helpers.resolve_target_project_ids(store, ["no-id"]) - assert excinfo.value.error_code == ErrorCode.CONFIG_ERROR + scope_helpers.resolve_target_project_ids(store, "prod", ["no-id"]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT -class TestItemStatus: - def test_extracts_scope_fields_from_meta(self) -> None: - item = _child_item( - "semantic-dataset", - "d1", - {"name": "fact_pnl"}, - meta={ - "scope": "targeted", - "targetProjectIds": [1, 2], - "scopeElevationRequestedAt": "2026-08-28T00:00:00Z", - "projectId": 5725, - }, - ) - status = scope_helpers.item_status(item) - assert status == { - "id": "d1", - "type": "semantic-dataset", - "name": "fact_pnl", - "scope": "targeted", - "target_project_ids": [1, 2], - "scope_elevation_requested_at": "2026-08-28T00:00:00Z", - "project_id": 5725, - } - - def test_defaults_to_project_scope_without_meta(self) -> None: - item = _child_item("semantic-metric", "m1", {"name": "rev"}) - status = scope_helpers.item_status(item) - assert status["scope"] == "project" - assert status["target_project_ids"] is None - - def test_glossary_falls_back_to_term(self) -> None: - item = _child_item("semantic-glossary", "g1", {"term": "GMV"}) - assert scope_helpers.item_status(item)["name"] == "GMV" - - -class TestGrantTargetProjects: +def _item(meta: dict[str, Any]) -> dict[str, Any]: + return {"type": "semantic-dataset", "id": "d1", "attributes": {"name": "x"}, "meta": meta} + + +OWNED = {"scope": "targeted", "projectId": 5725, "targetProjectIds": [1, 2]} + + +class TestSetTargetProjects: def test_replace_sends_exact_set_without_reading_current(self) -> None: client = MagicMock() - client.get_item.return_value = _child_item( - "semantic-dataset", "d1", {"name": "x"}, meta={"scope": "targeted"} - ) - scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", replace=[2, 1, 2]) - client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 2]) - # replace mode never reads the current grant set before writing. - assert client.get_item.call_count == 1 # only the post-write status re-fetch + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects(client, "semantic-dataset", "d1", replace=[9, 3, 9]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [3, 9]) + assert client.get_item.call_count == 1 # only the report re-read - def test_add_merges_with_current_grants(self) -> None: + def test_clear_via_empty_replace(self) -> None: client = MagicMock() - client.get_item.return_value = _child_item( - "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": [1, 2]} + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects(client, "semantic-dataset", "d1", replace=[]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", []) + + @pytest.mark.parametrize( + ("add", "remove", "expected"), + [([3], [], [1, 2, 3]), ([], [1], [2]), ([3], [1], [2, 3])], + ) + def test_owner_merge_applies_the_delta(self, add, remove, expected) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=add, remove=remove, caller_project_id=5725 ) - scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", add=[3]) - client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 2, 3]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", expected) - def test_remove_merges_with_current_grants(self) -> None: + @pytest.mark.parametrize( + "meta", + [ + # org admin from another project: the server hides the grants from a non-owner + {"scope": "targeted", "projectId": 5725}, + {"scope": "targeted", "projectId": 5725, "targetProjectIds": [1]}, + ], + ) + def test_non_owner_merge_is_refused_instead_of_wiping_grants(self, meta) -> None: client = MagicMock() - client.get_item.return_value = _child_item( - "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": [1, 2, 3]} + client.get_item.return_value = _item(meta) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=999 + ) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert "scope set --target-project" in excinfo.value.message + client.put_target_projects.assert_not_called() + + +class TestElevation: + @pytest.mark.parametrize( + ("helper", "client_method"), + [ + (scope_helpers.request_elevation, "request_scope_elevation"), + (scope_helpers.withdraw_elevation, "withdraw_scope_elevation"), + (scope_helpers.elevate_to_organization, "elevate_to_organization"), + ], + ) + def test_reports_grants_from_a_reread_not_the_endpoint_response( + self, helper, client_method + ) -> None: + client = MagicMock() + getattr(client, client_method).return_value = {"meta": {"scope": "targeted"}} # no grants + client.get_item.return_value = _item(OWNED) + assert helper(client, "semantic-dataset", "d1")["target_project_ids"] == [1, 2] + + def test_dry_run_elevation_never_patches(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + result = scope_helpers.elevate_to_organization( + client, "semantic-dataset", "d1", dry_run=True ) - scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", remove=[2]) - client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1, 3]) + client.elevate_to_organization.assert_not_called() + assert result["dry_run"] is True + assert result["would_set_scope"] == "organization" - def test_clear_via_empty_replace(self) -> None: + @pytest.mark.parametrize(("returned", "has_more"), [(3, False), (4, True)]) + def test_request_list_pages_by_fetching_one_extra_row(self, returned, has_more) -> None: client = MagicMock() - client.get_item.return_value = _child_item("semantic-dataset", "d1", {"name": "x"}) - scope_helpers.grant_target_projects(client, "semantic-dataset", "d1", replace=[]) - client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", []) + client.list_organization_items.return_value = [_item({}) for _ in range(returned)] + page = scope_helpers.list_pending_elevations(client, "semantic-dataset", limit=3, offset=6) + client.list_organization_items.assert_called_once_with( + "semantic-dataset", pending_elevation_only=True, limit=4, offset=6 + ) + assert (page["limit"], page["offset"], page["has_more"]) == (3, 6, has_more) + assert len(page["items"]) == 3 # --------------------------------------------------------------------------- -# SemanticLayerService.scope_* -- orchestration +# SemanticLayerService.scope_* # --------------------------------------------------------------------------- class TestServiceScopeMethods: - def test_resolve_target_project_ids_public_wrapper(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, _mock = _make_service(store) - assert service.resolve_target_project_ids(["analytics"]) == [1234] - - def test_resolve_scope_type_rejects_unknown_kind(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, _mock = _make_service(store) + def test_unknown_type_is_invalid_argument(self, tmp_path: Path) -> None: + service, _ = _make_service(_make_store(tmp_path)) with pytest.raises(KeboolaApiError) as excinfo: - service.scope_status("prod", "bogus-kind", "id-1") - assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + service.scope_get("prod", "table", "x") + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - def test_scope_status_delegates_to_client_get_item(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _child_item( - "semantic-dataset", "d1", {"name": "x"}, meta={"scope": "organization"} - ) - result = service.scope_status("prod", "dataset", "d1") - mock.get_item.assert_called_once_with("semantic-dataset", "d1") - assert result["scope"] == "organization" - - def test_scope_request_elevation_delegates(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _child_item( - "semantic-metric", "m1", {"name": "x"}, meta={"scopeElevationRequestedAt": "t"} - ) - result = service.scope_request_elevation("prod", "metric", "m1") - mock.request_scope_elevation.assert_called_once_with("semantic-metric", "m1") - assert result["scope_elevation_requested_at"] == "t" - - def test_scope_withdraw_elevation_delegates(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _child_item("semantic-metric", "m1", {"name": "x"}) - service.scope_withdraw_elevation("prod", "metric", "m1") - mock.withdraw_scope_elevation.assert_called_once_with("semantic-metric", "m1") - - def test_scope_elevate_delegates(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _child_item( - "semantic-metric", "m1", {"name": "x"}, meta={"scope": "organization"} + def test_update_targets_resolves_targets_and_passes_the_callers_project( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + service.scope_update_targets("prod", "dataset", "d1", add=["analytics,5678"]) + mock.put_target_projects.assert_called_once_with( + "semantic-dataset", "d1", [1, 2, 1234, 5678] ) - result = service.scope_elevate("prod", "metric", "m1") - mock.elevate_to_organization.assert_called_once_with("semantic-metric", "m1") - assert result["scope"] == "organization" @pytest.mark.parametrize( - ("method", "client_call"), + "kwargs", [ - ("scope_request_elevation", "request_scope_elevation"), - ("scope_withdraw_elevation", "withdraw_scope_elevation"), - ("scope_elevate", "elevate_to_organization"), + {}, # nothing + {"scope": "organization", "target_projects": ["analytics"]}, + {"scope": "organization", "clear": True}, + {"target_projects": ["analytics"], "clear": True}, + {"scope": "project"}, # scope only ever moves to organization ], ) - def test_elevation_ops_report_grants_from_a_reread_not_the_endpoint_response( - self, tmp_path: Path, method: str, client_call: str - ) -> None: - """Grants must survive in the REPORTED status, not just on the server. - - Verified live against metastore.us-east4: the - ``scope-elevation-request`` / ``PATCH`` responses omit - ``meta.targetProjectIds`` even though the item keeps its grants, so - rendering that response printed ``target_project_ids: null`` right - after a `scope status` had shown ``[5024]`` -- i.e. it looked like - requesting elevation had silently wiped every grant. - """ - store = _make_store(tmp_path) - service, mock = _make_service(store) - # What the mutating endpoint returns: grants missing from meta. - getattr(mock, client_call).return_value = _child_item( - "semantic-metric", "m1", {"name": "x"}, meta={"scope": "targeted"} - ) - # What the item actually looks like when re-read. - mock.get_item.return_value = _child_item( - "semantic-metric", - "m1", - {"name": "x"}, - meta={"scope": "targeted", "targetProjectIds": [5024]}, - ) - result = getattr(service, method)("prod", "metric", "m1") - assert result["target_project_ids"] == [5024] - mock.get_item.assert_called_with("semantic-metric", "m1") - - def test_scope_pending_delegates_with_pagination(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.list_organization_items.return_value = [ - _child_item("semantic-dataset", "d1", {"name": "x"}, meta={}) - ] - result = service.scope_pending("prod", "dataset", limit=5, offset=1) - mock.list_organization_items.assert_called_once_with( - "semantic-dataset", pending_elevation_only=True, limit=5, offset=1 + def test_set_requires_exactly_one_mode(self, tmp_path: Path, kwargs) -> None: + service, mock = _make_service(_make_store(tmp_path)) + with pytest.raises(KeboolaApiError) as excinfo: + service.scope_set("prod", "dataset", "d1", **kwargs) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.put_target_projects.assert_not_called() + mock.elevate_to_organization.assert_not_called() + + def test_set_modes_reach_the_right_metastore_call(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + service.scope_set("prod", "dataset", "d1", target_projects=["analytics"]) + mock.put_target_projects.assert_called_with("semantic-dataset", "d1", [1234]) + service.scope_set("prod", "dataset", "d1", clear=True) + mock.put_target_projects.assert_called_with("semantic-dataset", "d1", []) + service.scope_set("prod", "dataset", "d1", scope="organization") + mock.elevate_to_organization.assert_called_once_with("semantic-dataset", "d1") + + def test_set_dry_run_writes_nothing(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + result = service.scope_set( + "prod", "dataset", "d1", target_projects=["analytics"], dry_run=True ) - assert len(result) == 1 + assert result["would_set_target_project_ids"] == [1234] + mock.put_target_projects.assert_not_called() - def test_scope_grant_resolves_aliases_before_calling_client(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _child_item( - "semantic-dataset", "d1", {"name": "x"}, meta={"targetProjectIds": []} - ) - service.scope_grant("prod", "dataset", "d1", add=["analytics"]) - mock.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [1234]) + def test_request_list_defaults(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.list_organization_items.return_value = [] + page = service.scope_request_list("prod", "dataset") + assert page == {"items": [], "limit": 50, "offset": 0, "has_more": False} # --------------------------------------------------------------------------- -# Regression: edit (DELETE+POST) must preserve the original item's scope +# Creating with a scope / inheriting the model's scope # --------------------------------------------------------------------------- -class TestEditPreservesScope: - def test_edit_metric_preserves_targeted_scope_and_grants(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - original = _child_item( - "semantic-metric", - "m1", - {"name": "rev", "sql": "1"}, - meta={"scope": "targeted", "targetProjectIds": [1234]}, - ) +def _service_with_model(tmp_path: Path, model_meta: dict[str, Any] | None): + service, mock = _make_service(_make_store(tmp_path)) + model = _model_item("U", "m") + if model_meta is not None: + model["meta"] = model_meta + mock.list_items.return_value = [model] + mock.get_item.return_value = model + mock.post_item.return_value = {"id": "new", "attributes": {}} + return service, mock - def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: - if item_type == "semantic-model": - return [_model_item()] - if item_type == "semantic-metric": - return [original] - return [] - mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} +class TestChildScopeInheritance: + @pytest.mark.parametrize( + ("model_meta", "scope", "targets", "expected_scope", "expected_ids"), + [ + (None, None, None, "project", None), + ({"scope": "organization"}, None, None, "organization", None), + ({"scope": "targeted", "targetProjectIds": [5, 6]}, None, None, "targeted", [5, 6]), + ({"scope": "organization"}, "project", None, "project", None), # explicit wins + (None, "targeted", ["analytics,5678"], "targeted", [1234, 5678]), + (None, "organization", None, "organization", None), + ], + ) + def test_add_glossary_scope( + self, tmp_path: Path, model_meta, scope, targets, expected_scope, expected_ids + ) -> None: + service, mock = _service_with_model(tmp_path, model_meta) + service.add_glossary( + "prod", None, term="t", definition="d", scope=scope, target_projects=targets + ) + kwargs = mock.post_item.call_args.kwargs + assert kwargs["scope"] == expected_scope + assert kwargs["target_project_ids"] == expected_ids + + def test_model_create_defaults_to_project_and_resolves_targets(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, None) + service.create_model("prod", "m") + assert mock.post_item.call_args.kwargs["scope"] == "project" + service.create_model("prod", "m2", scope="targeted", target_projects=["analytics"]) + assert mock.post_item.call_args.kwargs["target_project_ids"] == [1234] + + def test_unknown_target_alias_fails_before_any_write(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, None) + with pytest.raises(KeboolaApiError) as excinfo: + service.create_model("prod", "m", scope="targeted", target_projects=["ghost"]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.post_item.assert_not_called() + - service.edit_metric("prod", None, current_name="rev", new_description="updated") +# --------------------------------------------------------------------------- +# Overwrite paths keep an item's scope (PUT in place, never DELETE+POST) +# --------------------------------------------------------------------------- - mock.post_item.assert_called_once() - _args, kwargs = mock.post_item.call_args - assert kwargs["scope"] == "targeted" - assert kwargs["target_project_ids"] == [1234] - def test_edit_dataset_preserves_organization_scope(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - original = _child_item( +class TestEditAndOverwriteKeepScope: + def test_edit_dataset_puts_in_place_with_the_items_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + dataset = _child_item( "semantic-dataset", "d1", - {"name": "fact_pnl", "tableId": "out.c-fin.fact_pnl"}, - meta={"scope": "organization"}, + {"name": "x", "tableId": "a.b.c"}, + {"scope": "targeted", "targetProjectIds": [1, 2]}, ) - - def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: - if item_type == "semantic-model": - return [_model_item()] - if item_type == "semantic-dataset": - return [original] - return [] - - mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "d_new", "attributes": {"name": "fact_pnl"}} - - service.edit_dataset("prod", None, current_name="fact_pnl", new_grain="daily") - - _args, kwargs = mock.post_item.call_args - assert kwargs["scope"] == "organization" - assert kwargs["target_project_ids"] is None - - def test_edit_metric_defaults_to_project_scope_when_no_meta(self, tmp_path: Path) -> None: - """Pre-PSGO-140 items (no meta.scope) edit as plain project scope, unchanged.""" + mock.list_items.side_effect = lambda t, m=None: ( + [_model_item("U", "m")] if t == "semantic-model" else [dataset] + ) + mock.put_item.return_value = {"id": "d1", "attributes": {"name": "x"}} + service.edit_dataset("prod", None, current_name="x", new_description="d") + args, kwargs = mock.put_item.call_args + assert args[:2] == ("semantic-dataset", "d1") + assert kwargs == {"scope": "targeted"} + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() + + def test_promote_overwrite_keeps_the_target_items_scope(self, tmp_path: Path) -> None: store = _make_store(tmp_path) - service, mock = _make_service(store) - original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) - - def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: - if item_type == "semantic-model": - return [_model_item()] - if item_type == "semantic-metric": - return [original] - return [] - - mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} - - service.edit_metric("prod", None, current_name="rev", new_sql="2") - - _args, kwargs = mock.post_item.call_args - assert kwargs["scope"] == "project" - assert kwargs["target_project_ids"] is None + src, tgt = MagicMock(), MagicMock() + for m in (src, tgt): + m.__enter__ = MagicMock(return_value=m) + m.__exit__ = MagicMock(return_value=False) + clients = iter([src, tgt]) + service = SemanticLayerService( + config_store=store, metastore_client_factory=lambda url, token: next(clients) + ) + src.list_items.side_effect = lambda t, m=None: ( + [_model_item("U_S", "src")] + if t == "semantic-model" + else [_child_item("semantic-metric", "m1", {"name": "b", "sql": "NEW"})] + if t == "semantic-metric" + else [] + ) + existing = _child_item( + "semantic-metric", "tm1", {"name": "b", "sql": "OLD"}, {"scope": "organization"} + ) + tgt.list_items.side_effect = lambda t, m=None: ( + [_model_item("U_T", "tgt")] + if t == "semantic-model" + else [existing] + if t == "semantic-metric" + else [] + ) + service.promote_model(from_project="prod", to_project="analytics") + args, kwargs = tgt.put_item.call_args + assert args[:3] == ("semantic-metric", "tm1", "b") + assert kwargs == {"scope": "organization"} + tgt.delete_item.assert_not_called() + tgt.post_item.assert_not_called() diff --git a/tests/test_semantic_layer_scope_cli.py b/tests/test_semantic_layer_scope_cli.py index 8e584e978..ad17a182d 100644 --- a/tests/test_semantic_layer_scope_cli.py +++ b/tests/test_semantic_layer_scope_cli.py @@ -19,6 +19,7 @@ from keboola_agent_cli.cli import app from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.constants import EXIT_PERMISSION_DENIED +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError from keboola_agent_cli.models import ProjectConfig from keboola_agent_cli.services.config_service import ConfigService from keboola_agent_cli.services.job_service import JobService @@ -78,454 +79,333 @@ def store(cfg_dir: Path) -> ConfigStore: ) +def _sl(*args: str) -> list[str]: + return ["--json", "semantic-layer", *args] + + +ITEM = ["--project", "prod", "--type", "dataset", "--context-id", "d1"] + + # --------------------------------------------------------------------------- # --scope / --target-project on model create / add # --------------------------------------------------------------------------- -class TestModelCreateScope: - def test_default_scope_is_project_and_unrequested(self, store: ConfigStore) -> None: - """No --scope passed: create_model still gets scope='project', no picker touched.""" +class TestCreateScopeFlags: + def test_omitted_scope_is_passed_as_none_so_the_service_inherits( + self, store: ConfigStore + ) -> None: mock = MagicMock() - mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + mock.add_metric.return_value = {"id": "m", "attributes": {"name": "n"}} result = _invoke( - ["--json", "semantic-layer", "model", "create", "--project", "prod", "--name", "n"], + _sl( + *("add", "metric", "--project", "prod", "--name", "n", "--sql", "1"), + *("--dataset", "t", "--yes"), + ), store=store, sl_mock=mock, ) assert result.exit_code == 0, result.output - _args, kwargs = mock.create_model.call_args - assert kwargs["scope"] == "project" - assert kwargs["target_project_ids"] is None + kwargs = mock.add_metric.call_args.kwargs + assert kwargs["scope"] is None + assert kwargs["target_projects"] is None - def test_targeted_scope_with_explicit_target_project(self, store: ConfigStore) -> None: + def test_model_create_with_targeted_scope_passes_aliases_through( + self, store: ConfigStore + ) -> None: mock = MagicMock() mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} - mock.resolve_target_project_ids.return_value = [1234] result = _invoke( [ - "--json", - "semantic-layer", - "model", - "create", - "--project", - "prod", - "--name", - "n", - "--scope", - "targeted", + *_sl( + *("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted") + ), "--target-project", - "analytics", + "analytics,5678", ], store=store, sl_mock=mock, ) assert result.exit_code == 0, result.output - mock.resolve_target_project_ids.assert_called_once_with(["analytics"]) - _args, kwargs = mock.create_model.call_args + kwargs = mock.create_model.call_args.kwargs assert kwargs["scope"] == "targeted" - assert kwargs["target_project_ids"] == [1234] + assert kwargs["target_projects"] == ["analytics,5678"] - def test_targeted_scope_without_target_project_fails_fast_non_tty( - self, store: ConfigStore - ) -> None: - """No --target-project + CliRunner's non-TTY stdin/stdout -> hard fail, never a hang.""" + def test_targeted_scope_without_target_fails_fast_non_tty(self, store: ConfigStore) -> None: mock = MagicMock() result = _invoke( - [ - "--json", - "semantic-layer", - "model", - "create", - "--project", - "prod", - "--name", - "n", - "--scope", - "targeted", - ], + _sl(*("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted")), store=store, sl_mock=mock, ) assert result.exit_code == 2, result.output mock.create_model.assert_not_called() - body = json.loads(result.output) - assert body["error"]["code"] == "INVALID_ARGUMENT" + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" - def test_organization_scope_ignores_absent_target_project(self, store: ConfigStore) -> None: - """--scope organization never needs --target-project; no picker, no failure.""" + @pytest.mark.parametrize( + "scope_args", [[], ["--scope", "project"], ["--scope", "organization"]] + ) + def test_target_project_without_targeted_scope_is_a_usage_error( + self, store: ConfigStore, scope_args: list[str] + ) -> None: mock = MagicMock() - mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} result = _invoke( [ - "--json", - "semantic-layer", - "model", - "create", - "--project", - "prod", - "--name", - "n", - "--scope", - "organization", + *_sl("model", "create", "--project", "prod", "--name", "n", *scope_args), + "--target-project", + "analytics", ], store=store, sl_mock=mock, ) - assert result.exit_code == 0, result.output - _args, kwargs = mock.create_model.call_args - assert kwargs["scope"] == "organization" - assert kwargs["target_project_ids"] is None - mock.resolve_target_project_ids.assert_not_called() - + assert result.exit_code == 2, result.output + mock.create_model.assert_not_called() + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" -class TestAddMetricScope: - def test_add_metric_targeted_scope(self, store: ConfigStore) -> None: + def test_unknown_target_alias_is_exit_2_not_a_traceback(self, store: ConfigStore) -> None: mock = MagicMock() - mock.add_metric.return_value = {"id": "m1", "attributes": {"name": "rev"}} - mock.resolve_target_project_ids.return_value = [1234] + mock.create_model.side_effect = KeboolaApiError( + message="--target-project 'ghost' is neither a registered project alias nor an ID", + error_code=ErrorCode.INVALID_ARGUMENT, + ) result = _invoke( [ - "--json", - "semantic-layer", - "add", - "metric", - "--project", - "prod", - "--name", - "rev", - "--sql", - "SUM(x)", - "--dataset", - "out.c-foo.bar", - "--scope", - "targeted", + *_sl( + *("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted") + ), "--target-project", - "analytics", + "ghost", ], store=store, sl_mock=mock, ) - assert result.exit_code == 0, result.output - _args, kwargs = mock.add_metric.call_args - assert kwargs["scope"] == "targeted" - assert kwargs["target_project_ids"] == [1234] - - -# --------------------------------------------------------------------------- -# semantic-layer scope -# --------------------------------------------------------------------------- - + assert result.exit_code == 2, result.output + assert "--target-project" in json.loads(result.output)["error"]["message"] -class TestScopeStatus: - def test_status_success(self, store: ConfigStore) -> None: - mock = MagicMock() - mock.scope_status.return_value = { - "id": "d1", - "scope": "targeted", - "target_project_ids": [1234], - "scope_elevation_requested_at": None, - } + def test_bad_scope_value_is_a_usage_error(self, store: ConfigStore) -> None: result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "status", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - ], + _sl("model", "create", "--project", "prod", "--name", "n", "--scope", "everyone"), store=store, - sl_mock=mock, + sl_mock=MagicMock(), ) - assert result.exit_code == 0, result.output - mock.scope_status.assert_called_once_with(alias="prod", kind="dataset", item_id="d1") - assert json.loads(result.output)["data"]["scope"] == "targeted" - - -class TestScopeGrant: - def test_grant_add_default_merge(self, store: ConfigStore) -> None: + assert result.exit_code == 2 + + @pytest.mark.parametrize( + ("flags", "blocked"), + [ + (["--deny-destructive"], True), + (["--deny-writes"], True), # any create is a write + ], + ) + def test_organization_scope_create_is_gated_per_command( + self, store: ConfigStore, flags: list[str], blocked: bool + ) -> None: + for command in ( + ["model", "create", "--name", "n"], + ["add", "glossary", "--term", "t"], + ["add", "dataset", "--name", "n", "--table-id", "a.b.c"], + ): + mock = MagicMock() + result = _invoke( + [*flags, *_sl(*command, "--project", "prod", "--scope", "organization")], + store=store, + sl_mock=mock, + ) + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + + def test_deny_destructive_still_allows_project_scope_create(self, store: ConfigStore) -> None: mock = MagicMock() - mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": [1234]} + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "grant", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - "--target-project", - "analytics", - ], + ["--deny-destructive", *_sl("model", "create", "--project", "prod", "--name", "n")], store=store, sl_mock=mock, ) assert result.exit_code == 0, result.output - mock.scope_grant.assert_called_once_with( - alias="prod", - kind="dataset", - item_id="d1", - add=["analytics"], - remove=None, - replace=None, - ) - def test_grant_replace(self, store: ConfigStore) -> None: + +# --------------------------------------------------------------------------- +# semantic-layer scope +# --------------------------------------------------------------------------- + + +class TestScopeVerbs: + @pytest.mark.parametrize( + ("argv", "method", "expected"), + [ + (["get"], "scope_get", {}), + ( + ["add", "--target-project", "analytics", "--target-project", "5678"], + "scope_update_targets", + {"add": ["analytics", "5678"]}, + ), + ( + ["remove", "--target-project", "analytics,5678"], + "scope_update_targets", + {"remove": ["analytics,5678"]}, + ), + (["request-create"], "scope_request_create", {}), + (["request-delete"], "scope_request_delete", {}), + ], + ) + def test_verb_calls_the_service( + self, store: ConfigStore, argv: list[str], method: str, expected: dict + ) -> None: mock = MagicMock() - mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": [1234]} - result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "grant", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - "--target-project", - "analytics", - "--replace", - ], - store=store, - sl_mock=mock, - ) + getattr(mock, method).return_value = {"scope": "targeted", "target_project_ids": []} + result = _invoke(_sl("scope", *argv[:1], *ITEM, *argv[1:]), store=store, sl_mock=mock) assert result.exit_code == 0, result.output - _args, kwargs = mock.scope_grant.call_args - assert kwargs["replace"] == ["analytics"] + kwargs = getattr(mock, method).call_args.kwargs + assert kwargs["alias"] == "prod" + assert kwargs["kind"] == "dataset" + assert kwargs["context_id"] == "d1" + assert {k: kwargs[k] for k in expected} == expected + + @pytest.mark.parametrize("verb", ["add", "remove"]) + def test_add_and_remove_require_a_target(self, store: ConfigStore, verb: str) -> None: + mock = MagicMock() + result = _invoke(_sl("scope", verb, *ITEM), store=store, sl_mock=mock) + assert result.exit_code == 2, result.output + mock.scope_update_targets.assert_not_called() - def test_grant_clear(self, store: ConfigStore) -> None: + def test_empty_target_list_is_printed_not_hidden(self, store: ConfigStore) -> None: + """A targeted item with no grants ([] = owner only) must still show the field.""" mock = MagicMock() - mock.scope_grant.return_value = {"scope": "targeted", "target_project_ids": []} + mock.scope_get.return_value = {"scope": "targeted", "target_project_ids": []} + with patch("keboola_agent_cli.commands._semantic_layer_scope.get_formatter") as gf: + fmt = MagicMock(json_mode=False) + gf.return_value = fmt + _invoke(["semantic-layer", "scope", "get", *ITEM], store=store, sl_mock=mock) + human = fmt.output.call_args.args[1] + console = MagicMock() + human(console, {"scope": "targeted", "target_project_ids": []}) + assert any("target_project_ids" in str(c) for c in console.print.call_args_list) + + def test_bad_type_is_a_usage_error(self, store: ConfigStore) -> None: result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "grant", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - "--clear", - ], + _sl("scope", "get", "--project", "prod", "--type", "table", "--context-id", "x"), store=store, - sl_mock=mock, + sl_mock=MagicMock(), ) - assert result.exit_code == 0, result.output - _args, kwargs = mock.scope_grant.call_args - assert kwargs["replace"] == [] + assert result.exit_code == 2 - def test_clear_combined_with_target_project_is_usage_error(self, store: ConfigStore) -> None: + def test_service_usage_error_exits_2(self, store: ConfigStore) -> None: mock = MagicMock() + mock.scope_set.side_effect = KeboolaApiError( + message="scope set takes exactly one of ...", error_code=ErrorCode.INVALID_ARGUMENT + ) result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "grant", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - "--clear", - "--target-project", - "analytics", - ], + _sl("scope", "set", *ITEM, "--clear", "--target-project", "analytics"), store=store, sl_mock=mock, ) assert result.exit_code == 2, result.output - mock.scope_grant.assert_not_called() -class TestScopeElevation: - def test_request_elevation(self, store: ConfigStore) -> None: +class TestScopeSet: + @pytest.mark.parametrize( + ("extra", "expected"), + [ + (["--target-project", "analytics,5678"], {"target_projects": ["analytics,5678"]}), + (["--clear"], {"clear": True}), + (["--scope", "organization", "--yes"], {"scope": "organization"}), + (["--scope", "organization", "--dry-run"], {"scope": "organization", "dry_run": True}), + ], + ) + def test_modes(self, store: ConfigStore, extra: list[str], expected: dict) -> None: mock = MagicMock() - mock.scope_request_elevation.return_value = {"scope_elevation_requested_at": "t"} - result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "request-elevation", - "--project", - "prod", - "--type", - "metric", - "--id", - "m1", - ], - store=store, - sl_mock=mock, - ) + mock.scope_set.return_value = {"scope": "organization"} + result = _invoke(_sl("scope", "set", *ITEM, *extra), store=store, sl_mock=mock) assert result.exit_code == 0, result.output - mock.scope_request_elevation.assert_called_once_with( - alias="prod", kind="metric", item_id="m1" - ) + kwargs = mock.scope_set.call_args.kwargs + assert {k: kwargs[k] for k in expected} == expected - def test_withdraw_elevation(self, store: ConfigStore) -> None: - mock = MagicMock() - mock.scope_withdraw_elevation.return_value = {"scope_elevation_requested_at": None} + def test_bad_scope_value_is_a_usage_error(self, store: ConfigStore) -> None: result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "withdraw-elevation", - "--project", - "prod", - "--type", - "metric", - "--id", - "m1", - ], - store=store, - sl_mock=mock, - ) - assert result.exit_code == 0, result.output - mock.scope_withdraw_elevation.assert_called_once_with( - alias="prod", kind="metric", item_id="m1" + _sl("scope", "set", *ITEM, "--scope", "project"), store=store, sl_mock=MagicMock() ) + assert result.exit_code == 2 - def test_elevate_with_yes_skips_prompt(self, store: ConfigStore) -> None: + def test_human_mode_declining_the_prompt_aborts_without_calling( + self, store: ConfigStore + ) -> None: mock = MagicMock() - mock.scope_elevate.return_value = {"scope": "organization"} - result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "elevate", - "--project", - "prod", - "--type", - "metric", - "--id", - "m1", - "--yes", - ], - store=store, - sl_mock=mock, - ) - assert result.exit_code == 0, result.output - mock.scope_elevate.assert_called_once_with(alias="prod", kind="metric", item_id="m1") - - def test_pending(self, store: ConfigStore) -> None: + with patch("typer.confirm", return_value=False): + result = _invoke( + ["semantic-layer", "scope", "set", *ITEM, "--scope", "organization"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + mock.scope_set.assert_not_called() + + +class TestScopeRequestList: + def test_defaults_and_pagination_envelope(self, store: ConfigStore) -> None: mock = MagicMock() - mock.scope_pending.return_value = [ - {"id": "d1", "name": "x", "scope_elevation_requested_at": "t"} - ] + mock.scope_request_list.return_value = { + "items": [{"id": "d1", "name": "x"}], + "limit": 50, + "offset": 0, + "has_more": False, + } result = _invoke( - [ - "--json", - "semantic-layer", - "scope", - "pending", - "--project", - "prod", - "--type", - "dataset", - "--limit", - "5", - ], + _sl("scope", "request-list", "--project", "prod", "--type", "dataset"), store=store, sl_mock=mock, ) assert result.exit_code == 0, result.output - mock.scope_pending.assert_called_once_with( - alias="prod", kind="dataset", limit=5, offset=None - ) + kwargs = mock.scope_request_list.call_args.kwargs + assert (kwargs["limit"], kwargs["offset"]) == (50, 0) + data = json.loads(result.output)["data"] + assert {"items", "limit", "offset", "has_more"} <= set(data) # --------------------------------------------------------------------------- # Permission gating # --------------------------------------------------------------------------- +WRITE_VERBS = [ + ["add", "--target-project", "analytics"], + ["remove", "--target-project", "analytics"], + ["set", "--clear"], + ["request-create"], + ["request-delete"], +] +READ_VERBS = [["get", *ITEM], ["request-list", "--project", "prod", "--type", "dataset"]] + class TestScopePermissions: - def test_deny_writes_blocks_grant(self, store: ConfigStore) -> None: + @pytest.mark.parametrize("verb", WRITE_VERBS, ids=lambda v: v[0]) + def test_deny_writes_blocks_write_verbs(self, store: ConfigStore, verb: list[str]) -> None: mock = MagicMock() result = _invoke( - [ - "--deny-writes", - "--json", - "semantic-layer", - "scope", - "grant", - "--project", - "prod", - "--type", - "dataset", - "--id", - "d1", - "--target-project", - "analytics", - ], - store=store, - sl_mock=mock, + ["--deny-writes", *_sl("scope", verb[0], *ITEM, *verb[1:])], store=store, sl_mock=mock ) - assert result.exit_code == EXIT_PERMISSION_DENIED - mock.scope_grant.assert_not_called() + assert result.exit_code == EXIT_PERMISSION_DENIED, result.output - def test_deny_destructive_blocks_elevate(self, store: ConfigStore) -> None: + @pytest.mark.parametrize("verb", READ_VERBS, ids=lambda v: v[0]) + def test_deny_writes_allows_read_verbs(self, store: ConfigStore, verb: list[str]) -> None: mock = MagicMock() - result = _invoke( - [ - "--deny-destructive", - "--json", - "semantic-layer", - "scope", - "elevate", - "--project", - "prod", - "--type", - "metric", - "--id", - "m1", - "--yes", - ], - store=store, - sl_mock=mock, - ) - assert result.exit_code == EXIT_PERMISSION_DENIED - mock.scope_elevate.assert_not_called() + result = _invoke(["--deny-writes", *_sl("scope", *verb)], store=store, sl_mock=mock) + assert result.exit_code == 0, result.output - def test_deny_writes_does_not_block_status(self, store: ConfigStore) -> None: - """--deny-writes must not block the read-only `scope status` leaf.""" + @pytest.mark.parametrize( + ("extra", "blocked"), + [ + (["--scope", "organization", "--yes"], True), + (["--scope", "organization", "--dry-run"], False), # preview changes nothing + (["--clear"], False), + (["--target-project", "analytics"], False), + ], + ) + def test_deny_destructive_blocks_only_the_elevation( + self, store: ConfigStore, extra: list[str], blocked: bool + ) -> None: mock = MagicMock() - mock.scope_status.return_value = {"scope": "project"} + mock.scope_set.return_value = {"scope": "organization"} result = _invoke( - [ - "--deny-writes", - "--json", - "semantic-layer", - "scope", - "status", - "--project", - "prod", - "--type", - "metric", - "--id", - "m1", - ], - store=store, - sl_mock=mock, + ["--deny-destructive", *_sl("scope", "set", *ITEM, *extra)], store=store, sl_mock=mock ) - assert result.exit_code == 0, result.output - mock.scope_status.assert_called_once() + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + assert mock.scope_set.called is (not blocked) diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index 2e0c917d3..6957b8b71 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -1499,7 +1499,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.side_effect = [ + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c_new", "attributes": dict(c_attrs, name="rev_warning")}, ] @@ -1512,8 +1512,9 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] assume_yes=True, ) assert result["updated"]["id"] == "m_new" - # delete called twice: once for the old metric, once for the cascade - assert mock.delete_item.call_count == 2 + # in-place PUTs: the metric and its cascaded constraint, nothing deleted + assert mock.put_item.call_count == 2 + mock.delete_item.assert_not_called() # cascade list populated assert result["cascaded_constraints"][0]["status"] == "updated" @@ -1530,7 +1531,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} + mock.put_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} result = service.edit_metric( "prod", @@ -1539,8 +1540,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] new_description="updated desc", ) assert result["updated"]["id"] == "m_new" - # Only one delete: the metric itself. - assert mock.delete_item.call_count == 1 + mock.delete_item.assert_not_called() assert result["cascaded_constraints"] == [] def test_metric_not_found(self, tmp_path: Path) -> None: @@ -1557,8 +1557,8 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] service.edit_metric("prod", None, current_name="ghost", new_sql="1") assert excinfo.value.error_code == ErrorCode.NOT_FOUND - def test_rollback_on_post_failure(self, tmp_path: Path) -> None: - """When POST after DELETE fails, the original item is re-POSTed.""" + def test_failed_put_leaves_original_untouched(self, tmp_path: Path) -> None: + """Edit is an in-place PUT: a failure never deletes anything, so nothing needs rolling back.""" store = _make_store(tmp_path) service, mock = _make_service(store) original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) @@ -1571,17 +1571,34 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - # First POST (the actual edit) fails; second POST (rollback) succeeds. - mock.post_item.side_effect = [ - KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR), - {"id": "restored", "attributes": {"name": "rev"}}, - ] + mock.put_item.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) - with pytest.raises(KeboolaApiError) as excinfo: + with pytest.raises(KeboolaApiError): service.edit_metric("prod", None, current_name="rev", new_sql="2") - details = excinfo.value.details or {} - rollback = details.get("rollback") or {} - assert rollback.get("status") == "succeeded" + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() + + def test_edit_keeps_scope_and_item_id(self, tmp_path: Path) -> None: + """PUT in place: the item keeps its id; the schemaVersion follows the item's scope.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + original["meta"] = {"scope": "organization"} + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [original] if item_type == "semantic-metric" else [] + + mock.list_items.side_effect = _list + mock.put_item.return_value = {"id": "m1", "attributes": {"name": "rev"}} + service.edit_metric("prod", None, current_name="rev", new_sql="2") + args, kwargs = mock.put_item.call_args + assert args[:3] == ("semantic-metric", "m1", "rev") + assert kwargs == {"scope": "organization"} + mock.post_item.assert_not_called() def test_partial_state_false_when_cascade_succeeds(self, tmp_path: Path) -> None: """Envelope carries partial_state=False + recovery_hint=None on full success (issue #294).""" @@ -1601,7 +1618,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.side_effect = [ + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c_new", "attributes": dict(c_attrs, name="rev_warning")}, ] @@ -1630,16 +1647,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - # POST sequence: - # 1. metric rename POST -> succeeds - # 2. c1 cascade POST -> succeeds - # 3. c2 cascade POST -> fails - # 4. c2 rollback POST -> succeeds (per-item rollback) - mock.post_item.side_effect = [ + # PUT sequence: metric rename ok, c1 cascade ok, c2 cascade fails. + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c1_new", "attributes": {"name": "rev_ok", "metrics": ["revenue"]}}, KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR), - {"id": "c2_restored", "attributes": {"name": "rev_fail", "metrics": ["rev"]}}, ] result = service.edit_metric( "prod", None, current_name="rev", new_name="revenue", assume_yes=True @@ -1720,7 +1732,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = { + mock.put_item.return_value = { "id": "r2", "attributes": {"name": "fact_to_dim", "from": "out.c.fact_v2"}, } @@ -1730,11 +1742,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] current_name="fact_to_dim", new_from="out.c.fact_v2", ) - mock.delete_item.assert_called_once_with("semantic-relationship", "r1") + mock.delete_item.assert_not_called() # The POST payload retains the unchanged endpoints but rewrites `from`. - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["from"] == "out.c.fact_v2" - assert post_kwargs["data"]["to"] == "out.c.dim" + put_args = mock.put_item.call_args.args + assert put_args[3]["from"] == "out.c.fact_v2" + assert put_args[3]["to"] == "out.c.dim" assert result["rollback"] is None assert result["cascaded_constraints"] == [] @@ -1785,7 +1797,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = { + mock.put_item.return_value = { "id": "g2", "attributes": {"term": "MRR", "definition": "Updated def"}, } @@ -1795,10 +1807,10 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] current_term="MRR", new_definition="Updated def", ) - mock.delete_item.assert_called_once_with("semantic-glossary", "g1") - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["term"] == "MRR" - assert post_kwargs["data"]["definition"] == "Updated def" + mock.delete_item.assert_not_called() + put_args = mock.put_item.call_args.args + assert put_args[3]["term"] == "MRR" + assert put_args[3]["definition"] == "Updated def" assert result["rollback"] is None def test_rename_term(self, tmp_path: Path) -> None: @@ -1814,16 +1826,16 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}} + mock.put_item.return_value = {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}} service.edit_glossary( "prod", None, current_term="MRR", new_term="RECURRING_REVENUE", ) - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["term"] == "RECURRING_REVENUE" - assert post_kwargs["name"] == "RECURRING_REVENUE" + put_args = mock.put_item.call_args.args + assert put_args[3]["term"] == "RECURRING_REVENUE" + assert put_args[2] == "RECURRING_REVENUE" def test_not_found(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -2002,7 +2014,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] assert result["imported"]["datasets"]["created"] == 0 mock.post_item.assert_not_called() - def test_overwrite_deletes_then_posts(self, tmp_path: Path) -> None: + def test_overwrite_puts_in_place_keeping_scope(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) @@ -2010,11 +2022,13 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] if item_type == "semantic-model": return [_model_item("U", "target")] if item_type == "semantic-dataset": - return [_child_item("semantic-dataset", "d1", {"name": "fact_x"})] + existing = _child_item("semantic-dataset", "d1", {"name": "fact_x"}) + existing["meta"] = {"scope": "organization"} + return [existing] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "new"} + mock.put_item.return_value = {"id": "d1"} snap = tmp_path / "s.json" _write_snapshot( snap, @@ -2024,8 +2038,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] ) result = service.import_snapshot("prod", snap, overwrite=True) assert result["imported"]["datasets"]["overwritten"] == 1 - mock.delete_item.assert_called_with("semantic-dataset", "d1") - mock.post_item.assert_called() + args, kwargs = mock.put_item.call_args + assert args[:3] == ("semantic-dataset", "d1", "fact_x") + assert kwargs == {"scope": "organization"} + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() def test_dry_run_no_writes(self, tmp_path: Path) -> None: store = _make_store(tmp_path) diff --git a/tests/test_server_router_calls.py b/tests/test_server_router_calls.py index 9ad34bc90..fb6844ad3 100644 --- a/tests/test_server_router_calls.py +++ b/tests/test_server_router_calls.py @@ -833,6 +833,152 @@ def test_reference_data_delete_route(tmp_path: Path) -> None: sl.delete_reference_data.assert_called_once_with(alias=PROJECT, record_id="r1") +# --------------------------------------------------------------------------- +# semantic_layer.py scope routes + scope fields on create -> SemanticLayerService +# --------------------------------------------------------------------------- + +SCOPE_ITEM = {"project": PROJECT, "type": "dataset"} + + +@pytest.mark.parametrize( + ("method", "path", "kwargs", "service_call", "expected"), + [ + ( + "get", + "/semantic-layer/scope/d1", + {"params": SCOPE_ITEM}, + "scope_get", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "post", + "/semantic-layer/scope/d1/target-projects", + {"json": {**SCOPE_ITEM, "target_projects": ["a", "5"]}}, + "scope_update_targets", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1", "add": ["a", "5"]}, + ), + ( + "delete", + "/semantic-layer/scope/d1/target-projects", + {"params": {**SCOPE_ITEM, "target_project": ["a", "5"]}}, + "scope_update_targets", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1", "remove": ["a", "5"]}, + ), + ( + "put", + "/semantic-layer/scope/d1", + {"json": {**SCOPE_ITEM, "scope": "organization"}}, + "scope_set", + { + "alias": PROJECT, + "kind": "dataset", + "context_id": "d1", + "scope": "organization", + "target_projects": None, + "clear": False, + "dry_run": False, + }, + ), + ( + "put", + "/semantic-layer/scope/d1", + {"json": {**SCOPE_ITEM, "clear": True}}, + "scope_set", + { + "alias": PROJECT, + "kind": "dataset", + "context_id": "d1", + "scope": None, + "target_projects": None, + "clear": True, + "dry_run": False, + }, + ), + ( + "put", + "/semantic-layer/scope/d1/elevation-request", + {"params": SCOPE_ITEM}, + "scope_request_create", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "delete", + "/semantic-layer/scope/d1/elevation-request", + {"params": SCOPE_ITEM}, + "scope_request_delete", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "get", + "/semantic-layer/scope/elevation-requests", + {"params": {**SCOPE_ITEM, "limit": 5, "offset": 10}}, + "scope_request_list", + {"alias": PROJECT, "kind": "dataset", "limit": 5, "offset": 10}, + ), + ], + ids=lambda v: v if isinstance(v, str) and v.startswith("scope_") else None, +) +def test_scope_routes(tmp_path: Path, method, path, kwargs, service_call, expected) -> None: + sl = MagicMock() + getattr(sl, service_call).return_value = {"scope": "project"} + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + resp = getattr(TestClient(app), method)(path, headers=AUTH, **kwargs) + assert resp.status_code == 200, resp.text + getattr(sl, service_call).assert_called_once_with(**expected) + + +def test_scope_set_body_rejects_an_unsupported_scope(tmp_path: Path) -> None: + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=MagicMock())) + resp = TestClient(app).put( + "/semantic-layer/scope/d1", json={**SCOPE_ITEM, "scope": "project"}, headers=AUTH + ) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "fields", + [ + {"target_projects": ["a"]}, + {"scope": "project", "target_projects": ["a"]}, + {"scope": "targeted"}, + ], +) +def test_create_model_scope_fields_are_validated(tmp_path: Path, fields) -> None: + """target_projects needs scope=targeted and vice versa -- same rule as the CLI.""" + sl = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + resp = TestClient(app).post( + "/semantic-layer/models", json={"project": PROJECT, "name": "m", **fields}, headers=AUTH + ) + assert resp.status_code == 422 + sl.create_model.assert_not_called() + + +def test_create_model_and_add_item_pass_scope_through(tmp_path: Path) -> None: + sl = MagicMock() + sl.create_model.return_value = {"model": {}} + sl.add_glossary.return_value = {"id": "g"} + client = TestClient(_make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl))) + scope = {"scope": "targeted", "target_projects": ["analytics"]} + assert ( + client.post( + "/semantic-layer/models", json={"project": PROJECT, "name": "m", **scope}, headers=AUTH + ).status_code + == 200 + ) + assert sl.create_model.call_args.kwargs["scope"] == "targeted" + assert sl.create_model.call_args.kwargs["target_projects"] == ["analytics"] + assert ( + client.post( + "/semantic-layer/items/glossary", json={"project": PROJECT, "term": "t"}, headers=AUTH + ).status_code + == 200 + ) + kwargs = sl.add_glossary.call_args.kwargs + assert kwargs["scope"] is None # omitted -> the service inherits the model's scope + assert kwargs["target_projects"] is None + + # --------------------------------------------------------------------------- # flows.py POST /flows/validate + GET /flows/{project}/schema # New in 0.57.0 -- mirror `flow validate` / `flow schema --full`. From 1052004ace9c5b25281ced26d34377bed2798043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 2 Oct 2026 15:37:49 +0200 Subject: [PATCH 5/6] fix(AI-3790): address re-review (default schemaVersion, gate inherited org scope) Blocking: - Create no longer sends schemaVersion. The metastore stores the version it resolves and checks elevation against the STORED version; pinning 1.0.0 on a plain create made every item created without --scope impossible to elevate. Removes _schema_version() and the version constants. - An inherited organization scope (add without --scope under an org-level model) is now permission-gated like a typed --scope organization, so --deny-destructive no longer lets it through. A non-org-admin 403 on an inherited scope gets a hint to pass --scope project. Non-blocking / nits: - scope add|remove: ownership is decided by meta.projectId only, so the owner of a targeted item with no grants can add again; an unknown own project ID says to run `kbagent project refresh`. - put_item no longer sends schemaVersion or takes a scope argument; a rename to a taken name maps to ALREADY_EXISTS. - scope set --scope organization --dry-run is gated too (same as sync push --force). - E2E: scope get/request-create/request-delete/dry-run on an item created without --scope, and an edit that keeps the item id. - Remove stale DELETE+POST wording in docstrings, comments and docs. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 3 +- .../skills/kbagent/references/gotchas.md | 34 +++++---- .../references/metastore-scope-workflow.md | 11 +-- .../references/semantic-layer-workflow.md | 5 +- .../commands/_semantic_layer_crud.py | 5 ++ .../commands/_semantic_layer_helpers.py | 16 ++++- .../commands/_semantic_layer_scope.py | 3 +- src/keboola_agent_cli/commands/context.py | 4 +- src/keboola_agent_cli/metastore_client.py | 72 ++++++++----------- .../services/_semantic_layer_crud.py | 15 +--- .../services/_semantic_layer_internals.py | 13 +--- .../services/_semantic_layer_scope.py | 20 ++++-- .../services/semantic_layer_service.py | 45 +++++++++--- tests/test_e2e.py | 46 ++++++++++-- tests/test_metastore_client.py | 40 +++++------ tests/test_semantic_layer_scope.py | 54 +++++++++++++- tests/test_semantic_layer_scope_cli.py | 31 +++++++- tests/test_semantic_layer_service.py | 6 +- .../test_server_semantic_layer_routes_e2e.py | 6 +- 19 files changed, 287 insertions(+), 142 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index cebc8c4e6..61425ae91 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1097,7 +1097,8 @@ kbagent semantic-layer token --encrypt --project P --component-id C # explicit --target-project grants). --scope/--target-project are also accepted by `model # create` and every `add ` above. --scope omitted: `model create` makes a "project" # item, `add ` INHERITS its model's scope (an org-level model gets org-level children; -# a targeted model, its target projects). --target-project takes a registered alias OR a +# a targeted model, its target projects); an inherited organization scope is permission-gated +# like a typed one, and a non-org-admin gets a 403 on it (pass --scope project). --target-project takes a registered alias OR a # numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack); # without --scope targeted it exits 2. With --scope targeted and no --target-project: a real # terminal launches an interactive picker over the other projects on the stack; --json fails diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index ef481485e..467d9cd4f 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -5781,20 +5781,23 @@ sub-app (also `GET/PUT /semantic-layer/scope/...` over `kbagent serve`). See [metastore-scope-workflow.md](metastore-scope-workflow.md) for worked examples. Surprises worth knowing before you touch this: -- **A write that changes scope sends metastore schema `1.1.0`.** Every - `semantic-*` schema's `x-metastore.scope.supported` is `["project"]` ONLY at - `1.0.0` -- `1.1.0` adds `organization`/`targeted` (purely additive ACL/scope - blocks, no `data` shape change). The server resolves the EXACT version - string sent and never upgrades it, so kbagent sends `1.1.0` only when the - scope is not `project` (create) or the item being edited is not `project` - scoped (edit); a plain project-scope write keeps sending `1.0.0`, so a stack - without the 1.1.0 schemas is unaffected until someone uses a scope. +- **kbagent never sends `schemaVersion` on create.** The metastore stores + the version it resolved for the item, and checks a later elevation (`PATCH + {"scope":"organization"}`, the elevation request) against that STORED + version. Every `semantic-*` schema supports only `scope=project` at `1.0.0` + (later schema versions add `organization`/`targeted`), so pinning `1.0.0` on a plain + create would make the item impossible to elevate later. With the key + omitted, the server uses the stack's default schema (`1.1.0` or later where + the PSGO-140 migrations ran). `PUT` (edit) never changes the stored version. - **`--scope` omitted means "inherit" for child items.** `add metric|dataset|relationship|constraint|glossary` take their model's scope (and, for `targeted`, its target projects) when `--scope` is not given; `model create` defaults to `project`. Pass `--scope` to override. Without this an org-level model would show up in consumer projects with no - datasets or metrics. + datasets or metrics. An INHERITED `organization` scope is gated exactly like + a typed `--scope organization` (`--deny-destructive` blocks it), and a + project-admin token that is not an org admin gets a 403 on it, with a hint + to pass `--scope project`. - **`--target-project` takes an alias or a numeric project ID** (repeatable or comma-separated), so the target need not be registered in kbagent. An alias must be on the owner project's stack (a project ID only means the same @@ -5818,11 +5821,14 @@ examples. Surprises worth knowing before you touch this: re-creating the item as project-scoped (losing its revision history). The permission engine escalates `--scope organization` on `scope set`, `model create` and every `add ` (`FLAG_ESCALATIONS`), so `--deny-destructive` - blocks them; a `scope set --scope organization --dry-run` is not blocked. -- **Elevating items created before this release can fail.** The server checks - `PATCH {"scope":"organization"}` and the elevation request against the - item's STORED `schemaVersion`; an item created by an older kbagent has - `1.0.0`, which supports only `project`. *Not yet verified on a live stack.* + blocks them, `scope set --scope organization --dry-run` included (same + as `sync push --force`). +- **Elevating items created by an older kbagent can fail.** Those were created + pinned to `schemaVersion 1.0.0`, which supports only `project`, and the + server checks elevation against the STORED version, so `scope set --scope + organization` / `scope request-create` fail with a scope-not-supported error + until the metastore's schema sweep moves the item to the default version. + *Read from the server source, not yet verified on a live stack.* - **A 403 vs a 404 on a scope call encodes something deliberate.** The metastore returns 404 (not 403) when the caller cannot see the object at all -- not its owner, not a granted project, not viewing an diff --git a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md index 5737aa44d..ed15e43ad 100644 --- a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md @@ -15,7 +15,8 @@ constraint, glossary term) carries a visibility **scope**: `add ` without `--scope` **inherits its model's scope** (and target projects), so an org-level model gets org-level children. Pass `--scope` to -override. +override (a project-admin token that is not an org admin must pass +`--scope project` under an org-level model, or it gets a 403). For one-line command reference, see [commands-reference.md](commands-reference.md#scope--target-project-grants--elevation-scope-sub-app-since-vnext-psgo-140). @@ -125,14 +126,16 @@ where the owner and the admin are different people/tokens. `--scope organization` is gated as **destructive** by the permission engine (`--deny-destructive` blocks it, here and on `model create` / `add `); -`--dry-run` is not blocked. +`--dry-run` is gated too (same as `sync push --force`). An `add ` that +would INHERIT `organization` from its model is gated the same way. There is **no bulk-elevate endpoint**. "Elevate an existing project's objects" as a migration means one `request-create` + `set --scope organization` call per item, run deliberately for objects the user has named -- never loop this over every object in a project speculatively. Items -created by a kbagent older than this feature may refuse elevation (their -stored schema version only supports `project`); if so, re-create them. +created by a kbagent older than this feature may refuse elevation (they were +pinned to schema `1.0.0`, which only supports `project`); if so, re-create +them. ## Workflow 4 -- Editing a scoped item diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index 278c7bc3e..8124b9ddf 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -509,9 +509,8 @@ For the live-validated metastore contract surprises -- where the Quick reminders: -- **POST envelope**: `{name, data, branch: "main", schemaVersion: - "1.0.0", scope: "project"}` (`schemaVersion: "1.1.0"` when the scope is - `organization`/`targeted`) -> 201 with `{data: {type, id, +- **POST envelope**: `{name, data, branch: "main", scope: "project"}` + (no `schemaVersion`: the server stores the stack's default) -> 201 with `{data: {type, id, attributes, meta}}`. kbagent handles this. - **Duplicate-name POST -> 409 Conflict** (post go-monorepo PR #513) with `"Object with this name already exists in this project"`, or **500** with diff --git a/src/keboola_agent_cli/commands/_semantic_layer_crud.py b/src/keboola_agent_cli/commands/_semantic_layer_crud.py index 0756fb01b..9cba3429e 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_crud.py @@ -94,6 +94,7 @@ def add_metric( scope=scope, target_project=target_project, owner_alias=project, + inherit_from_model=(model,), ) result = _handle_service_call( ctx, @@ -152,6 +153,7 @@ def add_dataset( scope=scope, target_project=target_project, owner_alias=project, + inherit_from_model=(model,), ) result = _handle_service_call( ctx, @@ -193,6 +195,7 @@ def add_relationship( scope=scope, target_project=target_project, owner_alias=project, + inherit_from_model=(model,), ) result = _handle_service_call( ctx, @@ -260,6 +263,7 @@ def add_constraint( scope=scope, target_project=target_project, owner_alias=project, + inherit_from_model=(model,), ) result = _handle_service_call( ctx, @@ -296,6 +300,7 @@ def add_glossary( scope=scope, target_project=target_project, owner_alias=project, + inherit_from_model=(model,), ) result = _handle_service_call( ctx, diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py index e39d5ac11..78080bb71 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -80,13 +80,16 @@ def resolve_scope_targets( scope: str | None, target_project: list[str] | None, owner_alias: str, + inherit_from_model: tuple[str | None] | None = None, ) -> list[str] | None: """Validate ``--scope`` / ``--target-project`` of a create command. Returns the ``--target-project`` values (aliases or IDs; the service resolves them) for ``--scope targeted``, else ``None``. ``--scope organization`` is gated here as destructive (``operation`` is the command's - permission key, e.g. ``semantic-layer.add.dataset``). ``--target-project`` + permission key, e.g. ``semantic-layer.add.dataset``); so is an omitted + ``--scope`` that would inherit ``organization`` from the model + (``inherit_from_model=(model,)`` on the ``add`` commands). ``--target-project`` without ``--scope targeted`` is a usage error, never silently ignored. With ``--scope targeted`` and no target, this is the "ask when uncertain" @@ -95,6 +98,17 @@ def resolve_scope_targets( context it hard-fails instead of silently defaulting -- widening an object's visibility across projects is not a guess this CLI makes. """ + engine = ctx.obj.get("permission_engine") + if scope is None and inherit_from_model is not None and engine is not None and engine.active: + # `add ` without --scope takes its model's scope: an inherited + # organization scope widens visibility just like a typed one, so it goes + # through the same permission gate. + service = get_service(ctx, "semantic_layer_service") + inherited = _handle_service_call( + ctx, service.child_scope, alias=owner_alias, model_name_or_uuid=inherit_from_model[0] + ) + if inherited == "organization": + check_cli_operation(ctx, f"{operation} --scope organization") if scope == "organization": check_cli_operation(ctx, f"{operation} --scope organization") if scope != "targeted": diff --git a/src/keboola_agent_cli/commands/_semantic_layer_scope.py b/src/keboola_agent_cli/commands/_semantic_layer_scope.py index e68646b16..286d5f261 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_scope.py @@ -182,8 +182,7 @@ def scope_set( formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") if scope == "organization": - if not dry_run: - check_cli_operation(ctx, "semantic-layer.scope.set --scope organization") + check_cli_operation(ctx, "semantic-layer.scope.set --scope organization") if not (yes or dry_run or formatter.json_mode) and not typer.confirm( f"Elevate {type_} {context_id!r} to organization scope? This is IRREVERSIBLE " "and makes it visible to every project in the organization. Continue?" diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 7a2feb631..cccb49b71 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1946,7 +1946,9 @@ has_more}}. The org-admin's discovery queue. Child items (`add metric|dataset|...`) with --scope omitted INHERIT their - model's scope and target projects; pass --scope to override. + model's scope and target projects; pass --scope to override. An inherited + organization scope is gated as destructive like a typed one, and a + non-org-admin token gets a 403 on it (pass --scope project). Elevating an EXISTING project's semantic-layer objects in bulk: there is no bulk-elevate endpoint -- each object needs its own `scope request-create` + diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 37a62ab1b..e47edbc6a 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -10,7 +10,7 @@ - ``GET /api/v1/repository/{type}`` → 200 with body ``{"data": [item, ...]}``. - ``POST /api/v1/repository/{type}`` → 201 with body ``{"data": {type, id, - attributes, meta}}``. Envelope: ``{name, data, branch, schemaVersion, scope}``. + attributes, meta}}``. Envelope: ``{name, data, branch, scope}``. - ``DELETE /api/v1/repository/{type}/{id}`` → 204 empty body. Missing ID → 404 with the standard error envelope. - Duplicate ``name`` on POST → **409 Conflict** with message ``"Object with @@ -110,26 +110,15 @@ # Envelope fields kept constant across every POST (per metastore contract). _ENVELOPE_BRANCH = "main" -# Every semantic-* schema's x-metastore.scope.supported is -# ["project"] ONLY at 1.0.0 -- 1.1.0 is what adds "organization"/"targeted" -# (go-monorepo migrations/schema/semantic-*_schema_1.1.0.json, diffed against -# 1.0.0 at commit e4f62941: purely additive x-metastore.acl/scope blocks, no -# `data` schema change, so this is safe for every existing caller). Sending -# scope="organization"/"targeted" against 1.0.0 gets a clean 400 -# ErrScopeNotSupported from prepareCreateSchema -- the server resolves the -# EXACT version string sent, never silently upgrades it. -# Only ``scope != "project"`` needs the 1.1.0 ACL schemas (additive; every -# ``semantic-*`` type supports ``scope=project`` at 1.0.0). Project-scope writes -# keep sending 1.0.0 so a stack without the 1.1.0 schemas is unaffected. -_ENVELOPE_SCHEMA_VERSION = "1.0.0" -_ACL_ENVELOPE_SCHEMA_VERSION = "1.1.0" +# No ``schemaVersion`` is sent on create: for an empty version the metastore +# resolves the stack's DEFAULT schema and stores that version on the item. The +# server checks a later scope elevation (PATCH / elevation request) against the +# STORED version, and every ``semantic-*`` schema supports only ``scope=project`` +# at 1.0.0 (the later schema versions add "organization"/"targeted"), so pinning "1.0.0" here would +# make every item created without ``--scope`` impossible to elevate. _DEFAULT_SCOPE: ObjectScope = "project" -def _schema_version(scope: str) -> str: - return _ENVELOPE_SCHEMA_VERSION if scope == "project" else _ACL_ENVELOPE_SCHEMA_VERSION - - class MetastoreClient(BaseHttpClient): """HTTP client for the Keboola Metastore (semantic layer repository). @@ -321,7 +310,6 @@ def post_item( "name": name, "data": data, "branch": _ENVELOPE_BRANCH, - "schemaVersion": _schema_version(scope), "scope": scope, } if target_project_ids is not None: @@ -366,38 +354,40 @@ def put_item( item_id: str, name: str, data: dict[str, Any], - *, - scope: ObjectScope = _DEFAULT_SCOPE, ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). - Unlike the DELETE+POST pattern the higher-level ``edit`` operations - use, ``PUT`` updates the record in place and increments + ``PUT`` updates the record in place (``edit``, ``import --overwrite`` and + ``promote`` use it) and increments ``meta.revision`` server-side, preserving the metastore's revision history. ``data`` is the inner ``attributes`` payload; the outer envelope is added here. - ``scope`` is the item's CURRENT scope and only selects the envelope - ``schemaVersion`` (an organization/targeted item needs 1.1.0); it is - never sent as a field. Deliberately carries no - ``scope``/``targetProjectIds`` -- the - server's request struct for this endpoint (``MetaObjectUpdatePutRequest``) - has no such fields, so scope/grants are untouched by a plain PUT. + Deliberately carries no ``scope``/``targetProjectIds``/``schemaVersion`` + -- the server's request struct for this endpoint + (``MetaObjectUpdatePutRequest``) has only ``name`` and ``data``, so + scope, grants and the stored schema version are untouched by a plain PUT. Use :meth:`elevate_to_organization` / :meth:`put_target_projects`. - Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. + Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404 and + ``ALREADY_EXISTS`` on 409 (renaming to a name that is already taken). """ - envelope = { - "name": name, - "data": data, - "branch": _ENVELOPE_BRANCH, - "schemaVersion": _schema_version(scope), - } - response = self._do_request( - "PUT", - f"/api/v1/repository/{item_type}/{item_id}", - json=envelope, - ) + envelope = {"name": name, "data": data, "branch": _ENVELOPE_BRANCH} + try: + response = self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}", + json=envelope, + ) + except KeboolaApiError as exc: + if exc.status_code == 409: + raise KeboolaApiError( + message=f"{item_type} with name {name!r} already exists. Pick another name.", + status_code=409, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) from exc + raise body = response.json() return body.get("data", body) if isinstance(body, dict) else body diff --git a/src/keboola_agent_cli/services/_semantic_layer_crud.py b/src/keboola_agent_cli/services/_semantic_layer_crud.py index d94368881..be63ef491 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/services/_semantic_layer_crud.py @@ -23,7 +23,6 @@ from typing import TYPE_CHECKING, Any from ..errors import ErrorCode, KeboolaApiError -from ._semantic_layer_scope import item_scope if TYPE_CHECKING: from ..metastore_client import MetastoreClient, SemanticType @@ -82,7 +81,6 @@ def edit_metric_with_cascade( ) original_attrs = dict(target.get("attributes") or {}) old_id = target["id"] - target_scope, _ = item_scope(target) effective_new_name = new_name if new_name is not None else current_name cascade_required: list[dict[str, Any]] = [] @@ -126,9 +124,7 @@ def edit_metric_with_cascade( if new_description is not None: new_attrs["description"] = new_description - new_item = client.put_item( - "semantic-metric", old_id, effective_new_name, new_attrs, scope=target_scope - ) + new_item = client.put_item("semantic-metric", old_id, effective_new_name, new_attrs) # Cascade constraints individually (each is independent -- # report per-constraint success/failure). @@ -140,9 +136,7 @@ def edit_metric_with_cascade( cattrs["metrics"] = cmetrics cname = cattrs.get("name", "") try: - cascaded_item = client.put_item( - "semantic-constraint", c["id"], cname, cattrs, scope=item_scope(c)[0] - ) + cascaded_item = client.put_item("semantic-constraint", c["id"], cname, cattrs) cascaded.append({"constraint": cname, "status": "updated", "id": cascaded_item["id"]}) except KeboolaApiError as exc: cascaded.append( @@ -288,10 +282,7 @@ def edit_simple( if v is not None: new_attrs[k] = v effective_new = new_attrs.get(id_key) or current_key - target_scope, _ = item_scope(target) - new_item = client.put_item( - item_type, target["id"], effective_new, new_attrs, scope=target_scope - ) + new_item = client.put_item(item_type, target["id"], effective_new, new_attrs) return { "updated": new_item, "cascaded_constraints": [], diff --git a/src/keboola_agent_cli/services/_semantic_layer_internals.py b/src/keboola_agent_cli/services/_semantic_layer_internals.py index 25e07c4f1..22d29b268 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_internals.py +++ b/src/keboola_agent_cli/services/_semantic_layer_internals.py @@ -40,7 +40,6 @@ from ..errors import ConfigError, ErrorCode, KeboolaApiError from ._semantic_layer_fqn import append_fqn_mismatch_warnings -from ._semantic_layer_scope import item_scope logger = logging.getLogger(__name__) @@ -445,10 +444,7 @@ def run_import_loop( per_type["overwritten"] += 1 continue try: - existing = existing_by_name[key] - client.put_item( - type_slug, existing["id"], key, attrs, scope=item_scope(existing)[0] - ) + client.put_item(type_slug, existing_by_name[key]["id"], key, attrs) per_type["overwritten"] += 1 except KeboolaApiError as exc: per_type["failed"].append({"name": key, "reason": exc.message}) @@ -478,7 +474,7 @@ def run_promote_loop( ) -> dict[str, Any]: """Run the additive + overwrite promote loop. - NEW: POST to target. CHANGED: DELETE+POST. IDENTICAL: skip. + NEW: POST to target. CHANGED: in-place PUT. IDENTICAL: skip. Items only in target are never deleted (additive only). Returns ``{plural: {new, overwritten, identical, failed, changes}}``. """ @@ -520,10 +516,7 @@ def run_promote_loop( stats["changes"].append({id_key: key, "diff_keys": diff_keys}) continue try: - existing = tgt_by_key[key] - target_client.put_item( - type_slug, existing["id"], key, src_attrs, scope=item_scope(existing)[0] - ) + target_client.put_item(type_slug, tgt_by_key[key]["id"], key, src_attrs) stats["overwritten"] += 1 stats["changes"].append({id_key: key, "diff_keys": diff_keys}) except KeboolaApiError as exc: diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py index c44b502a9..a118372ff 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -116,15 +116,23 @@ def _merge_targets( ) -> list[int]: """Apply an add/remove delta to the item's grants, refusing when they cannot be read. - The server returns ``meta.targetProjectIds`` only to the owning project, so - a non-owner (an org admin from elsewhere) would read ``None``, apply the - delta to an empty list, and PUT it -- silently replacing every grant. + The server returns ``meta.targetProjectIds`` only to the owning project (and + omits it when the list is empty), so a non-owner (an org admin from + elsewhere) would read ``None``, apply the delta to an empty list, and PUT it + -- silently replacing every grant. The owner check is on ``meta.projectId``, + never on the list being empty: an owner with no grants is a valid merge base. """ + if caller_project_id is None: + raise KeboolaApiError( + message=( + "This project's own ID is unknown, so ownership of the item cannot be checked. " + "Run `kbagent project refresh` (or `project add`) first." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) meta = item.get("meta") or {} owner_id = meta.get("projectId") - if (owner_id is not None and owner_id != caller_project_id) or ( - meta.get("scope", "project") == "targeted" and meta.get("targetProjectIds") is None - ): + if owner_id is not None and owner_id != caller_project_id: raise KeboolaApiError( message=( "This project does not own the item, so its current target projects cannot " diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 9d145dff0..d114446b1 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -686,6 +686,16 @@ def _target_ids( return None return _scope.resolve_target_project_ids(self._config_store, alias, target_projects) or None + def child_scope(self, alias: str, model_name_or_uuid: str | None) -> ObjectScope: + """The scope ``add `` gives a child when ``--scope`` is omitted: its model's own. + + Exposed so the command layer can gate an INHERITED ``organization`` scope + exactly like a typed ``--scope organization``. + """ + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _scope.inherited_scope(client, model_uuid)[0] + def _post_scoped( self, client: MetastoreClient, @@ -698,13 +708,28 @@ def _post_scoped( target_projects: list[str] | None, ) -> dict[str, Any]: """POST a child item; with ``scope`` omitted it takes the scope of its model.""" - if scope is None: + inherited = scope is None + if inherited: scope, target_ids = _scope.inherited_scope(client, data["modelUUID"]) else: target_ids = self._target_ids(alias, scope, target_projects) - return client.post_item( - item_type, name=name, data=data, scope=scope, target_project_ids=target_ids - ) + try: + return client.post_item( + item_type, name=name, data=data, scope=scope, target_project_ids=target_ids + ) + except KeboolaApiError as exc: + if not (inherited and scope != "project" and exc.error_code == ErrorCode.ACCESS_DENIED): + raise + raise KeboolaApiError( + message=( + f"{exc.message} The model is {scope!r}-scoped, so this child inherits that " + "scope; creating at it needs an organization-admin token. Pass " + "`--scope project` to create it project-only, or use an org-admin token." + ), + status_code=exc.status_code, + error_code=exc.error_code, + retryable=False, + ) from exc def create_model( self, @@ -1207,7 +1232,7 @@ def edit_metric( is_tty: bool = False, confirm_cb: Callable[[str], bool] | None = None, ) -> dict[str, Any]: - """Edit a metric via DELETE+POST with rename-cascade on constraints. + """Edit a metric in place (PUT) with rename-cascade on constraints. Returns: ``{updated: item, cascaded_constraints: [...], rollback: None|{...}}``. @@ -1243,7 +1268,7 @@ def edit_dataset( new_description: str | None = None, new_grain: str | None = None, ) -> dict[str, Any]: - """Edit a dataset (DELETE+POST). Renames do NOT cascade for datasets.""" + """Edit a dataset (in-place PUT). Renames do NOT cascade for datasets.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) @@ -1273,7 +1298,7 @@ def edit_constraint( new_severity: str | None = None, new_metrics: list[str] | None = None, ) -> dict[str, Any]: - """Edit a constraint (DELETE+POST). Validates new attrs locally first.""" + """Edit a constraint (in-place PUT). Validates new attrs locally first.""" _validate_constraint_attrs( name_re=CONSTRAINT_NAME_RE, constraint_types=CONSTRAINT_TYPES, @@ -1323,7 +1348,7 @@ def edit_relationship( new_on: str | None = None, new_type: str | None = None, ) -> dict[str, Any]: - """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally. + """Edit a relationship (in-place PUT). Validates ``--new-type`` locally. Relationships are not referenced by any other entity, so no cascade is needed -- the result is shaped identically to @@ -1362,7 +1387,7 @@ def edit_glossary( new_term: str | None = None, new_definition: str | None = None, ) -> dict[str, Any]: - """Edit a glossary term (DELETE+POST). + """Edit a glossary term (in-place PUT). Renaming via ``--new-term`` is destructive for downstream consumers that join on the literal term string (the term IS the @@ -1497,7 +1522,7 @@ def import_snapshot( types: Filter to a subset of types. ``None`` = all types. dry_run: When True, plan and return the action counts without hitting any write API. - overwrite: When True, DELETE+POST conflicting items by name. + overwrite: When True, update conflicting items by name in place (PUT). Default (False) skips conflicts. Returns: diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 42a7321bd..c4f48784d 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -4479,7 +4479,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: "--yes", ) new_metric_id = data["data"]["updated"]["id"] - # Replace tracking: the old metric was DELETE+POSTed + # Replace tracking: edits are in-place PUTs (same id), re-tracked defensively created_items = [ (t, i) for (t, i) in created_items @@ -4490,7 +4490,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: assert any(c["status"] == "updated" for c in cascaded), ( f"Expected at least one cascaded constraint, got: {cascaded}" ) - # The constraint id changed (DELETE+POST). Re-fetch the list. + # Re-fetch the list (the cascade updated the constraint in place). data = self._run_ok( "semantic-layer", "show", @@ -11423,6 +11423,42 @@ def _direct_delete(item_type: str, item_id: str) -> None: ) created_items.append(("semantic-dataset", ds2["data"]["id"])) + # scope commands on an item created WITHOUT --scope (it must stay + # elevatable: no schemaVersion is pinned on create), and the in-place + # PUT edit keeping the item id. + ds1_id = ds1["data"]["id"] + scope_args = ("--project", self.alias, "--type", "dataset", "--context-id", ds1_id) + data = self._run_ok("semantic-layer", "scope", "get", *scope_args) + assert data["data"]["scope"] == "project" + data = self._run_ok("semantic-layer", "scope", "request-create", *scope_args) + assert data["data"]["scope_elevation_requested_at"] + data = self._run_ok("semantic-layer", "scope", "request-delete", *scope_args) + assert not data["data"]["scope_elevation_requested_at"] + data = self._run_ok( + "semantic-layer", + "scope", + "set", + *scope_args, + "--scope", + "organization", + "--dry-run", + ) + assert data["data"]["would_set_scope"] == "organization" + data = self._run_ok( + "semantic-layer", + "edit", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_a", + "--new-description", + "edited in place", + ) + assert data["data"]["updated"]["id"] == ds1_id + m1 = self._run_ok( "semantic-layer", "add", @@ -11597,7 +11633,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: assert any(c["status"] == "updated" for c in cascaded), ( f"Expected constraint cascade, got: {cascaded}" ) - # DELETE+POST changed the constraint id -- refresh tracking + # refresh tracking (cascade updated the constraint in place) data = self._run_ok( "semantic-layer", "show", @@ -11614,7 +11650,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: # ---------- NB-5: edit + remove relationship / glossary ---------- - _step(5.1, "edit relationship --new-on -- DELETE+POST") + _step(5.1, "edit relationship --new-on -- in-place PUT") data = self._run_ok( "semantic-layer", "edit", @@ -11636,7 +11672,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: ] created_items.append(("semantic-relationship", new_rel_id)) - _step(5.2, "edit glossary --new-definition -- DELETE+POST") + _step(5.2, "edit glossary --new-definition -- in-place PUT") data = self._run_ok( "semantic-layer", "edit", diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 5db3272b1..f0910f24f 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -169,7 +169,7 @@ def test_post_envelope(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "rev" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" + assert "schemaVersion" not in body # the stack default is resolved server-side assert body["scope"] == "project" assert body["data"]["sql"] == "SUM(x)" assert body["data"]["modelUUID"] == "u" @@ -304,7 +304,7 @@ def test_put_envelope_and_url(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "chart_of_accounts" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" + assert "schemaVersion" not in body assert "scope" not in body assert body["data"]["dimensionName"] == "chart_of_accounts" assert body["data"]["members"] == [] @@ -543,13 +543,10 @@ def test_empty_target_project_ids_still_rejected_without_targeted_scope( ) assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR - @pytest.mark.parametrize( - ("scope", "expected"), - [("project", "1.0.0"), ("organization", "1.1.0"), ("targeted", "1.1.0")], - ) - def test_schema_version_is_1_1_0_only_for_acl_scopes( - self, httpx_mock, metastore_client, scope, expected - ) -> None: + @pytest.mark.parametrize("scope", ["project", "organization", "targeted"]) + def test_create_never_pins_a_schema_version(self, httpx_mock, metastore_client, scope) -> None: + """A pinned 1.0.0 would make the item impossible to elevate later (scope check is on + the STORED version), so the stack default is left to the server for every scope.""" httpx_mock.add_response( url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, @@ -562,21 +559,18 @@ def test_schema_version_is_1_1_0_only_for_acl_scopes( scope=scope, target_project_ids=[1] if scope == "targeted" else None, ) - assert json.loads(httpx_mock.get_requests()[0].content)["schemaVersion"] == expected + assert "schemaVersion" not in json.loads(httpx_mock.get_requests()[0].content) - def test_put_item_schema_version_follows_the_items_scope( - self, httpx_mock, metastore_client - ) -> None: - for _ in range(2): - httpx_mock.add_response( - method="PUT", - url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric/abc", - json={"data": {"type": "semantic-metric", "id": "abc", "attributes": {}}}, - ) - metastore_client.put_item("semantic-metric", "abc", "x", {}) - metastore_client.put_item("semantic-metric", "abc", "x", {}, scope="organization") - versions = [json.loads(r.content)["schemaVersion"] for r in httpx_mock.get_requests()] - assert versions == ["1.0.0", "1.1.0"] + def test_put_item_duplicate_name_is_already_exists(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric/abc", + status_code=409, + json={"error": "Object with this name already exists in this project"}, + ) + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.put_item("semantic-metric", "abc", "taken", {}) + assert excinfo.value.error_code == ErrorCode.ALREADY_EXISTS def test_rejects_target_project_ids_without_targeted_scope(self, metastore_client) -> None: with pytest.raises(KeboolaApiError) as excinfo: diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index cbce2c8e9..587e7aba4 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -157,6 +157,25 @@ def test_owner_merge_applies_the_delta(self, add, remove, expected) -> None: ) client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", expected) + def test_owner_with_no_grants_can_still_add(self) -> None: + """The server omits an empty list; that is the owner's state after `scope set --clear`.""" + client = MagicMock() + client.get_item.return_value = _item({"scope": "targeted", "projectId": 5725}) + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=5725 + ) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [3]) + + def test_unknown_own_project_id_is_refused_with_a_refresh_hint(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=None + ) + assert "kbagent project refresh" in excinfo.value.message + client.put_target_projects.assert_not_called() + @pytest.mark.parametrize( "meta", [ @@ -321,6 +340,37 @@ def test_add_glossary_scope( assert kwargs["scope"] == expected_scope assert kwargs["target_project_ids"] == expected_ids + def test_inherited_org_scope_denied_gets_a_hint(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, {"scope": "organization"}) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", definition="d") + assert "--scope project" in excinfo.value.message + assert "organization" in excinfo.value.message + + def test_explicit_scope_denied_is_not_rewritten(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, {"scope": "organization"}) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", scope="organization") + assert excinfo.value.message == "Insufficient permissions" + + @pytest.mark.parametrize( + ("meta", "expected"), + [ + (None, "project"), + ({"scope": "organization"}, "organization"), + ({"scope": "targeted"}, "targeted"), + ], + ) + def test_child_scope_reports_the_models_scope(self, tmp_path: Path, meta, expected) -> None: + service, _ = _service_with_model(tmp_path, meta) + assert service.child_scope("prod", None) == expected + def test_model_create_defaults_to_project_and_resolves_targets(self, tmp_path: Path) -> None: service, mock = _service_with_model(tmp_path, None) service.create_model("prod", "m") @@ -357,7 +407,7 @@ def test_edit_dataset_puts_in_place_with_the_items_scope(self, tmp_path: Path) - service.edit_dataset("prod", None, current_name="x", new_description="d") args, kwargs = mock.put_item.call_args assert args[:2] == ("semantic-dataset", "d1") - assert kwargs == {"scope": "targeted"} + assert kwargs == {} mock.delete_item.assert_not_called() mock.post_item.assert_not_called() @@ -391,6 +441,6 @@ def test_promote_overwrite_keeps_the_target_items_scope(self, tmp_path: Path) -> service.promote_model(from_project="prod", to_project="analytics") args, kwargs = tgt.put_item.call_args assert args[:3] == ("semantic-metric", "tm1", "b") - assert kwargs == {"scope": "organization"} + assert kwargs == {} # a PUT cannot carry scope, so it is untouched tgt.delete_item.assert_not_called() tgt.post_item.assert_not_called() diff --git a/tests/test_semantic_layer_scope_cli.py b/tests/test_semantic_layer_scope_cli.py index ad17a182d..11394e864 100644 --- a/tests/test_semantic_layer_scope_cli.py +++ b/tests/test_semantic_layer_scope_cli.py @@ -213,6 +213,35 @@ def test_organization_scope_create_is_gated_per_command( ) assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + @pytest.mark.parametrize( + ("model_scope", "blocked"), + [("organization", True), ("targeted", False), ("project", False)], + ) + def test_inherited_scope_is_gated_like_a_typed_one( + self, store: ConfigStore, model_scope: str, blocked: bool + ) -> None: + """`add` without --scope takes the model's scope; an inherited organization scope must not + bypass --deny-destructive.""" + mock = MagicMock() + mock.child_scope.return_value = model_scope + mock.add_glossary.return_value = {"id": "g", "attributes": {"term": "t"}} + result = _invoke( + ["--deny-destructive", *_sl("add", "glossary", "--project", "prod", "--term", "t")], + store=store, + sl_mock=mock, + ) + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + assert mock.add_glossary.called is (not blocked) + + def test_no_model_lookup_when_no_permission_policy_is_active(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_glossary.return_value = {"id": "g", "attributes": {"term": "t"}} + result = _invoke( + _sl("add", "glossary", "--project", "prod", "--term", "t"), store=store, sl_mock=mock + ) + assert result.exit_code == 0, result.output + mock.child_scope.assert_not_called() + def test_deny_destructive_still_allows_project_scope_create(self, store: ConfigStore) -> None: mock = MagicMock() mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} @@ -394,7 +423,7 @@ def test_deny_writes_allows_read_verbs(self, store: ConfigStore, verb: list[str] ("extra", "blocked"), [ (["--scope", "organization", "--yes"], True), - (["--scope", "organization", "--dry-run"], False), # preview changes nothing + (["--scope", "organization", "--dry-run"], True), # same gate as `sync push --force` (["--clear"], False), (["--target-project", "analytics"], False), ], diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index 6957b8b71..28774d4f7 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -1581,7 +1581,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] mock.post_item.assert_not_called() def test_edit_keeps_scope_and_item_id(self, tmp_path: Path) -> None: - """PUT in place: the item keeps its id; the schemaVersion follows the item's scope.""" + """PUT in place: the item keeps its id and (untouched by a PUT) its scope.""" store = _make_store(tmp_path) service, mock = _make_service(store) original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) @@ -1597,7 +1597,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] service.edit_metric("prod", None, current_name="rev", new_sql="2") args, kwargs = mock.put_item.call_args assert args[:3] == ("semantic-metric", "m1", "rev") - assert kwargs == {"scope": "organization"} + assert kwargs == {} mock.post_item.assert_not_called() def test_partial_state_false_when_cascade_succeeds(self, tmp_path: Path) -> None: @@ -2040,7 +2040,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] assert result["imported"]["datasets"]["overwritten"] == 1 args, kwargs = mock.put_item.call_args assert args[:3] == ("semantic-dataset", "d1", "fact_x") - assert kwargs == {"scope": "organization"} + assert kwargs == {} mock.delete_item.assert_not_called() mock.post_item.assert_not_called() diff --git a/tests/test_server_semantic_layer_routes_e2e.py b/tests/test_server_semantic_layer_routes_e2e.py index 86d5860f5..823f219c3 100644 --- a/tests/test_server_semantic_layer_routes_e2e.py +++ b/tests/test_server_semantic_layer_routes_e2e.py @@ -342,15 +342,15 @@ def test_put_items_edit_metric(http_session: dict[str, Any]) -> None: assert res.status_code == 200, res.text body = res.json() new_id = body["updated"]["id"] - # Rename leaves a fresh metric id and DELETE+POSTs the constraint — - # refresh tracking so cleanup hits the right rows. + # Edits are in-place PUTs, so ids are normally unchanged; re-track anyway so + # cleanup hits the right rows whichever ids come back. http_session["created_items"] = [ (t, i) for (t, i) in http_session["created_items"] if not (t == "semantic-metric" and i == http_session["metric_rev_id"]) ] http_session["created_items"].append(("semantic-metric", new_id)) - # Refresh constraint id (cascade DELETE+POST). + # Re-read the constraint ids (cascade update). show = http_session["client"].get( f"/semantic-layer/show?project={_PROJECT_ALIAS}" f"&model={http_session['model_name']}&type=constraint", From 3724194502260444eb916756d83e7b206f861ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 2 Oct 2026 17:23:29 +0200 Subject: [PATCH 6/6] fix(AI-3790): limit the org-admin hint to inherited organization scope A project admin may create a targeted item, so a 403 on an inherited targeted scope has another cause and the "needs an org-admin token" hint was wrong advice. Only add it when the inherited scope is organization. Co-Authored-By: Claude Sonnet 5.5 --- .../services/semantic_layer_service.py | 4 +++- tests/test_semantic_layer_scope.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index d114446b1..1a37f9881 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -718,7 +718,9 @@ def _post_scoped( item_type, name=name, data=data, scope=scope, target_project_ids=target_ids ) except KeboolaApiError as exc: - if not (inherited and scope != "project" and exc.error_code == ErrorCode.ACCESS_DENIED): + if not ( + inherited and scope == "organization" and exc.error_code == ErrorCode.ACCESS_DENIED + ): raise raise KeboolaApiError( message=( diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index 587e7aba4..42ab64243 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -350,6 +350,18 @@ def test_inherited_org_scope_denied_gets_a_hint(self, tmp_path: Path) -> None: assert "--scope project" in excinfo.value.message assert "organization" in excinfo.value.message + def test_inherited_targeted_scope_denied_gets_no_org_admin_hint(self, tmp_path: Path) -> None: + """A project admin may create `targeted`, so a 403 there has another cause.""" + service, mock = _service_with_model( + tmp_path, {"scope": "targeted", "targetProjectIds": [5]} + ) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", definition="d") + assert excinfo.value.message == "Insufficient permissions" + def test_explicit_scope_denied_is_not_rewritten(self, tmp_path: Path) -> None: service, mock = _service_with_model(tmp_path, {"scope": "organization"}) mock.post_item.side_effect = KeboolaApiError(