diff --git a/CLAUDE.md b/CLAUDE.md index bcc70fb5..61425ae9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1054,7 +1054,7 @@ kbagent dev-portal deprecate --app VENDOR.APP_ID [--identity A] [--dry-run] kbagent encrypt values --project ALIAS --component-id ID --input JSON|@file|- [--output-file PATH] kbagent semantic-layer model list --project P -kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] +kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS|ID ...] kbagent semantic-layer model delete --project P --model M [--yes] kbagent semantic-layer show --project P [--model M] [--type dataset|metric|relationship|constraint|glossary] kbagent semantic-layer search-context --project P [--pattern G ...] [--type model|dataset|metric|relationship|constraint|glossary|all] [--limit N] @@ -1092,19 +1092,55 @@ kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,. kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes] kbagent semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH] kbagent semantic-layer token --encrypt --project P --component-id C +# scope (PSGO-140, new): visibility scope for a semantic-layer item -- "project" +# (owner only), "organization" (every project in the org), or "targeted" (owner + +# explicit --target-project grants). --scope/--target-project are also accepted by `model +# create` and every `add ` above. --scope omitted: `model create` makes a "project" +# item, `add ` INHERITS its model's scope (an org-level model gets org-level children; +# a targeted model, its target projects); an inherited organization scope is permission-gated +# like a typed one, and a non-org-admin gets a 403 on it (pass --scope project). --target-project takes a registered alias OR a +# numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack); +# without --scope targeted it exits 2. With --scope targeted and no --target-project: a real +# terminal launches an interactive picker over the other projects on the stack; --json fails +# fast (exit 2). A bad alias/ID/--scope/--type is a usage error (exit 2), never a traceback. +# Creating directly at --scope organization requires the organization-admin ROLE (403 +# otherwise) -- an ordinary token instead uses `scope request-create` + an org-admin's +# `scope set --scope organization`. Elevation is ONE-WAY (no downgrade endpoint), which is why +# every `--scope organization` (here and on `model create` / `add `) is destructive-class +# (FLAG_ESCALATIONS). `edit`, `import --overwrite` and `promote` update in place (PUT): an item +# keeps its id, scope, grants and pending elevation request. `scope add|remove` merge into the +# grants client-side (not atomic; refused from a project that does not own the item, because +# the server hides the grants from a non-owner -- use `scope set --target-project` there). +# Verbs follow the CLI spec (#791): get / add / remove / set / request-create / request-delete / +# request-list; `scope set` takes exactly ONE of --scope organization, --target-project (replace +# the whole list) or --clear. +kbagent semantic-layer scope get --project P --type model|dataset|metric|relationship|constraint|glossary --context-id ID +kbagent semantic-layer scope add --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] +kbagent semantic-layer scope remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] +kbagent semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID [...] | --clear) [--dry-run] [--yes] +kbagent semantic-layer scope request-create --project P --type T --context-id ID +kbagent semantic-layer scope request-delete --project P --type T --context-id ID +kbagent semantic-layer scope request-list --project P --type T [--limit N] [--offset N] +# scope over `kbagent serve`: GET/PUT /semantic-layer/scope/{context_id}, POST/DELETE +# .../target-projects, PUT/DELETE .../elevation-request, GET /semantic-layer/scope/elevation-requests; +# POST /semantic-layer/models and /items/{kind} take `scope` + `target_projects`. kbagent semantic-layer reference-data list --project P [--model M] kbagent semantic-layer reference-data get --project P (--id ID | --dimension D) kbagent semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X] kbagent semantic-layer reference-data delete --project P --id ID [--yes] # Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. -# semantic-layer REQUIRES A MASTER (project admin) Storage token (#711): the Metastore's -# auth gate -- unlike the Storage API -- rejects every valid non-master token with an opaque -# 401 "Failed to create project scope" (the underlying MasterTokenRequiredError is swallowed -# server-side). kbagent reclassifies exactly that 401 to MISSING_MASTER_TOKEN (exit 3) with -# the remedy; other unexplained 401s anywhere map to AUTH_REJECTED instead of the false -# "Invalid or expired token" (INVALID_TOKEN stays for 401s that DO blame the credential). -# Version gate for this entry lives in gotchas.md -- `(since vNEXT)` cannot be written on -# these `# ` comment lines (check_version_gates.py parses them as ATX headings). +# semantic-layer READS work with any valid, non-disabled, non-expired Storage token +# (vNEXT, PSGO-282): the Metastore no longer requires a master token for GET/List. WRITES +# (add/edit/remove/import/promote/build/scope add|remove|set) still need a project-admin token +# (master token or any admin-role user token) -- a non-admin token 403s on the write. Before +# PSGO-282 the Metastore's auth gate rejected EVERY valid non-master token, including reads, +# with an opaque 401 "Failed to create project scope" (#711, MasterTokenRequiredError swallowed +# server-side); kbagent still reclassifies exactly that 401 to MISSING_MASTER_TOKEN (exit 3) as +# a safety net for a deployment that predates the fix. Other unexplained 401s anywhere map to +# AUTH_REJECTED instead of the false "Invalid or expired token" (INVALID_TOKEN stays for 401s +# that DO blame the credential). Version gate for this entry lives in gotchas.md -- `(since +# vNEXT)` cannot be written on these `# ` comment lines (check_version_gates.py parses them as +# ATX headings). kbagent http get PATH [--timeout SECONDS] kbagent http post PATH [--body JSON|@file|-] [--timeout SECONDS] diff --git a/docs/error-codes.md b/docs/error-codes.md index 6977cd7a..f37d951b 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -19,7 +19,7 @@ of `ErrorCode` in `src/keboola_agent_cli/errors.py`. | `INVALID_TOKEN` | Storage API token is invalid or expired | | `ACCESS_DENIED` | Token lacks the required permission for this API call | | `PERMISSION_DENIED` | Operation blocked by the active kbagent permission policy | -| `MISSING_MASTER_TOKEN` | Operation requires a master (admin) Storage token (e.g. `token create`, `config oauth-url` pre-flight, and every `semantic-layer` command -- the Metastore's opaque 401 `Failed to create project scope` is its master-token gate and is reclassified to this code); maps to exit 3 | +| `MISSING_MASTER_TOKEN` | Operation requires a master (admin) Storage token (e.g. `token create`, `config oauth-url` pre-flight, and a `semantic-layer` WRITE against a non-admin token -- `semantic-layer` reads work with any valid token since PSGO-282; the Metastore's opaque 401 `Failed to create project scope` was its master-token gate pre-PSGO-282 and is reclassified to this code as a safety net); maps to exit 3 | | `UNAUTHORIZED` | `kbagent serve` rejected the request's Bearer token | | `AUTH_REJECTED` | Upstream answered HTTP 401 but its own error text did not blame the credential. Same exit 3 as `INVALID_TOKEN`; rotating the token is not the fix. (The Metastore's `Failed to create project scope` 401 has a known cause and lands on `MISSING_MASTER_TOKEN` instead) | diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 95710a58..774f65de 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -9,7 +9,7 @@ auth, and the concepts behind these routes live in [`web-server.md`](web-server.md); a running server serves the same spec interactively at `/docs` (Swagger) and `/openapi.json`. -**248 operations** across **216 paths** and **31 routers**. +**255 operations** across **220 paths** and **31 routers**. Paths are shown as the server registers them. Reaching them through the Node BFF (or single-process `--ui` mode) prefixes every path with `/api`. @@ -388,7 +388,7 @@ Build and query cross-project data lineage (table-level and column-level). Mirro | `GET` | `/lineage/walk` | Walk lineage graph from a node | | `GET` | `/lineage/mermaid` | Render lineage as Mermaid | -### `semantic-layer` (21 operations) +### `semantic-layer` (28 operations) Model, validate, import/export, diff, promote, and build semantic layer artifacts (datasets, metrics, relationships, constraints, glossary). Mirrors `kbagent semantic-layer *`. @@ -415,6 +415,13 @@ Model, validate, import/export, diff, promote, and build semantic layer artifact | `PUT` | `/semantic-layer/reference-data` | Create or replace a reference-data record | | `GET` | `/semantic-layer/reference-data/{record_id}` | Get one reference-data record | | `DELETE` | `/semantic-layer/reference-data/{record_id}` | Delete a reference-data record | +| `GET` | `/semantic-layer/scope/elevation-requests` | List scope-elevation requests | +| `GET` | `/semantic-layer/scope/{context_id}` | Get an item's scope | +| `PUT` | `/semantic-layer/scope/{context_id}` | Set an item's scope or target projects | +| `POST` | `/semantic-layer/scope/{context_id}/target-projects` | Add target projects | +| `DELETE` | `/semantic-layer/scope/{context_id}/target-projects` | Remove target projects | +| `PUT` | `/semantic-layer/scope/{context_id}/elevation-request` | Request scope elevation | +| `DELETE` | `/semantic-layer/scope/{context_id}/elevation-request` | Withdraw scope elevation request | ## AI & Tools diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index f5ffbb8f..24e510bf 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -151,8 +151,9 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` -- uses the `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` the scheduler injects; `http get /openapi.json` discovers endpoints | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated) | `curl $KBAGENT_SERVE_URL/...` by hand (loses the auth header, error mapping and JSON formatting `kbagent http` adds) | | Launch the web UI for an end-user | `kbagent serve --ui [--port PORT]` -- one FastAPI process mounts the bundled React SPA and sets an HttpOnly `kbagent_session` cookie, so SSE works with no token in the URL or JS heap | the legacy Vite + Node BFF dev flow (`web/README.md`) | inventing a `--token-in-url` flag; running uvicorn directly against the dist (the cookie bootstrap only fires from `serve --ui`) | | Schedule / manage Agent Tasks | `kbagent agent ` -- CRUD, `run [--stream]`, history, `test`/`cron-preview`/`prompt-improve`. Local-only; cron firing needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` inside scheduled subprocesses | hand-editing `agents.json` | -| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. The WHOLE `semantic-layer` family needs a MASTER token: a valid non-master token gets `MISSING_MASTER_TOKEN` (0.92.0+, #711; the Metastore's opaque 401 "Failed to create project scope") -- register a master token, do not escalate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | -| ANY semantic-layer write (add / edit / remove / import / promote / build) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (vNEXT, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | +| ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | @@ -250,11 +251,13 @@ its absence is NOT a promise the entry is version-independent (see §1 Rule 6). **A 401 is not always `INVALID_TOKEN` (0.92.0+)** - A 401 whose body names something other than a bad credential now raises `AUTH_REJECTED`; the semantic-layer family's opaque metastore 401 - reclassifies to `MISSING_MASTER_TOKEN` instead (see its matrix row). - Exit code is unchanged (3) for all three -- code checking - `error.code == "INVALID_TOKEN"` alone now misses two real cases; accept - all three before concluding the credential itself is bad. 401/403/404 - now also carry `[exceptionId: ...]` -- quote it when escalating. gotchas.md. + reclassified to `MISSING_MASTER_TOKEN` on a write against a non-admin + token (see its matrix row) -- since PSGO-282 (vNEXT) a plain read never + hits this at all, any valid token passes. Exit code is unchanged (3) for + all three -- code checking `error.code == "INVALID_TOKEN"` alone still + misses two real cases; accept all three before concluding the credential + itself is bad. 401/403/404 now also carry `[exceptionId: ...]` -- quote it + when escalating. gotchas.md. **Finding an existing Storage token** - `kbagent token list -p P` (0.86.0+) -- the only source of the `--token-id` diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 2d0fe1a1..233e4cfb 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -309,8 +309,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Add a glossary term | `kbagent semantic-layer add glossary --project PROJECT --term TERM` | | Edit a metric. | `kbagent semantic-layer edit metric --project PROJECT --name NAME` | | Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent semantic-layer edit dataset --project PROJECT --name NAME` | -| Edit a constraint (DELETE+POST, with local validators) | `kbagent semantic-layer edit constraint --project PROJECT --name NAME` | -| Edit a relationship (DELETE+POST). | `kbagent semantic-layer edit relationship --project PROJECT --name NAME` | +| Edit a constraint (in-place update, with local validators) | `kbagent semantic-layer edit constraint --project PROJECT --name NAME` | +| Edit a relationship (in-place update). | `kbagent semantic-layer edit relationship --project PROJECT --name NAME` | | Edit a glossary term. | `kbagent semantic-layer edit glossary --project PROJECT --term TERM` | | Remove a metric. | `kbagent semantic-layer remove metric --project PROJECT --name NAME` | | Remove a dataset | `kbagent semantic-layer remove dataset --project PROJECT --name NAME` | @@ -321,6 +321,13 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent semantic-layer reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent semantic-layer reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent semantic-layer reference-data delete --project PROJECT --id ID-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent semantic-layer scope get --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Add target projects to a targeted-scope item (merges with the current grants) | `kbagent semantic-layer scope add --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Remove target projects from a targeted-scope item (merges with the current grants) | `kbagent semantic-layer scope remove --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Write an item's scope: elevate to organization, or replace/clear its target projects | `kbagent semantic-layer scope set --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent semantic-layer scope request-create --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Withdraw a pending scope-elevation request. | `kbagent semantic-layer scope request-delete --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent semantic-layer scope request-list --project PROJECT --type TYPE-` | | Encrypt the project's storage token for transformation `user_properties` | `kbagent sl token --project PROJECT --component-id COMPONENT-ID` | | Build a semantic-layer model from a list of storage tables (non-interactive) | `kbagent sl build --project PROJECT` | | Promote a model from one project to another (NEW + overwrite CHANGED; never deletes) | `kbagent sl promote --from-project FROM-PROJECT --to-project TO-PROJECT` | @@ -342,8 +349,8 @@ When working inside a git repository or project directory, run `kbagent init` (o | Add a glossary term | `kbagent sl add glossary --project PROJECT --term TERM` | | Edit a metric. | `kbagent sl edit metric --project PROJECT --name NAME` | | Edit a dataset (no cascade — metric.dataset uses tableId, not name) | `kbagent sl edit dataset --project PROJECT --name NAME` | -| Edit a constraint (DELETE+POST, with local validators) | `kbagent sl edit constraint --project PROJECT --name NAME` | -| Edit a relationship (DELETE+POST). | `kbagent sl edit relationship --project PROJECT --name NAME` | +| Edit a constraint (in-place update, with local validators) | `kbagent sl edit constraint --project PROJECT --name NAME` | +| Edit a relationship (in-place update). | `kbagent sl edit relationship --project PROJECT --name NAME` | | Edit a glossary term. | `kbagent sl edit glossary --project PROJECT --term TERM` | | Remove a metric. | `kbagent sl remove metric --project PROJECT --name NAME` | | Remove a dataset | `kbagent sl remove dataset --project PROJECT --name NAME` | @@ -354,6 +361,13 @@ When working inside a git repository or project directory, run `kbagent init` (o | Fetch one record (all members) by ``--id`` or by ``--dimension`` | `kbagent sl reference-data get --project PROJECT` | | Create or replace a reference-data record (keyed by dimension) | `kbagent sl reference-data set --project PROJECT --dimension DIMENSION --members-file MEMBERS-FILE` | | Delete a reference-data record by UUID (server-side soft-delete) | `kbagent sl reference-data delete --project PROJECT --id ID-` | +| Show an item's current scope, target-project grants, and pending elevation | `kbagent sl scope get --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Add target projects to a targeted-scope item (merges with the current grants) | `kbagent sl scope add --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Remove target projects from a targeted-scope item (merges with the current grants) | `kbagent sl scope remove --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Write an item's scope: elevate to organization, or replace/clear its target projects | `kbagent sl scope set --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent sl scope request-create --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| Withdraw a pending scope-elevation request. | `kbagent sl scope request-delete --project PROJECT --type TYPE- --context-id CONTEXT-ID` | +| List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent sl scope request-list --project PROJECT --type TYPE-` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | @@ -447,6 +461,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | Reading synced data | [reading-synced-data](references/reading-synced-data.md) | | SQL migration (input mapping removal) | [sql-migration-workflow](references/sql-migration-workflow.md) | | **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | +| **Share a semantic-layer item across projects / make it org-wide** -- `--scope`, `--target-project`, `scope get\|add\|remove\|set\|request-*` | [metastore-scope-workflow](references/metastore-scope-workflow.md) | | **Developer Portal** (identity CRUD, list/get apps, create/patch/upload-icon/publish/deprecate; TTY-confirm on writes) | [dev-portal-workflow](references/dev-portal-workflow.md) | | **Config metadata** (list/get/set/delete arbitrary key-value metadata on a configuration) | [config-metadata-workflow](references/config-metadata-workflow.md) | | **Storage descriptions** (describe bucket / table / column, batch from YAML) | [storage-describe-workflow](references/storage-describe-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 2b53faa1..cebb8caa 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -406,7 +406,7 @@ MFA login sends `challenge: SOFTWARE_TOKEN_MFA` explicitly to fix a 404 on perso ## Semantic Layer (Metastore) -Reads `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` env vars. The scheduler auto-injects these (plus `KBAGENT_CONFIG_DIR`) into every AI-agent / `cli_command` subprocess. Outside a serve subprocess context the command refuses to run with exit code 2. **Inside a scheduled-agent task, prefer `kbagent http get /openapi.json` then a typed call over forking another `kbagent` CLI -- the HTTP path always sees the operator's live config (not the global `~/.config/keboola-agent-cli/` one).** -Manage Keboola metastore models -- datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from the stack URL by replacing `connection.` with `metastore.` (cloud/region-agnostic). Auth: same `X-StorageApi-Token` as Storage, **but the metastore accepts only a MASTER (project admin) token** -- a valid non-master token gets its opaque 401 `Failed to create project scope` on every call, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711; see [gotchas.md](gotchas.md)). Hidden alias: `kbagent sl ...` is equivalent to `kbagent semantic-layer ...`. See [semantic-layer-workflow.md](semantic-layer-workflow.md) for full recipes. +Manage Keboola metastore models -- datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from the stack URL by replacing `connection.` with `metastore.` (cloud/region-agnostic). Auth: same `X-StorageApi-Token` as Storage -- **reads work with any valid, non-disabled, non-expired token** (vNEXT, PSGO-282); **writes** still need a project-admin token (master, or any other admin-role user token) -- a non-admin token gets the opaque 401 `Failed to create project scope` on a write, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711; see [gotchas.md](gotchas.md)). Hidden alias: `kbagent sl ...` is equivalent to `kbagent semantic-layer ...`. See [semantic-layer-workflow.md](semantic-layer-workflow.md) for full recipes. - `semantic-layer model list --project P` -- list all models in a project. Output: `{models: [{id, name, sql_dialect, description}, ...]}`. Use to disambiguate when `--model` is required and the project has more than one model. - `semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake]` -- create a new model. `--sql-dialect` defaults to `Snowflake`. Returns the new model UUID; subsequent commands accept either name or UUID via `--model`. @@ -423,28 +423,41 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer add relationship --project P [--model M] --name N --from TABLE_ID --to TABLE_ID --on EXPR [--type left|inner]` -- add a join relationship. `--type` defaults to `left`. - `semantic-layer add constraint --project P [--model M] --name N --constraint-type T --rule "EXPR" --metrics M1,M2 [--severity error|warning|info]` -- add a constraint. `--constraint-type` is the closed enum `inequality|equality|range|composition|exclusion|temporal|conditional`. `--rule` is a **STRING expression** (e.g. `"value >= 0"`), NEVER a `{bounds: {min, max}}` object (sl-builder docs are wrong -- see [gotchas.md](gotchas.md)). `--metrics` is a comma-separated list of metric names that must already exist in the model. `--severity` defaults to `warning`. Name regex `^[a-z][a-z0-9_]*$`; the 4-band health convention lives in the name suffix `_critical / _warning / _healthy / _review`, distinct from the 3-value API `severity`. - `semantic-layer add glossary --project P [--model M] --term TERM [--definition D]` -- add a glossary term. -- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. The metastore has NO PATCH endpoint, so this is DELETE+POST. Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. +- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. Updates the metric in place (`PUT`: same id, scope and grants kept). Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. - `semantic-layer edit dataset --project P [--model M] --name N [--new-name N2] [--new-description D] [--new-grain G]` -- edit a dataset. No cascade -- metrics reference the dataset's tableId, not its name. -- `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (DELETE+POST). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. -- `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (DELETE+POST). No constraint cascade. Rollback on POST failure. -- `semantic-layer edit glossary --project P [--model M] --term TERM [--new-term T2] [--new-definition D]` -- edit a glossary entry (DELETE+POST). `--new-term` is a destructive cascade through any downstream consumer that joins on the term; warns but allows. +- `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (in-place `PUT`). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. +- `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (in-place `PUT`). No constraint cascade. Rollback on POST failure. +- `semantic-layer edit glossary --project P [--model M] --term TERM [--new-term T2] [--new-definition D]` -- edit a glossary entry (in-place `PUT`). `--new-term` is a destructive cascade through any downstream consumer that joins on the term; warns but allows. - `semantic-layer remove metric --project P [--model M] --name N [--yes]` -- destructive. Pre-deletion scan lists every constraint whose `metrics[]` includes the target name; warning is ALWAYS printed (even with `--yes`) about the resulting orphan + dangling `DIM_METRIC_THRESHOLD` reference. Non-TTY without `--yes` refuses with exit 2. - `semantic-layer remove dataset --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. - `semantic-layer remove constraint --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. - `semantic-layer remove relationship --project P [--model M] --name N [--yes]` -- destructive. Relationships aren't referenced by other entities; no orphan-check. - `semantic-layer remove glossary --project P [--model M] --term TERM [--yes]` -- destructive. Glossary entries aren't referenced by other entities; no orphan-check. -- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into DELETE+POST for conflicting items. `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. +- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into an in-place update of conflicting items (they keep their scope). `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. - `semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes]` -- cross-project model copy with `modelUUID` rewrite to the target model's UUID. Classifies items NEW / IDENTICAL / CHANGED (deep-equality after stripping `modelUUID` + timestamps). Additive + overwrite only: NEVER deletes target items absent from source. Holds two MetastoreClients in try/finally. Response: per-type counts + `changes[]` with `diff_keys` and `failed[]`. - `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN = the table's Storage location, *(since 0.95.0)* no longer a hardcoded `"KEBOOLA"` database; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). **Rollback on push failure**: every successfully-POSTed child is DELETEd in reverse PUSH_ORDER, and the model itself is DELETEd if we created it during this call. The wrapped `KeboolaApiError` carries `details.rollback={attempted, posted_children, deleted, failed_deletes, model_created_here, model_deleted, model_uuid}` so operators get full diagnostics. Pass `--keep-on-failure` to preserve the partial state for forensic inspection (mirrors `data-app create --keep-on-failure`); the wrapped error then carries `details.rollback.attempted=False, reason='keep_on_failure'`. - `semantic-layer token --encrypt --project P --component-id C` -- encrypt the project's storage token for a transformation's `user_properties`. Builds `{"#metastore_token": }` from the project's already-stored Storage API token and delegates to the existing EncryptService. `--encrypt` is currently required; other modes are refused with `USAGE_ERROR` (exit 2). Output (human): the raw envelope ready to paste. JSON: full `{encrypted, component_id, project}`. +### Scope / target-project grants / elevation (`scope` sub-app, PSGO-140) + +*(since vNEXT)* Every semantic-layer item carries a visibility `scope`: `project` (owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above. With `--scope` omitted, `model create` makes a `project` model and `add ` **inherits its model's scope and target projects**. `--target-project` takes a registered alias or a numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack) and needs `--scope targeted` (else exit 2). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other projects on the stack; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2). **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule; `--scope organization` is irreversible and gated as destructive. `edit`, `import --overwrite` and `promote` update the item in place (PUT), so an organization/targeted item keeps its scope, grants, id and pending elevation request. + +- `semantic-layer scope get --project P --type model|dataset|metric|relationship|constraint|glossary --context-id ID` -- show an item's current `scope`, `target_project_ids`, and any pending `scope_elevation_requested_at`. +- `semantic-layer scope add|remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...]` -- attach / detach target projects of a `--scope targeted` item (alias or numeric ID; repeatable or comma-separated). A client-side **merge** (read the grants, apply the delta, PUT) -- **not atomic** against a concurrent grant change. Refused (exit 2) from a project that does not own the item -- the server hides the grants from a non-owner, so use `scope set --target-project` there. +- `semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID ... | --clear) [--dry-run] [--yes]` -- exactly ONE of: `--target-project` (REPLACES the whole list), `--clear` (revokes every grant), or `--scope organization` (the only accepted value; elevation). A mixed request exits 2. Elevation needs the **organization-admin role** (403 otherwise), is **ONE-WAY** (no downgrade), is gated as destructive, and prompts unless `--yes`/`--json`; `--dry-run` only shows the change. +- `semantic-layer scope request-create --project P --type T --context-id ID` -- owner-only. Flags a project-scoped item as awaiting an organization admin's step-up decision. Idempotent: calling again just refreshes the timestamp. +- `semantic-layer scope request-delete --project P --type T --context-id ID` -- owner-only. Clears a pending elevation request. Idempotent no-op if none is pending. +- `semantic-layer scope request-list --project P --type T [--limit N] [--offset N]` -- one page (default `--limit 50`) of items of `--type` awaiting an elevation decision, across the whole organization; returns `{items, limit, offset, has_more}`. The org-admin's discovery queue. + +There is no bulk-elevate endpoint: promoting every semantic-layer object in an existing project to organization/targeted scope ("elevate existing projects") means one `scope request-create` + `scope set --scope organization` call per item, deliberately, only for objects the user has named -- never loop this speculatively over an entire project. + ### Reference data (dimension members, e.g. a Chart of Accounts) `semantic-reference-data` is a per-dimension member store: ONE record per dimension holding the full member list in a `members[]` array. The driving use case is a Chart of Accounts (the account list + all attributes) held in the metastore instead of a hardcoded Storage table. It is deliberately kept **outside** `build` / `export` / `diff` / cascade / `PUSH_ORDER` — its members come from `DIM_COA`, not from AI generation — so it has its own self-contained CRUD surface. Member field names mirror the `DIM_COA` columns 1:1 (snake_case: `account_code`, `account_name`, `parent_code`, `is_leaf`, `level_1_code`, `cf_category`, …). - `semantic-layer reference-data list --project P [--model M]` -- list dimension records (summaries: `id`, `dimension_name`, `model_uuid`, `dataset_id`, `member_count`). `--model` filters to one model. Members are omitted from the summary; use `get` for them. - `semantic-layer reference-data get --project P (--id ID | --dimension D)` -- fetch one record with all members. Resolve by record UUID (`--id`) or by `--dimension`. The dimension is **unique per project**, so the lookup is project-wide and needs no model. Passing both `--id` and `--dimension` (or neither) is a usage error (exit 2). Returns `{id, dimension_name, model_uuid, dataset_id, member_count, revision, members[]}`. -- `semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X]` -- create-or-replace. `--members-file` is a JSON array of member objects (`-` reads stdin). **Idempotent** on `dimension`: the lookup is project-wide (the envelope `name` = dimension is unique per project per type), so an existing record is replaced in place via `PUT` (the metastore increments `meta.revision`, preserving history) — distinct from the DELETE+POST used by `edit` — regardless of which `--model` is passed (the resolved model is stored on the record); otherwise a new record is `POST`-ed. Response: `{id, dimension_name, member_count, action: "created"|"updated"}`. +- `semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X]` -- create-or-replace. `--members-file` is a JSON array of member objects (`-` reads stdin). **Idempotent** on `dimension`: the lookup is project-wide (the envelope `name` = dimension is unique per project per type), so an existing record is replaced in place via `PUT` (the metastore increments `meta.revision`, preserving history) — like `edit` — regardless of which `--model` is passed (the resolved model is stored on the record); otherwise a new record is `POST`-ed. Response: `{id, dimension_name, member_count, action: "created"|"updated"}`. - `semantic-layer reference-data delete --project P --id ID [--yes]` -- delete by UUID (server-side soft-delete; the record stays in revision history). Non-TTY without `--yes` refuses with exit 2. ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index ecc76fd9..467d9cd4 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -180,18 +180,31 @@ Versioning convention: verbatim instead of asserting the token is invalid. **Rotating the token does not fix an `AUTH_REJECTED`**; verify the token against another endpoint on the same stack, then escalate with the exceptionId. -- **The reported trigger case has its own, more specific mapping: every - `semantic-layer` / `sl` command needs a MASTER token.** The Metastore's - 401 `{"exception": "Failed to create project scope"}` is its master-token - gate: unlike the Storage API, the metastore accepts only a master (project - admin) Storage token, and this opaque 401 is how it answers a valid - non-master token -- which blocked every `semantic-layer` command while - `project status` reported the token healthy (issue #711; the underlying - `MasterTokenRequiredError` is swallowed server-side). kbagent reclassifies - exactly that 401 to `ErrorCode.MISSING_MASTER_TOKEN` with the remedy in - the message. Fix: check `kbagent project info` -> `is_master_token`, and - register a master token (`kbagent project edit --token ...`). Do NOT - escalate this one to support -- it is by design. +- **The reported trigger case has its own, more specific mapping: a + `semantic-layer` / `sl` WRITE needs a project-admin token.** The + Metastore's 401 `{"exception": "Failed to create project scope"}` is its + admin-token gate: writes (add/edit/remove/import/promote/build/scope + add|remove|set|request-*) require the token's Storage Admin role to be + `admin` -- a master token qualifies, so does any other project-admin + user's token -- and this opaque 401 was how it answered a valid non-admin + token, on every call, while `project status` reported the token healthy + (issue #711; the underlying `MasterTokenRequiredError` was swallowed + server-side). kbagent reclassifies exactly that 401 to + `ErrorCode.MISSING_MASTER_TOKEN` with the remedy in the message. Fix: + check `kbagent project info` -> `is_master_token`, and register a + project-admin token (`kbagent project edit --token ...`). Do NOT escalate + this one to support -- it is by design. + - **(updated vNEXT -- PSGO-282, go-monorepo#596):** the "every call" + part above is now history. The Metastore no longer requires a master + token for a plain read (`show`, `model list`, `search-context`, + `get-context`, `validate`, `diff`, `scope get`, `scope request-list`) -- + any valid, non-disabled, non-expired Storage token works. Only writes + still hit this gate. `MISSING_MASTER_TOKEN` on a read now means either + an outdated kbagent replaying the old client-side reclassification, or + a metastore deployment that predates the fix -- not a real requirement. + See [Metastore no longer requires a master token for reads + (PSGO-282)](#metastore-no-longer-requires-a-master-token-for-reads-psgo-282) + below for the full writeup. - **Exit code is unchanged: 3, same as `INVALID_TOKEN`.** All three are authentication-class failures; only the diagnosis differs. A script branching on `$?` sees nothing new -- one branching on @@ -1168,7 +1181,7 @@ record per member. Consequences an agent must internalize: and `set` the whole thing back. There is no per-member endpoint. - **It uses the metastore's real `PUT`** (revisioned update, `meta.revision` increments, history preserved) when a record for that dimension already - exists — distinct from the DELETE+POST that `edit metric|…` uses. + exists — as `edit metric|…` does since vNEXT (before, those used DELETE+POST). A brand-new dimension is `POST`-ed. - **The envelope `name` is the dimension, unique per project per type, so the `set`/`get` lookup is project-wide.** Because the dimension name is the @@ -1594,11 +1607,12 @@ events and emits a final `done` SSE frame mirroring the same record. ## Metric rename auto-cascades through `CODE_METRIC` -- `kbagent semantic-layer edit metric --new-name NEW` does DELETE+POST - on the metric and ALSO DELETE+POST on every constraint whose - `metrics[]` referenced the old name (POST new with `metrics[]` - updated to the new name). The metastore has no PATCH endpoint, so - every "edit" is a delete-then-create. +- `kbagent semantic-layer edit metric --new-name NEW` updates the metric + and ALSO every constraint whose `metrics[]` referenced the old name + (`metrics[]` rewritten to the new name). *(updated vNEXT)* Each is an + in-place `PUT` -- same id, scope, grants and revision history, and a failed + update changes nothing; before, every "edit" was a delete-then-create with + a rollback re-POST. - The `CODE_METRIC` derived value (used in downstream SQL joins on `DIM_METRIC_THRESHOLD` / `FACT_METRIC_*` lookups) is computed via ```python @@ -1620,8 +1634,8 @@ events and emits a final `done` SSE frame mirroring the same record. is left in a partial state -- surface that to the operator and recommend running `semantic-layer validate` immediately. - **Partial-state envelope signal (updated v0.41.10 -- closes #294)**: - the cascade has per-item rollback only (each constraint DELETE+POST - rolls back individually), NOT whole-operation atomicity. If the + the cascade is per-item only (each constraint is updated on its own; + *since vNEXT* a PUT, so there is nothing to roll back), NOT whole-operation atomicity. If the metric rename succeeds but M of N dependent constraints fail to repoint, the response envelope sets `partial_state: true` and `recovery_hint: ""` @@ -1629,8 +1643,8 @@ events and emits a final `done` SSE frame mirroring the same record. buried inside `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Atomic two-phase commit was intentionally NOT - implemented: the metastore has no PATCH endpoint, so every - cascade 'stage' is itself a DELETE+POST that can fail; true + implemented: every + cascade 'stage' is itself a write that can fail; true atomicity would require side-staging every cascade item, which is disproportionate for a rename. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then @@ -5730,3 +5744,104 @@ workspaces". (also `--force`) keeps it and reports it as `skipped`; only `--theirs` deletes it. A `kbc` manifest save removes the key too (`kbc` writes back only the keys it knows). + +## Metastore no longer requires a master token for reads (PSGO-282) + +*(since vNEXT)* Every `semantic-layer` read (`show`, `model list`, +`search-context`, `get-context`, `validate`, `diff`, `scope get`, `scope +request-list`) works with any valid, non-disabled, non-expired Storage token -- +programmatic (`kbc_at_*`/`kbc_pat_*`) included. Before this, the metastore +rejected every non-master token with an opaque 401 `"Failed to create +project scope"` on every single endpoint, including plain reads (#711); a +valid non-master token used to get reclassified client-side to +`MISSING_MASTER_TOKEN` (0.92.0+, #717) because there was no other way to +succeed. That reclassification still fires, but now only on a genuine +authorization failure: + +- **Writes still need a project-admin token.** `add`/`edit`/`remove`/ + `import`/`promote`/`build`/`scope add|remove|set`/`scope request-create` + all require the token's Storage `Admin.Role` to be `admin` -- a master + token qualifies, and so does any other project-admin user's token. A + non-admin token gets 403 (or `MISSING_MASTER_TOKEN` if the install is still + on the old reclassification path) on the write, never on a preceding read. +- **Do not pre-flight `isMasterToken` before a read.** That workaround + predates the fix and now rejects tokens the server happily accepts -- + drop it if you see it in older automation. +- **401 body is honest now.** The metastore relays the Storage API's real + error message instead of the generic "Failed to create project scope", + so an actually-invalid/expired token surfaces as such rather than looking + like a master-token gate. + +## Metastore `scope` / `--target-project` / `scope set --scope organization` (PSGO-140) + +*(since vNEXT)* `model create` / `add ` gained `--scope +project|organization|targeted` + `--target-project ALIAS|ID ...`, and a new +`semantic-layer scope ` +sub-app (also `GET/PUT /semantic-layer/scope/...` over `kbagent serve`). See +[metastore-scope-workflow.md](metastore-scope-workflow.md) for worked +examples. Surprises worth knowing before you touch this: + +- **kbagent never sends `schemaVersion` on create.** The metastore stores + the version it resolved for the item, and checks a later elevation (`PATCH + {"scope":"organization"}`, the elevation request) against that STORED + version. Every `semantic-*` schema supports only `scope=project` at `1.0.0` + (later schema versions add `organization`/`targeted`), so pinning `1.0.0` on a plain + create would make the item impossible to elevate later. With the key + omitted, the server uses the stack's default schema (`1.1.0` or later where + the PSGO-140 migrations ran). `PUT` (edit) never changes the stored version. +- **`--scope` omitted means "inherit" for child items.** `add + metric|dataset|relationship|constraint|glossary` take their model's scope + (and, for `targeted`, its target projects) when `--scope` is not given; + `model create` defaults to `project`. Pass `--scope` to override. Without + this an org-level model would show up in consumer projects with no + datasets or metrics. An INHERITED `organization` scope is gated exactly like + a typed `--scope organization` (`--deny-destructive` blocks it), and a + project-admin token that is not an org admin gets a 403 on it, with a hint + to pass `--scope project`. +- **`--target-project` takes an alias or a numeric project ID** (repeatable or + comma-separated), so the target need not be registered in kbagent. An alias + must be on the owner project's stack (a project ID only means the same + project on its own stack; a bare ID cannot be checked and is trusted). + `--target-project` without `--scope targeted`, an unknown alias, or a bad + `--scope`/`--type` value exits 2 -- never a traceback or a silent ignore. +- **`scope set --target-project` is a REPLACE on the wire; `scope add|remove` + are a client-side merge.** The underlying `PUT .../target-projects` replaces + the *whole* list (an empty array clears every grant); add/remove read the + current grants, apply the delta and PUT the result -- **not atomic** against + a concurrent grant change (last write wins). The server returns + `meta.targetProjectIds` only to the OWNING project, so add/remove from any + other project (an org admin elsewhere) is refused with exit 2 instead of + silently replacing every grant -- use `scope set --target-project` there. +- **Creating directly at `--scope organization` needs the organization-admin + ROLE**, the same role `scope set --scope organization` needs -- a normal + project token gets `ACCESS_DENIED` (403) on both. `--scope targeted` needs a + project-admin token (the same gate as every write). +- **Elevation is one-way, so `--scope organization` is destructive-class.** + There is no downgrade endpoint: the only way "back" is deleting and + re-creating the item as project-scoped (losing its revision history). The + permission engine escalates `--scope organization` on `scope set`, `model + create` and every `add ` (`FLAG_ESCALATIONS`), so `--deny-destructive` + blocks them, `scope set --scope organization --dry-run` included (same + as `sync push --force`). +- **Elevating items created by an older kbagent can fail.** Those were created + pinned to `schemaVersion 1.0.0`, which supports only `project`, and the + server checks elevation against the STORED version, so `scope set --scope + organization` / `scope request-create` fail with a scope-not-supported error + until the metastore's schema sweep moves the item to the default version. + *Read from the server source, not yet verified on a live stack.* +- **A 403 vs a 404 on a scope call encodes something deliberate.** The + metastore returns 404 (not 403) when the caller cannot see the object at + all -- not its owner, not a granted project, not viewing an + organization-scoped object -- so a 403 never leaks "an object owned by + some other project exists." Do not collapse the two when surfacing errors. +- **`edit`, `import --overwrite` and `promote` update in place (PUT).** The + item keeps its id, scope, grants and pending elevation request. They used to + DELETE+POST, which reset an organization/targeted item to `project` scope + (`import --overwrite`, `promote`) or could lose the item when the re-create + was refused (`edit`). A failed PUT changes nothing. Note that `promote`'s + diff compares `attributes` only, so a scope difference between source and + target is neither shown nor copied. +- **No bulk-elevate.** There is no "promote every object in this project" + endpoint -- each item needs its own `request-create` + `set --scope + organization` call. Never loop this over a whole project's objects without + the user having named which ones should become org-wide. diff --git a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md new file mode 100644 index 00000000..ed15e43a --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md @@ -0,0 +1,154 @@ +# Metastore Scope Workflow -- Sharing & Organization-Wide Objects (PSGO-140) + +Every metastore item (semantic-layer model, dataset, metric, relationship, +constraint, glossary term) carries a visibility **scope**: + +- `project` -- visible only to the owning project. A `model create` without + `--scope` creates a project-scoped model. +- `targeted` -- owner project + an explicit **target-project grant list** + (repeatable `--target-project`, alias or numeric project ID). Creating at + this scope needs a **project-admin** token, like every metastore write. +- `organization` -- visible to **every** project in the organization. + Creating directly at this scope, or *elevating* an existing item to it, + requires the **organization-admin** role -- a normal project token gets + `ACCESS_DENIED` (403). + +`add ` without `--scope` **inherits its model's scope** (and target +projects), so an org-level model gets org-level children. Pass `--scope` to +override (a project-admin token that is not an org admin must pass +`--scope project` under an org-level model, or it gets a 403). + +For one-line command reference, see +[commands-reference.md](commands-reference.md#scope--target-project-grants--elevation-scope-sub-app-since-vnext-psgo-140). +For the schema-version / replace-vs-merge / 403-vs-404 surprises, see +[gotchas.md](gotchas.md). + +**Hard rule for AI agents**: never pass `--scope organization|targeted`, and +never run `scope set --scope organization` / `scope request-create`, without +the user having explicitly named which project(s) should gain visibility. +Widening an object's visibility is a security-relevant decision, and +organization-scope elevation has **no downgrade endpoint** -- it cannot be +undone via the API. When in doubt, create/leave the item at `project` scope +and ask. + +## When to use what + +| Goal | Command | +|------|---------| +| Check an item's current scope/grants/pending elevation | `semantic-layer scope get --type T --context-id ID` | +| Create a new item visible to a couple of named projects | `model create` / `add ` with `--scope targeted --target-project ALIAS\|ID ...` | +| Add / remove target projects on an existing targeted item | `scope add --target-project X` / `scope remove --target-project X` (from the owning project) | +| Replace the whole target-project list in one call | `scope set --target-project X --target-project Y` | +| Revoke every grant (item becomes owner-only again) | `scope set --clear` | +| Ask an org-admin to make an item org-wide | `scope request-create` (owner-only) | +| Cancel a pending elevation request | `scope request-delete` (owner-only) | +| Actually make an item org-wide (org-admin token) | `scope set --scope organization --yes` (preview with `--dry-run`) | +| Find items awaiting an elevation decision | `scope request-list --type T` (org-admin token) | + +--- + +## Workflow 1 -- Create an item shared with specific projects + +```bash +# Ask the user which project(s) should see this metric BEFORE running this. +kbagent --json semantic-layer add metric \ + --project prod --model core_model \ + --name gross_margin --sql "SUM(revenue) - SUM(cogs)" --dataset out.c-fin.fact_pnl \ + --scope targeted --target-project analytics-prod,5678 +``` + +`--target-project` is repeatable or comma-separated and takes a registered +project **alias** or a numeric project **ID** (so the target need not be +registered). An alias must be on the owner project's stack. Omitting +`--target-project` with `--scope targeted`: + +- On a real terminal: launches an interactive checkbox picker over every + *other* registered project on the same stack. +- In `--json` / non-interactive mode: fails fast with `INVALID_ARGUMENT` + (exit 2) instead of guessing. There is always an explicit target list -- + never a silent default. + +`--target-project` without `--scope targeted`, or an unknown alias, also +exits 2. + +## Workflow 2 -- Adjust grants on an existing targeted item + +```bash +# See what's granted today +kbagent --json semantic-layer scope get --project prod --type metric --context-id +# -> {"scope": "targeted", "target_project_ids": [1234, 5678], ...} + +# Attach / detach one project (client-side merge -- NOT atomic against a +# concurrent grant change; only from the OWNING project) +kbagent semantic-layer scope add --project prod --type metric --context-id \ + --target-project new-team-prod +kbagent semantic-layer scope remove --project prod --type metric --context-id \ + --target-project 5678 + +# Replace the whole set in one round trip (the API's native semantics) +kbagent semantic-layer scope set --project prod --type metric --context-id \ + --target-project analytics-prod,5678 + +# Revoke every grant -- item becomes owner-only again +kbagent semantic-layer scope set --project prod --type metric --context-id --clear +``` + +These only work on an item already created with `scope="targeted"` -- they +400 against a project- or organization-scoped item. `scope add|remove` from a +project that does not own the item (an org admin elsewhere) are refused +(exit 2): the server hides the current grants from a non-owner, so a merge +would overwrite them. Use `scope set --target-project` in that case. + +## Workflow 3 -- Make an item organization-wide + +Two-step, deliberately: the owner requests, an organization-admin decides. + +```bash +# 1. Owner project flags the item (idempotent -- re-running just refreshes +# the timestamp) +kbagent semantic-layer scope request-create --project prod --type dataset --context-id + +# 2. An organization-admin discovers the queue... +kbagent --json semantic-layer scope request-list --project prod --type dataset --limit 50 +# -> {"items": [{"id": "", "name": "fact_pnl", ...}], "limit": 50, "offset": 0, "has_more": false} + +# 3. ...previews, then elevates. IRREVERSIBLE -- confirmation prompt unless --yes/--json. +kbagent semantic-layer scope set --project prod --type dataset --context-id \ + --scope organization --dry-run +kbagent semantic-layer scope set --project prod --type dataset --context-id \ + --scope organization --yes +``` + +A caller who already holds the organization-admin role can run step 3 +directly (skipping step 1) -- `request-create` exists for the common case +where the owner and the admin are different people/tokens. +`scope request-delete` cancels a pending request before an admin acts on it. + +`--scope organization` is gated as **destructive** by the permission engine +(`--deny-destructive` blocks it, here and on `model create` / `add `); +`--dry-run` is gated too (same as `sync push --force`). An `add ` that +would INHERIT `organization` from its model is gated the same way. + +There is **no bulk-elevate endpoint**. "Elevate an existing project's +objects" as a migration means one `request-create` + `set --scope +organization` call per item, run deliberately for objects the user has named +-- never loop this over every object in a project speculatively. Items +created by a kbagent older than this feature may refuse elevation (they were +pinned to schema `1.0.0`, which only supports `project`); if so, re-create +them. + +## Workflow 4 -- Editing a scoped item + +`semantic-layer edit `, `import --overwrite` and `promote` update the +item **in place** (PUT): it keeps its id, scope, target projects and any +pending elevation request, and a failed update changes nothing. Nothing to do +here; this is just so you don't have to re-grant after every rename. + +## Over `kbagent serve` + +`GET|PUT /semantic-layer/scope/{context_id}`, `POST|DELETE +.../target-projects`, `PUT|DELETE .../elevation-request` and `GET +/semantic-layer/scope/elevation-requests` mirror the commands above; +`POST /semantic-layer/models` and `/items/{kind}` accept `scope` + +`target_projects`. Like the other semantic-layer routes they do not consult the +`permissions` policy -- the serve bearer token is the gate. diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index fd165e76..8124b9dd 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -4,12 +4,17 @@ The Keboola semantic layer (aka **metastore**) is a project-scoped catalogue of datasets, metrics, relationships, constraints, and glossary terms. It is served from a separate API at `metastore.` (derived from `connection.` by string-substitution; cloud/region-agnostic). Auth is -the same `X-StorageApi-Token` as Storage, **with one extra requirement: it -must be a MASTER (project admin) token**. The metastore's auth gate rejects -every valid non-master token with an opaque 401 `Failed to create project -scope`, which kbagent reclassifies to `MISSING_MASTER_TOKEN` with the remedy -(since 0.92.0, #711). Pre-flight: `kbagent --json project info --project P` --> `is_master_token`. +the same `X-StorageApi-Token` as Storage: **reads work with any valid, +non-disabled, non-expired token** (vNEXT, PSGO-282); **writes** (add / +edit / remove / import / promote / build / scope add|remove|set) still need +a project-admin token -- a master token qualifies, but so does any other +project-admin user's token. Before PSGO-282, the metastore's auth gate +rejected every valid non-master token -- including reads -- with an opaque +401 `Failed to create project scope`, which kbagent reclassifies to +`MISSING_MASTER_TOKEN` with the remedy (since 0.92.0, #711); that +reclassification now only fires on a write against a non-admin token, or +against a metastore deployment that predates the fix. Pre-flight for a +write: `kbagent --json project info --project P` -> `is_master_token`. `kbagent semantic-layer ...` (alias `kbagent sl ...`, hidden) wraps the metastore so AI agents and CI scripts don't roll their own `urllib` loops. @@ -18,6 +23,10 @@ For one-line command reference, see For the live-validated metastore contract surprises (constraint rule shape, name regex, CODE_METRIC cascade), see [gotchas.md](gotchas.md#semantic-layer-constraint-rule-is-a-string-not-an-object-since-v0340). +To share an item with specific other projects, or make it organization-wide, +see [metastore-scope-workflow.md](metastore-scope-workflow.md) -- **always +ask the user which project(s) first**, this is a security-relevant, mostly +irreversible decision. ## When to use what @@ -34,6 +43,7 @@ name regex, CODE_METRIC cascade), see | Promote dev -> prod | `semantic-layer promote --from-project dev --to-project prod` | | Bootstrap a model from storage tables | `semantic-layer build --tables ...` (heuristic) | | Encrypt the storage token for a Python container | `semantic-layer token --encrypt` | +| Share an item with named projects, or make it org-wide | see [metastore-scope-workflow.md](metastore-scope-workflow.md) | --- @@ -217,7 +227,7 @@ kbagent semantic-layer edit metric \ --new-name revenue_growth_qoq # Will print something like: # CODE_METRIC: REVENUE_GROWTH -> REVENUE_GROWTH_QOQ -# Will DELETE+POST the following constraints (cascade): +# Will update the following constraints in place (cascade): # - revenue_growth_minimum_warning # - revenue_growth_band_review # Proceed? [y/N]: @@ -235,14 +245,13 @@ kbagent --json semantic-layer show --project prod --model core_model --type cons # These joins WILL silently start returning empty rows otherwise. ``` -If the new POST fails (e.g. the new name collides), the service -re-POSTs `original_attrs` to roll back and reports rollback -success/failure in the response envelope's `rollback` field. If the -rollback itself fails, the model is left in a partial state -- run -`semantic-layer validate` immediately. +Edits are an in-place `PUT`, so if the update fails (e.g. the new name +collides) nothing was deleted and nothing needs rolling back -- the item is +exactly as it was (the envelope's `rollback` field is always `null`). Items +keep their id, scope and revision history across an edit. -**Partial cascade state**: the cascade has per-item -rollback only -- each constraint DELETE+POST rolls back individually. +**Partial cascade state**: each cascaded constraint is +updated independently (an in-place PUT, no rollback needed). If the metric rename succeeds but M of N dependent constraints fail to repoint, the envelope sets `partial_state: true` at the top level and a `recovery_hint` string. Human-mode CLI prints a bright red @@ -257,10 +266,9 @@ kbagent semantic-layer edit constraint --project prod --model core_model \ --name revenue_growth_minimum_warning --new-metrics revenue_growth_qoq ``` -Atomic two-phase commit was rejected as disproportionate: the -metastore has no PATCH endpoint, so every cascade 'stage' is itself a -DELETE+POST that can fail; true atomicity would require side-staging -every cascade item. +Atomic two-phase commit was rejected as disproportionate: every cascade +'stage' is itself a PUT that can fail; true atomicity would require +side-staging every cascade item. --- @@ -318,7 +326,7 @@ kbagent --json semantic-layer promote \ | jq '.metrics, .constraints' # Inspect: # metrics.new -- count of items the promote will POST -# metrics.overwritten -- count of CHANGED items that will be DELETE+POSTed +# metrics.overwritten -- count of CHANGED items that will be updated in place # metrics.identical -- count of items skipped (already match) # metrics.changes[] -- per-item diff with diff_keys # metrics.failed[] -- per-item failures (e.g. dry-run validators) @@ -501,8 +509,8 @@ For the live-validated metastore contract surprises -- where the Quick reminders: -- **POST envelope**: `{name, data, branch: "main", schemaVersion: - "1.0.0", scope: "project"}` -> 201 with `{data: {type, id, +- **POST envelope**: `{name, data, branch: "main", scope: "project"}` + (no `schemaVersion`: the server stores the stack's default) -> 201 with `{data: {type, id, attributes, meta}}`. kbagent handles this. - **Duplicate-name POST -> 409 Conflict** (post go-monorepo PR #513) with `"Object with this name already exists in this project"`, or **500** with @@ -511,10 +519,11 @@ Quick reminders: the fix-deployed path avoids the `MAX_RETRIES` round-trips the 500 path still pays. - **DELETE -> 204** empty body. -- **No PATCH endpoint** -- every "edit" is DELETE+POST. kbagent's - `edit metric / dataset / relationship / constraint / glossary` - (five sub-subcommands matching `add`) rolls back on POST failure. - Same five types are available under `remove`. +- **Data edits are a `PUT`** (revisioned, in place; `PATCH` only changes + the scope). kbagent's `edit metric / dataset / relationship / constraint + / glossary` (five sub-subcommands matching `add`) keep the item's id, + scope and grants, and a failed PUT changes nothing. Same five types are + available under `remove`. - **`X-StorageApi-Token` is the only auth** -- no separate metastore token. `kbagent semantic-layer token --encrypt` encrypts the STORAGE token for a `user_properties` slot named `#metastore_token`, diff --git a/src/keboola_agent_cli/commands/_semantic_layer_crud.py b/src/keboola_agent_cli/commands/_semantic_layer_crud.py index 18e1d84f..9cba3429 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_crud.py @@ -18,7 +18,21 @@ get_formatter, get_service, ) -from ._semantic_layer_helpers import _handle_service_call, _is_stdin_tty +from ._semantic_layer_helpers import ( + ScopeChoice, + _handle_service_call, + _is_stdin_tty, + resolve_scope_targets, +) + +_SCOPE_HELP = ( + "Visibility: 'project' (owner only), 'organization' (every project in the org), or " + "'targeted' (owner + --target-project grants). Default: the model's own scope." +) +_TARGET_PROJECT_HELP = ( + "Project alias or ID to grant visibility to (repeatable or comma-separated; " + "--scope targeted only)." +) # --------------------------------------------------------------------------- # semantic-layer add -- one sub-subcommand per entity type @@ -68,10 +82,20 @@ def add_metric( dataset: str = typer.Option(..., "--dataset", help="Dataset tableId this metric belongs to"), description: str = typer.Option("", "--description", help="Optional description"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the dataset-mismatch warning"), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a metric to a semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.metric", + scope=scope, + target_project=target_project, + owner_alias=project, + inherit_from_model=(model,), + ) result = _handle_service_call( ctx, service.add_metric, @@ -84,6 +108,8 @@ def add_metric( assume_yes=yes, is_tty=_is_stdin_tty(), confirm_cb=typer.confirm, + scope=scope, + target_projects=target_projects, ) formatter.output(result, _print_item_added("metric")) @@ -115,10 +141,20 @@ def add_dataset( 'location from Storage, e.g. \'"DB"."out.c-bucket"."table"\'.' ), ), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a dataset (FQN read from the table's Storage location).""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.dataset", + scope=scope, + target_project=target_project, + owner_alias=project, + inherit_from_model=(model,), + ) result = _handle_service_call( ctx, service.add_dataset, @@ -131,6 +167,8 @@ def add_dataset( primary_key=primary_key, deep_fields=deep_fields, fqn=fqn, + scope=scope, + target_projects=target_projects, ) formatter.output(result, _print_item_added("dataset")) @@ -145,10 +183,20 @@ def add_relationship( to: str = typer.Option(..., "--to", help="Target dataset tableId"), on: str = typer.Option(..., "--on", help="Join condition"), type_: str = typer.Option("left", "--type", help="Join type: 'left' or 'inner'."), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a relationship between two datasets.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.relationship", + scope=scope, + target_project=target_project, + owner_alias=project, + inherit_from_model=(model,), + ) result = _handle_service_call( ctx, service.add_relationship, @@ -159,6 +207,8 @@ def add_relationship( to=to, on=on, type_=type_, + scope=scope, + target_projects=target_projects, ) formatter.output(result, _print_item_added("relationship")) @@ -194,6 +244,8 @@ def add_constraint( severity: str = typer.Option( "warning", "--severity", help="One of: error|warning|info (the 3-level API enum)." ), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a constraint.""" formatter = get_formatter(ctx) @@ -205,6 +257,14 @@ def add_constraint( error_code=ErrorCode.VALIDATION_ERROR, ) raise typer.Exit(code=2) + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.constraint", + scope=scope, + target_project=target_project, + owner_alias=project, + inherit_from_model=(model,), + ) result = _handle_service_call( ctx, service.add_constraint, @@ -215,6 +275,8 @@ def add_constraint( rule=rule, metrics=metrics_list, severity=severity, + scope=scope, + target_projects=target_projects, ) formatter.output(result, _print_item_added("constraint")) @@ -226,10 +288,20 @@ def add_glossary( model: str | None = typer.Option(None, "--model", help="Model name or UUID"), term: str = typer.Option(..., "--term", help="Glossary term"), definition: str = typer.Option("", "--definition", help="Optional definition"), + scope: ScopeChoice | None = typer.Option(None, "--scope", help=_SCOPE_HELP), + target_project: list[str] = typer.Option([], "--target-project", help=_TARGET_PROJECT_HELP), ) -> None: """Add a glossary term.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.add.glossary", + scope=scope, + target_project=target_project, + owner_alias=project, + inherit_from_model=(model,), + ) result = _handle_service_call( ctx, service.add_glossary, @@ -237,12 +309,14 @@ def add_glossary( model_name_or_uuid=model, term=term, definition=definition, + scope=scope, + target_projects=target_projects, ) formatter.output(result, _print_item_added("glossary")) # --------------------------------------------------------------------------- -# semantic-layer edit -- DELETE+POST with rollback + rename cascade +# semantic-layer edit -- in-place PUT + rename cascade # --------------------------------------------------------------------------- @@ -250,7 +324,7 @@ def add_glossary( name="edit", help=( "Edit a metric, dataset, constraint, relationship, or glossary term " - "(DELETE+POST with rollback)." + "(in-place update; keeps scope and id)." ), no_args_is_help=True, ) @@ -383,7 +457,7 @@ def edit_constraint( None, "--new-metrics", help="Comma-separated list of metric names" ), ) -> None: - """Edit a constraint (DELETE+POST, with local validators).""" + """Edit a constraint (in-place update, with local validators).""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") metrics_list = ( @@ -420,7 +494,7 @@ def edit_relationship( None, "--new-type", help="Replace join type (left | inner)" ), ) -> None: - """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally.""" + """Edit a relationship (in-place update). Validates ``--new-type`` locally.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") result = _handle_service_call( diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py index b48bb42c..78080bb7 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -9,11 +9,38 @@ from __future__ import annotations import sys +from enum import StrEnum import typer from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ._helpers import get_formatter, map_error_to_exit_code +from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select +from ._helpers import check_cli_operation, get_formatter, get_service, map_error_to_exit_code + + +class ScopeChoice(StrEnum): + """Fixed values of ``--scope`` on the create commands (rendered as a click Choice).""" + + PROJECT = "project" + ORGANIZATION = "organization" + TARGETED = "targeted" + + +class ElevationScope(StrEnum): + """``scope set --scope`` accepts only ``organization`` (scope changes are one-way).""" + + ORGANIZATION = "organization" + + +class ItemType(StrEnum): + """Semantic item kinds addressable by ``scope`` commands (``--type``).""" + + MODEL = "model" + DATASET = "dataset" + METRIC = "metric" + RELATIONSHIP = "relationship" + CONSTRAINT = "constraint" + GLOSSARY = "glossary" def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ignore[no-untyped-def] @@ -36,9 +63,119 @@ def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ig retryable=exc.retryable, details=exc.details, ) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + # A bad option value is a usage error (exit 2), not a general failure. + is_usage_error = exc.error_code == ErrorCode.INVALID_ARGUMENT + raise typer.Exit(code=2 if is_usage_error else map_error_to_exit_code(exc)) from None def _is_stdin_tty() -> bool: """Return ``True`` when stdin is attached to a TTY (interactive shell).""" return hasattr(sys.stdin, "isatty") and sys.stdin.isatty() + + +def resolve_scope_targets( + ctx: typer.Context, + *, + operation: str, + scope: str | None, + target_project: list[str] | None, + owner_alias: str, + inherit_from_model: tuple[str | None] | None = None, +) -> list[str] | None: + """Validate ``--scope`` / ``--target-project`` of a create command. + + Returns the ``--target-project`` values (aliases or IDs; the service + resolves them) for ``--scope targeted``, else ``None``. ``--scope + organization`` is gated here as destructive (``operation`` is the command's + permission key, e.g. ``semantic-layer.add.dataset``); so is an omitted + ``--scope`` that would inherit ``organization`` from the model + (``inherit_from_model=(model,)`` on the ``add`` commands). ``--target-project`` + without ``--scope targeted`` is a usage error, never silently ignored. + + With ``--scope targeted`` and no target, this is the "ask when uncertain" + mechanism: on a real terminal it launches the checkbox picker over every + OTHER registered project on the owner's stack; in a non-TTY or ``--json`` + context it hard-fails instead of silently defaulting -- widening an + object's visibility across projects is not a guess this CLI makes. + """ + engine = ctx.obj.get("permission_engine") + if scope is None and inherit_from_model is not None and engine is not None and engine.active: + # `add ` without --scope takes its model's scope: an inherited + # organization scope widens visibility just like a typed one, so it goes + # through the same permission gate. + service = get_service(ctx, "semantic_layer_service") + inherited = _handle_service_call( + ctx, service.child_scope, alias=owner_alias, model_name_or_uuid=inherit_from_model[0] + ) + if inherited == "organization": + check_cli_operation(ctx, f"{operation} --scope organization") + if scope == "organization": + check_cli_operation(ctx, f"{operation} --scope organization") + if scope != "targeted": + if target_project: + get_formatter(ctx).error( + message="--target-project requires --scope targeted.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + return None + if target_project: + return list(target_project) + + formatter = get_formatter(ctx) + project_service = get_service(ctx, "project_service") + projects = project_service.list_projects() + owner_stack = next((p["stack_url"] for p in projects if p["alias"] == owner_alias), None) + candidates = [ + p + for p in projects + if p["alias"] != owner_alias and (owner_stack is None or p["stack_url"] == owner_stack) + ] + if not candidates: + formatter.error( + message=( + "--scope targeted needs at least one other registered project to " + "target. Register one with `kbagent project add`, pass " + "--target-project, or use --scope project." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + if formatter.json_mode: + formatter.error( + message=( + "--scope targeted requires --target-project (repeatable) in " + "--json mode -- there is no terminal to pick from." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + items = [ + CheckboxItem( + label=p["alias"], + hint=f"{p.get('project_name', '')} ({p.get('project_id', '?')})", + ) + for p in candidates + ] + try: + selected = checkbox_select( + items, title="Select project(s) to grant visibility to (targeted scope):" + ) + except CheckboxUnavailable: + formatter.error( + message=( + "--scope targeted requires --target-project (repeatable) -- no " + "interactive terminal available to pick from." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + if not selected: + formatter.error( + message="No target project selected. Pass --target-project or use --scope project.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + return [candidates[i]["alias"] for i in selected] diff --git a/src/keboola_agent_cli/commands/_semantic_layer_scope.py b/src/keboola_agent_cli/commands/_semantic_layer_scope.py new file mode 100644 index 00000000..286d5f26 --- /dev/null +++ b/src/keboola_agent_cli/commands/_semantic_layer_scope.py @@ -0,0 +1,252 @@ +"""Typer sub-app for ``kbagent semantic-layer scope`` (PSGO-140). + +Extracted from :mod:`commands.semantic_layer` for the same LOC-ceiling reason +as ``_semantic_layer_crud.py``. Thin per the 3-layer architecture: all scope +resolution and metastore calls live in +:meth:`SemanticLayerService.scope_*` / :mod:`services._semantic_layer_scope`. + +Verbs follow the CLI spec (#791): ``get`` reads one attribute, ``add`` / +``remove`` attach / detach a target project, ``set`` writes the scope, +``request-create`` / ``request-delete`` / ``request-list`` manage the +elevation request. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.table import Table + +from ..errors import ErrorCode +from ..services._semantic_layer_scope import DEFAULT_REQUEST_LIST_LIMIT +from ._helpers import check_cli_operation, check_cli_permission, get_formatter, get_service +from ._semantic_layer_helpers import ElevationScope, ItemType, _handle_service_call + +scope_app = typer.Typer( + name="scope", + help=( + "Manage an item's visibility scope (project/organization/targeted), " + "target-project grants, and organization-scope elevation requests." + ), + no_args_is_help=True, +) + +_PROJECT = typer.Option(..., "--project", help="Owning project alias") +_TYPE = typer.Option(..., "--type", help="Semantic item type") +_CONTEXT_ID = typer.Option(..., "--context-id", help="Item UUID") +_TARGET_PROJECT = typer.Option( + [], + "--target-project", + help="Project alias or ID (repeatable, or comma-separated).", +) + + +@scope_app.callback(invoke_without_command=True) +def _scope_permission_check(ctx: typer.Context) -> None: + """Permission check for the ``scope`` sub-app. + + Composes ``semantic-layer.scope.{subcommand}``: ``get`` / ``request-list`` + are ``read``, the rest are ``write``. ``set --scope organization`` is + escalated to ``destructive`` in the command body (irreversible, widens + visibility org-wide) -- see ``permissions.FLAG_ESCALATIONS``. + """ + check_cli_permission(ctx, "semantic-layer.scope") + + +def _print_scope_status(console: Console, data: dict) -> None: + if data.get("dry_run"): + console.print("[yellow]Dry run -- nothing was changed.[/yellow]") + console.print(f"[bold]scope:[/bold] {data.get('scope', 'project')}") + targets = data.get("target_project_ids") + if targets is not None: + console.print(f"[bold]target_project_ids:[/bold] {targets}") + if data.get("would_set_scope"): + console.print(f"[bold]would set scope:[/bold] {data['would_set_scope']}") + if data.get("would_set_target_project_ids") is not None: + console.print( + f"[bold]would set target_project_ids:[/bold] {data['would_set_target_project_ids']}" + ) + pending = data.get("scope_elevation_requested_at") + if pending: + console.print(f"[bold]scope_elevation_requested_at:[/bold] {pending}") + + +def _print_request_table(console: Console, data: dict) -> None: + items = data.get("items", []) + if not items: + console.print("[dim]No items awaiting scope elevation.[/dim]") + return + table = Table(title="Pending scope-elevation requests") + table.add_column("Name", style="bold cyan") + table.add_column("ID", style="dim") + table.add_column("Requested at") + for item in items: + table.add_row( + item.get("name") or "", + item.get("id") or "", + str(item.get("scope_elevation_requested_at") or ""), + ) + console.print(table) + if data.get("has_more"): + console.print( + f"[dim]More results: re-run with --offset {data['offset'] + data['limit']}.[/dim]" + ) + + +@scope_app.command("get") +def scope_get( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, +) -> None: + """Show an item's current scope, target-project grants, and pending elevation.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_get, alias=project, kind=type_, context_id=context_id + ) + formatter.output(result, _print_scope_status) + + +def _update_targets(ctx: typer.Context, project: str, type_: str, context_id: str, **delta) -> None: # type: ignore[no-untyped-def] + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, + service.scope_update_targets, + alias=project, + kind=type_, + context_id=context_id, + **delta, + ) + formatter.output(result, _print_scope_status) + + +def _require_targets(ctx: typer.Context, target_project: list[str]) -> None: + if not target_project: + get_formatter(ctx).error( + message="--target-project is required.", error_code=ErrorCode.INVALID_ARGUMENT + ) + raise typer.Exit(code=2) + + +@scope_app.command("add") +def scope_add( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + target_project: list[str] = _TARGET_PROJECT, +) -> None: + """Add target projects to a targeted-scope item (merges with the current grants).""" + _require_targets(ctx, target_project) + _update_targets(ctx, project, type_, context_id, add=target_project) + + +@scope_app.command("remove") +def scope_remove( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + target_project: list[str] = _TARGET_PROJECT, +) -> None: + """Remove target projects from a targeted-scope item (merges with the current grants).""" + _require_targets(ctx, target_project) + _update_targets(ctx, project, type_, context_id, remove=target_project) + + +@scope_app.command("set") +def scope_set( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, + scope: ElevationScope | None = typer.Option( + None, + "--scope", + help="Step the item up to organization scope. Requires org-admin; ONE-WAY, no downgrade.", + ), + target_project: list[str] = _TARGET_PROJECT, + clear: bool = typer.Option(False, "--clear", help="Clear every target project."), + dry_run: bool = typer.Option(False, "--dry-run", help="Show the change without applying it."), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the elevation confirmation."), +) -> None: + """Write an item's scope: elevate to organization, or replace/clear its target projects. + + Pass exactly one of --scope organization, --target-project (replaces the + whole list) or --clear. Elevating gives every project in the organization + read access to the item and its full revision history, and cannot be undone. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + if scope == "organization": + check_cli_operation(ctx, "semantic-layer.scope.set --scope organization") + if not (yes or dry_run or formatter.json_mode) and not typer.confirm( + f"Elevate {type_} {context_id!r} to organization scope? This is IRREVERSIBLE " + "and makes it visible to every project in the organization. Continue?" + ): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + result = _handle_service_call( + ctx, + service.scope_set, + alias=project, + kind=type_, + context_id=context_id, + scope=scope, + target_projects=target_project, + clear=clear, + dry_run=dry_run, + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("request-create") +def scope_request_create( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, +) -> None: + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_request_create, alias=project, kind=type_, context_id=context_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("request-delete") +def scope_request_delete( + ctx: typer.Context, + project: str = _PROJECT, + type_: ItemType = _TYPE, + context_id: str = _CONTEXT_ID, +) -> None: + """Withdraw a pending scope-elevation request. Idempotent no-op if none is pending.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_request_delete, alias=project, kind=type_, context_id=context_id + ) + formatter.output(result, _print_scope_status) + + +@scope_app.command("request-list") +def scope_request_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias (org-admin token)"), + type_: ItemType = _TYPE, + limit: int = typer.Option(DEFAULT_REQUEST_LIST_LIMIT, "--limit", min=1, help="Max results"), + offset: int = typer.Option(0, "--offset", min=0, help="Skip this many results"), +) -> None: + """List items of --type awaiting an org-admin's elevation decision, across the org.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "semantic_layer_service") + result = _handle_service_call( + ctx, service.scope_request_list, alias=project, kind=type_, limit=limit, offset=offset + ) + formatter.output(result, _print_request_table) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 2b7593fb..cccb49b7 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1751,18 +1751,34 @@ Manage Keboola metastore models: datasets, metrics, relationships, constraints, glossary terms. Metastore URL derived from stack URL by replacing `connection.` -with `metastore.`. Auth: same `X-StorageApi-Token` as Storage, but it MUST be a -MASTER (project admin) token -- the metastore rejects valid non-master tokens -with an opaque 401 "Failed to create project scope", reclassified by kbagent to -MISSING_MASTER_TOKEN (exit 3) with the remedy (#711). Pre-flight: +with `metastore.`. Auth: same `X-StorageApi-Token` as Storage. READS work with +any valid, non-disabled, non-expired token (PSGO-282); WRITES (add, edit, +remove, import, promote, build, scope add|remove|set|request-*) need a +project-admin token (master, or any admin-role user's). A non-admin write is +the metastore's opaque 401 "Failed to create project scope", reclassified by +kbagent to MISSING_MASTER_TOKEN (exit 3) with the remedy (#711). Pre-flight: `kbagent --json project info --project P` -> is_master_token. Alias: `kbagent sl ...` (hidden) is equivalent to `kbagent semantic-layer ...`. kbagent semantic-layer model list --project P List all semantic-layer models in a project. - kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] - Create a new model (default sql-dialect: Snowflake). + kbagent semantic-layer model create --project P --name N [--description D] [--sql-dialect Snowflake] [--scope project|organization|targeted] [--target-project ALIAS|ID ...] + Create a new model (default sql-dialect: Snowflake). --scope omitted = + "project" (owner-only). "targeted" = owner + explicit --target-project + grants (alias or numeric project ID; repeatable or comma-separated; an + alias must be on the owner's stack; a project-admin token is needed). + "organization" = visible to every project in the org from creation -- the + schema's ACL requires the organization-admin ROLE to create directly at + this scope (a normal project token gets 403 ACCESS_DENIED); an ordinary + caller instead creates at project/targeted scope and uses + `scope request-create` + an org-admin's `scope set --scope organization` + (below). `--scope organization` is irreversible and is gated as + destructive. ASK THE USER which project(s) before ever passing --scope + organization|targeted -- never guess. --target-project without --scope + targeted exits 2. With --scope targeted and no --target-project: on a + real terminal this launches an interactive picker over the other projects + on the stack; in --json/non-interactive it fails fast (exit 2). kbagent semantic-layer model delete --project P --model M [--yes] Delete a model. Fails if the model still has child entities. @@ -1829,14 +1845,16 @@ else dimension). Constraint name regex `^[a-z][a-z0-9_]*$`, severity is error|warning|info (the 4-band health convention lives in the NAME suffix `_critical/_warning/_healthy/_review`, not the API severity). `--rule` is - a STRING expression (e.g. "value >= 0"), NEVER an object. + a STRING expression (e.g. "value >= 0"), NEVER an object. --scope / + --target-project: same semantics as `model create --scope` above -- + ASK THE USER before ever passing --scope organization|targeted. kbagent semantic-layer edit metric|dataset|constraint|relationship|glossary ... - DELETE+POST (no PATCH on metastore). Metric rename cascades through every - constraint referencing the old name (DELETE old + POST new with updated + In-place PUT (same id, scope and grants). Metric rename cascades through + every constraint referencing the old name (each PUT with updated metrics[]); CODE_METRIC warning shown - (re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")). On POST failure, - rollback re-POSTs original_attrs and reports success/failure explicitly. + (re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")). A failed PUT + changes nothing (the `rollback` field is always null). --yes skips the confirm prompt. `edit relationship` accepts --new-from / --new-to / --new-on / --new-type (left|inner). `edit glossary` accepts --new-term (destructive cascade; requires --yes in non-TTY) / --new-definition. @@ -1847,6 +1865,10 @@ --new-metrics ...`. Human-mode CLI prints a red `PARTIAL STATE` banner above the per-entry list. `edit_simple` (no-cascade variants) carries `partial_state: false, recovery_hint: null` for envelope uniformity. + Edits update the item in place (PUT): it keeps its id, scope, + target-project grants (`scope get` below), pending elevation request and + revision history -- editing an organization/targeted item never + downgrades it. A failed edit changes nothing. kbagent semantic-layer remove metric|dataset|constraint|relationship|glossary ... Destructive. `remove metric` pre-scans constraints whose metrics[] includes @@ -1858,7 +1880,7 @@ kbagent semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] Replay a snapshot. Default: skip on conflict. --overwrite opts into - DELETE+POST. Dependency-ordered push (datasets -> metrics -> relationships + an in-place update (keeps the existing item's scope). Dependency-ordered push (datasets -> metrics -> relationships -> glossary -> constraints). kbagent semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types ...] [--dry-run] [--yes] @@ -1883,6 +1905,63 @@ Builds {{"#metastore_token": }} and delegates to EncryptService. --encrypt is currently required; other modes refused with USAGE_ERROR. + kbagent semantic-layer scope get --project P --type T --context-id ID + Show an item's current scope ("project"|"organization"|"targeted"), + target_project_ids, and any pending scope_elevation_requested_at. + --type is one of model|dataset|metric|relationship|constraint|glossary. + + kbagent semantic-layer scope add --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] + kbagent semantic-layer scope remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...] + Attach / detach target projects of a --scope targeted item (alias or + numeric ID; repeatable or comma-separated). A client-side merge (read the + grants, apply the delta, PUT) -- NOT atomic against a concurrent grant + change. Refused (exit 2) from a project that does not own the item: the + server hides the grants from a non-owner, so a merge would overwrite them + -- use `scope set --target-project` there. The item must have been created + with scope="targeted" (400 otherwise). + + kbagent semantic-layer scope set --project P --type T --context-id ID (--scope organization | --target-project ALIAS|ID ... | --clear) [--dry-run] [--yes] + Write the scope. Exactly ONE of: --target-project (REPLACES the whole + list), --clear (revokes every grant; owner-only again), or --scope + organization (elevate; the only value --scope accepts). A mixed + request exits 2. Elevating REQUIRES the organization-admin ROLE (403 + otherwise), is ONE-WAY (no downgrade), and is gated as destructive; + it prompts unless --yes/--json, and --dry-run shows the change without + applying it. NEVER elevate without the user explicitly naming the item + -- it makes the item (and its full revision history) visible to every + project in the organization, irreversibly. + + kbagent semantic-layer scope request-create --project P --type T --context-id ID + Owner-only. Flags a project-scoped item as awaiting an organization + admin's step-up decision (`scope set --scope organization`). Idempotent -- + calling again just refreshes the timestamp. + + kbagent semantic-layer scope request-delete --project P --type T --context-id ID + Owner-only. Clears a pending elevation request. Idempotent no-op if + none is pending. + + kbagent semantic-layer scope request-list --project P --type T [--limit N] [--offset N] + One page (default --limit 50) of items of --type awaiting an elevation + decision, across the whole organization; returns {{items, limit, offset, + has_more}}. The org-admin's discovery queue. + + Child items (`add metric|dataset|...`) with --scope omitted INHERIT their + model's scope and target projects; pass --scope to override. An inherited + organization scope is gated as destructive like a typed one, and a + non-org-admin token gets a 403 on it (pass --scope project). + + Elevating an EXISTING project's semantic-layer objects in bulk: there is no + bulk-elevate endpoint -- each object needs its own `scope request-create` + + an org-admin's `scope set --scope organization`, one call per item. Do this + deliberately, one object at a time, only when the user has named which + objects should become org-wide; never loop this over every object in a + project speculatively. + + Over `kbagent serve`: GET/PUT /semantic-layer/scope/{{context_id}}, + POST/DELETE .../target-projects, PUT/DELETE .../elevation-request, + GET /semantic-layer/scope/elevation-requests; POST /semantic-layer/models + and /items/{{kind}} take `scope` + `target_projects`. + ### Self-call HTTP (inside `kbagent serve` subprocesses) diff --git a/src/keboola_agent_cli/commands/semantic_layer.py b/src/keboola_agent_cli/commands/semantic_layer.py index 1a3748ab..ff3197f6 100644 --- a/src/keboola_agent_cli/commands/semantic_layer.py +++ b/src/keboola_agent_cli/commands/semantic_layer.py @@ -24,8 +24,9 @@ get_service, ) from ._semantic_layer_crud import add_app, edit_app, remove_app -from ._semantic_layer_helpers import _handle_service_call +from ._semantic_layer_helpers import ScopeChoice, _handle_service_call, resolve_scope_targets from ._semantic_layer_reference_data import reference_data_app +from ._semantic_layer_scope import scope_app semantic_layer_app = typer.Typer( name="semantic-layer", @@ -112,10 +113,34 @@ def model_create( sql_dialect: str = typer.Option( "Snowflake", "--sql-dialect", help="SQL dialect (default: Snowflake)" ), + scope: ScopeChoice | None = typer.Option( + None, + "--scope", + help=( + "Visibility: 'project' (default, owner only), 'organization' " + "(every project in the org), or 'targeted' (owner + explicit " + "--target-project grants)." + ), + ), + target_project: list[str] = typer.Option( + [], + "--target-project", + help=( + "Project alias or ID to grant visibility to (repeatable or comma-separated; " + "--scope targeted only)." + ), + ), ) -> None: """Create a new semantic-layer model.""" formatter = get_formatter(ctx) service = get_service(ctx, "semantic_layer_service") + target_projects = resolve_scope_targets( + ctx, + operation="semantic-layer.model.create", + scope=scope, + target_project=target_project, + owner_alias=project, + ) result = _handle_service_call( ctx, service.create_model, @@ -123,6 +148,8 @@ def model_create( name=name, description=description, sql_dialect=sql_dialect, + scope=scope, + target_projects=target_projects, ) formatter.output( result, @@ -143,6 +170,7 @@ def model_create( semantic_layer_app.add_typer(edit_app, name="edit") semantic_layer_app.add_typer(remove_app, name="remove") semantic_layer_app.add_typer(reference_data_app, name="reference-data") +semantic_layer_app.add_typer(scope_app, name="scope") @model_app.command("delete") @@ -531,7 +559,7 @@ def semantic_layer_promote( """Promote a model from one project to another (NEW + overwrite CHANGED; never deletes). Default behaviour: NEW items are POSTed, CHANGED items are - DELETE+POSTed, IDENTICAL items are skipped. Items only present in + updated in place (scope kept), IDENTICAL items are skipped. Items only present in the target are never touched (additive-only). """ formatter = get_formatter(ctx) @@ -584,7 +612,7 @@ def semantic_layer_import( False, "--dry-run", help="Plan the import without calling any write API" ), overwrite: bool = typer.Option( - False, "--overwrite", help="DELETE+POST conflicting items (default: skip)" + False, "--overwrite", help="Update conflicting items in place (default: skip)" ), yes: bool = typer.Option( False, "--yes", "-y", help="Skip confirmation (alias for default SKIP behavior)" diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 7e92a362..e47edbc6 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -10,7 +10,7 @@ - ``GET /api/v1/repository/{type}`` → 200 with body ``{"data": [item, ...]}``. - ``POST /api/v1/repository/{type}`` → 201 with body ``{"data": {type, id, - attributes, meta}}``. Envelope: ``{name, data, branch, schemaVersion, scope}``. + attributes, meta}}``. Envelope: ``{name, data, branch, scope}``. - ``DELETE /api/v1/repository/{type}/{id}`` → 204 empty body. Missing ID → 404 with the standard error envelope. - Duplicate ``name`` on POST → **409 Conflict** with message ``"Object with @@ -20,6 +20,31 @@ :data:`ErrorCode.ALREADY_EXISTS`. - Error envelope has top-level ``error``, ``code``, ``exception``, ``status``, ``context.path``, and an ``errors[]`` list for 422 validation failures. + +Scope/target-project contract (grounded in go-monorepo commit ``e4f62941``, +services/metastore -- Linear PSGO-140; read from source, **not yet probed +live**): + +- Every item's ``scope`` (``"project"`` | ``"organization"`` | ``"targeted"``), + ``targetProjectIds``, and ``scopeElevationRequestedAt`` live under the + response's ``meta`` block, not ``attributes`` (server: + ``MetaObjectResponse.JSONAPIMeta``). +- ``POST`` accepts ``scope``/``targetProjectIds`` in the create envelope. + ``PUT /{id}`` (plain update) does **not** -- its server-side request struct + (``MetaObjectUpdatePutRequest``) has no such fields, so scope/grants can + only be set at creation or through the dedicated endpoints below. +- Elevating to organization scope is ``PATCH /{id}`` with body + ``{"scope": "organization"}`` **only** -- it cannot be combined with a + name/data change in the same request, is one-way (no downgrade endpoint + exists), and requires the organization-admin role. +- ``PUT /{id}/target-projects`` **replaces the whole grant set** (not + additive); only valid for an object created with ``scope="targeted"``. + 204 on success, no body. +- ``PUT``/``DELETE /{id}/scope-elevation-request`` self-service "request a + step-up" flag: empty body, 200 with the updated item. Idempotent. +- ``GET /{type}/organization`` lists organization-visible items across + projects; supports the same generic ``field[op]=value`` filter query + language as every other list endpoint, plus bare ``limit=N``/``offset=N``. """ import logging @@ -34,15 +59,24 @@ logger = logging.getLogger(__name__) -# The metastore's auth middleware collapses EVERY project-scope resolution -# failure into this one opaque 401 string (go-monorepo +# Pre-PSGO-282, the metastore's auth middleware collapsed EVERY project-scope +# resolution failure into this one opaque 401 string (go-monorepo # ``services/metastore/internal/middleware/auth.go``, ``resolveProjectScope`` -# -- the underlying error is logged server-side and discarded). The dominant -# cause is its master-token gate: ``NewProjectDeps`` is called without -# ``WithoutMasterToken()``, so unlike the Storage API the metastore accepts -# ONLY a master (project admin) Storage token, and every valid non-master -# token lands here (issue #711; A/B-verified live on us-east4.gcp: non-master -# token -> this 401 on every call, master token on the same stack -> 200). +# -- the underlying error was logged server-side and discarded), because +# ``NewProjectDeps`` was called without ``WithoutMasterToken()``: unlike the +# Storage API, the metastore accepted ONLY a master (project admin) Storage +# token and every valid non-master token landed here (issue #711; +# A/B-verified live on us-east4.gcp: non-master token -> this 401 on every +# call, master token on the same stack -> 200). +# +# Fixed server-side in go-monorepo#596 (PSGO-282, merged 2026-08-31): reads +# now work with any valid, non-disabled, non-expired Storage token, writes +# still require the token to belong to a project admin, and a 401 now relays +# the Storage API's real error message instead of this generic string. A +# fixed metastore will not emit this exact phrase again -- this +# reclassification is kept only as a safety net for a deployment that has +# not rolled the fix out yet (or an older regional stack); the message below +# no longer claims every call needs a master token. _PROJECT_SCOPE_401_EXCEPTION = "Failed to create project scope" # Re-extracts the ``[exceptionId: ...]`` suffix `_raise_api_error` appended, @@ -72,10 +106,17 @@ ) +ObjectScope = Literal["project", "organization", "targeted"] + # Envelope fields kept constant across every POST (per metastore contract). _ENVELOPE_BRANCH = "main" -_ENVELOPE_SCHEMA_VERSION = "1.0.0" -_ENVELOPE_SCOPE = "project" +# No ``schemaVersion`` is sent on create: for an empty version the metastore +# resolves the stack's DEFAULT schema and stores that version on the item. The +# server checks a later scope elevation (PATCH / elevation request) against the +# STORED version, and every ``semantic-*`` schema supports only ``scope=project`` +# at 1.0.0 (the later schema versions add "organization"/"targeted"), so pinning "1.0.0" here would +# make every item created without ``--scope`` impossible to elevate. +_DEFAULT_SCOPE: ObjectScope = "project" class MetastoreClient(BaseHttpClient): @@ -87,11 +128,12 @@ class MetastoreClient(BaseHttpClient): no in-memory caching. All such concerns live in the service layer. """ - # The metastore answers a VALID non-master token with 401 (its master-token - # gate, #711), so a bearer 401 here is not an expired token. Opt out of the - # refresh-and-retry: otherwise every non-master session call burns a - # refresh-token rotation and can report SESSION_EXPIRED instead of the real - # MISSING_MASTER_TOKEN. See ``_do_request`` below and ``BearerAuth``. + # The metastore answers a VALID non-admin token with 401 on a WRITE (its + # project-admin gate, #711 / PSGO-282), so a bearer 401 here is not an + # expired token. Opt out of the refresh-and-retry: otherwise every + # non-admin session write burns a refresh-token rotation and can report + # SESSION_EXPIRED instead of the real MISSING_MASTER_TOKEN. See + # ``_do_request`` below and ``BearerAuth``. BEARER_REFRESH_ON_401 = False def __init__(self, stack_url: str, token: str, *, http_auth: httpx.Auth | None = None) -> None: @@ -111,14 +153,18 @@ def __exit__(self, *args: object) -> None: def _do_request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: """Funnel every metastore call through the project-scope-401 diagnosis. - A 401 carrying ``"Failed to create project scope"`` is the metastore's - master-token gate rejecting a valid non-master token (see - :data:`_PROJECT_SCOPE_401_EXCEPTION`), not a bad credential and not a - server fault to escalate -- so it is reclassified from the generic - 401 mapping into :data:`ErrorCode.MISSING_MASTER_TOKEN` with the - actual remedy, mirroring the ``token create`` / ``config oauth-url`` - pre-flight guards (#599). Kept at the request funnel (rather than one - try/except per verb method) so no endpoint can miss it. + A 401 carrying ``"Failed to create project scope"`` was the + metastore's pre-PSGO-282 master-token gate rejecting a valid + non-master token (see :data:`_PROJECT_SCOPE_401_EXCEPTION`), not a + bad credential and not a server fault to escalate -- so it is + reclassified from the generic 401 mapping into + :data:`ErrorCode.MISSING_MASTER_TOKEN`, mirroring the + ``token create`` / ``config oauth-url`` pre-flight guards (#599). + Kept at the request funnel (rather than one try/except per verb + method) so no endpoint can miss it, but since PSGO-282 a fixed + metastore only 401s this way on a WRITE against a non-admin token + (reads succeed for any valid token) -- the remedy below reflects + that, not a blanket master-token requirement. """ try: return super()._do_request(method, path, **kwargs) @@ -131,25 +177,27 @@ def _do_request(self, method: str, path: str, **kwargs: Any) -> httpx.Response: # Session (browser-login) project: the token is a bearer, so the # masked-token note and `project edit --token` remedy do not apply. remedy = ( - "This browser-login session is not a project master. Register " - "the project with a static MASTER Storage token " + "This browser-login session is not a project admin. Register " + "the project with a static project-admin Storage token " "(`kbagent project add`), or sign in as a project admin, to " - "use semantic-layer commands." + "use semantic-layer write commands." ) else: remedy = ( - f"The token ({self._masked_token}) is not a master token. Check " + f"The token ({self._masked_token}) is not a project admin. Check " f"`kbagent project info` -> is_master_token, and register a " - f"master token (`kbagent project edit --token ...`) to use " - f"semantic-layer commands." + f"project-admin token (`kbagent project edit --token ...`) to " + f"use semantic-layer write commands." ) raise KeboolaApiError( message=( f"The Metastore API (semantic layer) rejected the request with " - f"HTTP 401 {_PROJECT_SCOPE_401_EXCEPTION!r}. Unlike the Storage " - f"API, the metastore accepts only a MASTER (project admin) " - f"Storage token, and this is how it answers a valid non-master " - f"token. {remedy}{id_suffix}" + f"HTTP 401 {_PROJECT_SCOPE_401_EXCEPTION!r}. This is the " + f"metastore's project-admin gate: writes need a token whose " + f"Storage Admin role is `admin` on this project (a master " + f"token qualifies, but so does any other project-admin " + f"user's token); reads work with any valid token. " + f"{remedy}{id_suffix}" ), status_code=exc.status_code, error_code=ErrorCode.MISSING_MASTER_TOKEN, @@ -223,24 +271,49 @@ def post_item( item_type: SemanticType, name: str, data: dict[str, Any], + *, + scope: ObjectScope = _DEFAULT_SCOPE, + target_project_ids: list[int] | None = None, ) -> dict[str, Any]: """Create an item. Returns the server's stored representation. ``data`` is the inner ``attributes`` payload (including ``modelUUID`` for non-model types). The outer envelope is added here. + ``scope`` defaults to ``"project"`` (today's only behavior, unchanged + for every existing caller). ``target_project_ids`` is only meaningful + with ``scope="targeted"``; the server rejects the combination + otherwise (``meta_object_repository.go`` ``Create``: not a typed + sentinel error there, so it may surface as a 500 rather than a clean + 400 -- checked client-side below instead of relying on that). Neither + can be changed later via :meth:`put_item` -- use + :meth:`elevate_to_organization` / :meth:`put_target_projects`. + Normalizes the duplicate-name conflict into a clean :data:`ErrorCode.ALREADY_EXISTS`, accepting both shapes the metastore has used: HTTP 409 (post go-monorepo PR #513) and HTTP 500 with ``"Failed to create meta object"`` (legacy / pre-fix deployments). """ - envelope = { + if scope not in ("project", "organization", "targeted"): + raise KeboolaApiError( + message=f"scope must be one of 'project'|'organization'|'targeted', got {scope!r}.", + error_code=ErrorCode.VALIDATION_ERROR, + ) + if target_project_ids is not None and scope != "targeted": + raise KeboolaApiError( + message=( + f"target_project_ids is only valid with scope='targeted', got scope={scope!r}." + ), + error_code=ErrorCode.VALIDATION_ERROR, + ) + envelope: dict[str, Any] = { "name": name, "data": data, "branch": _ENVELOPE_BRANCH, - "schemaVersion": _ENVELOPE_SCHEMA_VERSION, - "scope": _ENVELOPE_SCOPE, + "scope": scope, } + if target_project_ids is not None: + envelope["targetProjectIds"] = target_project_ids try: response = self._do_request( "POST", @@ -284,26 +357,37 @@ def put_item( ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). - Unlike the DELETE+POST pattern the higher-level ``edit`` operations - use, ``PUT`` updates the record in place and increments + ``PUT`` updates the record in place (``edit``, ``import --overwrite`` and + ``promote`` use it) and increments ``meta.revision`` server-side, preserving the metastore's revision history. ``data`` is the inner ``attributes`` payload; the outer - envelope is added here (identical shape to :meth:`post_item`). + envelope is added here. - Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. + Deliberately carries no ``scope``/``targetProjectIds``/``schemaVersion`` + -- the server's request struct for this endpoint + (``MetaObjectUpdatePutRequest``) has only ``name`` and ``data``, so + scope, grants and the stored schema version are untouched by a plain PUT. + Use :meth:`elevate_to_organization` / :meth:`put_target_projects`. + + Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404 and + ``ALREADY_EXISTS`` on 409 (renaming to a name that is already taken). """ - envelope = { - "name": name, - "data": data, - "branch": _ENVELOPE_BRANCH, - "schemaVersion": _ENVELOPE_SCHEMA_VERSION, - "scope": _ENVELOPE_SCOPE, - } - response = self._do_request( - "PUT", - f"/api/v1/repository/{item_type}/{item_id}", - json=envelope, - ) + envelope = {"name": name, "data": data, "branch": _ENVELOPE_BRANCH} + try: + response = self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}", + json=envelope, + ) + except KeboolaApiError as exc: + if exc.status_code == 409: + raise KeboolaApiError( + message=f"{item_type} with name {name!r} already exists. Pick another name.", + status_code=409, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) from exc + raise body = response.json() return body.get("data", body) if isinstance(body, dict) else body @@ -313,3 +397,97 @@ def delete_item(self, item_type: SemanticType, item_id: str) -> None: Raises :class:`KeboolaApiError` with ``error_code=NOT_FOUND`` on 404. """ self._do_request("DELETE", f"/api/v1/repository/{item_type}/{item_id}") + + # ------------------------------------------------------------------ + # Scope / target-project primitives (PSGO-140) + # ------------------------------------------------------------------ + + def elevate_to_organization(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Step an item up from project/targeted scope to organization scope. + + ``PATCH /{id}`` with ``{"scope": "organization"}`` only -- the server + rejects combining a scope change with name/data in the same request, + so this method sends nothing else. One-way: there is no downgrade + endpoint. Requires the caller to hold the organization-admin role; + a caller without it gets ``ACCESS_DENIED`` (403). + """ + response = self._do_request( + "PATCH", + f"/api/v1/repository/{item_type}/{item_id}", + json={"scope": "organization"}, + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def put_target_projects( + self, + item_type: SemanticType, + item_id: str, + target_project_ids: list[int], + ) -> None: + """Replace the full set of projects granted access to a targeted-scope item. + + This is a **replace**, not a merge -- an empty list clears every + grant. Only the owning project or an organization admin may call + this; only valid for an item created with ``scope="targeted"`` + (the server 400s otherwise, including for organization-scoped + items, where targeting is meaningless). 204 on success, no body. + """ + self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}/target-projects", + json={"targetProjectIds": target_project_ids}, + ) + + def request_scope_elevation(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Flag a project-scoped item as awaiting an org-admin's step-up decision. + + Owner-only, idempotent (repeating just refreshes the timestamp). No + request body. Returns the updated item. + """ + response = self._do_request( + "PUT", + f"/api/v1/repository/{item_type}/{item_id}/scope-elevation-request", + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def withdraw_scope_elevation(self, item_type: SemanticType, item_id: str) -> dict[str, Any]: + """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" + response = self._do_request( + "DELETE", + f"/api/v1/repository/{item_type}/{item_id}/scope-elevation-request", + ) + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + + def list_organization_items( + self, + item_type: SemanticType, + *, + pending_elevation_only: bool = False, + limit: int | None = None, + offset: int | None = None, + ) -> list[dict[str, Any]]: + """List organization-visible items of ``item_type`` across projects. + + ``GET /{type}/organization``. ``pending_elevation_only`` applies the + server's documented discovery filter for an org-admin's queue + (``scope_elevation_requested_at[not][null]=true``); the generic + filter/limit/offset query language is shared with every other list + endpoint in this API. + """ + params: dict[str, str] = {} + if pending_elevation_only: + params["scope_elevation_requested_at[not][null]"] = "true" + if limit is not None: + params["limit"] = str(limit) + if offset is not None: + params["offset"] = str(offset) + response = self._do_request( + "GET", + f"/api/v1/repository/{item_type}/organization", + params=params or None, + ) + body = response.json() + return body.get("data", []) if isinstance(body, dict) else [] diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 48968526..a52aa8a7 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -357,6 +357,20 @@ "semantic-layer.reference-data.get": "read", "semantic-layer.reference-data.set": "write", "semantic-layer.reference-data.delete": "destructive", + # `scope` sub-app: visibility scope / target-project grants / elevation + # requests (PSGO-140). Parent key at the least-privileged level (read), + # same pattern as `reference-data` above. `set --scope organization` is + # escalated to `destructive` (FLAG_ESCALATIONS): it is one-way (no + # downgrade endpoint exists) and widens an item's visibility to every + # project in the organization. + "semantic-layer.scope": "read", + "semantic-layer.scope.get": "read", + "semantic-layer.scope.request-list": "read", + "semantic-layer.scope.add": "write", + "semantic-layer.scope.remove": "write", + "semantic-layer.scope.set": "write", + "semantic-layer.scope.request-create": "write", + "semantic-layer.scope.request-delete": "write", # Raw HTTP client against `kbagent serve` (used by AI subprocesses). # Categorised by the underlying HTTP method under the taxonomy at the top # of this registry: GET = read, POST/PATCH = write (they create/modify), @@ -440,6 +454,15 @@ FLAG_ESCALATIONS: dict[str, str] = { "auth.logout --remove-projects": "admin", "sync.push --force": "destructive", + # `--scope organization` makes an item (and its revision history) visible to + # every project in the org with no downgrade endpoint -- irreversible. + "semantic-layer.scope.set --scope organization": "destructive", + "semantic-layer.model.create --scope organization": "destructive", + "semantic-layer.add.metric --scope organization": "destructive", + "semantic-layer.add.dataset --scope organization": "destructive", + "semantic-layer.add.relationship --scope organization": "destructive", + "semantic-layer.add.constraint --scope organization": "destructive", + "semantic-layer.add.glossary --scope organization": "destructive", } # Operations that exist ONLY on the `kbagent serve` REST surface. They are real diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 42eb22bd..e45441de 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -29,6 +29,11 @@ ("DELETE", "/notifications/{project}/{subscription_id}"): "notification delete", ("DELETE", "/projects/{alias}"): "project remove", ("DELETE", "/semantic-layer/items/{kind}/{name}"): "", + ( + "DELETE", + "/semantic-layer/scope/{context_id}/elevation-request", + ): "semantic-layer scope request-delete", + ("DELETE", "/semantic-layer/scope/{context_id}/target-projects"): "semantic-layer scope remove", ("DELETE", "/semantic-layer/models/{model}"): "semantic-layer model delete", ( "DELETE", @@ -130,6 +135,8 @@ ("GET", "/semantic-layer/reference-data"): "semantic-layer reference-data list", ("GET", "/semantic-layer/reference-data/{record_id}"): "semantic-layer reference-data get", ("GET", "/semantic-layer/schema"): "semantic-layer schema", + ("GET", "/semantic-layer/scope/elevation-requests"): "semantic-layer scope request-list", + ("GET", "/semantic-layer/scope/{context_id}"): "semantic-layer scope get", ("GET", "/semantic-layer/search-context"): "semantic-layer search-context", ("GET", "/semantic-layer/show"): "semantic-layer show", ("GET", "/semantic-layer/validate"): "semantic-layer validate", @@ -245,6 +252,7 @@ ("POST", "/semantic-layer/items/{kind}"): "", ("POST", "/semantic-layer/models"): "semantic-layer model create", ("POST", "/semantic-layer/promote"): "semantic-layer promote", + ("POST", "/semantic-layer/scope/{context_id}/target-projects"): "semantic-layer scope add", ("POST", "/semantic-layer/token/encrypt"): "semantic-layer token", ("POST", "/sharing/{project}/link"): "sharing link", ("POST", "/sharing/{project}/share"): "sharing share", @@ -289,4 +297,9 @@ ("PUT", "/projects/{alias}/description"): "project description-set", ("PUT", "/semantic-layer/items/{kind}/{name}"): "", ("PUT", "/semantic-layer/reference-data"): "semantic-layer reference-data set", + ("PUT", "/semantic-layer/scope/{context_id}"): "semantic-layer scope set", + ( + "PUT", + "/semantic-layer/scope/{context_id}/elevation-request", + ): "semantic-layer scope request-create", } diff --git a/src/keboola_agent_cli/server/routers/semantic_layer.py b/src/keboola_agent_cli/server/routers/semantic_layer.py index ad3f4b14..4ee8cfd7 100644 --- a/src/keboola_agent_cli/server/routers/semantic_layer.py +++ b/src/keboola_agent_cli/server/routers/semantic_layer.py @@ -1,4 +1,4 @@ -"""Semantic-layer endpoints — 18 routes covering 30 CLI subcommands. +"""Semantic-layer endpoints — 25 routes covering 37 CLI subcommands. Mirrors the per-subcommand surface of :class:`keboola_agent_cli.services.semantic_layer_service.SemanticLayerService`. @@ -39,7 +39,26 @@ # ── Pydantic body models ─────────────────────────────────────────── -class ModelCreate(BaseModel): +class ScopeFields(BaseModel): + """``--scope`` / ``--target-project`` of the create commands. + + ``scope`` omitted: a model is created ``project``-scoped, a child item takes + its model's scope. ``target_projects`` (alias or numeric ID) needs ``targeted``. + """ + + scope: Literal["project", "organization", "targeted"] | None = None + target_projects: list[str] | None = None + + @model_validator(mode="after") + def _targets_need_targeted_scope(self) -> ScopeFields: + if self.target_projects and self.scope != "targeted": + raise ValueError("target_projects requires scope='targeted'.") + if self.scope == "targeted" and not self.target_projects: + raise ValueError("scope='targeted' requires target_projects.") + return self + + +class ModelCreate(ScopeFields): project: str name: str description: str = "" @@ -71,7 +90,7 @@ def _exactly_one_per_side(cls, data: Any) -> Any: return data -class AddMetric(BaseModel): +class AddMetric(ScopeFields): project: str model: str | None = None name: str @@ -80,7 +99,7 @@ class AddMetric(BaseModel): description: str = "" -class AddDataset(BaseModel): +class AddDataset(ScopeFields): project: str model: str | None = None name: str @@ -92,7 +111,7 @@ class AddDataset(BaseModel): fqn: str | None = None -class AddRelationship(BaseModel): +class AddRelationship(ScopeFields): project: str model: str | None = None name: str @@ -104,7 +123,7 @@ class AddRelationship(BaseModel): model_config = {"populate_by_name": True} -class AddConstraint(BaseModel): +class AddConstraint(ScopeFields): project: str model: str | None = None name: str @@ -114,7 +133,7 @@ class AddConstraint(BaseModel): severity: str = "warning" -class AddGlossary(BaseModel): +class AddGlossary(ScopeFields): project: str model: str | None = None term: str @@ -235,6 +254,8 @@ def create_model( name=body.name, description=body.description, sql_dialect=body.sql_dialect, + scope=body.scope, + target_projects=body.target_projects, ) @@ -426,6 +447,8 @@ def add_item( description=m.description, assume_yes=True, is_tty=False, + scope=m.scope, + target_projects=m.target_projects, ) if kind == "dataset": d = AddDataset.model_validate(body) @@ -439,6 +462,8 @@ def add_item( primary_key=d.primary_key, deep_fields=d.deep_fields, fqn=d.fqn, + scope=d.scope, + target_projects=d.target_projects, ) if kind == "relationship": r = AddRelationship.model_validate(body) @@ -450,6 +475,8 @@ def add_item( to=r.to, on=r.on, type_=r.type_, + scope=r.scope, + target_projects=r.target_projects, ) if kind == "constraint": c = AddConstraint.model_validate(body) @@ -461,6 +488,8 @@ def add_item( rule=c.rule, metrics=c.metrics, severity=c.severity, + scope=c.scope, + target_projects=c.target_projects, ) if kind == "glossary": g = AddGlossary.model_validate(body) @@ -469,6 +498,8 @@ def add_item( model_name_or_uuid=g.model, term=g.term, definition=g.definition, + scope=g.scope, + target_projects=g.target_projects, ) raise HTTPException( status_code=404, @@ -678,5 +709,120 @@ def delete_reference_data( return registry.semantic_layer.delete_reference_data(alias=project, record_id=record_id) +# ── scope (visibility, target-project grants, org-elevation requests) ── +# Mirrors `kbagent semantic-layer scope *`. A write to `organization` scope is +# irreversible; like every router outside /auth, these do not consult the +# permission engine -- the bearer token of `serve` is the gate. + +ScopeKind = Literal["model", "dataset", "metric", "relationship", "constraint", "glossary"] + + +class ScopeTargets(BaseModel): + project: str + type: ScopeKind + target_projects: list[str] = Field(min_length=1) + + +class ScopeSet(BaseModel): + """Exactly one of ``scope`` ('organization'), ``target_projects`` or ``clear``.""" + + project: str + type: ScopeKind + scope: Literal["organization"] | None = None + target_projects: list[str] | None = None + clear: bool = False + dry_run: bool = False + + +@router.get("/scope/elevation-requests", summary="List scope-elevation requests") +def scope_request_list( + project: str, + type: ScopeKind, + limit: int = Query(50, ge=1), + offset: int = Query(0, ge=0), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Items of ``type`` awaiting an org-admin's elevation decision (org-admin token).""" + return registry.semantic_layer.scope_request_list( + alias=project, kind=type, limit=limit, offset=offset + ) + + +@router.get("/scope/{context_id}", summary="Get an item's scope") +def scope_get( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_get(alias=project, kind=type, context_id=context_id) + + +@router.put("/scope/{context_id}", summary="Set an item's scope or target projects") +def scope_set( + context_id: str, + body: ScopeSet, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Elevate to organization (irreversible), or replace/clear the target projects.""" + return registry.semantic_layer.scope_set( + alias=body.project, + kind=body.type, + context_id=context_id, + scope=body.scope, + target_projects=body.target_projects, + clear=body.clear, + dry_run=body.dry_run, + ) + + +@router.post("/scope/{context_id}/target-projects", summary="Add target projects") +def scope_add( + context_id: str, + body: ScopeTargets, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_update_targets( + alias=body.project, kind=body.type, context_id=context_id, add=body.target_projects + ) + + +@router.delete("/scope/{context_id}/target-projects", summary="Remove target projects") +def scope_remove( + context_id: str, + project: str, + type: ScopeKind, + target_project: list[str] = Query(min_length=1), + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_update_targets( + alias=project, kind=type, context_id=context_id, remove=target_project + ) + + +@router.put("/scope/{context_id}/elevation-request", summary="Request scope elevation") +def scope_request_create( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_request_create( + alias=project, kind=type, context_id=context_id + ) + + +@router.delete("/scope/{context_id}/elevation-request", summary="Withdraw scope elevation request") +def scope_request_delete( + context_id: str, + project: str, + type: ScopeKind, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + return registry.semantic_layer.scope_request_delete( + alias=project, kind=type, context_id=context_id + ) + + # Re-export the closed set of kinds for tests / docs. __all__ = ["ErrorCode", "ItemKind", "router"] diff --git a/src/keboola_agent_cli/services/_semantic_layer_crud.py b/src/keboola_agent_cli/services/_semantic_layer_crud.py index 20b0ab31..be63ef49 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_crud.py +++ b/src/keboola_agent_cli/services/_semantic_layer_crud.py @@ -6,9 +6,11 @@ main module are thin orchestrators that resolve credentials + the model UUID, then delegate. +Edits are an in-place ``PUT`` (the item keeps its ID, scope, grants, pending +elevation request and revision history); a failed PUT leaves the original intact. + Helpers: -- :func:`delete_then_post` -- safe DELETE+POST with rollback - :func:`edit_metric_with_cascade` -- metric rename + constraint cascade - :func:`scan_orphan_constraints` -- pre-deletion orphan scan for metric - :data:`REMOVE_KINDS` -- accepted kinds for ``remove_item`` / @@ -25,6 +27,7 @@ if TYPE_CHECKING: from ..metastore_client import MetastoreClient, SemanticType + # Kinds accepted by `remove_item` / `preview_remove`. Relationship and # glossary were added in iter-4 (NB-5) -- neither is referenced by # other entities, so removal cannot orphan anything. @@ -46,52 +49,6 @@ def code_metric(name: str) -> str: return re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_") -def delete_then_post( - client: MetastoreClient, - item_type: SemanticType, - *, - old_id: str, - original_attrs: dict[str, Any], - new_name: str, - new_attrs: dict[str, Any], -) -> tuple[dict[str, Any], dict[str, Any] | None]: - """Run a safe DELETE+POST, rolling back to ``original_attrs`` on POST failure. - - Returns ``(new_item, rollback)`` where ``rollback`` is None on - success. On POST failure we re-POST the original payload; the - rollback dict records whether that re-POST succeeded. The original - exception is re-raised wrapped in a KeboolaApiError with the - rollback context. - """ - client.delete_item(item_type, old_id) - try: - new_item = client.post_item(item_type, name=new_name, data=new_attrs) - except KeboolaApiError as exc: - rollback_status: dict[str, Any] = { - "attempted": True, - "original_name": original_attrs.get("name") or original_attrs.get("term", ""), - "error": exc.message, - } - try: - old_name = original_attrs.get("name") or original_attrs.get("term", "") - restored = client.post_item(item_type, name=old_name, data=original_attrs) - rollback_status["status"] = "succeeded" - rollback_status["restored_id"] = restored.get("id", "") - except KeboolaApiError as rollback_exc: - rollback_status["status"] = "failed" - rollback_status["rollback_error"] = rollback_exc.message - raise KeboolaApiError( - message=( - f"edit failed (POST after DELETE raised): {exc.message}. " - f"Rollback: {rollback_status['status']}." - ), - error_code=exc.error_code, - status_code=exc.status_code, - details={"rollback": rollback_status}, - ) from exc - return new_item, None - - def edit_metric_with_cascade( client: MetastoreClient, *, @@ -108,8 +65,8 @@ def edit_metric_with_cascade( """Body of :meth:`SemanticLayerService.edit_metric`. Resolves the target metric, computes the constraint cascade list, - enforces TTY/--yes guards, then DELETE+POSTs the metric and - DELETE+POSTs each cascaded constraint individually so per-item + enforces TTY/--yes guards, then PUTs the metric and + PUTs each cascaded constraint individually so per-item failures don't poison the rest. """ metrics = client.list_items("semantic-metric", model_uuid) @@ -167,14 +124,7 @@ def edit_metric_with_cascade( if new_description is not None: new_attrs["description"] = new_description - new_item, rollback = delete_then_post( - client, - "semantic-metric", - old_id=old_id, - original_attrs=original_attrs, - new_name=effective_new_name, - new_attrs=new_attrs, - ) + new_item = client.put_item("semantic-metric", old_id, effective_new_name, new_attrs) # Cascade constraints individually (each is independent -- # report per-constraint success/failure). @@ -186,14 +136,7 @@ def edit_metric_with_cascade( cattrs["metrics"] = cmetrics cname = cattrs.get("name", "") try: - cascaded_item, _ = delete_then_post( - client, - "semantic-constraint", - old_id=c["id"], - original_attrs=c.get("attributes") or {}, - new_name=cname, - new_attrs=cattrs, - ) + cascaded_item = client.put_item("semantic-constraint", c["id"], cname, cattrs) cascaded.append({"constraint": cname, "status": "updated", "id": cascaded_item["id"]}) except KeboolaApiError as exc: cascaded.append( @@ -201,7 +144,6 @@ def edit_metric_with_cascade( "constraint": cname, "status": "failed", "error": exc.message, - "rollback": (exc.details or {}).get("rollback"), } ) @@ -210,7 +152,7 @@ def edit_metric_with_cascade( return { "updated": new_item, "cascaded_constraints": cascaded, - "rollback": rollback, + "rollback": None, "partial_state": partial_state, "recovery_hint": ( ( @@ -340,18 +282,11 @@ def edit_simple( if v is not None: new_attrs[k] = v effective_new = new_attrs.get(id_key) or current_key - new_item, rollback = delete_then_post( - client, - item_type, - old_id=target["id"], - original_attrs=original_attrs, - new_name=effective_new, - new_attrs=new_attrs, - ) + new_item = client.put_item(item_type, target["id"], effective_new, new_attrs) return { "updated": new_item, "cascaded_constraints": [], - "rollback": rollback, + "rollback": None, "partial_state": False, "recovery_hint": None, } diff --git a/src/keboola_agent_cli/services/_semantic_layer_internals.py b/src/keboola_agent_cli/services/_semantic_layer_internals.py index 035eb37a..22d29b26 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_internals.py +++ b/src/keboola_agent_cli/services/_semantic_layer_internals.py @@ -444,8 +444,7 @@ def run_import_loop( per_type["overwritten"] += 1 continue try: - client.delete_item(type_slug, existing_by_name[key]["id"]) - client.post_item(type_slug, name=key, data=attrs) + client.put_item(type_slug, existing_by_name[key]["id"], key, attrs) per_type["overwritten"] += 1 except KeboolaApiError as exc: per_type["failed"].append({"name": key, "reason": exc.message}) @@ -475,7 +474,7 @@ def run_promote_loop( ) -> dict[str, Any]: """Run the additive + overwrite promote loop. - NEW: POST to target. CHANGED: DELETE+POST. IDENTICAL: skip. + NEW: POST to target. CHANGED: in-place PUT. IDENTICAL: skip. Items only in target are never deleted (additive only). Returns ``{plural: {new, overwritten, identical, failed, changes}}``. """ @@ -517,8 +516,7 @@ def run_promote_loop( stats["changes"].append({id_key: key, "diff_keys": diff_keys}) continue try: - target_client.delete_item(type_slug, tgt_by_key[key]["id"]) - target_client.post_item(type_slug, name=key, data=src_attrs) + target_client.put_item(type_slug, tgt_by_key[key]["id"], key, src_attrs) stats["overwritten"] += 1 stats["changes"].append({id_key: key, "diff_keys": diff_keys}) except KeboolaApiError as exc: diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py new file mode 100644 index 00000000..a118372f --- /dev/null +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -0,0 +1,221 @@ +"""Scope / target-project / elevation logic for ``semantic-layer scope`` (PSGO-140). + +Composes the :class:`MetastoreClient` scope primitives +(``elevate_to_organization``, ``put_target_projects``, +``request_scope_elevation``, ``withdraw_scope_elevation``, +``list_organization_items``) into the operations +:class:`SemanticLayerService` exposes as ``scope_*``. Alias<->numeric +``project_id`` resolution happens here (service layer), not in the client, +following this project's client/service split -- see CLAUDE.md +"Architecture: 3-Layer Design". + +Split out (like the other ``_semantic_layer_*.py`` helpers) so the +orchestrator class stays under the CONTRIBUTING.md services LOC ceiling. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from ..errors import ErrorCode, KeboolaApiError + +if TYPE_CHECKING: + from ..config_store import ConfigStore + from ..metastore_client import MetastoreClient, ObjectScope, SemanticType + +DEFAULT_REQUEST_LIST_LIMIT = 50 + + +def parse_target_projects(values: list[str] | None) -> list[str]: + """Flatten repeatable and comma-separated ``--target-project`` values, de-duplicated.""" + seen: dict[str, None] = {} + for value in values or []: + for part in value.split(","): + if part.strip(): + seen.setdefault(part.strip()) + return list(seen) + + +def resolve_target_project_ids( + config_store: ConfigStore, owner_alias: str, targets: list[str] | None +) -> list[int]: + """Resolve ``--target-project`` values (alias or numeric project ID) to project IDs. + + A registered alias wins; otherwise an all-digit value is taken as a project + ID, so a target does not have to be registered in kbagent. An alias must be + on the owner project's stack -- a project ID is only meaningful there, and + an ID cannot be checked, so it is trusted as being on that stack. Any bad + value is ``INVALID_ARGUMENT`` (a usage error) naming the option. + """ + owner = config_store.get_project(owner_alias) + ids: list[int] = [] + for target in parse_target_projects(targets): + project = config_store.get_project(target) + if project is None: + if not target.isdigit(): + raise KeboolaApiError( + message=( + f"--target-project {target!r} is neither a registered project alias " + "nor a numeric project ID. See `kbagent project list`." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + ids.append(int(target)) + continue + if owner is not None and project.stack_url.rstrip("/") != owner.stack_url.rstrip("/"): + raise KeboolaApiError( + message=( + f"--target-project {target!r} is on a different stack than {owner_alias!r}; " + "a project ID only means the same project on its own stack." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + if project.project_id is None: + raise KeboolaApiError( + message=( + f"--target-project {target!r} has no numeric project_id on record " + "(re-run `kbagent project refresh` or `project add`)." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + ids.append(project.project_id) + return list(dict.fromkeys(ids)) + + +def item_scope(item: dict[str, Any]) -> tuple[ObjectScope, list[int] | None]: + """``(scope, target_project_ids)`` from a raw item's ``meta`` block (``project`` if absent).""" + meta = item.get("meta") or {} + return meta.get("scope", "project"), meta.get("targetProjectIds") + + +def inherited_scope( + client: MetastoreClient, model_uuid: str +) -> tuple[ObjectScope, list[int] | None]: + """The scope a child item takes when ``--scope`` is omitted: its model's own.""" + scope, targets = item_scope(client.get_item("semantic-model", model_uuid)) + return scope, (targets or None) if scope == "targeted" else None + + +def item_status(item: dict[str, Any]) -> dict[str, Any]: + """Extract the scope/grant/elevation-request fields from a raw item for display.""" + meta = item.get("meta") or {} + attrs = item.get("attributes") or {} + return { + "id": item.get("id"), + "type": item.get("type"), + "name": attrs.get("name") or attrs.get("term"), + "scope": meta.get("scope", "project"), + "target_project_ids": meta.get("targetProjectIds"), + "scope_elevation_requested_at": meta.get("scopeElevationRequestedAt"), + "project_id": meta.get("projectId"), + } + + +def _merge_targets( + item: dict[str, Any], caller_project_id: int | None, add: list[int], remove: list[int] +) -> list[int]: + """Apply an add/remove delta to the item's grants, refusing when they cannot be read. + + The server returns ``meta.targetProjectIds`` only to the owning project (and + omits it when the list is empty), so a non-owner (an org admin from + elsewhere) would read ``None``, apply the delta to an empty list, and PUT it + -- silently replacing every grant. The owner check is on ``meta.projectId``, + never on the list being empty: an owner with no grants is a valid merge base. + """ + if caller_project_id is None: + raise KeboolaApiError( + message=( + "This project's own ID is unknown, so ownership of the item cannot be checked. " + "Run `kbagent project refresh` (or `project add`) first." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + meta = item.get("meta") or {} + owner_id = meta.get("projectId") + if owner_id is not None and owner_id != caller_project_id: + raise KeboolaApiError( + message=( + "This project does not own the item, so its current target projects cannot " + "be read and add/remove would overwrite them. Use `scope set --target-project` " + "to replace the whole list from the owning project's view." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + return sorted((set(meta.get("targetProjectIds") or []) | set(add)) - set(remove)) + + +def set_target_projects( + client: MetastoreClient, + item_type: SemanticType, + item_id: str, + *, + replace: list[int] | None = None, + add: list[int] | None = None, + remove: list[int] | None = None, + caller_project_id: int | None = None, +) -> dict[str, Any]: + """Update a targeted-scope item's grants: replace the list, or merge an add/remove delta. + + ``replace`` is the server's native replace-only PUT (``[]`` clears). A merge + is read-modify-write and not atomic against a concurrent grant change. + """ + if replace is not None: + new_ids = sorted(set(replace)) + else: + current = client.get_item(item_type, item_id) + new_ids = _merge_targets(current, caller_project_id, add or [], remove or []) + client.put_target_projects(item_type, item_id, new_ids) + return item_status(client.get_item(item_type, item_id)) + + +def request_elevation( + client: MetastoreClient, item_type: SemanticType, item_id: str +) -> dict[str, Any]: + """Flag an item as awaiting an org-admin's step-up decision. + + Re-reads the item instead of rendering the endpoint's own response: + the ``scope-elevation-request`` (and ``PATCH``) response bodies omit + ``meta.targetProjectIds``, so reporting them directly prints + ``target_project_ids: null`` for an item whose grants are actually + intact. Same re-read pattern as :func:`set_target_projects`. + """ + client.request_scope_elevation(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) + + +def withdraw_elevation( + client: MetastoreClient, item_type: SemanticType, item_id: str +) -> dict[str, Any]: + """Clear a pending elevation request. Re-reads for the reason in :func:`request_elevation`.""" + client.withdraw_scope_elevation(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) + + +def elevate_to_organization( + client: MetastoreClient, item_type: SemanticType, item_id: str, *, dry_run: bool = False +) -> dict[str, Any]: + """Step an item up to organization scope (one-way). ``dry_run`` only reports the plan.""" + if dry_run: + status = item_status(client.get_item(item_type, item_id)) + return {**status, "dry_run": True, "would_set_scope": "organization"} + client.elevate_to_organization(item_type, item_id) + return item_status(client.get_item(item_type, item_id)) + + +def list_pending_elevations( + client: MetastoreClient, + item_type: SemanticType, + *, + limit: int = DEFAULT_REQUEST_LIST_LIMIT, + offset: int = 0, +) -> dict[str, Any]: + """One page of elevation requests; ``has_more`` comes from fetching one row past ``limit``.""" + items = client.list_organization_items( + item_type, pending_elevation_only=True, limit=limit + 1, offset=offset + ) + return { + "items": [item_status(i) for i in items[:limit]], + "limit": limit, + "offset": offset, + "has_more": len(items) > limit, + } diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 67061b55..1a37f988 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -24,13 +24,13 @@ from ..auth.sentinel import require_static_token from ..config_store import ConfigStore from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..metastore_client import MetastoreClient, SemanticType +from ..metastore_client import MetastoreClient, ObjectScope, SemanticType from ..models import ProjectConfig from . import _semantic_layer_reference_data as _refdata +from . import _semantic_layer_scope as _scope from ._semantic_layer_cascade import cascade_delete_model as _cascade_delete_model_impl from ._semantic_layer_crud import REMOVE_KINDS as _REMOVE_KINDS_HELPER from ._semantic_layer_crud import code_metric as _code_metric_helper -from ._semantic_layer_crud import delete_then_post as _delete_then_post_helper from ._semantic_layer_crud import edit_metric_with_cascade as _edit_metric_helper from ._semantic_layer_crud import edit_simple as _edit_simple_helper from ._semantic_layer_crud import find_target_for_remove as _find_target_for_remove @@ -678,12 +678,70 @@ def _collect_side( # Phase 4 — Model lifecycle (create / delete) # ------------------------------------------------------------------ + def _target_ids( + self, alias: str, scope: ObjectScope, target_projects: list[str] | None + ) -> list[int] | None: + """Resolve ``--target-project`` values for a ``targeted`` write (``None`` otherwise).""" + if scope != "targeted": + return None + return _scope.resolve_target_project_ids(self._config_store, alias, target_projects) or None + + def child_scope(self, alias: str, model_name_or_uuid: str | None) -> ObjectScope: + """The scope ``add `` gives a child when ``--scope`` is omitted: its model's own. + + Exposed so the command layer can gate an INHERITED ``organization`` scope + exactly like a typed ``--scope organization``. + """ + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + model_uuid, _ = self._resolve_model(client, model_name_or_uuid) + return _scope.inherited_scope(client, model_uuid)[0] + + def _post_scoped( + self, + client: MetastoreClient, + alias: str, + item_type: SemanticType, + *, + name: str, + data: dict[str, Any], + scope: ObjectScope | None, + target_projects: list[str] | None, + ) -> dict[str, Any]: + """POST a child item; with ``scope`` omitted it takes the scope of its model.""" + inherited = scope is None + if inherited: + scope, target_ids = _scope.inherited_scope(client, data["modelUUID"]) + else: + target_ids = self._target_ids(alias, scope, target_projects) + try: + return client.post_item( + item_type, name=name, data=data, scope=scope, target_project_ids=target_ids + ) + except KeboolaApiError as exc: + if not ( + inherited and scope == "organization" and exc.error_code == ErrorCode.ACCESS_DENIED + ): + raise + raise KeboolaApiError( + message=( + f"{exc.message} The model is {scope!r}-scoped, so this child inherits that " + "scope; creating at it needs an organization-admin token. Pass " + "`--scope project` to create it project-only, or use an org-admin token." + ), + status_code=exc.status_code, + error_code=exc.error_code, + retryable=False, + ) from exc + def create_model( self, alias: str, name: str, description: str = "", sql_dialect: str = "Snowflake", + *, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a semantic-layer model and return the server-stored item.""" project = self._resolve_one_project(alias) @@ -691,7 +749,14 @@ def create_model( data: dict[str, Any] = {"name": name, "sql_dialect": sql_dialect} if description: data["description"] = description - created = client.post_item("semantic-model", name=name, data=data) + scope = scope or "project" + created = client.post_item( + "semantic-model", + name=name, + data=data, + scope=scope, + target_project_ids=self._target_ids(alias, scope, target_projects), + ) return {"project": alias, "model": created} def delete_model( @@ -735,6 +800,8 @@ def add_metric( assume_yes: bool = False, is_tty: bool = False, confirm_cb: Callable[[str], bool] | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a metric. The ``dataset`` argument is a tableId. @@ -774,7 +841,15 @@ def add_metric( } if description: data["description"] = description - return client.post_item("semantic-metric", name=name, data=data) + return self._post_scoped( + client, + alias, + "semantic-metric", + name=name, + data=data, + scope=scope, + target_projects=target_projects, + ) def add_dataset( self, @@ -788,6 +863,8 @@ def add_dataset( primary_key: list[str] | None = None, deep_fields: bool = False, fqn: str | None = None, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a dataset. @@ -829,7 +906,15 @@ def add_dataset( fields = _synthesize_role_classified_fields(detail, _classify_field_role) if fields: data["fields"] = fields - return client.post_item("semantic-dataset", name=name, data=data) + return self._post_scoped( + client, + alias, + "semantic-dataset", + name=name, + data=data, + scope=scope, + target_projects=target_projects, + ) def add_relationship( self, @@ -841,6 +926,8 @@ def add_relationship( to: str, on: str, type_: str, + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a relationship. ``from``/``to`` are tableIds; ``type``='left'|'inner'.""" if type_ not in ("left", "inner"): @@ -859,7 +946,15 @@ def add_relationship( "type": type_, "modelUUID": model_uuid, } - return client.post_item("semantic-relationship", name=name, data=data) + return self._post_scoped( + client, + alias, + "semantic-relationship", + name=name, + data=data, + scope=scope, + target_projects=target_projects, + ) def add_constraint( self, @@ -871,6 +966,8 @@ def add_constraint( rule: str, metrics: list[str], severity: str = "warning", + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a constraint after validating every field locally.""" _validate_constraint_attrs( @@ -905,7 +1002,15 @@ def add_constraint( "severity": severity, "modelUUID": model_uuid, } - return client.post_item("semantic-constraint", name=name, data=data) + return self._post_scoped( + client, + alias, + "semantic-constraint", + name=name, + data=data, + scope=scope, + target_projects=target_projects, + ) def add_glossary( self, @@ -914,6 +1019,8 @@ def add_glossary( *, term: str, definition: str = "", + scope: ObjectScope | None = None, + target_projects: list[str] | None = None, ) -> dict[str, Any]: """Create a glossary term. Outer envelope ``name`` must equal ``term``.""" project = self._resolve_one_project(alias) @@ -922,7 +1029,124 @@ def add_glossary( data: dict[str, Any] = {"term": term, "modelUUID": model_uuid} if definition: data["definition"] = definition - return client.post_item("semantic-glossary", name=term, data=data) + return self._post_scoped( + client, + alias, + "semantic-glossary", + name=term, + data=data, + scope=scope, + target_projects=target_projects, + ) + + # ------------------------------------------------------------------ + # Phase 5 — Scope / target-project / elevation (PSGO-140) + # ------------------------------------------------------------------ + + def _scope_item_type(self, kind: str) -> SemanticType: + item_type = SCHEMA_TYPE_ALIAS.get(kind) + if item_type is None: + raise KeboolaApiError( + message=f"--type must be one of {sorted(SCHEMA_TYPE_ALIAS)}, got {kind!r}.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + return item_type + + def scope_get(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: + """Show the current scope/grants/pending-elevation state of one item.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.item_status(client.get_item(item_type, context_id)) + + def scope_update_targets( + self, + alias: str, + kind: str, + context_id: str, + *, + add: list[str] | None = None, + remove: list[str] | None = None, + ) -> dict[str, Any]: + """Merge ``add``/``remove`` target projects (alias or ID) into a targeted item's grants.""" + item_type = self._scope_item_type(kind) + project = self._resolve_one_project(alias) + resolve = _scope.resolve_target_project_ids + with self._new_metastore_client(project) as client: + return _scope.set_target_projects( + client, + item_type, + context_id, + add=resolve(self._config_store, alias, add), + remove=resolve(self._config_store, alias, remove), + caller_project_id=project.project_id, + ) + + def scope_set( + self, + alias: str, + kind: str, + context_id: str, + *, + scope: str | None = None, + target_projects: list[str] | None = None, + clear: bool = False, + dry_run: bool = False, + ) -> dict[str, Any]: + """Write the scope: elevate to ``organization``, or replace/clear the target projects. + + Exactly one of ``scope`` / ``target_projects`` / ``clear`` is allowed -- the + metastore changes scope only to ``organization`` and accepts a target list + only for a ``targeted`` item, so a mixed request is refused before any call. + """ + modes = [scope is not None, bool(target_projects), clear] + if sum(modes) != 1: + raise KeboolaApiError( + message="scope set takes exactly one of --scope organization, " + "--target-project, or --clear.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + if scope is not None and scope != "organization": + raise KeboolaApiError( + message="--scope can only be 'organization' (scope is one-way).", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + item_type = self._scope_item_type(kind) + project = self._resolve_one_project(alias) + replace = _scope.resolve_target_project_ids(self._config_store, alias, target_projects) + with self._new_metastore_client(project) as client: + if scope is not None: + return _scope.elevate_to_organization( + client, item_type, context_id, dry_run=dry_run + ) + if dry_run: + status = _scope.item_status(client.get_item(item_type, context_id)) + return {**status, "dry_run": True, "would_set_target_project_ids": replace} + return _scope.set_target_projects(client, item_type, context_id, replace=replace) + + def scope_request_create(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: + """Flag a project-scoped item as awaiting an org-admin's step-up decision.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.request_elevation(client, item_type, context_id) + + def scope_request_delete(self, alias: str, kind: str, context_id: str) -> dict[str, Any]: + """Clear a pending scope-elevation request. Idempotent no-op if none is pending.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.withdraw_elevation(client, item_type, context_id) + + def scope_request_list( + self, + alias: str, + kind: str, + *, + limit: int = _scope.DEFAULT_REQUEST_LIST_LIMIT, + offset: int = 0, + ) -> dict[str, Any]: + """One page of ``kind`` items awaiting an org-admin's elevation decision.""" + item_type = self._scope_item_type(kind) + with self._new_metastore_client(self._resolve_one_project(alias)) as client: + return _scope.list_pending_elevations(client, item_type, limit=limit, offset=offset) # ------------------------------------------------------------------ # Reference data — dimension-member records (e.g. a Chart of Accounts). @@ -995,7 +1219,6 @@ def delete_reference_data(self, alias: str, record_id: str) -> dict[str, Any]: # Thin delegates to ._semantic_layer_crud helpers -- bodies live # there so this orchestrator stays under the services budget. _code_metric = staticmethod(_code_metric_helper) - _delete_then_post = staticmethod(_delete_then_post_helper) def edit_metric( self, @@ -1011,7 +1234,7 @@ def edit_metric( is_tty: bool = False, confirm_cb: Callable[[str], bool] | None = None, ) -> dict[str, Any]: - """Edit a metric via DELETE+POST with rename-cascade on constraints. + """Edit a metric in place (PUT) with rename-cascade on constraints. Returns: ``{updated: item, cascaded_constraints: [...], rollback: None|{...}}``. @@ -1047,7 +1270,7 @@ def edit_dataset( new_description: str | None = None, new_grain: str | None = None, ) -> dict[str, Any]: - """Edit a dataset (DELETE+POST). Renames do NOT cascade for datasets.""" + """Edit a dataset (in-place PUT). Renames do NOT cascade for datasets.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: model_uuid, _ = self._resolve_model(client, model_name_or_uuid) @@ -1077,7 +1300,7 @@ def edit_constraint( new_severity: str | None = None, new_metrics: list[str] | None = None, ) -> dict[str, Any]: - """Edit a constraint (DELETE+POST). Validates new attrs locally first.""" + """Edit a constraint (in-place PUT). Validates new attrs locally first.""" _validate_constraint_attrs( name_re=CONSTRAINT_NAME_RE, constraint_types=CONSTRAINT_TYPES, @@ -1127,7 +1350,7 @@ def edit_relationship( new_on: str | None = None, new_type: str | None = None, ) -> dict[str, Any]: - """Edit a relationship (DELETE+POST). Validates ``--new-type`` locally. + """Edit a relationship (in-place PUT). Validates ``--new-type`` locally. Relationships are not referenced by any other entity, so no cascade is needed -- the result is shaped identically to @@ -1166,7 +1389,7 @@ def edit_glossary( new_term: str | None = None, new_definition: str | None = None, ) -> dict[str, Any]: - """Edit a glossary term (DELETE+POST). + """Edit a glossary term (in-place PUT). Renaming via ``--new-term`` is destructive for downstream consumers that join on the literal term string (the term IS the @@ -1301,7 +1524,7 @@ def import_snapshot( types: Filter to a subset of types. ``None`` = all types. dry_run: When True, plan and return the action counts without hitting any write API. - overwrite: When True, DELETE+POST conflicting items by name. + overwrite: When True, update conflicting items by name in place (PUT). Default (False) skips conflicts. Returns: diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 42a7321b..c4f48784 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -4479,7 +4479,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: "--yes", ) new_metric_id = data["data"]["updated"]["id"] - # Replace tracking: the old metric was DELETE+POSTed + # Replace tracking: edits are in-place PUTs (same id), re-tracked defensively created_items = [ (t, i) for (t, i) in created_items @@ -4490,7 +4490,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: assert any(c["status"] == "updated" for c in cascaded), ( f"Expected at least one cascaded constraint, got: {cascaded}" ) - # The constraint id changed (DELETE+POST). Re-fetch the list. + # Re-fetch the list (the cascade updated the constraint in place). data = self._run_ok( "semantic-layer", "show", @@ -11423,6 +11423,42 @@ def _direct_delete(item_type: str, item_id: str) -> None: ) created_items.append(("semantic-dataset", ds2["data"]["id"])) + # scope commands on an item created WITHOUT --scope (it must stay + # elevatable: no schemaVersion is pinned on create), and the in-place + # PUT edit keeping the item id. + ds1_id = ds1["data"]["id"] + scope_args = ("--project", self.alias, "--type", "dataset", "--context-id", ds1_id) + data = self._run_ok("semantic-layer", "scope", "get", *scope_args) + assert data["data"]["scope"] == "project" + data = self._run_ok("semantic-layer", "scope", "request-create", *scope_args) + assert data["data"]["scope_elevation_requested_at"] + data = self._run_ok("semantic-layer", "scope", "request-delete", *scope_args) + assert not data["data"]["scope_elevation_requested_at"] + data = self._run_ok( + "semantic-layer", + "scope", + "set", + *scope_args, + "--scope", + "organization", + "--dry-run", + ) + assert data["data"]["would_set_scope"] == "organization" + data = self._run_ok( + "semantic-layer", + "edit", + "dataset", + "--project", + self.alias, + "--model", + model_name, + "--name", + f"{tag}_ds_a", + "--new-description", + "edited in place", + ) + assert data["data"]["updated"]["id"] == ds1_id + m1 = self._run_ok( "semantic-layer", "add", @@ -11597,7 +11633,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: assert any(c["status"] == "updated" for c in cascaded), ( f"Expected constraint cascade, got: {cascaded}" ) - # DELETE+POST changed the constraint id -- refresh tracking + # refresh tracking (cascade updated the constraint in place) data = self._run_ok( "semantic-layer", "show", @@ -11614,7 +11650,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: # ---------- NB-5: edit + remove relationship / glossary ---------- - _step(5.1, "edit relationship --new-on -- DELETE+POST") + _step(5.1, "edit relationship --new-on -- in-place PUT") data = self._run_ok( "semantic-layer", "edit", @@ -11636,7 +11672,7 @@ def _direct_delete(item_type: str, item_id: str) -> None: ] created_items.append(("semantic-relationship", new_rel_id)) - _step(5.2, "edit glossary --new-definition -- DELETE+POST") + _step(5.2, "edit glossary --new-definition -- in-place PUT") data = self._run_ok( "semantic-layer", "edit", diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index b6d99141..f0910f24 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -169,10 +169,11 @@ def test_post_envelope(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "rev" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" + assert "schemaVersion" not in body # the stack default is resolved server-side assert body["scope"] == "project" assert body["data"]["sql"] == "SUM(x)" assert body["data"]["modelUUID"] == "u" + assert "targetProjectIds" not in body @pytest.fixture @@ -303,8 +304,8 @@ def test_put_envelope_and_url(self, httpx_mock) -> None: body = json.loads(request.content) assert body["name"] == "chart_of_accounts" assert body["branch"] == "main" - assert body["schemaVersion"] == "1.0.0" - assert body["scope"] == "project" + assert "schemaVersion" not in body + assert "scope" not in body assert body["data"]["dimensionName"] == "chart_of_accounts" assert body["data"]["members"] == [] @@ -342,12 +343,15 @@ def test_semantic_types_complete(self) -> None: class TestProjectScope401Reclassification: - """The metastore's master-token gate 401 becomes MISSING_MASTER_TOKEN. - - The metastore auth middleware collapses every project-scope resolution - failure into a 401 ``"Failed to create project scope"``; for a valid - non-master token that IS the master-token gate firing (issue #711). - The client funnels every verb through the reclassification. + """The metastore's project-admin gate 401 becomes MISSING_MASTER_TOKEN. + + Pre-PSGO-282, the metastore auth middleware collapsed every project-scope + resolution failure into a 401 ``"Failed to create project scope"``, + including plain reads from a valid non-master token (issue #711). Since + PSGO-282 (go-monorepo#596) a fixed metastore only emits this 401 for a + WRITE against a token that is not a project admin -- reads succeed for + any valid token. The client still funnels every verb through the + reclassification as a safety net for a not-yet-upgraded deployment. """ _SCOPE_401_BODY: ClassVar[dict] = { @@ -520,3 +524,204 @@ def test_default_bearer_would_refresh_reproducing_the_bug(self, httpx_mock) -> N client.close() assert spy.refresh_calls == 1 assert len(httpx_mock.get_requests()) == 2 + + +class TestPostItemScopeValidation: + """post_item validates scope/target_project_ids client-side (PSGO-140).""" + + def test_rejects_unknown_scope(self, metastore_client) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item("semantic-metric", name="x", data={}, scope="bogus") + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_empty_target_project_ids_still_rejected_without_targeted_scope( + self, metastore_client + ) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item( + "semantic-metric", name="x", data={}, scope="organization", target_project_ids=[] + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + @pytest.mark.parametrize("scope", ["project", "organization", "targeted"]) + def test_create_never_pins_a_schema_version(self, httpx_mock, metastore_client, scope) -> None: + """A pinned 1.0.0 would make the item impossible to elevate later (scope check is on + the STORED version), so the stack default is left to the server for every scope.""" + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "semantic-metric", + name="x", + data={}, + scope=scope, + target_project_ids=[1] if scope == "targeted" else None, + ) + assert "schemaVersion" not in json.loads(httpx_mock.get_requests()[0].content) + + def test_put_item_duplicate_name_is_already_exists(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric/abc", + status_code=409, + json={"error": "Object with this name already exists in this project"}, + ) + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.put_item("semantic-metric", "abc", "taken", {}) + assert excinfo.value.error_code == ErrorCode.ALREADY_EXISTS + + def test_rejects_target_project_ids_without_targeted_scope(self, metastore_client) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item( + "semantic-metric", name="x", data={}, scope="project", target_project_ids=[1] + ) + assert excinfo.value.error_code == ErrorCode.VALIDATION_ERROR + + def test_targeted_scope_sends_target_project_ids(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "semantic-metric", + name="x", + data={}, + scope="targeted", + target_project_ids=[123, 456], + ) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "targeted" + assert body["targetProjectIds"] == [123, 456] + + def test_organization_scope_omits_target_project_ids_key( + self, httpx_mock, metastore_client + ) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item("semantic-metric", name="x", data={}, scope="organization") + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "organization" + assert "targetProjectIds" not in body + + +class TestElevateToOrganization: + """PATCH /{type}/{id} with {"scope": "organization"} only.""" + + def test_sends_scope_only_patch(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PATCH", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc", + json={ + "data": { + "type": "semantic-dataset", + "id": "abc", + "attributes": {}, + "meta": {"scope": "organization"}, + } + }, + status_code=200, + ) + result = metastore_client.elevate_to_organization("semantic-dataset", "abc") + assert result["meta"]["scope"] == "organization" + request = httpx_mock.get_requests()[0] + assert request.method == "PATCH" + assert json.loads(request.content) == {"scope": "organization"} + + def test_403_maps_to_access_denied(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PATCH", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc", + status_code=403, + json={"error": "Insufficient permissions"}, + ) + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.elevate_to_organization("semantic-dataset", "abc") + assert excinfo.value.error_code == ErrorCode.ACCESS_DENIED + + +class TestPutTargetProjects: + """PUT /{type}/{id}/target-projects replaces the whole grant set; 204 no body.""" + + def test_replaces_target_projects(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/target-projects", + status_code=204, + ) + assert metastore_client.put_target_projects("semantic-dataset", "abc", [1, 2]) is None + request = httpx_mock.get_requests()[0] + assert json.loads(request.content) == {"targetProjectIds": [1, 2]} + + def test_clears_with_empty_list(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/target-projects", + status_code=204, + ) + metastore_client.put_target_projects("semantic-dataset", "abc", []) + request = httpx_mock.get_requests()[0] + assert json.loads(request.content) == {"targetProjectIds": []} + + +class TestScopeElevationRequest: + """PUT/DELETE .../scope-elevation-request: empty body, 200 with the updated item.""" + + def test_request_elevation(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/scope-elevation-request", + json={ + "data": { + "type": "semantic-dataset", + "id": "abc", + "attributes": {}, + "meta": {"scopeElevationRequestedAt": "2026-08-28T00:00:00Z"}, + } + }, + status_code=200, + ) + result = metastore_client.request_scope_elevation("semantic-dataset", "abc") + assert result["meta"]["scopeElevationRequestedAt"] + + def test_withdraw_elevation(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="DELETE", + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/abc/scope-elevation-request", + json={"data": {"type": "semantic-dataset", "id": "abc", "attributes": {}, "meta": {}}}, + status_code=200, + ) + result = metastore_client.withdraw_scope_elevation("semantic-dataset", "abc") + assert result["id"] == "abc" + + +class TestListOrganizationItems: + """GET /{type}/organization with the generic filter/limit/offset query language.""" + + def test_pending_elevation_filter_and_pagination(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=( + f"{METASTORE_URL_US}/api/v1/repository/semantic-dataset/organization" + "?scope_elevation_requested_at%5Bnot%5D%5Bnull%5D=true&limit=5&offset=10" + ), + json={"data": [{"type": "semantic-dataset", "id": "a", "attributes": {}}]}, + status_code=200, + ) + result = metastore_client.list_organization_items( + "semantic-dataset", pending_elevation_only=True, limit=5, offset=10 + ) + assert len(result) == 1 + assert result[0]["id"] == "a" + + def test_no_filters_sends_bare_request(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-model/organization", + json={"data": []}, + status_code=200, + ) + assert metastore_client.list_organization_items("semantic-model") == [] diff --git a/tests/test_project_ref.py b/tests/test_project_ref.py index 5dd5e60a..299dc877 100644 --- a/tests/test_project_ref.py +++ b/tests/test_project_ref.py @@ -148,6 +148,10 @@ def test_no_shadow_notice_for_aliases_of_the_same_project(self) -> None: "--project-ids": "already project IDs (org setup)", "--source-project-id": "already a project ID (sharing link)", "--target-project-ids": "already project IDs (sharing share)", + "--target-project": ( + "semantic-layer: alias OR numeric ID resolved by the service itself (a target need not " + "be registered); config clone's --target-project is in ALIAS_OPTIONS" + ), "--all-projects": "a boolean flag", "--register-projects": "a boolean flag", "--remove-projects": "a boolean flag", @@ -183,7 +187,7 @@ def _leaf_commands(group: Any, path: tuple[str, ...] = ()) -> dict[tuple[str, .. def _dummy_value(param: Any, existing_file: Path) -> str: """A value Click accepts for ``param`` (the command body never runs).""" type_name = type(param.type).__name__ - if type_name == "TyperChoice": + if type_name in ("TyperChoice", "Choice"): return str(param.type.choices[0]) if type_name == "TyperPath": return str(existing_file) diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py new file mode 100644 index 00000000..42ab6424 --- /dev/null +++ b/tests/test_semantic_layer_scope.py @@ -0,0 +1,458 @@ +"""Tests for metastore scope/target-project/elevation support (PSGO-140). + +Covers ``services._semantic_layer_scope`` (target resolution, grant merge, +elevation) and the service orchestration on top of it: ``scope_*``, scope +inheritance on child creates, and the overwrite paths (``import --overwrite``, +``promote``) keeping an item's scope. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services import _semantic_layer_scope as scope_helpers +from keboola_agent_cli.services.semantic_layer_service import SemanticLayerService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + + +def _make_store(tmp_path: Path) -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + store.add_project( + "analytics", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="analytics", + project_id=1234, + ), + ) + store.add_project( + "eu-project", + ProjectConfig( + stack_url="https://connection.europe-west3.gcp.keboola.com", + token=TEST_TOKEN, + project_name="eu", + project_id=77, + ), + ) + return store + + +def _make_service(store: ConfigStore, *, metastore_mock: MagicMock | None = None): + mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + service = SemanticLayerService( + config_store=store, + metastore_client_factory=lambda url, token: mock, + ) + return service, mock + + +def _model_item(uuid: str = "u-model", name: str = "default") -> dict[str, Any]: + return {"type": "semantic-model", "id": uuid, "attributes": {"name": name}} + + +def _child_item( + item_type: str, item_id: str, attrs: dict[str, Any], meta: dict[str, Any] | None = None +) -> dict[str, Any]: + item: dict[str, Any] = {"type": item_type, "id": item_id, "attributes": dict(attrs)} + if meta is not None: + item["meta"] = meta + return item + + +# --------------------------------------------------------------------------- +# services._semantic_layer_scope -- helpers +# --------------------------------------------------------------------------- + + +class TestResolveTargetProjectIds: + def test_aliases_ids_and_comma_lists_resolve_and_dedupe(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + resolved = scope_helpers.resolve_target_project_ids( + store, "prod", ["analytics,5678", "analytics", "9"] + ) + assert resolved == [1234, 5678, 9] + + @pytest.mark.parametrize( + ("target", "needle"), + [ + ("ghost", "neither a registered project alias nor a numeric project ID"), + ("eu-project", "different stack"), + ], + ) + def test_bad_target_is_invalid_argument_naming_the_option( + self, tmp_path: Path, target: str, needle: str + ) -> None: + store = _make_store(tmp_path) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.resolve_target_project_ids(store, "prod", [target]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert "--target-project" in excinfo.value.message + assert needle in excinfo.value.message + + def test_registered_project_without_id_is_invalid_argument(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + store.add_project( + "no-id", + ProjectConfig( + stack_url="https://connection.keboola.com", token=TEST_TOKEN, project_id=None + ), + ) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.resolve_target_project_ids(store, "prod", ["no-id"]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + + +def _item(meta: dict[str, Any]) -> dict[str, Any]: + return {"type": "semantic-dataset", "id": "d1", "attributes": {"name": "x"}, "meta": meta} + + +OWNED = {"scope": "targeted", "projectId": 5725, "targetProjectIds": [1, 2]} + + +class TestSetTargetProjects: + def test_replace_sends_exact_set_without_reading_current(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects(client, "semantic-dataset", "d1", replace=[9, 3, 9]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [3, 9]) + assert client.get_item.call_count == 1 # only the report re-read + + def test_clear_via_empty_replace(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects(client, "semantic-dataset", "d1", replace=[]) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", []) + + @pytest.mark.parametrize( + ("add", "remove", "expected"), + [([3], [], [1, 2, 3]), ([], [1], [2]), ([3], [1], [2, 3])], + ) + def test_owner_merge_applies_the_delta(self, add, remove, expected) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=add, remove=remove, caller_project_id=5725 + ) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", expected) + + def test_owner_with_no_grants_can_still_add(self) -> None: + """The server omits an empty list; that is the owner's state after `scope set --clear`.""" + client = MagicMock() + client.get_item.return_value = _item({"scope": "targeted", "projectId": 5725}) + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=5725 + ) + client.put_target_projects.assert_called_once_with("semantic-dataset", "d1", [3]) + + def test_unknown_own_project_id_is_refused_with_a_refresh_hint(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=None + ) + assert "kbagent project refresh" in excinfo.value.message + client.put_target_projects.assert_not_called() + + @pytest.mark.parametrize( + "meta", + [ + # org admin from another project: the server hides the grants from a non-owner + {"scope": "targeted", "projectId": 5725}, + {"scope": "targeted", "projectId": 5725, "targetProjectIds": [1]}, + ], + ) + def test_non_owner_merge_is_refused_instead_of_wiping_grants(self, meta) -> None: + client = MagicMock() + client.get_item.return_value = _item(meta) + with pytest.raises(KeboolaApiError) as excinfo: + scope_helpers.set_target_projects( + client, "semantic-dataset", "d1", add=[3], caller_project_id=999 + ) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert "scope set --target-project" in excinfo.value.message + client.put_target_projects.assert_not_called() + + +class TestElevation: + @pytest.mark.parametrize( + ("helper", "client_method"), + [ + (scope_helpers.request_elevation, "request_scope_elevation"), + (scope_helpers.withdraw_elevation, "withdraw_scope_elevation"), + (scope_helpers.elevate_to_organization, "elevate_to_organization"), + ], + ) + def test_reports_grants_from_a_reread_not_the_endpoint_response( + self, helper, client_method + ) -> None: + client = MagicMock() + getattr(client, client_method).return_value = {"meta": {"scope": "targeted"}} # no grants + client.get_item.return_value = _item(OWNED) + assert helper(client, "semantic-dataset", "d1")["target_project_ids"] == [1, 2] + + def test_dry_run_elevation_never_patches(self) -> None: + client = MagicMock() + client.get_item.return_value = _item(OWNED) + result = scope_helpers.elevate_to_organization( + client, "semantic-dataset", "d1", dry_run=True + ) + client.elevate_to_organization.assert_not_called() + assert result["dry_run"] is True + assert result["would_set_scope"] == "organization" + + @pytest.mark.parametrize(("returned", "has_more"), [(3, False), (4, True)]) + def test_request_list_pages_by_fetching_one_extra_row(self, returned, has_more) -> None: + client = MagicMock() + client.list_organization_items.return_value = [_item({}) for _ in range(returned)] + page = scope_helpers.list_pending_elevations(client, "semantic-dataset", limit=3, offset=6) + client.list_organization_items.assert_called_once_with( + "semantic-dataset", pending_elevation_only=True, limit=4, offset=6 + ) + assert (page["limit"], page["offset"], page["has_more"]) == (3, 6, has_more) + assert len(page["items"]) == 3 + + +# --------------------------------------------------------------------------- +# SemanticLayerService.scope_* +# --------------------------------------------------------------------------- + + +class TestServiceScopeMethods: + def test_unknown_type_is_invalid_argument(self, tmp_path: Path) -> None: + service, _ = _make_service(_make_store(tmp_path)) + with pytest.raises(KeboolaApiError) as excinfo: + service.scope_get("prod", "table", "x") + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + + def test_update_targets_resolves_targets_and_passes_the_callers_project( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + service.scope_update_targets("prod", "dataset", "d1", add=["analytics,5678"]) + mock.put_target_projects.assert_called_once_with( + "semantic-dataset", "d1", [1, 2, 1234, 5678] + ) + + @pytest.mark.parametrize( + "kwargs", + [ + {}, # nothing + {"scope": "organization", "target_projects": ["analytics"]}, + {"scope": "organization", "clear": True}, + {"target_projects": ["analytics"], "clear": True}, + {"scope": "project"}, # scope only ever moves to organization + ], + ) + def test_set_requires_exactly_one_mode(self, tmp_path: Path, kwargs) -> None: + service, mock = _make_service(_make_store(tmp_path)) + with pytest.raises(KeboolaApiError) as excinfo: + service.scope_set("prod", "dataset", "d1", **kwargs) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.put_target_projects.assert_not_called() + mock.elevate_to_organization.assert_not_called() + + def test_set_modes_reach_the_right_metastore_call(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + service.scope_set("prod", "dataset", "d1", target_projects=["analytics"]) + mock.put_target_projects.assert_called_with("semantic-dataset", "d1", [1234]) + service.scope_set("prod", "dataset", "d1", clear=True) + mock.put_target_projects.assert_called_with("semantic-dataset", "d1", []) + service.scope_set("prod", "dataset", "d1", scope="organization") + mock.elevate_to_organization.assert_called_once_with("semantic-dataset", "d1") + + def test_set_dry_run_writes_nothing(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _item(OWNED) + result = service.scope_set( + "prod", "dataset", "d1", target_projects=["analytics"], dry_run=True + ) + assert result["would_set_target_project_ids"] == [1234] + mock.put_target_projects.assert_not_called() + + def test_request_list_defaults(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.list_organization_items.return_value = [] + page = service.scope_request_list("prod", "dataset") + assert page == {"items": [], "limit": 50, "offset": 0, "has_more": False} + + +# --------------------------------------------------------------------------- +# Creating with a scope / inheriting the model's scope +# --------------------------------------------------------------------------- + + +def _service_with_model(tmp_path: Path, model_meta: dict[str, Any] | None): + service, mock = _make_service(_make_store(tmp_path)) + model = _model_item("U", "m") + if model_meta is not None: + model["meta"] = model_meta + mock.list_items.return_value = [model] + mock.get_item.return_value = model + mock.post_item.return_value = {"id": "new", "attributes": {}} + return service, mock + + +class TestChildScopeInheritance: + @pytest.mark.parametrize( + ("model_meta", "scope", "targets", "expected_scope", "expected_ids"), + [ + (None, None, None, "project", None), + ({"scope": "organization"}, None, None, "organization", None), + ({"scope": "targeted", "targetProjectIds": [5, 6]}, None, None, "targeted", [5, 6]), + ({"scope": "organization"}, "project", None, "project", None), # explicit wins + (None, "targeted", ["analytics,5678"], "targeted", [1234, 5678]), + (None, "organization", None, "organization", None), + ], + ) + def test_add_glossary_scope( + self, tmp_path: Path, model_meta, scope, targets, expected_scope, expected_ids + ) -> None: + service, mock = _service_with_model(tmp_path, model_meta) + service.add_glossary( + "prod", None, term="t", definition="d", scope=scope, target_projects=targets + ) + kwargs = mock.post_item.call_args.kwargs + assert kwargs["scope"] == expected_scope + assert kwargs["target_project_ids"] == expected_ids + + def test_inherited_org_scope_denied_gets_a_hint(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, {"scope": "organization"}) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", definition="d") + assert "--scope project" in excinfo.value.message + assert "organization" in excinfo.value.message + + def test_inherited_targeted_scope_denied_gets_no_org_admin_hint(self, tmp_path: Path) -> None: + """A project admin may create `targeted`, so a 403 there has another cause.""" + service, mock = _service_with_model( + tmp_path, {"scope": "targeted", "targetProjectIds": [5]} + ) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", definition="d") + assert excinfo.value.message == "Insufficient permissions" + + def test_explicit_scope_denied_is_not_rewritten(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, {"scope": "organization"}) + mock.post_item.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + with pytest.raises(KeboolaApiError) as excinfo: + service.add_glossary("prod", None, term="t", scope="organization") + assert excinfo.value.message == "Insufficient permissions" + + @pytest.mark.parametrize( + ("meta", "expected"), + [ + (None, "project"), + ({"scope": "organization"}, "organization"), + ({"scope": "targeted"}, "targeted"), + ], + ) + def test_child_scope_reports_the_models_scope(self, tmp_path: Path, meta, expected) -> None: + service, _ = _service_with_model(tmp_path, meta) + assert service.child_scope("prod", None) == expected + + def test_model_create_defaults_to_project_and_resolves_targets(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, None) + service.create_model("prod", "m") + assert mock.post_item.call_args.kwargs["scope"] == "project" + service.create_model("prod", "m2", scope="targeted", target_projects=["analytics"]) + assert mock.post_item.call_args.kwargs["target_project_ids"] == [1234] + + def test_unknown_target_alias_fails_before_any_write(self, tmp_path: Path) -> None: + service, mock = _service_with_model(tmp_path, None) + with pytest.raises(KeboolaApiError) as excinfo: + service.create_model("prod", "m", scope="targeted", target_projects=["ghost"]) + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.post_item.assert_not_called() + + +# --------------------------------------------------------------------------- +# Overwrite paths keep an item's scope (PUT in place, never DELETE+POST) +# --------------------------------------------------------------------------- + + +class TestEditAndOverwriteKeepScope: + def test_edit_dataset_puts_in_place_with_the_items_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + dataset = _child_item( + "semantic-dataset", + "d1", + {"name": "x", "tableId": "a.b.c"}, + {"scope": "targeted", "targetProjectIds": [1, 2]}, + ) + mock.list_items.side_effect = lambda t, m=None: ( + [_model_item("U", "m")] if t == "semantic-model" else [dataset] + ) + mock.put_item.return_value = {"id": "d1", "attributes": {"name": "x"}} + service.edit_dataset("prod", None, current_name="x", new_description="d") + args, kwargs = mock.put_item.call_args + assert args[:2] == ("semantic-dataset", "d1") + assert kwargs == {} + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() + + def test_promote_overwrite_keeps_the_target_items_scope(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + src, tgt = MagicMock(), MagicMock() + for m in (src, tgt): + m.__enter__ = MagicMock(return_value=m) + m.__exit__ = MagicMock(return_value=False) + clients = iter([src, tgt]) + service = SemanticLayerService( + config_store=store, metastore_client_factory=lambda url, token: next(clients) + ) + src.list_items.side_effect = lambda t, m=None: ( + [_model_item("U_S", "src")] + if t == "semantic-model" + else [_child_item("semantic-metric", "m1", {"name": "b", "sql": "NEW"})] + if t == "semantic-metric" + else [] + ) + existing = _child_item( + "semantic-metric", "tm1", {"name": "b", "sql": "OLD"}, {"scope": "organization"} + ) + tgt.list_items.side_effect = lambda t, m=None: ( + [_model_item("U_T", "tgt")] + if t == "semantic-model" + else [existing] + if t == "semantic-metric" + else [] + ) + service.promote_model(from_project="prod", to_project="analytics") + args, kwargs = tgt.put_item.call_args + assert args[:3] == ("semantic-metric", "tm1", "b") + assert kwargs == {} # a PUT cannot carry scope, so it is untouched + tgt.delete_item.assert_not_called() + tgt.post_item.assert_not_called() diff --git a/tests/test_semantic_layer_scope_cli.py b/tests/test_semantic_layer_scope_cli.py new file mode 100644 index 00000000..11394e86 --- /dev/null +++ b/tests/test_semantic_layer_scope_cli.py @@ -0,0 +1,440 @@ +"""CLI-layer tests for ``semantic-layer scope`` and the ``--scope``/ +``--target-project`` flags on ``model create`` / ``add `` (PSGO-140). + +Mirrors the test_semantic_layer_cli.py pattern: patch cli.py's service +factory so the runner sees a MagicMock SemanticLayerService, plus a REAL +ProjectService (built from the test ConfigStore) since ``resolve_scope_targets`` +reads project aliases through it for the interactive-picker fallback. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.constants import EXIT_PERMISSION_DENIED +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.config_service import ConfigService +from keboola_agent_cli.services.job_service import JobService +from keboola_agent_cli.services.project_service import ProjectService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + +runner = CliRunner() + + +def _setup_config(config_dir: Path, projects: dict[str, dict]) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + for alias, info in projects.items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info["token"], + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _invoke(args: list[str], *, store: ConfigStore, sl_mock: MagicMock): + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ProjectService") as MockProj, + patch("keboola_agent_cli.cli.ConfigService") as MockCfg, + patch("keboola_agent_cli.cli.JobService") as MockJob, + patch("keboola_agent_cli.cli.SemanticLayerService") as MockSL, + ): + MockStore.return_value = store + MockProj.return_value = ProjectService(config_store=store) + MockCfg.return_value = ConfigService(config_store=store) + MockJob.return_value = JobService(config_store=store) + MockSL.return_value = sl_mock + return runner.invoke(app, args) + + +@pytest.fixture +def cfg_dir(tmp_path: Path) -> Path: + d = tmp_path / "config" + d.mkdir() + return d + + +@pytest.fixture +def store(cfg_dir: Path) -> ConfigStore: + return _setup_config( + cfg_dir, + { + "prod": {"token": TEST_TOKEN, "project_id": 5725}, + "analytics": {"token": TEST_TOKEN, "project_id": 1234}, + }, + ) + + +def _sl(*args: str) -> list[str]: + return ["--json", "semantic-layer", *args] + + +ITEM = ["--project", "prod", "--type", "dataset", "--context-id", "d1"] + + +# --------------------------------------------------------------------------- +# --scope / --target-project on model create / add +# --------------------------------------------------------------------------- + + +class TestCreateScopeFlags: + def test_omitted_scope_is_passed_as_none_so_the_service_inherits( + self, store: ConfigStore + ) -> None: + mock = MagicMock() + mock.add_metric.return_value = {"id": "m", "attributes": {"name": "n"}} + result = _invoke( + _sl( + *("add", "metric", "--project", "prod", "--name", "n", "--sql", "1"), + *("--dataset", "t", "--yes"), + ), + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + kwargs = mock.add_metric.call_args.kwargs + assert kwargs["scope"] is None + assert kwargs["target_projects"] is None + + def test_model_create_with_targeted_scope_passes_aliases_through( + self, store: ConfigStore + ) -> None: + mock = MagicMock() + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + result = _invoke( + [ + *_sl( + *("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted") + ), + "--target-project", + "analytics,5678", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + kwargs = mock.create_model.call_args.kwargs + assert kwargs["scope"] == "targeted" + assert kwargs["target_projects"] == ["analytics,5678"] + + def test_targeted_scope_without_target_fails_fast_non_tty(self, store: ConfigStore) -> None: + mock = MagicMock() + result = _invoke( + _sl(*("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted")), + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.create_model.assert_not_called() + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" + + @pytest.mark.parametrize( + "scope_args", [[], ["--scope", "project"], ["--scope", "organization"]] + ) + def test_target_project_without_targeted_scope_is_a_usage_error( + self, store: ConfigStore, scope_args: list[str] + ) -> None: + mock = MagicMock() + result = _invoke( + [ + *_sl("model", "create", "--project", "prod", "--name", "n", *scope_args), + "--target-project", + "analytics", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + mock.create_model.assert_not_called() + assert json.loads(result.output)["error"]["code"] == "INVALID_ARGUMENT" + + def test_unknown_target_alias_is_exit_2_not_a_traceback(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.create_model.side_effect = KeboolaApiError( + message="--target-project 'ghost' is neither a registered project alias nor an ID", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + result = _invoke( + [ + *_sl( + *("model", "create", "--project", "prod", "--name", "n", "--scope", "targeted") + ), + "--target-project", + "ghost", + ], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + assert "--target-project" in json.loads(result.output)["error"]["message"] + + def test_bad_scope_value_is_a_usage_error(self, store: ConfigStore) -> None: + result = _invoke( + _sl("model", "create", "--project", "prod", "--name", "n", "--scope", "everyone"), + store=store, + sl_mock=MagicMock(), + ) + assert result.exit_code == 2 + + @pytest.mark.parametrize( + ("flags", "blocked"), + [ + (["--deny-destructive"], True), + (["--deny-writes"], True), # any create is a write + ], + ) + def test_organization_scope_create_is_gated_per_command( + self, store: ConfigStore, flags: list[str], blocked: bool + ) -> None: + for command in ( + ["model", "create", "--name", "n"], + ["add", "glossary", "--term", "t"], + ["add", "dataset", "--name", "n", "--table-id", "a.b.c"], + ): + mock = MagicMock() + result = _invoke( + [*flags, *_sl(*command, "--project", "prod", "--scope", "organization")], + store=store, + sl_mock=mock, + ) + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + + @pytest.mark.parametrize( + ("model_scope", "blocked"), + [("organization", True), ("targeted", False), ("project", False)], + ) + def test_inherited_scope_is_gated_like_a_typed_one( + self, store: ConfigStore, model_scope: str, blocked: bool + ) -> None: + """`add` without --scope takes the model's scope; an inherited organization scope must not + bypass --deny-destructive.""" + mock = MagicMock() + mock.child_scope.return_value = model_scope + mock.add_glossary.return_value = {"id": "g", "attributes": {"term": "t"}} + result = _invoke( + ["--deny-destructive", *_sl("add", "glossary", "--project", "prod", "--term", "t")], + store=store, + sl_mock=mock, + ) + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + assert mock.add_glossary.called is (not blocked) + + def test_no_model_lookup_when_no_permission_policy_is_active(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.add_glossary.return_value = {"id": "g", "attributes": {"term": "t"}} + result = _invoke( + _sl("add", "glossary", "--project", "prod", "--term", "t"), store=store, sl_mock=mock + ) + assert result.exit_code == 0, result.output + mock.child_scope.assert_not_called() + + def test_deny_destructive_still_allows_project_scope_create(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.create_model.return_value = {"project": "prod", "model": {"id": "u", "attributes": {}}} + result = _invoke( + ["--deny-destructive", *_sl("model", "create", "--project", "prod", "--name", "n")], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + + +# --------------------------------------------------------------------------- +# semantic-layer scope +# --------------------------------------------------------------------------- + + +class TestScopeVerbs: + @pytest.mark.parametrize( + ("argv", "method", "expected"), + [ + (["get"], "scope_get", {}), + ( + ["add", "--target-project", "analytics", "--target-project", "5678"], + "scope_update_targets", + {"add": ["analytics", "5678"]}, + ), + ( + ["remove", "--target-project", "analytics,5678"], + "scope_update_targets", + {"remove": ["analytics,5678"]}, + ), + (["request-create"], "scope_request_create", {}), + (["request-delete"], "scope_request_delete", {}), + ], + ) + def test_verb_calls_the_service( + self, store: ConfigStore, argv: list[str], method: str, expected: dict + ) -> None: + mock = MagicMock() + getattr(mock, method).return_value = {"scope": "targeted", "target_project_ids": []} + result = _invoke(_sl("scope", *argv[:1], *ITEM, *argv[1:]), store=store, sl_mock=mock) + assert result.exit_code == 0, result.output + kwargs = getattr(mock, method).call_args.kwargs + assert kwargs["alias"] == "prod" + assert kwargs["kind"] == "dataset" + assert kwargs["context_id"] == "d1" + assert {k: kwargs[k] for k in expected} == expected + + @pytest.mark.parametrize("verb", ["add", "remove"]) + def test_add_and_remove_require_a_target(self, store: ConfigStore, verb: str) -> None: + mock = MagicMock() + result = _invoke(_sl("scope", verb, *ITEM), store=store, sl_mock=mock) + assert result.exit_code == 2, result.output + mock.scope_update_targets.assert_not_called() + + def test_empty_target_list_is_printed_not_hidden(self, store: ConfigStore) -> None: + """A targeted item with no grants ([] = owner only) must still show the field.""" + mock = MagicMock() + mock.scope_get.return_value = {"scope": "targeted", "target_project_ids": []} + with patch("keboola_agent_cli.commands._semantic_layer_scope.get_formatter") as gf: + fmt = MagicMock(json_mode=False) + gf.return_value = fmt + _invoke(["semantic-layer", "scope", "get", *ITEM], store=store, sl_mock=mock) + human = fmt.output.call_args.args[1] + console = MagicMock() + human(console, {"scope": "targeted", "target_project_ids": []}) + assert any("target_project_ids" in str(c) for c in console.print.call_args_list) + + def test_bad_type_is_a_usage_error(self, store: ConfigStore) -> None: + result = _invoke( + _sl("scope", "get", "--project", "prod", "--type", "table", "--context-id", "x"), + store=store, + sl_mock=MagicMock(), + ) + assert result.exit_code == 2 + + def test_service_usage_error_exits_2(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_set.side_effect = KeboolaApiError( + message="scope set takes exactly one of ...", error_code=ErrorCode.INVALID_ARGUMENT + ) + result = _invoke( + _sl("scope", "set", *ITEM, "--clear", "--target-project", "analytics"), + store=store, + sl_mock=mock, + ) + assert result.exit_code == 2, result.output + + +class TestScopeSet: + @pytest.mark.parametrize( + ("extra", "expected"), + [ + (["--target-project", "analytics,5678"], {"target_projects": ["analytics,5678"]}), + (["--clear"], {"clear": True}), + (["--scope", "organization", "--yes"], {"scope": "organization"}), + (["--scope", "organization", "--dry-run"], {"scope": "organization", "dry_run": True}), + ], + ) + def test_modes(self, store: ConfigStore, extra: list[str], expected: dict) -> None: + mock = MagicMock() + mock.scope_set.return_value = {"scope": "organization"} + result = _invoke(_sl("scope", "set", *ITEM, *extra), store=store, sl_mock=mock) + assert result.exit_code == 0, result.output + kwargs = mock.scope_set.call_args.kwargs + assert {k: kwargs[k] for k in expected} == expected + + def test_bad_scope_value_is_a_usage_error(self, store: ConfigStore) -> None: + result = _invoke( + _sl("scope", "set", *ITEM, "--scope", "project"), store=store, sl_mock=MagicMock() + ) + assert result.exit_code == 2 + + def test_human_mode_declining_the_prompt_aborts_without_calling( + self, store: ConfigStore + ) -> None: + mock = MagicMock() + with patch("typer.confirm", return_value=False): + result = _invoke( + ["semantic-layer", "scope", "set", *ITEM, "--scope", "organization"], + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0 + mock.scope_set.assert_not_called() + + +class TestScopeRequestList: + def test_defaults_and_pagination_envelope(self, store: ConfigStore) -> None: + mock = MagicMock() + mock.scope_request_list.return_value = { + "items": [{"id": "d1", "name": "x"}], + "limit": 50, + "offset": 0, + "has_more": False, + } + result = _invoke( + _sl("scope", "request-list", "--project", "prod", "--type", "dataset"), + store=store, + sl_mock=mock, + ) + assert result.exit_code == 0, result.output + kwargs = mock.scope_request_list.call_args.kwargs + assert (kwargs["limit"], kwargs["offset"]) == (50, 0) + data = json.loads(result.output)["data"] + assert {"items", "limit", "offset", "has_more"} <= set(data) + + +# --------------------------------------------------------------------------- +# Permission gating +# --------------------------------------------------------------------------- + +WRITE_VERBS = [ + ["add", "--target-project", "analytics"], + ["remove", "--target-project", "analytics"], + ["set", "--clear"], + ["request-create"], + ["request-delete"], +] +READ_VERBS = [["get", *ITEM], ["request-list", "--project", "prod", "--type", "dataset"]] + + +class TestScopePermissions: + @pytest.mark.parametrize("verb", WRITE_VERBS, ids=lambda v: v[0]) + def test_deny_writes_blocks_write_verbs(self, store: ConfigStore, verb: list[str]) -> None: + mock = MagicMock() + result = _invoke( + ["--deny-writes", *_sl("scope", verb[0], *ITEM, *verb[1:])], store=store, sl_mock=mock + ) + assert result.exit_code == EXIT_PERMISSION_DENIED, result.output + + @pytest.mark.parametrize("verb", READ_VERBS, ids=lambda v: v[0]) + def test_deny_writes_allows_read_verbs(self, store: ConfigStore, verb: list[str]) -> None: + mock = MagicMock() + result = _invoke(["--deny-writes", *_sl("scope", *verb)], store=store, sl_mock=mock) + assert result.exit_code == 0, result.output + + @pytest.mark.parametrize( + ("extra", "blocked"), + [ + (["--scope", "organization", "--yes"], True), + (["--scope", "organization", "--dry-run"], True), # same gate as `sync push --force` + (["--clear"], False), + (["--target-project", "analytics"], False), + ], + ) + def test_deny_destructive_blocks_only_the_elevation( + self, store: ConfigStore, extra: list[str], blocked: bool + ) -> None: + mock = MagicMock() + mock.scope_set.return_value = {"scope": "organization"} + result = _invoke( + ["--deny-destructive", *_sl("scope", "set", *ITEM, *extra)], store=store, sl_mock=mock + ) + assert (result.exit_code == EXIT_PERMISSION_DENIED) is blocked, result.output + assert mock.scope_set.called is (not blocked) diff --git a/tests/test_semantic_layer_service.py b/tests/test_semantic_layer_service.py index 2e0c917d..28774d4f 100644 --- a/tests/test_semantic_layer_service.py +++ b/tests/test_semantic_layer_service.py @@ -1499,7 +1499,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.side_effect = [ + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c_new", "attributes": dict(c_attrs, name="rev_warning")}, ] @@ -1512,8 +1512,9 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] assume_yes=True, ) assert result["updated"]["id"] == "m_new" - # delete called twice: once for the old metric, once for the cascade - assert mock.delete_item.call_count == 2 + # in-place PUTs: the metric and its cascaded constraint, nothing deleted + assert mock.put_item.call_count == 2 + mock.delete_item.assert_not_called() # cascade list populated assert result["cascaded_constraints"][0]["status"] == "updated" @@ -1530,7 +1531,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} + mock.put_item.return_value = {"id": "m_new", "attributes": {"name": "rev"}} result = service.edit_metric( "prod", @@ -1539,8 +1540,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] new_description="updated desc", ) assert result["updated"]["id"] == "m_new" - # Only one delete: the metric itself. - assert mock.delete_item.call_count == 1 + mock.delete_item.assert_not_called() assert result["cascaded_constraints"] == [] def test_metric_not_found(self, tmp_path: Path) -> None: @@ -1557,8 +1557,8 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] service.edit_metric("prod", None, current_name="ghost", new_sql="1") assert excinfo.value.error_code == ErrorCode.NOT_FOUND - def test_rollback_on_post_failure(self, tmp_path: Path) -> None: - """When POST after DELETE fails, the original item is re-POSTed.""" + def test_failed_put_leaves_original_untouched(self, tmp_path: Path) -> None: + """Edit is an in-place PUT: a failure never deletes anything, so nothing needs rolling back.""" store = _make_store(tmp_path) service, mock = _make_service(store) original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) @@ -1571,17 +1571,34 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - # First POST (the actual edit) fails; second POST (rollback) succeeds. - mock.post_item.side_effect = [ - KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR), - {"id": "restored", "attributes": {"name": "rev"}}, - ] + mock.put_item.side_effect = KeboolaApiError( + message="boom", status_code=500, error_code=ErrorCode.API_ERROR + ) - with pytest.raises(KeboolaApiError) as excinfo: + with pytest.raises(KeboolaApiError): service.edit_metric("prod", None, current_name="rev", new_sql="2") - details = excinfo.value.details or {} - rollback = details.get("rollback") or {} - assert rollback.get("status") == "succeeded" + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() + + def test_edit_keeps_scope_and_item_id(self, tmp_path: Path) -> None: + """PUT in place: the item keeps its id and (untouched by a PUT) its scope.""" + store = _make_store(tmp_path) + service, mock = _make_service(store) + original = _child_item("semantic-metric", "m1", {"name": "rev", "sql": "1"}) + original["meta"] = {"scope": "organization"} + + def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]]: + if item_type == "semantic-model": + return [_model_item("U", "m")] + return [original] if item_type == "semantic-metric" else [] + + mock.list_items.side_effect = _list + mock.put_item.return_value = {"id": "m1", "attributes": {"name": "rev"}} + service.edit_metric("prod", None, current_name="rev", new_sql="2") + args, kwargs = mock.put_item.call_args + assert args[:3] == ("semantic-metric", "m1", "rev") + assert kwargs == {} + mock.post_item.assert_not_called() def test_partial_state_false_when_cascade_succeeds(self, tmp_path: Path) -> None: """Envelope carries partial_state=False + recovery_hint=None on full success (issue #294).""" @@ -1601,7 +1618,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.side_effect = [ + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c_new", "attributes": dict(c_attrs, name="rev_warning")}, ] @@ -1630,16 +1647,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - # POST sequence: - # 1. metric rename POST -> succeeds - # 2. c1 cascade POST -> succeeds - # 3. c2 cascade POST -> fails - # 4. c2 rollback POST -> succeeds (per-item rollback) - mock.post_item.side_effect = [ + # PUT sequence: metric rename ok, c1 cascade ok, c2 cascade fails. + mock.put_item.side_effect = [ {"id": "m_new", "attributes": {"name": "revenue"}}, {"id": "c1_new", "attributes": {"name": "rev_ok", "metrics": ["revenue"]}}, KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR), - {"id": "c2_restored", "attributes": {"name": "rev_fail", "metrics": ["rev"]}}, ] result = service.edit_metric( "prod", None, current_name="rev", new_name="revenue", assume_yes=True @@ -1720,7 +1732,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = { + mock.put_item.return_value = { "id": "r2", "attributes": {"name": "fact_to_dim", "from": "out.c.fact_v2"}, } @@ -1730,11 +1742,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] current_name="fact_to_dim", new_from="out.c.fact_v2", ) - mock.delete_item.assert_called_once_with("semantic-relationship", "r1") + mock.delete_item.assert_not_called() # The POST payload retains the unchanged endpoints but rewrites `from`. - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["from"] == "out.c.fact_v2" - assert post_kwargs["data"]["to"] == "out.c.dim" + put_args = mock.put_item.call_args.args + assert put_args[3]["from"] == "out.c.fact_v2" + assert put_args[3]["to"] == "out.c.dim" assert result["rollback"] is None assert result["cascaded_constraints"] == [] @@ -1785,7 +1797,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = { + mock.put_item.return_value = { "id": "g2", "attributes": {"term": "MRR", "definition": "Updated def"}, } @@ -1795,10 +1807,10 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] current_term="MRR", new_definition="Updated def", ) - mock.delete_item.assert_called_once_with("semantic-glossary", "g1") - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["term"] == "MRR" - assert post_kwargs["data"]["definition"] == "Updated def" + mock.delete_item.assert_not_called() + put_args = mock.put_item.call_args.args + assert put_args[3]["term"] == "MRR" + assert put_args[3]["definition"] == "Updated def" assert result["rollback"] is None def test_rename_term(self, tmp_path: Path) -> None: @@ -1814,16 +1826,16 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}} + mock.put_item.return_value = {"id": "g2", "attributes": {"term": "RECURRING_REVENUE"}} service.edit_glossary( "prod", None, current_term="MRR", new_term="RECURRING_REVENUE", ) - post_kwargs = mock.post_item.call_args.kwargs - assert post_kwargs["data"]["term"] == "RECURRING_REVENUE" - assert post_kwargs["name"] == "RECURRING_REVENUE" + put_args = mock.put_item.call_args.args + assert put_args[3]["term"] == "RECURRING_REVENUE" + assert put_args[2] == "RECURRING_REVENUE" def test_not_found(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -2002,7 +2014,7 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] assert result["imported"]["datasets"]["created"] == 0 mock.post_item.assert_not_called() - def test_overwrite_deletes_then_posts(self, tmp_path: Path) -> None: + def test_overwrite_puts_in_place_keeping_scope(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) @@ -2010,11 +2022,13 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] if item_type == "semantic-model": return [_model_item("U", "target")] if item_type == "semantic-dataset": - return [_child_item("semantic-dataset", "d1", {"name": "fact_x"})] + existing = _child_item("semantic-dataset", "d1", {"name": "fact_x"}) + existing["meta"] = {"scope": "organization"} + return [existing] return [] mock.list_items.side_effect = _list - mock.post_item.return_value = {"id": "new"} + mock.put_item.return_value = {"id": "d1"} snap = tmp_path / "s.json" _write_snapshot( snap, @@ -2024,8 +2038,11 @@ def _list(item_type: str, model_uuid: str | None = None) -> list[dict[str, Any]] ) result = service.import_snapshot("prod", snap, overwrite=True) assert result["imported"]["datasets"]["overwritten"] == 1 - mock.delete_item.assert_called_with("semantic-dataset", "d1") - mock.post_item.assert_called() + args, kwargs = mock.put_item.call_args + assert args[:3] == ("semantic-dataset", "d1", "fact_x") + assert kwargs == {} + mock.delete_item.assert_not_called() + mock.post_item.assert_not_called() def test_dry_run_no_writes(self, tmp_path: Path) -> None: store = _make_store(tmp_path) diff --git a/tests/test_server_router_calls.py b/tests/test_server_router_calls.py index 9ad34bc9..fb6844ad 100644 --- a/tests/test_server_router_calls.py +++ b/tests/test_server_router_calls.py @@ -833,6 +833,152 @@ def test_reference_data_delete_route(tmp_path: Path) -> None: sl.delete_reference_data.assert_called_once_with(alias=PROJECT, record_id="r1") +# --------------------------------------------------------------------------- +# semantic_layer.py scope routes + scope fields on create -> SemanticLayerService +# --------------------------------------------------------------------------- + +SCOPE_ITEM = {"project": PROJECT, "type": "dataset"} + + +@pytest.mark.parametrize( + ("method", "path", "kwargs", "service_call", "expected"), + [ + ( + "get", + "/semantic-layer/scope/d1", + {"params": SCOPE_ITEM}, + "scope_get", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "post", + "/semantic-layer/scope/d1/target-projects", + {"json": {**SCOPE_ITEM, "target_projects": ["a", "5"]}}, + "scope_update_targets", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1", "add": ["a", "5"]}, + ), + ( + "delete", + "/semantic-layer/scope/d1/target-projects", + {"params": {**SCOPE_ITEM, "target_project": ["a", "5"]}}, + "scope_update_targets", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1", "remove": ["a", "5"]}, + ), + ( + "put", + "/semantic-layer/scope/d1", + {"json": {**SCOPE_ITEM, "scope": "organization"}}, + "scope_set", + { + "alias": PROJECT, + "kind": "dataset", + "context_id": "d1", + "scope": "organization", + "target_projects": None, + "clear": False, + "dry_run": False, + }, + ), + ( + "put", + "/semantic-layer/scope/d1", + {"json": {**SCOPE_ITEM, "clear": True}}, + "scope_set", + { + "alias": PROJECT, + "kind": "dataset", + "context_id": "d1", + "scope": None, + "target_projects": None, + "clear": True, + "dry_run": False, + }, + ), + ( + "put", + "/semantic-layer/scope/d1/elevation-request", + {"params": SCOPE_ITEM}, + "scope_request_create", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "delete", + "/semantic-layer/scope/d1/elevation-request", + {"params": SCOPE_ITEM}, + "scope_request_delete", + {"alias": PROJECT, "kind": "dataset", "context_id": "d1"}, + ), + ( + "get", + "/semantic-layer/scope/elevation-requests", + {"params": {**SCOPE_ITEM, "limit": 5, "offset": 10}}, + "scope_request_list", + {"alias": PROJECT, "kind": "dataset", "limit": 5, "offset": 10}, + ), + ], + ids=lambda v: v if isinstance(v, str) and v.startswith("scope_") else None, +) +def test_scope_routes(tmp_path: Path, method, path, kwargs, service_call, expected) -> None: + sl = MagicMock() + getattr(sl, service_call).return_value = {"scope": "project"} + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + resp = getattr(TestClient(app), method)(path, headers=AUTH, **kwargs) + assert resp.status_code == 200, resp.text + getattr(sl, service_call).assert_called_once_with(**expected) + + +def test_scope_set_body_rejects_an_unsupported_scope(tmp_path: Path) -> None: + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=MagicMock())) + resp = TestClient(app).put( + "/semantic-layer/scope/d1", json={**SCOPE_ITEM, "scope": "project"}, headers=AUTH + ) + assert resp.status_code == 422 + + +@pytest.mark.parametrize( + "fields", + [ + {"target_projects": ["a"]}, + {"scope": "project", "target_projects": ["a"]}, + {"scope": "targeted"}, + ], +) +def test_create_model_scope_fields_are_validated(tmp_path: Path, fields) -> None: + """target_projects needs scope=targeted and vice versa -- same rule as the CLI.""" + sl = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl)) + resp = TestClient(app).post( + "/semantic-layer/models", json={"project": PROJECT, "name": "m", **fields}, headers=AUTH + ) + assert resp.status_code == 422 + sl.create_model.assert_not_called() + + +def test_create_model_and_add_item_pass_scope_through(tmp_path: Path) -> None: + sl = MagicMock() + sl.create_model.return_value = {"model": {}} + sl.add_glossary.return_value = {"id": "g"} + client = TestClient(_make_app_with_registry(tmp_path, _mock_registry(semantic_layer=sl))) + scope = {"scope": "targeted", "target_projects": ["analytics"]} + assert ( + client.post( + "/semantic-layer/models", json={"project": PROJECT, "name": "m", **scope}, headers=AUTH + ).status_code + == 200 + ) + assert sl.create_model.call_args.kwargs["scope"] == "targeted" + assert sl.create_model.call_args.kwargs["target_projects"] == ["analytics"] + assert ( + client.post( + "/semantic-layer/items/glossary", json={"project": PROJECT, "term": "t"}, headers=AUTH + ).status_code + == 200 + ) + kwargs = sl.add_glossary.call_args.kwargs + assert kwargs["scope"] is None # omitted -> the service inherits the model's scope + assert kwargs["target_projects"] is None + + # --------------------------------------------------------------------------- # flows.py POST /flows/validate + GET /flows/{project}/schema # New in 0.57.0 -- mirror `flow validate` / `flow schema --full`. diff --git a/tests/test_server_semantic_layer_routes_e2e.py b/tests/test_server_semantic_layer_routes_e2e.py index 86d5860f..823f219c 100644 --- a/tests/test_server_semantic_layer_routes_e2e.py +++ b/tests/test_server_semantic_layer_routes_e2e.py @@ -342,15 +342,15 @@ def test_put_items_edit_metric(http_session: dict[str, Any]) -> None: assert res.status_code == 200, res.text body = res.json() new_id = body["updated"]["id"] - # Rename leaves a fresh metric id and DELETE+POSTs the constraint — - # refresh tracking so cleanup hits the right rows. + # Edits are in-place PUTs, so ids are normally unchanged; re-track anyway so + # cleanup hits the right rows whichever ids come back. http_session["created_items"] = [ (t, i) for (t, i) in http_session["created_items"] if not (t == "semantic-metric" and i == http_session["metric_rev_id"]) ] http_session["created_items"].append(("semantic-metric", new_id)) - # Refresh constraint id (cascade DELETE+POST). + # Re-read the constraint ids (cascade update). show = http_session["client"].get( f"/semantic-layer/show?project={_PROJECT_ALIAS}" f"&model={http_session['model_name']}&type=constraint",