From 61c6cfbfc4ee498b91f152ce87fe572d5fd202a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:51:29 +0200 Subject: [PATCH 01/35] feat(CLI-17): extend MetastoreClient for org/targeted RLS policy scope post_item/put_item gain scope + target_project_ids params (default "project" preserves every existing semantic-layer caller); new put_target_projects() manages targeted-scope grants; rls-policy added as a recognized item type. New ErrorCode.INVALID_RLS_POLICY for candidate rls-policy bodies that fail validation. --- src/keboola_agent_cli/errors.py | 5 +++++ src/keboola_agent_cli/metastore_client.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index a09240ec..3f03f1bb 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -179,6 +179,11 @@ class ErrorCode(StrEnum): UPDATED_ACROSS_MAJOR = "UPDATED_ACROSS_MAJOR" UPDATED_ACROSS_BREAKING = "UPDATED_ACROSS_BREAKING" + # Row-level security (CLI-17). Same role INVALID_FLOW_DEFINITION plays + # for conditional flows: a candidate `rls-policy` body (table/dialect/ + # rules/condition) that fails op-name or JSON-Schema structural checks. + INVALID_RLS_POLICY = "INVALID_RLS_POLICY" + def mask_token(token: str) -> str: """Mask a Keboola Storage API token for safe display. diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 03a8941a..9cce5ffa 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -92,6 +92,7 @@ "semantic-constraint", "semantic-glossary", "semantic-reference-data", + "rls-policy", ] @@ -103,6 +104,7 @@ "semantic-constraint", "semantic-glossary", "semantic-reference-data", + "rls-policy", ) @@ -371,6 +373,9 @@ def put_item( item_id: str, name: str, data: dict[str, Any], + *, + scope: MetastoreScope = _ENVELOPE_SCOPE, + target_project_ids: list[str] | None = None, ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). From 50c417b60fc9ddb8a36b6def50827de412235e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:51:36 +0200 Subject: [PATCH 02/35] feat(CLI-17): add RlsService and condition primitive compiler RlsService (list/get/create/update/delete rls-policy objects) mirrors SemanticLayerService's metastore-CRUD shape. Every write is authored at organization scope by default, targeted scope when target project IDs are given -- never project scope, matching the RFC's org-admin-only authorship model. update_policy fetch-then-merges so a partial update never blanks untouched fields. _rls_condition.py holds the declarative condition shape (comparison, membership, nullness, and/or, true sentinel), local + live-schema (jsonschema) validation, and a preview-only condition-to-string renderer for --dry-run -- explicitly not the enforcement engine, which lives in keboola-mcp-server's query_data tool. --- .../services/_rls_condition.py | 176 ++++++++++ src/keboola_agent_cli/services/rls_service.py | 330 ++++++++++++++++++ 2 files changed, 506 insertions(+) create mode 100644 src/keboola_agent_cli/services/_rls_condition.py create mode 100644 src/keboola_agent_cli/services/rls_service.py diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py new file mode 100644 index 00000000..632ddbb8 --- /dev/null +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -0,0 +1,176 @@ +"""RLS condition primitives: shape constants, structural validation, preview. + +Pure functions: no HTTP, no ``ConfigStore`` -- trivially unit-testable, +mirroring ``flow_validation.py``'s split for the same reason. + +The condition/rules shape mirrored here is the one defined in +``keboola-mcp-server``'s RFC (``feature_spec/rls_query_tool/RFC.md``, "Rule +storage" section) -- kbagent never invents its own shape, it authors exactly +what the enforcement engine (that repo's ``rls.py::_compile_primitive``, +sqlglot-based) expects to read back from the metastore. + +``compile_condition_preview`` below is a **preview convenience only**: a +small recursive pure-Python string renderer used for ``--dry-run`` / +confirmation display before a write. It is deliberately NOT the enforcement +engine and never will be -- the actual compiler that decides what SQL a +query gets rewritten to lives in ``keboola-mcp-server`` (a different repo, +sqlglot-based). Drift between the two renderings is acceptable here because +this string is only ever shown to an admin for sanity-checking, never +executed. Adding sqlglot as a kbagent dependency just to produce this +confirmation string would be a heavier dependency than the six condition +shapes below warrant. +""" + +from __future__ import annotations + +from typing import Any + +import jsonschema + +# The RFC's `condition` schema supports these operators; kept as a frozenset +# so the CLI/service layer can validate an op before it ever reaches the +# structural (schema) check, giving a clearer error for the common typo case. +RLS_COMPARISON_OPS = frozenset({"eq", "ne", "gt", "gte", "lt", "lte"}) +RLS_MEMBERSHIP_OPS = frozenset({"in", "not_in"}) +RLS_NULLNESS_OPS = frozenset({"is_null", "is_not_null"}) +RLS_CONDITION_OPS = RLS_COMPARISON_OPS | RLS_MEMBERSHIP_OPS | RLS_NULLNESS_OPS + +# The metastore's `rls-policy` schema (RFC) restricts `dialect` to these two +# -- the same pair `rewrite_query()` on the enforcement side never transpiles +# between (predicates are pinned to one workspace dialect per policy). +RLS_DIALECTS: tuple[str, ...] = ("snowflake", "bigquery") + +_COMPARISON_SQL = { + "eq": "=", + "ne": "!=", + "gt": ">", + "gte": ">=", + "lt": "<", + "lte": "<=", +} + + +def validate_policy_structural(policy: dict[str, Any], schema: dict[str, Any]) -> list[str]: + """Draft7-validate a candidate ``rls-policy`` body against a live-fetched schema. + + ``policy`` is the full candidate body (``{"table", "dialect", "rules"}``). + Defense in depth -- the metastore backend validates on write too, per the + RFC. Returns human-readable error strings (empty = valid); never raises + and never reaches the network -- ``schema`` must already be fetched by + the caller (see ``RlsService.fetch_schema``). + """ + validator = jsonschema.Draft7Validator(schema) + errors: list[str] = [] + for err in sorted(validator.iter_errors(policy), key=lambda e: list(e.path)): + path = "/".join(str(p) for p in err.path) or "(root)" + errors.append(f"Schema error at {path}: {err.message}") + return errors + + +def validate_condition_ops(condition: Any) -> list[str]: + """Recursively check every operator name used in ``condition`` is known. + + A defensive pre-check ahead of the (possibly unavailable) live schema + fetch: an unknown ``op`` typo gets a specific error here instead of a + generic Draft7 "not valid under any of the given schemas" once it does + reach structural validation. Returns human-readable error strings. + """ + errors: list[str] = [] + if not isinstance(condition, dict): + return [f"condition must be an object, got {type(condition).__name__}"] + if "true" in condition: + return errors + if "and" in condition or "or" in condition: + clauses = condition.get("and", condition.get("or")) + if not isinstance(clauses, list) or len(clauses) < 2: + key = "and" if "and" in condition else "or" + errors.append(f"'{key}' requires at least 2 nested conditions") + return errors + for clause in clauses: + errors.extend(validate_condition_ops(clause)) + return errors + op = condition.get("op") + if op not in RLS_CONDITION_OPS: + errors.append(f"unknown condition op {op!r} (expected one of {sorted(RLS_CONDITION_OPS)})") + return errors + + +def validate_rules_local(rules: Any) -> list[str]: + """Semantic checks on ``rules`` that hold regardless of schema availability. + + Runs even when the live schema fetch degraded (see + ``RlsService.fetch_schema``) -- the one check this module can still make + with no network access at all: each rule names exactly one of + ``principal``/``principals`` (the RFC's ``oneOf``, which a Draft7 + validator would otherwise be the only thing checking) and every + ``condition`` uses a known operator (:func:`validate_condition_ops`). + Returns human-readable error strings (empty = valid). + """ + errors: list[str] = [] + if not isinstance(rules, list) or not rules: + return ["'rules' must be a non-empty array"] + for index, rule in enumerate(rules): + if not isinstance(rule, dict): + errors.append(f"rules[{index}] must be an object") + continue + principals_value = rule.get("principals") + has_principal = bool(rule.get("principal")) + has_principals = isinstance(principals_value, list) and bool(principals_value) + if has_principal == has_principals: # both or neither + errors.append(f"rules[{index}] must set exactly one of 'principal'/'principals'") + condition = rule.get("condition") + if condition is None: + errors.append(f"rules[{index}] is missing 'condition'") + continue + errors.extend(f"rules[{index}].{err}" for err in validate_condition_ops(condition)) + return errors + + +def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: + """Render ``condition`` as a human-readable ``WHERE``-clause-shaped string. + + Preview only -- see module docstring. Raises ``ValueError`` on a + condition shape this renderer doesn't recognize (callers should already + have run it through :func:`validate_condition_ops` / structural + validation first; this function stays strict rather than silently + rendering something misleading). + """ + if "true" in condition: + return "TRUE" + if "and" in condition: + clauses = condition["and"] + return " AND ".join(f"({compile_condition_preview(c, dialect)})" for c in clauses) + if "or" in condition: + clauses = condition["or"] + return " OR ".join(f"({compile_condition_preview(c, dialect)})" for c in clauses) + + column = condition.get("column") + op = condition.get("op") + if not column or op is None: + raise ValueError(f"unrecognized condition shape: {condition!r}") + + if op in _COMPARISON_SQL: + return f"{column} {_COMPARISON_SQL[op]} {_preview_literal(condition.get('value'))}" + if op in RLS_MEMBERSHIP_OPS: + values = condition.get("values") or [] + rendered = ", ".join(_preview_literal(v) for v in values) + keyword = "IN" if op == "in" else "NOT IN" + return f"{column} {keyword} ({rendered})" + if op == "is_null": + return f"{column} IS NULL" + if op == "is_not_null": + return f"{column} IS NOT NULL" + raise ValueError(f"unrecognized condition op: {op!r}") + + +def _preview_literal(value: Any) -> str: + """Render one scalar literal for the preview string. Not SQL-injection-safe + + on purpose -- this output is never executed, only displayed for human + review (see module docstring). + """ + if isinstance(value, str): + return "'" + value.replace("'", "''") + "'" + if value is None: + return "NULL" + return str(value) diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py new file mode 100644 index 00000000..2470f07c --- /dev/null +++ b/src/keboola_agent_cli/services/rls_service.py @@ -0,0 +1,330 @@ +"""Row-level security service -- business logic for ``kbagent rls``. + +Composes :class:`MetastoreClient` primitives into the ``rls-policy`` CRUD +operations exposed by the CLI, mirroring :class:`SemanticLayerService`'s +shape for the same kind of metastore-backed object type -- but scaled down +to one object type instead of seven. + +The one structural rule every write path in this module enforces: an +``rls-policy`` object is **never** created or updated at plain ``project`` +scope. Authorship is centralized at ``organization`` (default) or +``targeted`` scope only -- this command surface does not even have a +``--scope`` flag that could select ``project``, matching the backend's own +restriction (RFC "Scope is organization or targeted -- never plain +project, by design, not merely by convention"). + +The ``rls-policy`` metastore object type does not exist on any deployed +backend yet (go-monorepo companion work, not shipped -- see +``keboola-mcp-server``'s ``feature_spec/rls_query_tool/PLAN.md``). Every +method here is unit-tested against a mocked :class:`MetastoreClient`; a real +project will answer with a schema-fetch/list/get/post failure until that +backend work lands (see ``fetch_schema`` and the ``gotchas.md`` entry this +PR adds). +""" + +from __future__ import annotations + +import logging +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any + +from ..config_store import ConfigStore +from ..errors import ErrorCode, KeboolaApiError +from ..metastore_client import MetastoreClient, MetastoreScope +from ..models import ProjectConfig +from . import _rls_condition +from ._rls_condition import RLS_DIALECTS +from .base import BaseService, ClientFactory, make_session_aware_client_factory + +logger = logging.getLogger(__name__) + +RLS_ITEM_TYPE = "rls-policy" + +__all__ = ["RLS_DIALECTS", "RLS_ITEM_TYPE", "RlsSchemaFetch", "RlsService"] + +MetastoreClientFactory = Callable[[str, str], MetastoreClient] + + +@dataclass(frozen=True) +class RlsSchemaFetch: + """Result of fetching the live ``rls-policy`` JSON Schema. + + ``schema`` is ``None`` on any failure (network error, ``KeboolaApiError``, + malformed/empty schema -- most likely today: the object type simply isn't + registered on the backend yet). A ``None`` schema must NOT block a write; + callers degrade to the schema-independent checks in ``_rls_condition`` + and surface ``reason`` as a warning -- mirrors + ``FlowService._fetch_flow_schema`` / ``FlowSchemaFetch`` exactly. + """ + + schema: dict[str, Any] | None + reason: str | None + + +class RlsService(BaseService): + """Business logic for the ``rls`` command group. + + Inherits multi-project resolution (``resolve_projects``) from + :class:`BaseService`. Adds a dedicated :class:`MetastoreClient` factory, + same pattern as :class:`SemanticLayerService`, so command-layer + operations can target the metastore without polluting the Storage API + client. + """ + + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory | None = None, + metastore_client_factory: MetastoreClientFactory | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._metastore_factory: MetastoreClientFactory = ( + metastore_client_factory + or make_session_aware_client_factory(config_store, MetastoreClient) + ) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _resolve_one_project(self, alias: str) -> ProjectConfig: + return self.resolve_projects([alias])[alias] + + def _new_metastore_client(self, project: ProjectConfig) -> MetastoreClient: + """Build a fresh metastore client. Caller is responsible for ``close()``.""" + return self._metastore_factory(project.stack_url, project.token) + + def fetch_schema(self, alias: str) -> RlsSchemaFetch: + """Public schema fetch (used by ``rls schema`` and every write path).""" + project = self._resolve_one_project(alias) + return self._fetch_schema_for_project(project) + + def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: + """Fetch the live ``rls-policy`` schema. Never raises -- any failure + + (network error, ``KeboolaApiError``, empty/malformed schema -- most + likely today: the object type isn't registered on the backend yet) + degrades to ``schema=None`` + ``reason``, never blocks a caller. + """ + with self._new_metastore_client(project) as client: + try: + schema = client.get_schema(RLS_ITEM_TYPE) + except KeboolaApiError as exc: + return RlsSchemaFetch(schema=None, reason=exc.message) + except Exception as exc: # any fetch failure must degrade, never block a write + return RlsSchemaFetch(schema=None, reason=str(exc)) + if not schema: + return RlsSchemaFetch(schema=None, reason="metastore returned no schema for rls-policy") + return RlsSchemaFetch(schema=schema, reason=None) + + @staticmethod + def _row_from_item(item: dict[str, Any]) -> dict[str, Any]: + attrs = item.get("attributes") or {} + meta = item.get("meta") or {} + return { + "id": item.get("id", ""), + "table": attrs.get("table", ""), + "dialect": attrs.get("dialect", ""), + "rule_count": len(attrs.get("rules") or []), + "scope": meta.get("scope", ""), + "source_project_id": meta.get("sourceProjectId"), + "target_project_ids": meta.get("targetProjectIds") or [], + } + + def _resolve_scope(self, target_project_ids: list[str] | None) -> MetastoreScope: + """Never ``"project"`` -- ``targeted`` iff target projects were given.""" + return "targeted" if target_project_ids else "organization" + + def _validate_policy( + self, *, table: str, dialect: str, rules: list[dict[str, Any]], project: ProjectConfig + ) -> list[str]: + """Run every validation this module can, local checks first. + + Local (schema-independent) checks always run. Structural (Draft7, + against the live-fetched schema) checks run too when the fetch + succeeds -- when it doesn't, the caller gets a warning, not a block + (see :class:`RlsSchemaFetch`). + """ + errors: list[str] = [] + if dialect not in _rls_condition.RLS_DIALECTS: + errors.append(f"dialect must be one of {_rls_condition.RLS_DIALECTS}, got {dialect!r}") + errors.extend(_rls_condition.validate_rules_local(rules)) + if errors: + return errors + + fetch = self._fetch_schema_for_project(project) + if fetch.schema: + policy_body = {"table": table, "dialect": dialect, "rules": rules} + errors.extend(_rls_condition.validate_policy_structural(policy_body, fetch.schema)) + return errors + + def _raise_invalid(self, errors: list[str]) -> None: + raise KeboolaApiError( + message="RLS policy is invalid: " + "; ".join(errors), + status_code=400, + error_code=ErrorCode.INVALID_RLS_POLICY, + retryable=False, + ) + + # ------------------------------------------------------------------ + # Read + # ------------------------------------------------------------------ + + def list_policies(self, alias: str) -> dict[str, Any]: + """List every ``rls-policy`` object visible to ``alias``'s project.""" + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + raw = client.list_items(RLS_ITEM_TYPE) + return {"project": alias, "policies": [self._row_from_item(item) for item in raw]} + + def get_policy(self, alias: str, policy_id: str) -> dict[str, Any]: + """Fetch one ``rls-policy`` object's full attributes + meta.""" + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + item = client.get_item(RLS_ITEM_TYPE, policy_id) + row = self._row_from_item(item) + row["rules"] = (item.get("attributes") or {}).get("rules", []) + row["revision"] = (item.get("meta") or {}).get("revision") + return row + + # ------------------------------------------------------------------ + # Write + # ------------------------------------------------------------------ + + def create_policy( + self, + alias: str, + *, + table: str, + dialect: str, + rules: list[dict[str, Any]], + target_project_ids: list[str] | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Create one ``rls-policy`` object -- always at ``organization``/``targeted`` scope. + + ``dry_run=True`` returns the same preview shape a real write would + produce (compiled condition previews per rule) without calling the + metastore's write endpoint at all -- only the (read-only) schema + fetch used for validation happens. + """ + project = self._resolve_one_project(alias) + errors = self._validate_policy(table=table, dialect=dialect, rules=rules, project=project) + if errors: + self._raise_invalid(errors) + + scope = self._resolve_scope(target_project_ids) + preview = [ + { + "principal": rule.get("principal") or rule.get("principals"), + "condition": _rls_condition.compile_condition_preview(rule["condition"], dialect), + } + for rule in rules + ] + if dry_run: + return { + "project": alias, + "table": table, + "dialect": dialect, + "scope": scope, + "target_project_ids": target_project_ids or [], + "preview": preview, + "dry_run": True, + } + + with self._new_metastore_client(project) as client: + created = client.post_item( + RLS_ITEM_TYPE, + name=table, + data={"table": table, "dialect": dialect, "rules": rules}, + scope=scope, + target_project_ids=target_project_ids, + ) + if scope == "targeted" and target_project_ids: + client.put_target_projects(RLS_ITEM_TYPE, created.get("id", ""), target_project_ids) + row = self._row_from_item(created) + row["preview"] = preview + return row + + def update_policy( + self, + alias: str, + policy_id: str, + *, + table: str | None = None, + dialect: str | None = None, + rules: list[dict[str, Any]] | None = None, + target_project_ids: list[str] | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Update one ``rls-policy`` object. + + ``put_item`` is a whole-record replace -- fetch the current item + first and merge only the given overrides onto it, so an ``update`` + call that only changes ``rules`` never silently wipes ``table`` or + ``dialect`` (the failure mode this repo's own ``merge-request + resolve`` docs warn about for exactly this kind of PUT-based API). + """ + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + current = client.get_item(RLS_ITEM_TYPE, policy_id) + attrs = current.get("attributes") or {} + meta = current.get("meta") or {} + + merged_table = table if table is not None else attrs.get("table", "") + merged_dialect = dialect if dialect is not None else attrs.get("dialect", "") + merged_rules = rules if rules is not None else attrs.get("rules", []) + merged_targets = ( + target_project_ids if target_project_ids is not None else meta.get("targetProjectIds") + ) + + errors = self._validate_policy( + table=merged_table, dialect=merged_dialect, rules=merged_rules, project=project + ) + if errors: + self._raise_invalid(errors) + + scope = self._resolve_scope(merged_targets) + preview = [ + { + "principal": rule.get("principal") or rule.get("principals"), + "condition": _rls_condition.compile_condition_preview( + rule["condition"], merged_dialect + ), + } + for rule in merged_rules + ] + if dry_run: + return { + "project": alias, + "policy_id": policy_id, + "table": merged_table, + "dialect": merged_dialect, + "scope": scope, + "target_project_ids": merged_targets or [], + "preview": preview, + "dry_run": True, + } + + with self._new_metastore_client(project) as client: + updated = client.put_item( + RLS_ITEM_TYPE, + policy_id, + name=merged_table, + data={"table": merged_table, "dialect": merged_dialect, "rules": merged_rules}, + scope=scope, + target_project_ids=merged_targets, + ) + if scope == "targeted" and merged_targets: + client.put_target_projects(RLS_ITEM_TYPE, policy_id, merged_targets) + row = self._row_from_item(updated) + row["preview"] = preview + return row + + def delete_policy(self, alias: str, policy_id: str) -> dict[str, Any]: + """Delete one ``rls-policy`` object by id.""" + project = self._resolve_one_project(alias) + with self._new_metastore_client(project) as client: + client.delete_item(RLS_ITEM_TYPE, policy_id) + return {"project": alias, "policy_id": policy_id, "deleted": True} From ff0a917a55b37800bd4b34b4391062e7be56f011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:51:43 +0200 Subject: [PATCH 03/35] feat(CLI-17): add kbagent rls command group rls list|detail|schema|create|update|delete|setup. Reads are permission-classified "read"; every write (including setup, which performs the same create_policy write via a guided picker) is classified "admin", not merely "write", matching the RFC's org-level authorship requirement. rls setup is an interactive-terminal-only wizard: a checkbox table picker (reusing storage_service.list_tables) plus a guided column/op/value condition builder, with a --rules JSON escape hatch for the full and/or-nested schema. It refuses cleanly under --json or a non-TTY stdout, same carve-out as auth register-projects' picker. --- src/keboola_agent_cli/cli.py | 5 + src/keboola_agent_cli/commands/rls.py | 639 ++++++++++++++++++++++++++ 2 files changed, 644 insertions(+) create mode 100644 src/keboola_agent_cli/commands/rls.py diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 5b829c37..0d10bb06 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -34,6 +34,7 @@ from .commands.permissions import permissions_app from .commands.project import project_app from .commands.repl import repl_command +from .commands.rls import rls_app from .commands.schedule import schedule_app from .commands.search import search_command from .commands.semantic_layer import semantic_layer_app @@ -81,6 +82,7 @@ from .services.org_service import OrgService from .services.project_service import ProjectService from .services.repo_validate_service import RepoValidateService +from .services.rls_service import RlsService from .services.schedule_service import ScheduleService from .services.search_service import SearchService from .services.semantic_layer_service import SemanticLayerService @@ -166,6 +168,7 @@ app.add_typer(encrypt_app, name="encrypt", rich_help_panel=_DEV) app.add_typer(semantic_layer_app, name="semantic-layer", rich_help_panel=_DEV) app.add_typer(semantic_layer_app, name="sl", rich_help_panel=_DEV, hidden=True) +app.add_typer(rls_app, name="rls", rich_help_panel=_DEV) app.add_typer(http_app, name="http", rich_help_panel=_DEV) app.add_typer(agent_app, name="agent", rich_help_panel=_DEV) app.add_typer(dev_portal_app, name="dev-portal", rich_help_panel=_DEV) @@ -362,6 +365,7 @@ def main( data_app_service = DataAppService(config_store=config_store) data_app_git_service = DataAppGitService(config_store=config_store) semantic_layer_service = SemanticLayerService(config_store=config_store) + rls_service = RlsService(config_store=config_store) repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) docs_service = DocsService(config_store=config_store) @@ -423,6 +427,7 @@ def main( ctx.obj["data_app_service"] = data_app_service ctx.obj["data_app_git_service"] = data_app_git_service ctx.obj["semantic_layer_service"] = semantic_layer_service + ctx.obj["rls_service"] = rls_service ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service ctx.obj["docs_service"] = docs_service diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py new file mode 100644 index 00000000..6f76b434 --- /dev/null +++ b/src/keboola_agent_cli/commands/rls.py @@ -0,0 +1,639 @@ +"""Row-level security commands (CLI-17). + +Thin CLI layer over :class:`RlsService`. Seven subcommands: + +- ``rls list`` -- policies visible to a project. +- ``rls detail`` -- one policy's full rules. +- ``rls schema`` -- the live ``rls-policy`` JSON Schema from the metastore. +- ``rls create`` -- author a policy for one table (write). +- ``rls update`` -- fetch-then-merge update of an existing policy (write). +- ``rls delete`` -- remove a policy (destructive). +- ``rls setup`` -- guided, interactive-terminal-only wizard: pick tables via + a checkbox picker, build conditions, preview, then create one policy per + table via the same ``create_policy`` path ``create`` uses. + +**Every policy this group writes is authored at ``organization`` or +``targeted`` scope, never ``project``** -- there is deliberately no +``--scope`` flag that could select ``project``, matching the metastore +backend's own restriction (an RLS policy is centrally governed, never +authored by a project's own admin for its own tables -- see the RFC in +``keboola-mcp-server``'s ``feature_spec/rls_query_tool/RFC.md``). + +The ``rls-policy`` metastore object type does not exist on any deployed +backend yet -- see ``gotchas.md``'s "RLS backend not yet available" entry. +Every read/write command here will answer with a clean, classified error +against a real project until that backend work (tracked separately) lands. + +``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``. +``create``/``update``/``setup`` are gated as ``admin`` (organization-level, +not merely ``write``); ``delete`` is gated as ``admin`` too -- see +``rls.*`` in ``OPERATION_REGISTRY`` (``permissions.py``). +""" + +from __future__ import annotations + +import json +import logging +import sys +from typing import Any + +import typer +from rich.syntax import Syntax +from rich.table import Table + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..services.rls_service import RLS_DIALECTS +from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select +from ._helpers import ( + check_cli_permission, + get_formatter, + get_service, + map_error_to_exit_code, + parse_json_arg, +) + +logger = logging.getLogger(__name__) + +rls_app = typer.Typer( + help=( + "Manage row-level security policies (metastore-backed, org-admin-only). " + "'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' " + "are gated as admin -- see CONTRIBUTING.md." + ) +) + + +@rls_app.callback(invoke_without_command=True) +def _rls_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "rls") + + +def _is_stdout_tty() -> bool: + """True when stdout is an interactive terminal (picker eligibility check).""" + return hasattr(sys.stdout, "isatty") and sys.stdout.isatty() + + +# --------------------------------------------------------------------------- +# Shared helpers +# --------------------------------------------------------------------------- + + +def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None: + tbl = Table( + "ID", + "Table", + "Dialect", + "Rules", + "Scope", + "Source Project", + "Target Projects", + show_header=True, + header_style="bold cyan", + ) + for policy in policies: + tbl.add_row( + str(policy.get("id", "")), + str(policy.get("table", "")), + str(policy.get("dialect", "")), + str(policy.get("rule_count", 0)), + str(policy.get("scope", "")), + str(policy.get("source_project_id") or ""), + ", ".join(str(p) for p in policy.get("target_project_ids", [])), + ) + formatter.console.print(tbl) + + +def _print_policy(formatter: Any, row: dict[str, Any]) -> None: + formatter.console.print( + f"\n[bold]{row.get('table', '')}[/bold] [dim](id {row.get('id', '')})[/dim]" + ) + formatter.console.print(f" Dialect: {row.get('dialect', '')}") + formatter.console.print(f" Scope: {row.get('scope', '')}") + formatter.console.print(f" Source project: {row.get('source_project_id') or '(none)'}") + targets = row.get("target_project_ids") or [] + if targets: + formatter.console.print(f" Target projects: {', '.join(str(t) for t in targets)}") + rules = row.get("rules") or [] + formatter.console.print(f" Rules ({len(rules)}):") + for rule in rules: + principal = rule.get("principal") or ", ".join(rule.get("principals") or []) + formatter.console.print(f" - {principal}: {rule.get('condition')}") + + +def _print_preview(formatter: Any, result: dict[str, Any]) -> None: + formatter.console.print( + f"\n[bold]Preview[/bold] -- {result.get('table', '')} " + f"[dim]({result.get('dialect', '')}, scope={result.get('scope', '')})[/dim]" + ) + for entry in result.get("preview", []): + formatter.console.print(f" {entry.get('principal')}: WHERE {entry.get('condition')}") + + +def _parse_rules_arg(formatter: Any, raw: str) -> list[dict[str, Any]]: + try: + parsed = parse_json_arg(raw, label="--rules") + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.INVALID_ARGUMENT) + raise typer.Exit(code=2) from None + if not isinstance(parsed, list): + formatter.error( + message="--rules must be a JSON array of {principal|principals, condition} objects", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + return parsed + + +# --------------------------------------------------------------------------- +# rls list +# --------------------------------------------------------------------------- + + +@rls_app.command("list") +def rls_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), +) -> None: + """List row-level security policies visible to a project.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "rls_service") + + try: + result = service.list_policies(alias=project) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + return + + policies = result.get("policies", []) + if not policies: + formatter.console.print("[dim]No RLS policies found.[/dim]") + else: + _format_policy_table(formatter, policies) + + +# --------------------------------------------------------------------------- +# rls detail +# --------------------------------------------------------------------------- + + +@rls_app.command("detail") +def rls_detail( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), +) -> None: + """Show one RLS policy's full rule set.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "rls_service") + + try: + result = service.get_policy(alias=project, policy_id=policy_id) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + return + + _print_policy(formatter, result) + + +# --------------------------------------------------------------------------- +# rls schema +# --------------------------------------------------------------------------- + + +@rls_app.command("schema") +def rls_schema( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", help="Project alias -- fetch the live schema from this stack" + ), +) -> None: + """Print the live ``rls-policy`` JSON Schema fetched from the metastore. + + Unlike ``flow schema``, there is no offline bundled snapshot -- the + schema is authoritative only from the live metastore, and (until the + go-monorepo backend work lands) fetching it will fail with a clean, + classified error rather than returning a schema. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "rls_service") + + try: + fetch = service.fetch_schema(project) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if fetch.schema is None: + formatter.error( + message=f"Could not fetch the rls-policy schema: {fetch.reason}", + error_code=ErrorCode.NOT_FOUND, + ) + raise typer.Exit(code=4) + + if formatter.json_mode: + formatter.output({"format": "json-schema", "source": "live", "schema": fetch.schema}) + return + formatter.console.print( + Syntax(json.dumps(fetch.schema, indent=2), "json", theme="monokai", line_numbers=False) + ) + + +# --------------------------------------------------------------------------- +# rls create +# --------------------------------------------------------------------------- + + +@rls_app.command("create") +def rls_create( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", help="Project alias -- the project this policy protects" + ), + table: str = typer.Option(..., "--table", help="Table key, e.g. in.c-crm.invoices"), + dialect: str = typer.Option( + ..., "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" + ), + rules: str = typer.Option( + ..., + "--rules", + help=( + "JSON|@file|- array of {principal|principals, condition} objects " + "(see `rls schema` / rls-workflow.md for the condition shape)" + ), + ), + target_project: list[str] | None = typer.Option( + None, + "--target-project", + help=( + "Project ID this policy also applies to (repeatable). Omit for plain " + "organization scope -- still only applies where source_project_id / " + "target_project_ids match at read time, never by table-name text alone." + ), + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Preview the compiled condition without writing" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Create one RLS policy for one table. + + Always authored at ``organization`` scope, or ``targeted`` scope when + ``--target-project`` is given -- never plain ``project`` scope. + """ + formatter = get_formatter(ctx) + parsed_rules = _parse_rules_arg(formatter, rules) + + if dialect not in RLS_DIALECTS: + formatter.error( + message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + service = get_service(ctx, "rls_service") + + if not dry_run and not yes and not formatter.json_mode: + try: + preview = service.create_policy( + alias=project, + table=table, + dialect=dialect, + rules=parsed_rules, + target_project_ids=target_project, + dry_run=True, + ) + except (ConfigError, KeboolaApiError): + pass # validation/network error -- let the real call below report it properly + else: + _print_preview(formatter, preview) + confirmed = typer.confirm(f"Create RLS policy on {table}?") + if not confirmed: + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + try: + result = service.create_policy( + alias=project, + table=table, + dialect=dialect, + rules=parsed_rules, + target_project_ids=target_project, + dry_run=dry_run, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + return + + if dry_run: + _print_preview(formatter, result) + return + formatter.success(f"Created RLS policy {result.get('id', '')} on {table}") + _print_preview(formatter, result) + + +# --------------------------------------------------------------------------- +# rls update +# --------------------------------------------------------------------------- + + +@rls_app.command("update") +def rls_update( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), + table: str | None = typer.Option(None, "--table", help="New table key (unset = unchanged)"), + dialect: str | None = typer.Option( + None, "--dialect", help=f"New dialect: {' | '.join(RLS_DIALECTS)} (unset = unchanged)" + ), + rules: str | None = typer.Option( + None, "--rules", help="New JSON|@file|- rules array (unset = unchanged)" + ), + target_project: list[str] | None = typer.Option( + None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Preview the compiled condition without writing" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Update an existing RLS policy. + + Fetch-then-merge: only the flags you pass are changed -- an omitted flag + keeps the policy's current value, it is never silently blanked. + """ + formatter = get_formatter(ctx) + parsed_rules = _parse_rules_arg(formatter, rules) if rules is not None else None + + if dialect is not None and dialect not in RLS_DIALECTS: + formatter.error( + message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + service = get_service(ctx, "rls_service") + + if not dry_run and not yes and not formatter.json_mode: + confirmed = typer.confirm(f"Update RLS policy {policy_id}?") + if not confirmed: + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + try: + result = service.update_policy( + alias=project, + policy_id=policy_id, + table=table, + dialect=dialect, + rules=parsed_rules, + target_project_ids=target_project, + dry_run=dry_run, + ) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + return + + if dry_run: + _print_preview(formatter, result) + return + formatter.success(f"Updated RLS policy {policy_id}") + _print_preview(formatter, result) + + +# --------------------------------------------------------------------------- +# rls delete +# --------------------------------------------------------------------------- + + +@rls_app.command("delete") +def rls_delete( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Delete an RLS policy.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "rls_service") + + if not yes and not formatter.json_mode: + confirmed = typer.confirm(f"Delete RLS policy {policy_id}? This un-protects its table.") + if not confirmed: + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + try: + result = service.delete_policy(alias=project, policy_id=policy_id) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + if formatter.json_mode: + formatter.output(result) + else: + formatter.success(f"Deleted RLS policy {policy_id}") + + +# --------------------------------------------------------------------------- +# rls setup (guided, interactive-terminal-only -- no REST route, see +# CONTRIBUTING.md's "genuinely terminal-only" carve-out) +# --------------------------------------------------------------------------- + +_SETUP_HINT = ( + "rls setup is an interactive picker and needs a real terminal. " + "Use `rls create --project P --table T --dialect D --rules '[...]'` " + "directly instead (see rls-workflow.md)." +) + +_CONDITION_MENU = ( + " 1) column comparison (=, !=, >, >=, <, <=)\n" + " 2) column IN / NOT IN a list of values\n" + " 3) column IS NULL / IS NOT NULL\n" + " 4) always true (no filtering for this principal)" +) + +_COMPARISON_OPS = ("eq", "ne", "gt", "gte", "lt", "lte") + + +def _prompt_choice(text: str, choices: tuple[str, ...], default: str) -> str: + while True: + value = typer.prompt(text, default=default) + if value in choices: + return value + typer.echo(f"Enter one of: {', '.join(choices)}") + + +def _prompt_condition() -> dict[str, Any]: + typer.echo(_CONDITION_MENU) + choice = _prompt_choice("Choice", ("1", "2", "3", "4"), default="1") + if choice == "4": + return {"true": True} + column = typer.prompt("Column name") + if choice == "1": + op = _prompt_choice("Operator", _COMPARISON_OPS, default="eq") + value = typer.prompt("Value") + return {"column": column, "op": op, "value": value} + if choice == "2": + op = _prompt_choice("Operator", ("in", "not_in"), default="in") + raw_values = typer.prompt("Values (comma-separated)") + values = [v.strip() for v in raw_values.split(",") if v.strip()] + return {"column": column, "op": op, "values": values} + op = _prompt_choice("Operator", ("is_null", "is_not_null"), default="is_null") + return {"column": column, "op": op} + + +def _build_rules_interactively() -> list[dict[str, Any]]: + rules: list[dict[str, Any]] = [] + while True: + principal = typer.prompt("Principal (user email, or comma-separated list for a group)") + principals = [p.strip() for p in principal.split(",") if p.strip()] + condition = _prompt_condition() + rule: dict[str, Any] = {"condition": condition} + if len(principals) == 1: + rule["principal"] = principals[0] + else: + rule["principals"] = principals + rules.append(rule) + if not typer.confirm("Add another rule to this policy?", default=False): + return rules + + +@rls_app.command("setup") +def rls_setup( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", help="Project alias whose tables you're protecting" + ), + dialect: str | None = typer.Option( + None, "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" + ), + rules: str | None = typer.Option( + None, + "--rules", + help="JSON|@file|- rules array -- skips the interactive condition builder", + ), + target_project: list[str] | None = typer.Option( + None, "--target-project", help="Project ID to also share the policy with (repeatable)" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip the final confirmation"), +) -> None: + """Guided RLS setup: pick tables, build a condition, preview, then write. + + Interactive-terminal-only (no ``--json`` / non-TTY path -- use ``rls + create`` directly there). Creates one RLS policy per selected table, + all sharing the same rules, via the exact same write path ``rls + create`` uses. + """ + formatter = get_formatter(ctx) + if formatter.json_mode or not _is_stdout_tty(): + target_console = formatter.err_console if formatter.json_mode else formatter.console + target_console.print(_SETUP_HINT) + raise typer.Exit(code=2) + + storage_service = get_service(ctx, "storage_service") + try: + tables_result = storage_service.list_tables(aliases=[project]) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + tables = tables_result.get("tables", []) + if not tables: + formatter.console.print(f"[dim]No tables found in project '{project}'.[/dim]") + raise typer.Exit(code=0) + + items = [ + CheckboxItem(label=str(t.get("id", "")), hint=f"{t.get('rows_count', 0)} rows") + for t in tables + ] + try: + indices = checkbox_select(items, title=f"Select tables to protect with RLS in '{project}'") + except CheckboxUnavailable: + formatter.console.print(f"[yellow]{_SETUP_HINT}[/yellow]") + raise typer.Exit(code=2) from None + + selected = indices or [] + if not selected: + formatter.console.print("No tables selected.") + raise typer.Exit(code=0) + + resolved_dialect = dialect or _prompt_choice("Dialect", RLS_DIALECTS, default="snowflake") + + if rules is not None: + parsed_rules = _parse_rules_arg(formatter, rules) + else: + parsed_rules = _build_rules_interactively() + + service = get_service(ctx, "rls_service") + selected_tables = [str(tables[i]["id"]) for i in selected] + + for table_id in selected_tables: + try: + preview = service.create_policy( + alias=project, + table=table_id, + dialect=resolved_dialect, + rules=parsed_rules, + target_project_ids=target_project, + dry_run=True, + ) + except (ConfigError, KeboolaApiError) as exc: + formatter.console.print(f"[yellow]{table_id}: preview failed ({exc}).[/yellow]") + continue + _print_preview(formatter, preview) + + if not yes: + confirmed = typer.confirm( + f"Create {len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}?" + ) + if not confirmed: + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + for table_id in selected_tables: + try: + result = service.create_policy( + alias=project, + table=table_id, + dialect=resolved_dialect, + rules=parsed_rules, + target_project_ids=target_project, + ) + except (ConfigError, KeboolaApiError) as exc: + formatter.warning(f"{table_id}: {exc}") + continue + formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") From 04af5c12f777e96e5bac5d266226803514df403b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:51:49 +0200 Subject: [PATCH 04/35] feat(CLI-17): expose rls over kbagent serve REST API New /rls router mirrors the CLI 1:1 except setup (interactive-only, no REST equivalent, same carve-out as auth register-projects). Every route enforces the permission policy via require_permission(), unlike notifications.py's routes which currently enforce nothing over HTTP -- RLS is security-sensitive enough that CLI-gates-but-REST-doesn't would be a real hole. --- .../server/_serve_command_map.py | 6 + src/keboola_agent_cli/server/app.py | 14 ++ src/keboola_agent_cli/server/dependencies.py | 3 + src/keboola_agent_cli/server/routers/rls.py | 148 ++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 src/keboola_agent_cli/server/routers/rls.py diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 9c782919..61735e53 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -28,6 +28,7 @@ ("DELETE", "/flows/{project}/{config_id}/schedule"): "flow schedule-remove", ("DELETE", "/notifications/{project}/{subscription_id}"): "notification delete", ("DELETE", "/projects/{alias}"): "project remove", + ("DELETE", "/rls/{project}/{policy_id}"): "rls delete", ("DELETE", "/semantic-layer/items/{kind}/{name}"): "", ( "DELETE", @@ -126,6 +127,9 @@ ("GET", "/projects/status"): "project status", ("GET", "/projects/{alias}/description"): "project description-get", ("GET", "/projects/{alias}/info"): "project info", + ("GET", "/rls/{project}"): "rls list", + ("GET", "/rls/{project}/schema"): "rls schema", + ("GET", "/rls/{project}/{policy_id}"): "rls detail", ("GET", "/schedules"): "schedule list", ("GET", "/schedules/find/query"): "schedule find", ("GET", "/schedules/{project}/{schedule_id}"): "schedule detail", @@ -252,6 +256,7 @@ ("POST", "/projects"): "project add", ("POST", "/projects/bulk-delete"): "project remove", ("POST", "/projects/use/{alias}"): "project use", + ("POST", "/rls/{project}"): "rls create", ("POST", "/semantic-layer/build"): "semantic-layer build", ("POST", "/semantic-layer/diff"): "semantic-layer diff", ("POST", "/semantic-layer/import"): "semantic-layer import", @@ -301,6 +306,7 @@ ("PUT", "/configs/{project}/{component_id}/{config_id}/variables"): "config variables-set", ("PUT", "/data-apps/{project}/{app_id}/secrets"): "data-app secrets-set", ("PUT", "/projects/{alias}/description"): "project description-set", + ("PUT", "/rls/{project}/{policy_id}"): "rls update", ("PUT", "/semantic-layer/items/{kind}/{name}"): "", ("PUT", "/semantic-layer/reference-data"): "semantic-layer reference-data set", ("PUT", "/semantic-layer/scope/{context_id}"): "semantic-layer scope set", diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index 8baf21f0..f0bd0973 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -77,6 +77,7 @@ notifications, org, projects, + rls, schedules, search, semantic_layer, @@ -354,6 +355,18 @@ "Mirrors `kbagent semantic-layer *`." ), }, + { + "name": "rls", + "description": ( + "**Development.** " + "Row-level security policies (metastore-backed `rls-policy` " + "objects), always authored at `organization`/`targeted` scope -- " + "never `project` scope. Mirrors `kbagent rls list|detail|schema|" + "create|update|delete` (`rls setup`'s interactive picker has no " + "endpoint of its own, same carve-out as " + "`auth register-projects`)." + ), + }, # ---- AI & Tools ---- { "name": "kai", @@ -1016,6 +1029,7 @@ async def _generic_handler(request: Request, exc: Exception): app.include_router(encrypt.router) app.include_router(search.router) app.include_router(semantic_layer.router) + app.include_router(rls.router) app.include_router(transformation.router) app.include_router(docs.router) app.include_router(org.router) diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index 97920a90..8462b502 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -48,6 +48,7 @@ from ..services.org_service import OrgService from ..services.project_service import ProjectService from ..services.repo_validate_service import RepoValidateService +from ..services.rls_service import RlsService from ..services.schedule_service import ScheduleService from ..services.search_service import SearchService from ..services.semantic_layer_service import SemanticLayerService @@ -125,6 +126,7 @@ class ServiceRegistry: data_app_git: DataAppGitService = field(init=False) dev_portal: DeveloperPortalService = field(init=False) semantic_layer: SemanticLayerService = field(init=False) + rls: RlsService = field(init=False) repo_validate: RepoValidateService = field(init=False) kai: KaiService = field(init=False) encrypt: EncryptService = field(init=False) @@ -171,6 +173,7 @@ def __post_init__(self) -> None: # validate --deep + add dataset --deep-fields + build) and an # optional metastore_client_factory; the defaults work for both. self.semantic_layer = SemanticLayerService(config_store=cs) + self.rls = RlsService(config_store=cs) self.repo_validate = RepoValidateService(config_store=cs) self.kai = KaiService(config_store=cs) self.encrypt = EncryptService(config_store=cs) diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py new file mode 100644 index 00000000..42b7cd17 --- /dev/null +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -0,0 +1,148 @@ +"""Row-level security endpoints -- the REST mirror of ``kbagent rls *``. + +1:1 with the CLI group (CONTRIBUTING: every non-terminal command has a +route), except ``rls setup``: it is a genuinely terminal-only interactive +wizard (a checkbox table picker + guided condition prompts), the same +carve-out ``auth register-projects``' picker uses. Its data source (listing +a project's tables) is already covered by the existing ``storage`` router; +its write (create one policy per selected table) is the same +``POST /rls/{project}`` route below. + +**Every route enforces the permission policy** (``Depends(require_permission)``) +-- RLS is security-sensitive enough (org-admin-only authorship) that +CLI-gates-but-REST-doesn't would be a real hole, mirroring +``merge_requests.py`` rather than the (ungated) ``notifications.py``. + +The ``GET /{project}/schema`` route is registered before +``GET /{project}/{policy_id}`` so ``schema`` is never shadowed by the +parameterized route -- Starlette matches path operations in registration +order. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from ...errors import ErrorCode, KeboolaApiError +from ..dependencies import ServiceRegistry, get_registry, require_permission + +router = APIRouter(prefix="/rls", tags=["rls"]) + + +def _perm(operation: str) -> Any: + return Depends(require_permission(f"rls.{operation}")) + + +# -- Bodies -------------------------------------------------------------------------------------- + + +class RlsPolicyCreate(BaseModel): + table: str + dialect: str + rules: list[dict[str, Any]] + target_project_ids: list[str] | None = None + dry_run: bool = False + + +class RlsPolicyUpdate(BaseModel): + table: str | None = None + dialect: str | None = None + rules: list[dict[str, Any]] | None = None + target_project_ids: list[str] | None = None + dry_run: bool = False + + +# -- Reads ----------------------------------------------------------------------------------- + + +@router.get("/{project}", summary="List RLS policies", dependencies=[_perm("list")]) +def list_policies( + project: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """List row-level security policies visible to a project. Mirrors `kbagent rls list`.""" + return registry.rls.list_policies(project) + + +@router.get( + "/{project}/schema", + summary="Fetch the live rls-policy JSON Schema", + dependencies=[_perm("schema")], +) +def get_schema(project: str, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: + """Live schema from the metastore -- no offline bundled snapshot. Mirrors `kbagent rls schema`.""" + fetch = registry.rls.fetch_schema(project) + if fetch.schema is None: + raise KeboolaApiError( + message=f"Could not fetch the rls-policy schema: {fetch.reason}", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + return {"format": "json-schema", "source": "live", "schema": fetch.schema} + + +@router.get("/{project}/{policy_id}", summary="Get one RLS policy", dependencies=[_perm("detail")]) +def get_policy( + project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Full rule set for one policy. Mirrors `kbagent rls detail`.""" + return registry.rls.get_policy(project, policy_id) + + +# -- Writes ---------------------------------------------------------------------------------- + + +@router.post("/{project}", summary="Create an RLS policy", dependencies=[_perm("create")]) +def create_policy( + project: str, body: RlsPolicyCreate, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Create one policy for one table -- always `organization`/`targeted` scope, + + never `project` scope (no such option exists on this body). Mirrors + `kbagent rls create`. + """ + return registry.rls.create_policy( + project, + table=body.table, + dialect=body.dialect, + rules=body.rules, + target_project_ids=body.target_project_ids, + dry_run=body.dry_run, + ) + + +@router.put( + "/{project}/{policy_id}", summary="Update an RLS policy", dependencies=[_perm("update")] +) +def update_policy( + project: str, + policy_id: str, + body: RlsPolicyUpdate, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Fetch-then-merge: an omitted field keeps its current value, it is never + + silently blanked. Mirrors `kbagent rls update`. + """ + return registry.rls.update_policy( + project, + policy_id, + table=body.table, + dialect=body.dialect, + rules=body.rules, + target_project_ids=body.target_project_ids, + dry_run=body.dry_run, + ) + + +@router.delete( + "/{project}/{policy_id}", summary="Delete an RLS policy", dependencies=[_perm("delete")] +) +def delete_policy( + project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Delete a policy, un-protecting its table. Mirrors `kbagent rls delete`.""" + return registry.rls.delete_policy(project, policy_id) From cda24f8dc45f5d40e83db80e38909d2538cfe9b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:51:57 +0200 Subject: [PATCH 05/35] test(CLI-17): add RLS service/CLI/server/E2E test coverage test_rls_service.py, test_rls_cli.py, test_server_rls.py (new) cover scope resolution, fetch-then-merge updates, condition compilation, permission gating (cli:admin blocks writes, reads stay reachable), and router-to-service kwarg parity, all against a mocked MetastoreClient. test_metastore_client.py extended for the new scope/target_project_ids params and put_target_projects(). test_e2e.py adds an rls schema/list case that expects a clean, classified failure -- the rls-policy object type isn't registered on any real metastore backend yet (companion go-monorepo work, tracked separately). --- tests/test_e2e.py | 40 +++ tests/test_metastore_client.py | 88 ++++++ tests/test_rls_cli.py | 480 ++++++++++++++++++++++++++++++++ tests/test_rls_service.py | 483 +++++++++++++++++++++++++++++++++ tests/test_server_rls.py | 252 +++++++++++++++++ 5 files changed, 1343 insertions(+) create mode 100644 tests/test_rls_cli.py create mode 100644 tests/test_rls_service.py create mode 100644 tests/test_server_rls.py diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 131776f8..324cec87 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -677,6 +677,17 @@ def test_full_cli_e2e(self) -> None: _step(40, "storage delete-column", "dry-run + actual delete + verify") self._test_delete_column(table_id) + # ============================================================== + # PHASE 14.5: Row-level security (CLI-17) + # ============================================================== + + _step( + "40.5", + "rls schema", + "the rls-policy metastore object type isn't registered on any real stack yet", + ) + self._test_rls_commands() + # ============================================================== # PHASE 15: Cleanup # ============================================================== @@ -3710,6 +3721,35 @@ def _test_mcp_parity_commands(self) -> None: except Exception as exc: print(f" WARN: failed to delete transformation {tf_config_id}: {exc}") + def _test_rls_commands(self) -> None: + """`rls schema` -- the `rls-policy` metastore object type isn't + + registered on any real stack yet (companion go-monorepo work, CLI-17), + so a clean, classified failure here is the expected outcome, not a + kbagent bug -- same SKIP-on-failure shape as `_test_mcp_parity_commands`'s + `semantic-layer schema` case above. Full CRUD coverage (create/update/ + delete against a real `rls-policy` object) is exercised against a + mocked `MetastoreClient` in `tests/test_rls_service.py` / + `tests/test_rls_cli.py` and will get a live E2E round-trip once the + backend registers the type. + """ + result = self._run("rls", "schema", "--project", self.alias) + if result.exit_code != 0: + print(f" {_YELLOW}SKIP: rls schema (rls-policy backend not registered yet){_RESET}") + else: + data = _json_ok(result) + assert isinstance(data["data"]["schema"], dict) and data["data"]["schema"] + assert data["data"]["source"] == "live" + + # `rls list` must degrade the same way -- never a traceback -- on a + # stack without the object type registered. + result = self._run("rls", "list", "--project", self.alias) + if result.exit_code != 0: + print(f" {_YELLOW}SKIP: rls list (rls-policy backend not registered yet){_RESET}") + else: + data = _json_ok(result) + assert isinstance(data["data"]["policies"], list) + def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" # job list diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 17eee403..aa88a9b8 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -355,9 +355,97 @@ def test_semantic_types_complete(self) -> None: "semantic-constraint", "semantic-glossary", "semantic-reference-data", + "rls-policy", } +class TestPostItemScope: + """post_item/put_item default to project scope but accept organization/ + + targeted -- CLI-17: rls-policy is the first type that must never use + plain project scope. + """ + + def test_default_scope_is_still_project(self, httpx_mock, metastore_client) -> None: + """Every existing caller (semantic-layer) keeps its current behavior.""" + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", + json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item("semantic-metric", name="rev", data={"name": "rev"}) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "project" + assert "targetProjectIds" not in body + + def test_organization_scope(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy", + json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "rls-policy", + name="in.c-crm.invoices", + data={"table": "in.c-crm.invoices"}, + scope="organization", + ) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "organization" + assert "targetProjectIds" not in body + + def test_targeted_scope_includes_target_project_ids(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy", + json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, + status_code=201, + ) + metastore_client.post_item( + "rls-policy", + name="in.c-crm.invoices", + data={"table": "in.c-crm.invoices"}, + scope="targeted", + target_project_ids=["111", "222"], + ) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "targeted" + assert body["targetProjectIds"] == ["111", "222"] + + def test_put_item_accepts_scope_too(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1", + json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, + status_code=200, + ) + metastore_client.put_item( + "rls-policy", + "rls-1", + name="in.c-crm.invoices", + data={"table": "in.c-crm.invoices"}, + scope="organization", + ) + body = json.loads(httpx_mock.get_requests()[0].content) + assert body["scope"] == "organization" + + +class TestPutTargetProjects: + def test_puts_project_ids_body(self, httpx_mock, metastore_client) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", + json={"data": {"type": "rls-policy", "id": "rls-1"}}, + status_code=200, + ) + result = metastore_client.put_target_projects("rls-policy", "rls-1", ["111", "222"]) + assert result["id"] == "rls-1" + request = httpx_mock.get_requests()[0] + assert request.method == "PUT" + assert request.url.path.endswith("/rls-policy/rls-1/target-projects") + body = json.loads(request.content) + assert body["projectIds"] == ["111", "222"] + + class TestProjectScope401Reclassification: """The metastore's project-admin gate 401 becomes MISSING_MASTER_TOKEN. diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py new file mode 100644 index 00000000..aca4a70e --- /dev/null +++ b/tests/test_rls_cli.py @@ -0,0 +1,480 @@ +"""Tests for `kbagent rls` CLI commands via CliRunner -- CLI-17. + +Covers JSON output, human-mode rendering, exit codes for ConfigError / +KeboolaApiError / invalid arguments, --dry-run never triggering a write, and +`rls setup`'s non-TTY refusal (CliRunner's captured stdout is never a real +terminal, so it naturally exercises that path). +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig + +runner = CliRunner() +TEST_TOKEN = "999-token-abc" + + +def _setup_config(config_dir: Path, projects: dict[str, dict] | None = None) -> ConfigStore: + store = ConfigStore(config_dir=config_dir) + for alias, info in (projects or {}).items(): + store.add_project( + alias, + ProjectConfig( + stack_url=info.get("stack_url", "https://connection.keboola.com"), + token=info.get("token", TEST_TOKEN), + project_name=info.get("project_name", alias), + project_id=info.get("project_id", 1234), + ), + ) + return store + + +def _run( + args: list[str], + store: ConfigStore, + mock_service: MagicMock, + input: str | None = None, + storage_service: MagicMock | None = None, +) -> Any: + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.RlsService") as MockRS, + patch("keboola_agent_cli.cli.StorageService") as MockSS, + ): + MockStore.return_value = store + MockRS.return_value = mock_service + MockSS.return_value = storage_service or MagicMock() + return runner.invoke(app, args, input=input) + + +def _policy_row(**overrides: Any) -> dict[str, Any]: + row = { + "id": "p-1", + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "rule_count": 1, + "scope": "organization", + "source_project_id": "5725", + "target_project_ids": [], + } + row.update(overrides) + return row + + +# --------------------------------------------------------------------------- +# rls list +# --------------------------------------------------------------------------- + + +class TestRlsListCli: + def test_json_output(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.list_policies.return_value = {"project": "prod", "policies": [_policy_row()]} + + result = _run(["--json", "rls", "list", "--project", "prod"], store, service) + + assert result.exit_code == 0, result.output + payload = json.loads(result.output)["data"] + assert payload["policies"][0]["table"] == "in.c-crm.invoices" + + def test_human_output_shows_table(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.list_policies.return_value = {"project": "prod", "policies": [_policy_row()]} + + result = _run(["rls", "list", "--project", "prod"], store, service) + + assert result.exit_code == 0 + assert "in.c-crm.invoices" in result.output + + def test_empty_list(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.list_policies.return_value = {"project": "prod", "policies": []} + + result = _run(["rls", "list", "--project", "prod"], store, service) + + assert result.exit_code == 0 + assert "No RLS policies found" in result.output + + def test_config_error_exit_5(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {}) + service = MagicMock() + service.list_policies.side_effect = ConfigError("unknown alias 'prod'") + + result = _run(["rls", "list", "--project", "prod"], store, service) + + assert result.exit_code == 5 + + def test_api_error_mapped(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.list_policies.side_effect = KeboolaApiError( + message="boom", status_code=401, error_code=ErrorCode.INVALID_TOKEN + ) + + result = _run(["rls", "list", "--project", "prod"], store, service) + + assert result.exit_code == 3 + + +# --------------------------------------------------------------------------- +# rls detail +# --------------------------------------------------------------------------- + + +class TestRlsDetailCli: + def test_json_output(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.get_policy.return_value = { + **_policy_row(), + "rules": [{"principal": "a@x.com", "condition": {"true": True}}], + } + + result = _run( + ["--json", "rls", "detail", "--project", "prod", "--policy-id", "p-1"], store, service + ) + + assert result.exit_code == 0 + payload = json.loads(result.output)["data"] + assert payload["rules"][0]["principal"] == "a@x.com" + service.get_policy.assert_called_once_with(alias="prod", policy_id="p-1") + + +# --------------------------------------------------------------------------- +# rls schema +# --------------------------------------------------------------------------- + + +class TestRlsSchemaCli: + def test_success(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + fetch = MagicMock(schema={"type": "object"}, reason=None) + service.fetch_schema.return_value = fetch + + result = _run(["--json", "rls", "schema", "--project", "prod"], store, service) + + assert result.exit_code == 0 + payload = json.loads(result.output)["data"] + assert payload["schema"] == {"type": "object"} + assert payload["source"] == "live" + + def test_fetch_failure_is_exit_4_not_a_crash(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + fetch = MagicMock(schema=None, reason="404 not found -- backend not registered yet") + service.fetch_schema.return_value = fetch + + result = _run(["rls", "schema", "--project", "prod"], store, service) + + assert result.exit_code == 4 + assert "not registered yet" in result.output or "Could not fetch" in result.output + + +# --------------------------------------------------------------------------- +# rls create +# --------------------------------------------------------------------------- + + +class TestRlsCreateCli: + _RULES_JSON = '[{"principal": "a@x.com", "condition": {"true": true}}]' + + def test_json_output_no_confirm_needed(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + [ + "--json", + "rls", + "create", + "--project", + "prod", + "--table", + "in.c-crm.invoices", + "--dialect", + "snowflake", + "--rules", + self._RULES_JSON, + ], + store, + service, + ) + + assert result.exit_code == 0, result.output + service.create_policy.assert_called_once() + + def test_invalid_dialect_exit_2_before_service_call(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + [ + "--json", + "rls", + "create", + "--project", + "prod", + "--table", + "t", + "--dialect", + "postgres", + "--rules", + self._RULES_JSON, + ], + store, + service, + ) + + assert result.exit_code == 2 + service.create_policy.assert_not_called() + + def test_invalid_rules_json_exit_2(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + [ + "--json", + "rls", + "create", + "--project", + "prod", + "--table", + "t", + "--dialect", + "snowflake", + "--rules", + "not-json", + ], + store, + service, + ) + + assert result.exit_code == 2 + service.create_policy.assert_not_called() + + def test_dry_run_uses_dry_run_flag_and_needs_no_confirm(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "dry_run": True, "preview": []} + + result = _run( + [ + "rls", + "create", + "--project", + "prod", + "--table", + "t", + "--dialect", + "snowflake", + "--rules", + self._RULES_JSON, + "--dry-run", + ], + store, + service, + ) + + assert result.exit_code == 0, result.output + _, kwargs = service.create_policy.call_args + assert kwargs["dry_run"] is True + + def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + [ + "rls", + "create", + "--project", + "prod", + "--table", + "t", + "--dialect", + "snowflake", + "--rules", + self._RULES_JSON, + ], + store, + service, + input="n\n", + ) + + assert result.exit_code == 0 + assert "Aborted" in result.output + # Only the pre-confirm preview call (dry_run=True) may have happened; + # the real write must never fire after a "no". + for call in service.create_policy.call_args_list: + assert call.kwargs.get("dry_run") is True + + def test_config_error_exit_5(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {}) + service = MagicMock() + service.create_policy.side_effect = ConfigError("unknown alias") + + result = _run( + [ + "--json", + "rls", + "create", + "--project", + "prod", + "--table", + "t", + "--dialect", + "snowflake", + "--rules", + self._RULES_JSON, + ], + store, + service, + ) + + assert result.exit_code == 5 + + +# --------------------------------------------------------------------------- +# rls update +# --------------------------------------------------------------------------- + + +class TestRlsUpdateCli: + def test_json_output(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.update_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + [ + "--json", + "rls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--table", + "in.c-crm.invoices", + ], + store, + service, + ) + + assert result.exit_code == 0, result.output + _, kwargs = service.update_policy.call_args + assert kwargs["policy_id"] == "p-1" + assert kwargs["rules"] is None # untouched flags stay None, not overwritten + + +# --------------------------------------------------------------------------- +# rls delete +# --------------------------------------------------------------------------- + + +class TestRlsDeleteCli: + def test_yes_flag_skips_confirm(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.delete_policy.return_value = { + "project": "prod", + "policy_id": "p-1", + "deleted": True, + } + + result = _run( + ["rls", "delete", "--project", "prod", "--policy-id", "p-1", "--yes"], store, service + ) + + assert result.exit_code == 0, result.output + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1") + + def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + ["rls", "delete", "--project", "prod", "--policy-id", "p-1"], + store, + service, + input="n\n", + ) + + assert result.exit_code == 0 + assert "Aborted" in result.output + service.delete_policy.assert_not_called() + + def test_json_mode_needs_no_confirm(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.delete_policy.return_value = { + "project": "prod", + "policy_id": "p-1", + "deleted": True, + } + + result = _run( + ["--json", "rls", "delete", "--project", "prod", "--policy-id", "p-1"], store, service + ) + + assert result.exit_code == 0 + service.delete_policy.assert_called_once() + + +# --------------------------------------------------------------------------- +# rls setup (non-TTY refusal -- CliRunner's captured stdout is never a real +# terminal, so this exercises the exact same path a piped/non-interactive +# invocation would) +# --------------------------------------------------------------------------- + + +class TestRlsSetupCli: + def test_non_tty_refuses_and_hints_at_create(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + storage_service = MagicMock() + + result = _run( + ["rls", "setup", "--project", "prod"], + store, + service, + storage_service=storage_service, + ) + + assert result.exit_code == 2 + assert "rls create" in result.output + storage_service.list_tables.assert_not_called() + service.create_policy.assert_not_called() + + def test_json_mode_refuses_without_launching_picker(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + storage_service = MagicMock() + + result = _run( + ["--json", "rls", "setup", "--project", "prod"], + store, + service, + storage_service=storage_service, + ) + + assert result.exit_code == 2 + storage_service.list_tables.assert_not_called() + service.create_policy.assert_not_called() diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py new file mode 100644 index 00000000..a0853262 --- /dev/null +++ b/tests/test_rls_service.py @@ -0,0 +1,483 @@ +"""Service-layer tests for ``RlsService`` and the ``_rls_condition`` helpers. + +Each test injects a ``unittest.mock.MagicMock`` as the metastore client +factory so we verify orchestration (scope resolution, validation order, call +sequencing) without touching HTTP -- mirrors ``test_semantic_layer_service.py``. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services._rls_condition import ( + compile_condition_preview, + validate_condition_ops, + validate_policy_structural, + validate_rules_local, +) +from keboola_agent_cli.services.rls_service import RlsService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" + + +# --------------------------------------------------------------------------- +# Fixtures / helpers +# --------------------------------------------------------------------------- + + +def _make_store(tmp_path: Path, alias: str = "prod") -> ConfigStore: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + alias, + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name=alias, + project_id=5725, + ), + ) + return store + + +def _make_service( + store: ConfigStore, *, metastore_mock: MagicMock | None = None +) -> tuple[RlsService, MagicMock]: + mock = metastore_mock or MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + service = RlsService(config_store=store, metastore_client_factory=lambda url, token: mock) + return service, mock + + +def _policy_item( + item_id: str = "p-1", + table: str = "in.c-crm.invoices", + dialect: str = "snowflake", + rules: list[dict[str, Any]] | None = None, + scope: str = "organization", + source_project_id: str | None = "5725", + target_project_ids: list[str] | None = None, +) -> dict[str, Any]: + return { + "type": "rls-policy", + "id": item_id, + "attributes": { + "table": table, + "dialect": dialect, + "rules": rules + if rules is not None + else [{"principal": "a@x.com", "condition": {"true": True}}], + }, + "meta": { + "scope": scope, + "sourceProjectId": source_project_id, + "targetProjectIds": target_project_ids or [], + "revision": 1, + }, + } + + +_RULES = [{"principal": "a@x.com", "condition": {"column": "region", "op": "eq", "value": "EU"}}] + + +# --------------------------------------------------------------------------- +# Read +# --------------------------------------------------------------------------- + + +class TestListPolicies: + def test_returns_rows_from_list_items(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.list_items.return_value = [_policy_item()] + + result = service.list_policies("prod") + + mock.list_items.assert_called_once_with("rls-policy") + assert result["project"] == "prod" + assert result["policies"][0]["id"] == "p-1" + assert result["policies"][0]["rule_count"] == 1 + assert result["policies"][0]["scope"] == "organization" + + +class TestGetPolicy: + def test_returns_full_rules(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _policy_item() + + result = service.get_policy("prod", "p-1") + + mock.get_item.assert_called_once_with("rls-policy", "p-1") + assert result["rules"] == [{"principal": "a@x.com", "condition": {"true": True}}] + assert result["revision"] == 1 + + +class TestFetchSchema: + def test_success(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.return_value = {"type": "object"} + + fetch = service.fetch_schema("prod") + + assert fetch.schema == {"type": "object"} + assert fetch.reason is None + + def test_api_error_degrades(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + + fetch = service.fetch_schema("prod") + + assert fetch.schema is None + assert fetch.reason == "not found" + + def test_unexpected_error_degrades(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = RuntimeError("boom") + + fetch = service.fetch_schema("prod") + + assert fetch.schema is None + assert "boom" in (fetch.reason or "") + + def test_empty_schema_degrades(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.return_value = {} + + fetch = service.fetch_schema("prod") + + assert fetch.schema is None + assert fetch.reason is not None + + +# --------------------------------------------------------------------------- +# Write: create +# --------------------------------------------------------------------------- + + +class TestCreatePolicy: + def test_default_scope_is_organization_never_project(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.return_value = _policy_item() + + service.create_policy("prod", table="in.c-crm.invoices", dialect="snowflake", rules=_RULES) + + _, kwargs = mock.post_item.call_args + assert kwargs["scope"] == "organization" + assert kwargs["scope"] != "project" + mock.put_target_projects.assert_not_called() + + def test_target_projects_use_targeted_scope_and_manage_grants(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.return_value = _policy_item(scope="targeted", target_project_ids=["999"]) + + service.create_policy( + "prod", + table="in.c-crm.invoices", + dialect="snowflake", + rules=_RULES, + target_project_ids=["999"], + ) + + _, kwargs = mock.post_item.call_args + assert kwargs["scope"] == "targeted" + assert kwargs["target_project_ids"] == ["999"] + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["999"]) + + def test_invalid_dialect_rejected_before_any_write(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t", dialect="postgres", rules=_RULES) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() + mock.get_schema.assert_not_called() + + def test_unknown_condition_op_rejected_before_any_write(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + bad_rules = [ + {"principal": "a@x.com", "condition": {"column": "x", "op": "bogus", "value": 1}} + ] + + with pytest.raises(KeboolaApiError): + service.create_policy("prod", table="t", dialect="snowflake", rules=bad_rules) + + mock.post_item.assert_not_called() + + def test_rule_with_both_principal_and_principals_rejected(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + bad_rules = [ + {"principal": "a@x.com", "principals": ["b@x.com"], "condition": {"true": True}} + ] + + with pytest.raises(KeboolaApiError): + service.create_policy("prod", table="t", dialect="snowflake", rules=bad_rules) + + mock.post_item.assert_not_called() + + def test_structural_validation_runs_when_schema_available(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + # A schema that requires a `description` field the candidate body lacks. + mock.get_schema.return_value = { + "type": "object", + "required": ["table", "dialect", "rules", "description"], + } + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t", dialect="snowflake", rules=_RULES) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() + + def test_dry_run_never_calls_post_item(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + + result = service.create_policy( + "prod", table="t", dialect="snowflake", rules=_RULES, dry_run=True + ) + + mock.post_item.assert_not_called() + assert result["dry_run"] is True + assert result["preview"][0]["condition"] == "region = 'EU'" + + +# --------------------------------------------------------------------------- +# Write: update +# --------------------------------------------------------------------------- + + +class TestUpdatePolicy: + def test_partial_override_preserves_untouched_fields(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _policy_item(table="in.c-crm.invoices", dialect="snowflake") + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.put_item.return_value = _policy_item() + + service.update_policy("prod", "p-1", rules=_RULES) + + _, kwargs = mock.put_item.call_args + assert kwargs["data"]["table"] == "in.c-crm.invoices" # unchanged, not wiped + assert kwargs["data"]["dialect"] == "snowflake" # unchanged, not wiped + assert kwargs["data"]["rules"] == _RULES # the one field we changed + + def test_targeted_scope_derived_from_merged_targets(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _policy_item(scope="organization", target_project_ids=[]) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=["42"]) + + service.update_policy("prod", "p-1", target_project_ids=["42"]) + + _, kwargs = mock.put_item.call_args + assert kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["42"]) + + def test_invalid_override_rejected_before_write(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _policy_item() + + with pytest.raises(KeboolaApiError): + service.update_policy("prod", "p-1", dialect="postgres") + + mock.put_item.assert_not_called() + + def test_dry_run_never_calls_put_item(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_item.return_value = _policy_item() + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + + result = service.update_policy("prod", "p-1", rules=_RULES, dry_run=True) + + mock.put_item.assert_not_called() + assert result["dry_run"] is True + + +# --------------------------------------------------------------------------- +# Write: delete +# --------------------------------------------------------------------------- + + +class TestDeletePolicy: + def test_calls_delete_item(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + + result = service.delete_policy("prod", "p-1") + + mock.delete_item.assert_called_once_with("rls-policy", "p-1") + assert result == {"project": "prod", "policy_id": "p-1", "deleted": True} + + +# --------------------------------------------------------------------------- +# Pure condition helpers +# --------------------------------------------------------------------------- + + +class TestCompileConditionPreview: + def test_true_sentinel(self) -> None: + assert compile_condition_preview({"true": True}, "snowflake") == "TRUE" + + @pytest.mark.parametrize( + "op,sql", + [ + ("eq", "region = 'EU'"), + ("ne", "region != 'EU'"), + ("gt", "region > 'EU'"), + ("gte", "region >= 'EU'"), + ("lt", "region < 'EU'"), + ("lte", "region <= 'EU'"), + ], + ) + def test_comparison_ops(self, op: str, sql: str) -> None: + condition = {"column": "region", "op": op, "value": "EU"} + assert compile_condition_preview(condition, "snowflake") == sql + + def test_in_op(self) -> None: + condition = {"column": "region", "op": "in", "values": ["EU", "US"]} + assert compile_condition_preview(condition, "snowflake") == "region IN ('EU', 'US')" + + def test_not_in_op(self) -> None: + condition = {"column": "region", "op": "not_in", "values": ["EU"]} + assert compile_condition_preview(condition, "snowflake") == "region NOT IN ('EU')" + + def test_is_null(self) -> None: + condition = {"column": "deleted_at", "op": "is_null"} + assert compile_condition_preview(condition, "snowflake") == "deleted_at IS NULL" + + def test_is_not_null(self) -> None: + condition = {"column": "deleted_at", "op": "is_not_null"} + assert compile_condition_preview(condition, "snowflake") == "deleted_at IS NOT NULL" + + def test_and_nesting(self) -> None: + condition = { + "and": [ + {"column": "region", "op": "eq", "value": "EU"}, + {"column": "status", "op": "ne", "value": "draft"}, + ] + } + assert compile_condition_preview(condition, "snowflake") == ( + "(region = 'EU') AND (status != 'draft')" + ) + + def test_or_nesting(self) -> None: + condition = { + "or": [ + {"column": "region", "op": "eq", "value": "EU"}, + {"column": "region", "op": "eq", "value": "US"}, + ] + } + assert compile_condition_preview(condition, "snowflake") == ( + "(region = 'EU') OR (region = 'US')" + ) + + def test_numeric_and_null_literals_not_quoted(self) -> None: + condition = {"column": "amount", "op": "gt", "value": 100} + assert compile_condition_preview(condition, "snowflake") == "amount > 100" + + def test_unrecognized_shape_raises(self) -> None: + with pytest.raises(ValueError): + compile_condition_preview({"nonsense": True}, "snowflake") + + +class TestValidateConditionOps: + def test_known_op_passes(self) -> None: + assert validate_condition_ops({"column": "x", "op": "eq", "value": 1}) == [] + + def test_unknown_op_reported(self) -> None: + errors = validate_condition_ops({"column": "x", "op": "bogus", "value": 1}) + assert errors and "bogus" in errors[0] + + def test_true_sentinel_always_valid(self) -> None: + assert validate_condition_ops({"true": True}) == [] + + def test_and_requires_at_least_two(self) -> None: + errors = validate_condition_ops({"and": [{"true": True}]}) + assert errors + + def test_nested_and_recurses(self) -> None: + condition = { + "and": [ + {"column": "x", "op": "bogus", "value": 1}, + {"true": True}, + ] + } + errors = validate_condition_ops(condition) + assert errors and "bogus" in errors[0] + + +class TestValidateRulesLocal: + def test_empty_rules_rejected(self) -> None: + assert validate_rules_local([]) != [] + + def test_non_list_rejected(self) -> None: + assert validate_rules_local("not-a-list") != [] + + def test_valid_single_principal(self) -> None: + assert validate_rules_local([{"principal": "a@x.com", "condition": {"true": True}}]) == [] + + def test_valid_principals_list(self) -> None: + rules = [{"principals": ["a@x.com", "b@x.com"], "condition": {"true": True}}] + assert validate_rules_local(rules) == [] + + def test_neither_principal_nor_principals_rejected(self) -> None: + errors = validate_rules_local([{"condition": {"true": True}}]) + assert errors + + def test_both_principal_and_principals_rejected(self) -> None: + rules = [{"principal": "a@x.com", "principals": ["b@x.com"], "condition": {"true": True}}] + assert validate_rules_local(rules) + + def test_missing_condition_rejected(self) -> None: + errors = validate_rules_local([{"principal": "a@x.com"}]) + assert errors + + +class TestValidatePolicyStructural: + def test_valid_policy_passes(self) -> None: + schema = { + "type": "object", + "required": ["table", "dialect", "rules"], + "properties": { + "table": {"type": "string"}, + "dialect": {"enum": ["snowflake", "bigquery"]}, + "rules": {"type": "array", "minItems": 1}, + }, + } + policy = {"table": "in.c-x.y", "dialect": "snowflake", "rules": [{"condition": {}}]} + assert validate_policy_structural(policy, schema) == [] + + def test_missing_required_field_reported(self) -> None: + schema = {"type": "object", "required": ["table", "dialect", "rules"]} + errors = validate_policy_structural({"table": "t"}, schema) + assert errors diff --git a/tests/test_server_rls.py b/tests/test_server_rls.py new file mode 100644 index 00000000..07df51ff --- /dev/null +++ b/tests/test_server_rls.py @@ -0,0 +1,252 @@ +"""Tests for `server/routers/rls.py` -- router -> service call parity + permission gating. + +Two families, mirroring ``test_server_router_calls.py`` (kwarg parity via a +mocked ``ServiceRegistry``) and ``test_server_permissions.py`` (persisted +policy -> 403). RLS is classified ``admin`` for every write -- these tests +pin that ``cli:admin`` (not merely ``cli:write``) is what has to be denied +to block ``create``/``update``/``delete``, and that ``list``/``detail``/ +``schema`` stay reachable under that same denial. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +if importlib.util.find_spec("fastapi") is None: # pragma: no cover + pytest.skip( + "FastAPI not installed; run `uv pip install -e '.[server]'`", allow_module_level=True + ) + +from fastapi.testclient import TestClient + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import PermissionPolicy +from keboola_agent_cli.server import create_app +from keboola_agent_cli.server.dependencies import ServiceRegistry, get_registry + +AUTH = {"Authorization": "Bearer test-token"} +PROJECT = "prod" +POLICY_ID = "p-1" + + +def _make_app_with_registry(tmp_path: Path, registry: ServiceRegistry) -> Any: + app = create_app(config_dir=str(tmp_path), auth_token="test-token") + app.dependency_overrides[get_registry] = lambda: registry + return app + + +def _mock_registry(**services: Any) -> ServiceRegistry: + registry = ServiceRegistry.__new__(ServiceRegistry) + for name, mock in services.items(): + setattr(registry, name, mock) + return registry + + +def _persist_policy(config_dir: Path, policy: PermissionPolicy) -> None: + store = ConfigStore(config_dir=config_dir) + config = store.load() + config.permissions = policy + store.save(config) + + +# --------------------------------------------------------------------------- +# Router -> service call parity +# --------------------------------------------------------------------------- + + +def test_list_policies_calls_service(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.list_policies.return_value = {"project": PROJECT, "policies": []} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}", headers=AUTH) + + assert res.status_code == 200, res.text + rls_svc.list_policies.assert_called_once_with(PROJECT) + + +def test_get_policy_calls_service(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.get_policy.return_value = {"id": POLICY_ID} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 200, res.text + rls_svc.get_policy.assert_called_once_with(PROJECT, POLICY_ID) + + +def test_schema_success(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.fetch_schema.return_value = MagicMock(schema={"type": "object"}, reason=None) + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}/schema", headers=AUTH) + + assert res.status_code == 200, res.text + assert res.json()["schema"] == {"type": "object"} + + +def test_schema_fetch_failure_is_404_not_a_crash(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.fetch_schema.return_value = MagicMock(schema=None, reason="not registered yet") + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}/schema", headers=AUTH) + + assert res.status_code == 404, res.text + assert res.json()["error"]["code"] == "NOT_FOUND" + + +def test_schema_route_not_shadowed_by_policy_id_route(tmp_path: Path) -> None: + """`/rls/{project}/schema` must resolve to the schema route, never `detail` + + with policy_id='schema' -- the registration-order guard the router's + docstring promises.""" + rls_svc = MagicMock() + rls_svc.fetch_schema.return_value = MagicMock(schema={"type": "object"}, reason=None) + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + client.get(f"/rls/{PROJECT}/schema", headers=AUTH) + + rls_svc.fetch_schema.assert_called_once_with(PROJECT) + rls_svc.get_policy.assert_not_called() + + +def test_create_policy_passes_kwargs(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.create_policy.return_value = {"id": POLICY_ID} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + body = { + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "rules": [{"principal": "a@x.com", "condition": {"true": True}}], + "target_project_ids": ["999"], + "dry_run": False, + } + + with TestClient(app) as client: + res = client.post(f"/rls/{PROJECT}", headers=AUTH, json=body) + + assert res.status_code == 200, res.text + kwargs = rls_svc.create_policy.call_args.kwargs + assert kwargs["table"] == "in.c-crm.invoices" + assert kwargs["dialect"] == "snowflake" + assert kwargs["rules"] == body["rules"] + assert kwargs["target_project_ids"] == ["999"] + assert kwargs["dry_run"] is False + + +def test_update_policy_passes_kwargs(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.update_policy.return_value = {"id": POLICY_ID} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.put(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table": "new.table"}) + + assert res.status_code == 200, res.text + args, kwargs = rls_svc.update_policy.call_args + assert args == (PROJECT, POLICY_ID) + assert kwargs["table"] == "new.table" + assert kwargs["rules"] is None + + +def test_delete_policy_calls_service(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.delete_policy.return_value = {"deleted": True} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.delete(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 200, res.text + rls_svc.delete_policy.assert_called_once_with(PROJECT, POLICY_ID) + + +def test_keboola_api_error_from_service_maps_to_http_status(tmp_path: Path) -> None: + rls_svc = MagicMock() + rls_svc.get_policy.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 404, res.text + + +# --------------------------------------------------------------------------- +# Permission gating -- RLS writes are `admin`, not merely `write` +# --------------------------------------------------------------------------- + + +class TestPermissionGating: + def test_denying_cli_admin_blocks_create(self, tmp_path: Path) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) + rls_svc = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.post( + f"/rls/{PROJECT}", + headers=AUTH, + json={ + "table": "t", + "dialect": "snowflake", + "rules": [{"principal": "a@x.com", "condition": {"true": True}}], + }, + ) + + assert res.status_code == 403, res.text + assert res.json()["error"]["code"] == "PERMISSION_DENIED" + rls_svc.create_policy.assert_not_called() + + def test_denying_cli_admin_blocks_delete(self, tmp_path: Path) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) + rls_svc = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.delete(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 403, res.text + rls_svc.delete_policy.assert_not_called() + + def test_denying_cli_admin_still_allows_reads(self, tmp_path: Path) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) + rls_svc = MagicMock() + rls_svc.list_policies.return_value = {"project": PROJECT, "policies": []} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.get(f"/rls/{PROJECT}", headers=AUTH) + + assert res.status_code == 200, res.text + rls_svc.list_policies.assert_called_once() + + def test_denying_cli_write_also_blocks_admin_ops(self, tmp_path: Path) -> None: + """`cli:write` spans write+destructive+admin (see permissions.py) -- + + pins that RLS writes are not a hole in that broader firewall.""" + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:write"])) + rls_svc = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + + with TestClient(app) as client: + res = client.delete(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 403, res.text + rls_svc.delete_policy.assert_not_called() From 71b56d4c27a368fa7ae9f630e17d2f8565212b49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 17 Sep 2026 14:52:08 +0200 Subject: [PATCH 06/35] docs(CLI-17): document the rls command group across all doc surfaces CLAUDE.md '## All CLI Commands', commands/context.py AGENT_CONTEXT, commands-reference.md, and keboola-expert.md's tool-selection matrix per CONTRIBUTING's "Checklist: Adding a New CLI Command". New rls-workflow.md (guided setup, scripted creation, condition primitive cookbook, targeted-scope sharing, troubleshooting the not-yet-available backend). gotchas.md gains the RLS entry (since vNEXT). SKILL.md frontmatter triggers + workflow table + regenerated auto-command-table (trimmed elsewhere to stay under the 1024-char Claude Desktop limit). docs/error-codes.md documents INVALID_RLS_POLICY; web-server-endpoints.md regenerated for the new /rls router. No version bump, no changelog.py entry -- feature PRs never bump the version per this repo's binding convention; the release PR resolves the (since vNEXT) placeholder. --- CLAUDE.md | 30 ++ docs/error-codes.md | 6 + docs/web-server-endpoints.md | 13 + plugins/kbagent/skills/kbagent/SKILL.md | 4 +- .../kbagent/references/commands-reference.md | 12 + .../skills/kbagent/references/rls-workflow.md | 302 ++++++++++++++++++ src/keboola_agent_cli/errors.py | 1 + 7 files changed, 366 insertions(+), 2 deletions(-) create mode 100644 plugins/kbagent/skills/kbagent/references/rls-workflow.md diff --git a/CLAUDE.md b/CLAUDE.md index 7fb95e63..154a62fe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1382,6 +1382,36 @@ kbagent semantic-layer reference-data delete --project P --id ID [--yes] # 0.97.0)` cannot be written on these `# ` comment lines (check_version_gates.py parses them as # ATX headings). +kbagent rls list --project P +kbagent rls detail --project P --policy-id ID +kbagent rls schema --project P +kbagent rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] +kbagent rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes] +kbagent rls delete --project P --policy-id ID [--yes] +kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes] +# rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, +# one object per protected table). ORG-ADMIN-ONLY, STRUCTURALLY: every write (create/update/ +# setup) is authored at organization scope (default) or targeted scope (--target-project, +# repeatable), never project scope -- there is no --scope flag offering project at all, matching +# the backend's own restriction that a project's own admin can never author policy for its own +# tables. --rules is a JSON array of {principal|principals, condition} objects; condition is a +# declarative primitive tree (column/op/value comparisons, in/not_in, is_null/is_not_null, +# and/or nesting, or a {"true": true} sentinel) -- never a free-text predicate string, closing +# the injection surface a hand-written-SQL model would have. `--dry-run` previews the compiled +# condition via kbagent's OWN preview renderer, which is explicitly NOT the enforcement engine +# (that lives in keboola-mcp-server's `query_data`, a different repo/runtime) -- the preview is +# for admin sanity-checking only, drift between the two is acceptable since the preview string +# is never executed. `rls setup` is a guided, interactive-terminal-only wizard (checkbox table +# picker + guided condition builder, reusing `storage tables` and the same write path `rls +# create` uses) -- it refuses under --json or a non-TTY stdout with a hint to use `rls create` +# directly, same carve-out as `auth register-projects`'s picker, and has no REST route. +# THE METASTORE BACKEND DOES NOT REGISTER `rls-policy` ON ANY DEPLOYED STACK YET (a companion +# go-monorepo change, tracked separately) -- every command here answers a clean, classified +# error (schema fetch failure, NOT_FOUND) against a real project until that lands; this is +# expected, not a kbagent bug. New error code: INVALID_RLS_POLICY. Version gate for this whole +# entry lives in gotchas.md -- a `(since vNEXT)` tag cannot be written on these `# ` comment +# lines (check_version_gates.py parses them as ATX markdown headings). + kbagent http get PATH [--timeout SECONDS] kbagent http post PATH [--body JSON|@file|-] [--timeout SECONDS] kbagent http patch PATH [--body JSON|@file|-] [--timeout SECONDS] diff --git a/docs/error-codes.md b/docs/error-codes.md index ee85d523..ee4fb368 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -227,3 +227,9 @@ A command that fails with an error code exits with the code's `Exit` value below |---|---|---|---| | `UPDATED_ACROSS_MAJOR` | 6 | `authorization` | After an automatic update to a higher major version (for example from 0.x to 1.x), the first write or destructive command was refused once, with exit code 6. Every later command runs. Reads before that command run and get the same code as a `warnings[]` entry. Admin commands and an update started with `kbagent update --major` do not refuse. See [migrate-to-1.0.md](migrate-to-1.0.md) | | `UPDATED_ACROSS_BREAKING` | 1 | `api` | Only a `warnings[]` entry, in the first command after an update, never an error exit: the update crossed versions with `BREAKING:` changelog notes. `versions` lists them. `kbagent changelog --full` shows the notes | + +### Row-Level Security + +| Code | Exit | Type | Description | +|---|---|---|---| +| `INVALID_RLS_POLICY` | 1 | `api` | An `rls create`/`update` candidate body (`table`/`dialect`/`rules`/`condition`) failed validation -- either the schema-independent local checks (unknown op, `principal`/`principals` not exactly one) or the live JSON-Schema check against the metastore's `rls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 20f68e27..3136d733 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -428,6 +428,19 @@ Model, validate, import/export, diff, promote, and build semantic layer artifact | `PUT` | `/semantic-layer/scope/{context_id}/elevation-request` | Request scope elevation | | `DELETE` | `/semantic-layer/scope/{context_id}/elevation-request` | Withdraw scope elevation request | +### `rls` (6 operations) + +Row-level security policies (metastore-backed `rls-policy` objects), always authored at `organization`/`targeted` scope -- never `project` scope. Mirrors `kbagent rls list|detail|schema|create|update|delete` (`rls setup`'s interactive picker has no endpoint of its own, same carve-out as `auth register-projects`). + +| Method | Path | Summary | +|---|---|---| +| `GET` | `/rls/{project}` | List RLS policies | +| `POST` | `/rls/{project}` | Create an RLS policy | +| `GET` | `/rls/{project}/schema` | Fetch the live rls-policy JSON Schema | +| `GET` | `/rls/{project}/{policy_id}` | Get one RLS policy | +| `PUT` | `/rls/{project}/{policy_id}` | Update an RLS policy | +| `DELETE` | `/rls/{project}/{policy_id}` | Delete an RLS policy | + ## AI & Tools ### `kai` (6 operations) diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index c74e3e50..338c4a9f 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -8,7 +8,7 @@ description: > Storage tables/files/snapshots, large (200 GB) file uploads, data apps, flows/schedules, invitations, feature flags, OTLP data streams, scoped Storage tokens, semantic - layer, Developer Portal, + layer, row-level security, Developer Portal, first-time setup and logout. Triggers: kbagent, Keboola, keboola config, keboola job, keboola lineage, keboola sync, gitops, dev branch, @@ -18,7 +18,7 @@ description: > feature flag, flow schedule, invite member, SQL transformation edit, sync action, keboola docs, table snapshot, auth, login, sign in, PAYG credits, flow notifications, alert recipients, config trash, - restore config, zero-copy clone, workspace load type, + restore config, rls, zero-copy clone, workspace load type, multipart, import job, job-detail, upload-table no-wait, set up keboola, connect project, logout. --- diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 9f27277a..32837645 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -467,6 +467,18 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e - `semantic-layer reference-data set --project P [--model M] --dimension D --members-file PATH [--dataset-id T] [--description X]` -- create-or-replace. `--members-file` is a JSON array of member objects (`-` reads stdin). **Idempotent** on `dimension`: the lookup is project-wide (the envelope `name` = dimension is unique per project per type), so an existing record is replaced in place via `PUT` (the metastore increments `meta.revision`, preserving history) — like `edit` — regardless of which `--model` is passed (the resolved model is stored on the record); otherwise a new record is `POST`-ed. Response: `{id, dimension_name, member_count, action: "created"|"updated"}`. - `semantic-layer reference-data delete --project P --id ID [--yes]` -- delete by UUID (server-side soft-delete; the record stays in revision history). Non-TTY without `--yes` refuses with exit 2. +## Row-Level Security (RLS) + +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo). Same metastore/master-token requirements as Semantic Layer above (see [gotchas.md](gotchas.md)). **Every write is `organization`/`targeted` scope, never `project`** -- no `--scope project` option exists anywhere in this group, matching the backend's restriction that a project's own admin can never author policy for its own tables. **The metastore backend does not register `rls-policy` on any deployed stack yet** (companion `go-monorepo` work, tracked separately) -- every command answers a classified error against a real project until that lands; see [rls-workflow.md](rls-workflow.md) Workflow 7 for how to tell that apart from a real problem. + +- `rls list --project P` -- list policies visible to a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, source_project_id, target_project_ids}, ...]}`. An `organization`-scope row's `source_project_id` may not match the current project -- see `rls detail`/the workflow doc before assuming it applies here. +- `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals, condition}], revision}`. +- `rls schema --project P` -- live `rls-policy` JSON Schema fetched from the metastore. Unlike `flow schema`, there is no offline bundled snapshot -- always live-only, `NOT_FOUND` (exit 4) on fetch failure rather than a stale local copy. +- `rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or a `{"true": true}` sentinel) -- never a raw predicate string. Validated locally (op names, `principal`-xor-`principals`) always, plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any network write. `--target-project` (repeatable) shifts scope from `organization` to `targeted` and registers the grant via a follow-up `PUT .../target-projects` call. `--dry-run` runs every validation and returns the same compiled-condition preview a real write would, without writing -- see the preview-vs-enforcement caveat in [gotchas.md](gotchas.md). +- `rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes]` -- fetch-then-merge: `get_item` first, only the flags you pass overwrite the stored `table`/`dialect`/`rules`/target projects, everything else is carried forward untouched (the metastore's `PUT` is a whole-record replace, so this merge step is what stops a partial update from silently blanking fields). +- `rls delete --project P --policy-id ID [--yes]` -- delete a policy, un-protecting its table. Confirmation prompt unless `--yes`; `--json` skips it too. +- `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table (same write path, never a second one). Refuses cleanly under `--json` or a non-TTY stdout with a hint to use `rls create` directly -- there is no non-interactive path through `setup` itself, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. + ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) - `http get PATH [--timeout SECONDS]` -- GET an endpoint on the running `kbagent serve` - `http post PATH [--body JSON|@file|-] [--timeout SECONDS]` -- POST with optional JSON body diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md new file mode 100644 index 00000000..745bb624 --- /dev/null +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -0,0 +1,302 @@ +# RLS Workflow -- Row-Level Security Policies + +Row-level security in Keboola is authored as metastore `rls-policy` objects +-- one object per protected table, holding a list of per-principal condition +primitives (never a hand-written SQL predicate). `kbagent` only **authors** +these objects; enforcement (the actual SQL rewrite on a query) happens +inside `keboola-mcp-server`'s `query_data` tool, a different repo and a +different runtime. `kbagent rls` never executes a query and never decides +whether a real query gets filtered -- it only creates, reads, updates and +deletes the policy objects the enforcement engine reads. + +**Authorship is org-admin-only, structurally.** Every write (`create`, +`update`, `setup`) is issued at `organization` scope (default) or +`targeted` scope (`--target-project`, repeatable) -- there is no `--scope +project` option anywhere in this command group, matching the backend's own +restriction that a project's own admin can never author policy for its own +tables. `--project` on every command still names the project whose tables +you're working with (and, for reads, which metastore instance to query) -- +it does not grant that project's admin any extra authority over policy +authorship. + +Same metastore requirements as [semantic-layer-workflow.md](semantic-layer-workflow.md): +a MASTER (project admin) Storage token (`kbagent --json project info +--project P` -> `is_master_token`), same `MISSING_MASTER_TOKEN` reclassification +on a non-master token. Pre-flight before anything else in this file: + +```bash +kbagent --json project info --project P | jq '.is_master_token' +``` + +**The metastore backend does not register the `rls-policy` object type on +any deployed stack yet** (a companion `go-monorepo` change, tracked +separately from the kbagent work). Every command below will answer with a +classified `NOT_FOUND` (schema fetch failure) or similar error against a +real project until that lands -- see Workflow 7 for how to tell that apart +from an actual problem. All the examples here are still correct and worth +running once the backend ships; they are also fully covered by kbagent's own +test suite against a mocked metastore in the meantime. + +Full per-command flag reference: [commands-reference.md](commands-reference.md#row-level-security-rls). +Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). + +## When to use what + +| Goal | Command | +|------|---------| +| See what RLS policies already protect a project | `rls list --project P` | +| Inspect one policy's full rule set | `rls detail --project P --policy-id ID` | +| Check the live condition/rules shape before hand-writing JSON | `rls schema --project P` | +| Walk an org admin through authoring a new policy interactively | `rls setup --project P` | +| Script or automate policy creation (CI, batch onboarding) | `rls create --project P --table T --dialect D --rules '[...]'` | +| Change one field on an existing policy without touching the rest | `rls update --project P --policy-id ID --table/--dialect/--rules ...` | +| Share one policy across sibling projects | `rls create`/`rls update ... --target-project ID` | +| Preview the compiled condition before writing anything | any write command + `--dry-run` | +| Remove a policy (un-protect a table) | `rls delete --project P --policy-id ID` | + +--- + +## Workflow 1 -- Guided setup (the default path for a human org admin) + +`rls setup` is interactive-terminal-only -- it needs a real TTY for the +checkbox table picker and the condition-builder prompts. It refuses cleanly +under `--json` or a piped/non-TTY stdout, printing a one-line hint instead +of hanging or producing malformed JSON output. + +```bash +# 1. Run it directly in an interactive terminal -- NOT through kbagent http, +# NOT with --json, NOT from a scheduled agent task. +kbagent rls setup --project prod + +# 2. It shows a checkbox picker over `storage tables --project prod`'s +# result: arrows/j-k to move, space to toggle, 'a' to select all, +# enter to confirm, 'q'/esc/ctrl-c to cancel. + +# 3. For the rules, either: +# (a) build interactively -- it prompts for principal(s), then a small +# menu: comparison / IN-NOT_IN / IS NULL-IS NOT NULL / always-true, +# with an "add another rule?" loop, or +# (b) skip the builder entirely with a prepared --rules file for +# anything beyond a single simple comparison per principal: +kbagent rls setup --project prod --rules @rls_rules.json + +# 4. It prints a compiled-condition preview per selected table, then a +# single confirm (skip with --yes) before creating one `rls-policy` +# object per table -- via the exact same write path `rls create` uses, +# never a second, parallel one. +``` + +If `rls setup` is not usable (no TTY -- a CI job, a scheduled agent task, a +piped invocation), it tells you so and points at `rls create` -- go to +Workflow 2. + +## Workflow 2 -- Scripted / non-interactive policy creation + +The path for CI, batch onboarding, or any AI-agent-driven invocation. +`--rules` always takes the same `JSON|@file|-` form the rest of kbagent uses +for structured input (see e.g. `config update --configuration`). + +```bash +# 1. Write the rules array to a file (or build it inline for something +# this short) -- ALWAYS a declarative condition, never a predicate string. +cat > rls_rules.json <<'EOF' +[ + { + "principal": "eu-analyst@example.com", + "condition": {"column": "region", "op": "eq", "value": "EU"} + }, + { + "principals": ["us-analyst@example.com", "us-lead@example.com"], + "condition": {"column": "region", "op": "eq", "value": "US"} + } +] +EOF + +# 2. ALWAYS preview first -- --dry-run runs every validation (dialect, +# rule shape, live schema check when available) and compiles a +# human-readable condition preview, but writes nothing. +kbagent --json rls create \ + --project prod \ + --table in.c-crm.invoices \ + --dialect snowflake \ + --rules @rls_rules.json \ + --dry-run +# -> {"table": "in.c-crm.invoices", "dialect": "snowflake", +# "scope": "organization", "preview": [ +# {"principal": "eu-analyst@example.com", "condition": "region = 'EU'"}, +# {"principal": "us-analyst@example.com, us-lead@example.com", +# "condition": "region = 'US'"} +# ], "dry_run": true} + +# 3. Then the real write (still --json for a clean, single-document +# response -- no confirmation prompt in --json mode): +kbagent --json rls create \ + --project prod \ + --table in.c-crm.invoices \ + --dialect snowflake \ + --rules @rls_rules.json +``` + +A table with no rows matching ANY rule for a given caller means the query +is refused, not silently returned empty -- fail-closed is the enforcement +engine's whole design point (see the RFC in `keboola-mcp-server`). Before +protecting a table, make sure every principal who legitimately needs access +has a matching rule, including yourself if you plan to verify the result +via `query_data`. + +## Workflow 3 -- Inspect existing policies before making a change + +Always read before you write -- `rls update` merges onto the CURRENT stored +state, so knowing what's there first avoids surprises. + +```bash +# 1. What's protected in this project already? +kbagent --json rls list --project prod +# -> {"project": "prod", "policies": [ +# {"id": "...", "table": "in.c-crm.invoices", "dialect": "snowflake", +# "rule_count": 2, "scope": "organization", +# "source_project_id": "12345", "target_project_ids": []} +# ]} + +# 2. Full rule set for one policy (needed before a partial `rls update`): +kbagent --json rls detail --project prod --policy-id +``` + +`scope`/`source_project_id`/`target_project_ids` tell you how a policy +became visible here: `organization` scope is visible to every project in +the org, but only APPLIES where `source_project_id` matches the querying +project (or that project is in `target_project_ids` for a `targeted` +policy) -- a policy authored for a different project, with the same table +name, never applies here. Don't assume a listed policy protects the current +project just because it's in the listing; check `source_project_id`. + +## Workflow 4 -- Condition primitive cookbook + +Every `condition` is one of six shapes -- there is no free-text predicate +field anywhere in this schema, by design (closes the injection surface a +hand-written-SQL model would have). `rls schema --project P` returns the +live, authoritative JSON Schema; this is the quick-reference version. + +```jsonc +// Comparison: eq | ne | gt | gte | lt | lte +{"column": "status", "op": "eq", "value": "active"} + +// Membership: in | not_in +{"column": "region", "op": "in", "values": ["EU", "US"]} + +// Nullness: is_null | is_not_null +{"column": "deleted_at", "op": "is_null"} + +// Boolean composition (2+ nested conditions each) +{"and": [ + {"column": "region", "op": "eq", "value": "EU"}, + {"column": "status", "op": "ne", "value": "draft"} +]} +{"or": [ + {"column": "region", "op": "eq", "value": "EU"}, + {"column": "region", "op": "eq", "value": "US"} +]} + +// Always applies (no filtering for this principal -- use sparingly, this +// is the escape hatch for "this admin sees everything") +{"true": true} +``` + +`rls create --dry-run` (or `rls setup`'s preview step) renders any of these +as a `WHERE`-clause-shaped string for human sanity-checking. **That preview +is kbagent's own renderer, not the enforcement engine** -- the engine that +actually decides what a query returns lives in `keboola-mcp-server`'s +`rls.py` (sqlglot-based). Treat the preview as "does this look like what I +meant," never as proof of what will be enforced. + +## Workflow 5 -- Share a policy across sibling projects (targeted scope) + +`--target-project` (repeatable, on `create`/`update`/`setup`) shifts scope +from `organization` to `targeted` and registers each listed project as a +grant. Use this when one org-authored policy should apply identically +across specific sibling customer projects, not the whole org. + +```bash +kbagent --json rls create \ + --project prod \ + --table in.c-crm.invoices \ + --dialect snowflake \ + --rules @rls_rules.json \ + --target-project 22222 \ + --target-project 33333 \ + --dry-run +# -> {"scope": "targeted", "target_project_ids": ["22222", "33333"], ...} +``` + +Omitting `--target-project` on a later `rls update` call leaves the current +target list untouched (fetch-then-merge, see Workflow 6) -- pass the FULL +desired list to change it, not just an addition. + +## Workflow 6 -- Update or revoke a policy safely + +`rls update` is fetch-then-merge: only the flags you pass change, every +omitted flag keeps its current stored value -- a partial update can never +silently blank `table`/`dialect`/`rules`/target projects you didn't mean to +touch. + +```bash +# Change only the rules, leave table/dialect/scope exactly as they are: +kbagent --json rls update \ + --project prod \ + --policy-id \ + --rules @updated_rls_rules.json \ + --dry-run # preview first, same as create + +kbagent --json rls update --project prod --policy-id --rules @updated_rls_rules.json + +# Revoke entirely (un-protects the table -- confirm this is what you want, +# a project without the feature flag or without an applicable policy is +# simply unfiltered, not an error state on the enforcement side): +kbagent rls delete --project prod --policy-id +# --yes to skip the confirmation prompt in a script; --json also skips it. +``` + +## Workflow 7 -- Troubleshooting "backend not available" errors + +Until the `go-monorepo` companion work registers the `rls-policy` object +type, EVERY command in this group will fail against a real, live project -- +this is expected, not a kbagent bug. Tell it apart from a real problem: + +```bash +kbagent --json rls schema --project prod +``` + +- **`NOT_FOUND` / "Could not fetch the rls-policy schema: ..."** -- the + expected shape while the backend isn't registered yet. The commands are + fully implemented and unit-tested against a mocked metastore client + (`tests/test_rls_service.py`); nothing here is broken, the object type + just doesn't exist on the stack yet. +- **`MISSING_MASTER_TOKEN`** -- an actual, fixable problem: the registered + token for this project isn't a master token. `kbagent project info + --project P` -> `is_master_token`, then `project edit --project P + --token ...` with a master token. +- **`INVALID_RLS_POLICY`** -- your `--rules`/`--dialect`/`--table` failed + validation (either the schema-independent checks kbagent always runs, or + the live JSON-Schema check when the backend IS reachable). The error + message lists every violation found; nothing was written. +- **`CONFIG_ERROR` / exit 5** -- the `--project` alias isn't registered + locally. `kbagent project list`. + +--- + +## Reference: RLS contract gotchas + +- [gotchas.md](gotchas.md) -- search "rls" for the full list, version-tagged. +- One `rls-policy` object per protected table, never one blob per project. +- `organization` scope is visible org-wide but only APPLIES where + `source_project_id` (or `target_project_ids` for `targeted` scope) + matches the querying project -- never by table-name text alone. +- `--scope project` does not exist anywhere in this command group. This is + intentional, not a missing feature -- see the RFC in + `keboola-mcp-server`'s `feature_spec/rls_query_tool/RFC.md`. +- `rls setup` has no REST route (`kbagent serve`) -- it is genuinely + terminal-only, same carve-out as `auth register-projects`'s picker. +- kbagent's `--dry-run` preview and the real enforcement engine + (`keboola-mcp-server`'s `rls.py`) are two separate implementations by + design -- the preview is a convenience for human review, never the + source of truth for what a query actually returns. diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index 3f03f1bb..f5962dbc 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -493,6 +493,7 @@ class ErrorClass: # Flow ErrorCode.SCHEDULE_DELETE_FAILED: ErrorClass("api", 1), ErrorCode.INVALID_FLOW_DEFINITION: ErrorClass("api", 1), + ErrorCode.INVALID_RLS_POLICY: ErrorClass("api", 1), # Data apps ErrorCode.DATA_APP_BUILD_FAILED: ErrorClass("api", 1), ErrorCode.DATA_APP_DEPLOY_TIMEOUT: ErrorClass("api", 1), From a2f1c41d4a7b8ffc3b2703ed51f8dee1ac3d0267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 2 Oct 2026 12:10:25 +0200 Subject: [PATCH 07/35] feat(CLI-17): add kbagent cls command group for column-level security policies Adds list/detail/schema/create/update/delete for the metastore cls-policy object type, the column-level sibling of rls-policy: each rule names a principal and a visible_columns allowlist. Same scope rule as rls (organization/targeted, never project), fetch-then-merge update, admin permission class for writes and a /cls REST mirror. No setup wizard. - ClsService subclasses RlsService; item type, label, error code and the two rule hooks (local validation, dry-run preview) are the only differences. Shared principal check moves to _rls_condition.validate_principal_fields. - New error code INVALID_CLS_POLICY; cls-policy added to MetastoreClient's item-type literal. - Tests: service, CLI and server suites for cls; E2E schema/list case covers both rls and cls. - Docs: every hand-maintained surface (CLAUDE.md, AGENT_CONTEXT, commands reference, gotchas, rls-workflow, expert prompt, error codes) plus the regenerated SKILL.md and web-server-endpoints.md. - Correct stale RLS claims: the rls-policy/cls-policy schemas now exist in the metastore (a stack with an older metastore answers a classified NOT_FOUND), and the ACL reserves organization scope and cross-project grants for organization admins while the owning project's admin may author targeted policies. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 28 +- docs/error-codes.md | 3 +- docs/web-server-endpoints.md | 13 + plugins/kbagent/skills/kbagent/SKILL.md | 2 +- .../kbagent/references/commands-reference.md | 13 +- .../skills/kbagent/references/rls-workflow.md | 72 ++- src/keboola_agent_cli/cli.py | 5 + src/keboola_agent_cli/commands/cls.py | 320 +++++++++++++ src/keboola_agent_cli/commands/rls.py | 24 +- src/keboola_agent_cli/errors.py | 4 + src/keboola_agent_cli/metastore_client.py | 1 + .../server/_serve_command_map.py | 6 + src/keboola_agent_cli/server/app.py | 13 + src/keboola_agent_cli/server/dependencies.py | 3 + src/keboola_agent_cli/server/routers/cls.py | 134 ++++++ src/keboola_agent_cli/server/routers/rls.py | 2 +- .../services/_rls_condition.py | 20 +- src/keboola_agent_cli/services/cls_service.py | 69 +++ src/keboola_agent_cli/services/rls_service.py | 93 ++-- tests/test_cls_cli.py | 428 ++++++++++++++++++ tests/test_cls_service.py | 272 +++++++++++ tests/test_e2e.py | 46 +- tests/test_server_cls.py | 172 +++++++ 23 files changed, 1625 insertions(+), 118 deletions(-) create mode 100644 src/keboola_agent_cli/commands/cls.py create mode 100644 src/keboola_agent_cli/server/routers/cls.py create mode 100644 src/keboola_agent_cli/services/cls_service.py create mode 100644 tests/test_cls_cli.py create mode 100644 tests/test_cls_service.py create mode 100644 tests/test_server_cls.py diff --git a/CLAUDE.md b/CLAUDE.md index 154a62fe..9676c7fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1390,11 +1390,10 @@ kbagent rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect kbagent rls delete --project P --policy-id ID [--yes] kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes] # rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, -# one object per protected table). ORG-ADMIN-ONLY, STRUCTURALLY: every write (create/update/ +# one object per protected table). SCOPE: every write (create/update/ # setup) is authored at organization scope (default) or targeted scope (--target-project, -# repeatable), never project scope -- there is no --scope flag offering project at all, matching -# the backend's own restriction that a project's own admin can never author policy for its own -# tables. --rules is a JSON array of {principal|principals, condition} objects; condition is a +# repeatable), never project scope -- there is no --scope flag offering project at all (the metastore ACL +# reserves organization scope and cross-project grants for organization admins). --rules is a JSON array of {principal|principals, condition} objects; condition is a # declarative primitive tree (column/op/value comparisons, in/not_in, is_null/is_not_null, # and/or nesting, or a {"true": true} sentinel) -- never a free-text predicate string, closing # the injection surface a hand-written-SQL model would have. `--dry-run` previews the compiled @@ -1405,13 +1404,26 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # picker + guided condition builder, reusing `storage tables` and the same write path `rls # create` uses) -- it refuses under --json or a non-TTY stdout with a hint to use `rls create` # directly, same carve-out as `auth register-projects`'s picker, and has no REST route. -# THE METASTORE BACKEND DOES NOT REGISTER `rls-policy` ON ANY DEPLOYED STACK YET (a companion -# go-monorepo change, tracked separately) -- every command here answers a clean, classified -# error (schema fetch failure, NOT_FOUND) against a real project until that lands; this is -# expected, not a kbagent bug. New error code: INVALID_RLS_POLICY. Version gate for this whole +# A stack whose metastore predates the `rls-policy`/`cls-policy` schemas answers a clean, +# classified error (schema fetch failure, NOT_FOUND); expected there, not a kbagent bug. +# New error code: INVALID_RLS_POLICY. Version gate for this whole # entry lives in gotchas.md -- a `(since vNEXT)` tag cannot be written on these `# ` comment # lines (check_version_gates.py parses them as ATX markdown headings). +kbagent cls list --project P +kbagent cls detail --project P --policy-id ID +kbagent cls schema --project P +kbagent cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] +kbagent cls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes] +kbagent cls delete --project P --policy-id ID [--yes] +# cls: column-level security policy authoring (`cls-policy` object type, one object per +# protected table), sibling of `rls`: same organization/targeted-only scope, fetch-then-merge +# update, master-token requirement and permission class (`admin` for writes). --rules is a +# JSON array of {principal|principals, visible_columns: [col, ...]} objects -- an allowlist +# projection (masking is not supported). `--dry-run` prints each principal's projection. +# No `setup` wizard; the REST routes under /cls/{project} mirror /rls. +# New error code: INVALID_CLS_POLICY. Version gate lives in gotchas.md. + kbagent http get PATH [--timeout SECONDS] kbagent http post PATH [--body JSON|@file|-] [--timeout SECONDS] kbagent http patch PATH [--body JSON|@file|-] [--timeout SECONDS] diff --git a/docs/error-codes.md b/docs/error-codes.md index ee4fb368..8a6c05fb 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -228,8 +228,9 @@ A command that fails with an error code exits with the code's `Exit` value below | `UPDATED_ACROSS_MAJOR` | 6 | `authorization` | After an automatic update to a higher major version (for example from 0.x to 1.x), the first write or destructive command was refused once, with exit code 6. Every later command runs. Reads before that command run and get the same code as a `warnings[]` entry. Admin commands and an update started with `kbagent update --major` do not refuse. See [migrate-to-1.0.md](migrate-to-1.0.md) | | `UPDATED_ACROSS_BREAKING` | 1 | `api` | Only a `warnings[]` entry, in the first command after an update, never an error exit: the update crossed versions with `BREAKING:` changelog notes. `versions` lists them. `kbagent changelog --full` shows the notes | -### Row-Level Security +### Row-Level and Column-Level Security | Code | Exit | Type | Description | |---|---|---|---| +| `INVALID_CLS_POLICY` | 1 | `api` | A `cls create`/`update` candidate body (`table`/`dialect`/`rules`/`visible_columns`) failed validation -- either the schema-independent local checks (`principal`/`principals` not exactly one, empty or non-identifier `visible_columns`) or the live JSON-Schema check against the metastore's `cls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | | `INVALID_RLS_POLICY` | 1 | `api` | An `rls create`/`update` candidate body (`table`/`dialect`/`rules`/`condition`) failed validation -- either the schema-independent local checks (unknown op, `principal`/`principals` not exactly one) or the live JSON-Schema check against the metastore's `rls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 3136d733..81c8fab4 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -441,6 +441,19 @@ Row-level security policies (metastore-backed `rls-policy` objects), always auth | `PUT` | `/rls/{project}/{policy_id}` | Update an RLS policy | | `DELETE` | `/rls/{project}/{policy_id}` | Delete an RLS policy | +### `cls` (6 operations) + +Column-level security policies (metastore-backed `cls-policy` objects: a per-principal `visible_columns` allowlist), always authored at `organization`/`targeted` scope -- never `project` scope. Mirrors `kbagent cls list|detail|schema|create|update|delete`. + +| Method | Path | Summary | +|---|---|---| +| `GET` | `/cls/{project}` | List CLS policies | +| `POST` | `/cls/{project}` | Create a CLS policy | +| `GET` | `/cls/{project}/schema` | Fetch the live cls-policy JSON Schema | +| `GET` | `/cls/{project}/{policy_id}` | Get one CLS policy | +| `PUT` | `/cls/{project}/{policy_id}` | Update a CLS policy | +| `DELETE` | `/cls/{project}/{policy_id}` | Delete a CLS policy | + ## AI & Tools ### `kai` (6 operations) diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 338c4a9f..6ec60c0d 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -18,7 +18,7 @@ description: > feature flag, flow schedule, invite member, SQL transformation edit, sync action, keboola docs, table snapshot, auth, login, sign in, PAYG credits, flow notifications, alert recipients, config trash, - restore config, rls, zero-copy clone, workspace load type, + restore config, rls, cls, zero-copy clone, workspace load type, multipart, import job, job-detail, upload-table no-wait, set up keboola, connect project, logout. --- diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 32837645..44eab7bf 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,7 +469,7 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo). Same metastore/master-token requirements as Semantic Layer above (see [gotchas.md](gotchas.md)). **Every write is `organization`/`targeted` scope, never `project`** -- no `--scope project` option exists anywhere in this group, matching the backend's restriction that a project's own admin can never author policy for its own tables. **The metastore backend does not register `rls-policy` on any deployed stack yet** (companion `go-monorepo` work, tracked separately) -- every command answers a classified error against a real project until that lands; see [rls-workflow.md](rls-workflow.md) Workflow 7 for how to tell that apart from a real problem. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo). Same metastore/master-token requirements as Semantic Layer above (see [gotchas.md](gotchas.md)). **Every write is `organization`/`targeted` scope, never `project`** -- no `--scope project` option exists anywhere in this group. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. See [rls-workflow.md](rls-workflow.md) Workflow 7 for how to tell that apart from a real problem. - `rls list --project P` -- list policies visible to a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, source_project_id, target_project_ids}, ...]}`. An `organization`-scope row's `source_project_id` may not match the current project -- see `rls detail`/the workflow doc before assuming it applies here. - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals, condition}], revision}`. @@ -479,6 +479,17 @@ Author `rls-policy` metastore objects -- one object per protected table, each ho - `rls delete --project P --policy-id ID [--yes]` -- delete a policy, un-protecting its table. Confirmation prompt unless `--yes`; `--json` skips it too. - `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table (same write path, never a second one). Refuses cleanly under `--json` or a non-TTY stdout with a hint to use `rls create` directly -- there is no non-interactive path through `setup` itself, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. +## Column-Level Security (CLS) + +Author `cls-policy` metastore objects -- one object per protected table, each holding per-principal `visible_columns` allowlists (columns not listed are omitted from that principal's result; masking is not supported). Sibling of RLS above: same scope rule (**`organization`/`targeted` only, never `project`**), same fetch-then-merge `update`, same master-token requirement, no interactive `setup`. Enforcement (and RLS+CLS composition) happens in `keboola-mcp-server`'s `query_data`. See [rls-workflow.md](rls-workflow.md#column-level-security-cls). + +- `cls list --project P` -- list policies visible to a project (same row shape as `rls list`). +- `cls detail --project P --policy-id ID` -- full rule set: `{..., rules: [{principal|principals, visible_columns}], revision}`. +- `cls schema --project P` -- live `cls-policy` JSON Schema from the metastore (`NOT_FOUND`, exit 4, when the stack's metastore does not have the object type). +- `cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each principal's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. +- `cls update --project P --policy-id ID [--table ...] [--dialect ...] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes]` -- fetch-then-merge; omitted flags keep their current value. +- `cls delete --project P --policy-id ID [--yes]` -- delete a policy; the table's columns become unrestricted for those principals. + ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) - `http get PATH [--timeout SECONDS]` -- GET an endpoint on the running `kbagent serve` - `http post PATH [--body JSON|@file|-] [--timeout SECONDS]` -- POST with optional JSON body diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index 745bb624..ba60fdd3 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -1,4 +1,4 @@ -# RLS Workflow -- Row-Level Security Policies +# RLS / CLS Workflow -- Row- and Column-Level Security Policies Row-level security in Keboola is authored as metastore `rls-policy` objects -- one object per protected table, holding a list of per-principal condition @@ -9,15 +9,13 @@ different runtime. `kbagent rls` never executes a query and never decides whether a real query gets filtered -- it only creates, reads, updates and deletes the policy objects the enforcement engine reads. -**Authorship is org-admin-only, structurally.** Every write (`create`, -`update`, `setup`) is issued at `organization` scope (default) or -`targeted` scope (`--target-project`, repeatable) -- there is no `--scope -project` option anywhere in this command group, matching the backend's own -restriction that a project's own admin can never author policy for its own -tables. `--project` on every command still names the project whose tables -you're working with (and, for reads, which metastore instance to query) -- -it does not grant that project's admin any extra authority over policy -authorship. +**Scope is `organization` or `targeted`, never `project`.** Every write (`create`, +`update`, `setup`) is issued at `organization` scope (default) or `targeted` scope +(`--target-project`, repeatable) -- there is no `--scope project` option anywhere in this +command group. The metastore ACL reserves `organization` scope and cross-project grants +for organization admins; it additionally lets the owning project's admin author +`targeted` policies for that project. `--project` on every command names the project +whose tables you're working with (and which metastore instance to query). Same metastore requirements as [semantic-layer-workflow.md](semantic-layer-workflow.md): a MASTER (project admin) Storage token (`kbagent --json project info @@ -28,14 +26,8 @@ on a non-master token. Pre-flight before anything else in this file: kbagent --json project info --project P | jq '.is_master_token' ``` -**The metastore backend does not register the `rls-policy` object type on -any deployed stack yet** (a companion `go-monorepo` change, tracked -separately from the kbagent work). Every command below will answer with a -classified `NOT_FOUND` (schema fetch failure) or similar error against a -real project until that lands -- see Workflow 7 for how to tell that apart -from an actual problem. All the examples here are still correct and worth -running once the backend ships; they are also fully covered by kbagent's own -test suite against a mocked metastore in the meantime. +**Backend availability.** The `rls-policy` and `cls-policy` object types are registered by +the metastore's `2026-09-29` schema migrations. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. See Workflow 7. Full per-command flag reference: [commands-reference.md](commands-reference.md#row-level-security-rls). Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). @@ -258,19 +250,17 @@ kbagent rls delete --project prod --policy-id ## Workflow 7 -- Troubleshooting "backend not available" errors -Until the `go-monorepo` companion work registers the `rls-policy` object -type, EVERY command in this group will fail against a real, live project -- -this is expected, not a kbagent bug. Tell it apart from a real problem: +On a stack whose metastore predates the `rls-policy`/`cls-policy` schemas, +EVERY command in this group fails with a classified `NOT_FOUND` -- expected, +not a kbagent bug. Tell it apart from a real problem: ```bash kbagent --json rls schema --project prod ``` - **`NOT_FOUND` / "Could not fetch the rls-policy schema: ..."** -- the - expected shape while the backend isn't registered yet. The commands are - fully implemented and unit-tested against a mocked metastore client - (`tests/test_rls_service.py`); nothing here is broken, the object type - just doesn't exist on the stack yet. + object type isn't registered on this stack's metastore (same for + `cls schema`). Nothing here is broken; use a stack with a newer metastore. - **`MISSING_MASTER_TOKEN`** -- an actual, fixable problem: the registered token for this project isn't a master token. `kbagent project info --project P` -> `is_master_token`, then `project edit --project P @@ -300,3 +290,35 @@ kbagent --json rls schema --project prod (`keboola-mcp-server`'s `rls.py`) are two separate implementations by design -- the preview is a convenience for human review, never the source of truth for what a query actually returns. + +--- + +## Column-Level Security (CLS) + +`kbagent cls` authors `cls-policy` objects: one per protected table, each rule naming a +principal and the `visible_columns` that principal may read (an allowlist -- unlisted +columns are omitted from the result; masking is not supported). It mirrors `rls` +(`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same scope rule, +master-token requirement, `--dry-run`/`--yes` flags and fetch-then-merge `update`; +there is no `cls setup` wizard. Enforcement and RLS+CLS composition happen in +`keboola-mcp-server`'s `query_data`. + +```bash +kbagent --json cls schema --project prod # live shape +kbagent --json cls list --project prod # expect [] on a fresh project + +# Dry-run first: prints each principal's projection, writes nothing. +kbagent cls create --project prod --table in.c-crm.customers --dialect snowflake \ + --rules '[{"principal":"analyst@example.com","visible_columns":["id","region","amount"]}, + {"principals":["a@example.com","b@example.com"],"visible_columns":["id"]}]' --dry-run + +kbagent cls create --project prod --table in.c-crm.customers --dialect snowflake \ + --rules @cls_rules.json --yes +kbagent --json cls update --project prod --policy-id --rules @cls_rules_v2.json +kbagent cls delete --project prod --policy-id --yes +``` + +- A table can carry both an `rls-policy` and a `cls-policy`; author them separately. +- `INVALID_CLS_POLICY` -- `--rules` failed validation (exactly one of `principal`/`principals` + per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, plus the live JSON Schema). + Nothing was written. diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 0d10bb06..479af683 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -13,6 +13,7 @@ from .commands.billing import billing_app from .commands.branch import branch_app from .commands.changelog import changelog_command +from .commands.cls import cls_app from .commands.component import component_app from .commands.config import config_app from .commands.context import context_command @@ -62,6 +63,7 @@ from .services.auth_service import AuthService from .services.billing_service import BillingService from .services.branch_service import BranchService +from .services.cls_service import ClsService from .services.component_service import ComponentService from .services.config_service import ConfigService from .services.data_app_git_service import DataAppGitService @@ -169,6 +171,7 @@ app.add_typer(semantic_layer_app, name="semantic-layer", rich_help_panel=_DEV) app.add_typer(semantic_layer_app, name="sl", rich_help_panel=_DEV, hidden=True) app.add_typer(rls_app, name="rls", rich_help_panel=_DEV) +app.add_typer(cls_app, name="cls", rich_help_panel=_DEV) app.add_typer(http_app, name="http", rich_help_panel=_DEV) app.add_typer(agent_app, name="agent", rich_help_panel=_DEV) app.add_typer(dev_portal_app, name="dev-portal", rich_help_panel=_DEV) @@ -366,6 +369,7 @@ def main( data_app_git_service = DataAppGitService(config_store=config_store) semantic_layer_service = SemanticLayerService(config_store=config_store) rls_service = RlsService(config_store=config_store) + cls_service = ClsService(config_store=config_store) repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) docs_service = DocsService(config_store=config_store) @@ -428,6 +432,7 @@ def main( ctx.obj["data_app_git_service"] = data_app_git_service ctx.obj["semantic_layer_service"] = semantic_layer_service ctx.obj["rls_service"] = rls_service + ctx.obj["cls_service"] = cls_service ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service ctx.obj["docs_service"] = docs_service diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py new file mode 100644 index 00000000..b3e94669 --- /dev/null +++ b/src/keboola_agent_cli/commands/cls.py @@ -0,0 +1,320 @@ +"""Column-level security commands. + +Thin CLI layer over :class:`ClsService`, the column-level sibling of +``kbagent rls``. Six subcommands (no interactive ``setup`` wizard -- a CLS +rule is just a principal plus a column allowlist, so ``create`` covers it): + +- ``cls list`` -- policies visible to a project. +- ``cls detail`` -- one policy's full rules. +- ``cls schema`` -- the live ``cls-policy`` JSON Schema from the metastore. +- ``cls create`` -- author a policy for one table (write). +- ``cls update`` -- fetch-then-merge update of an existing policy (write). +- ``cls delete`` -- remove a policy (destructive). + +Each rule is ``{principal|principals, visible_columns: [...]}``: the listed +columns are the only ones that principal can read (allowlist projection; +masking is not supported). As with ``rls``, every write is authored at +``organization`` or ``targeted`` scope, never ``project`` -- there is no +``--scope`` flag. Enforcement happens in ``keboola-mcp-server``'s +``query_data``; kbagent only authors policies. + +``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``; +``create``/``update``/``delete`` are gated as ``admin`` (``cls.*`` in +``OPERATION_REGISTRY``). +""" + +from __future__ import annotations + +import contextlib +import json +from collections.abc import Callable +from typing import Any + +import typer +from rich.syntax import Syntax + +from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..services.rls_service import RLS_DIALECTS +from ._helpers import check_cli_permission, get_formatter, get_service, map_error_to_exit_code +from .rls import _format_policy_table, _parse_rules_arg + +cls_app = typer.Typer( + help=( + "Manage column-level security policies (metastore-backed, per-principal column " + "allowlist). 'list'/'detail'/'schema' are read-only; 'create'/'update'/'delete' " + "are gated as admin -- see CONTRIBUTING.md." + ) +) + +_RULES_SHAPE = "{principal|principals, visible_columns}" + + +@cls_app.callback(invoke_without_command=True) +def _cls_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "cls") + + +def _call(formatter: Any, fn: Callable[..., dict[str, Any]], **kwargs: Any) -> dict[str, Any]: + """Run a service call, turning its errors into a formatted message + exit code.""" + try: + return fn(**kwargs) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None + + +def _print_policy(formatter: Any, row: dict[str, Any]) -> None: + formatter.console.print( + f"\n[bold]{row.get('table', '')}[/bold] [dim](id {row.get('id', '')})[/dim]" + ) + formatter.console.print(f" Dialect: {row.get('dialect', '')}") + formatter.console.print(f" Scope: {row.get('scope', '')}") + formatter.console.print(f" Source project: {row.get('source_project_id') or '(none)'}") + targets = row.get("target_project_ids") or [] + if targets: + formatter.console.print(f" Target projects: {', '.join(str(t) for t in targets)}") + rules = row.get("rules") or [] + formatter.console.print(f" Rules ({len(rules)}):") + for rule in rules: + principal = rule.get("principal") or ", ".join(rule.get("principals") or []) + formatter.console.print( + f" - {principal}: {', '.join(rule.get('visible_columns') or [])}" + ) + + +def _print_preview(formatter: Any, result: dict[str, Any]) -> None: + formatter.console.print( + f"\n[bold]Preview[/bold] -- {result.get('table', '')} " + f"[dim]({result.get('dialect', '')}, scope={result.get('scope', '')})[/dim]" + ) + for entry in result.get("preview", []): + formatter.console.print( + f" {entry.get('principal')}: SELECT {', '.join(entry.get('visible_columns') or [])}" + ) + + +def _check_dialect(formatter: Any, dialect: str | None) -> None: + if dialect is not None and dialect not in RLS_DIALECTS: + formatter.error( + message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + +def _confirm_or_exit(formatter: Any, question: str) -> None: + if not typer.confirm(question): + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + +@cls_app.command("list") +def cls_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), +) -> None: + """List column-level security policies visible to a project.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "cls_service") + result = _call(formatter, service.list_policies, alias=project) + + if formatter.json_mode: + formatter.output(result) + elif not result.get("policies"): + formatter.console.print("[dim]No CLS policies found.[/dim]") + else: + _format_policy_table(formatter, result["policies"]) + + +@cls_app.command("detail") +def cls_detail( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), +) -> None: + """Show one CLS policy's full rule set.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "cls_service") + result = _call(formatter, service.get_policy, alias=project, policy_id=policy_id) + + if formatter.json_mode: + formatter.output(result) + else: + _print_policy(formatter, result) + + +@cls_app.command("schema") +def cls_schema( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", help="Project alias -- fetch the live schema from this stack" + ), +) -> None: + """Print the live ``cls-policy`` JSON Schema fetched from the metastore. + + Live-only (no bundled snapshot): if the metastore of that stack does not + have the ``cls-policy`` object type registered, this fails with a clean, + classified ``NOT_FOUND`` error. + """ + formatter = get_formatter(ctx) + service = get_service(ctx, "cls_service") + try: + fetch = service.fetch_schema(project) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + + if fetch.schema is None: + formatter.error( + message=f"Could not fetch the cls-policy schema: {fetch.reason}", + error_code=ErrorCode.NOT_FOUND, + ) + raise typer.Exit(code=4) + + if formatter.json_mode: + formatter.output({"format": "json-schema", "source": "live", "schema": fetch.schema}) + return + formatter.console.print( + Syntax(json.dumps(fetch.schema, indent=2), "json", theme="monokai", line_numbers=False) + ) + + +@cls_app.command("create") +def cls_create( + ctx: typer.Context, + project: str = typer.Option( + ..., "--project", help="Project alias -- the project this policy protects" + ), + table: str = typer.Option(..., "--table", help="Table key, e.g. in.c-crm.invoices"), + dialect: str = typer.Option( + ..., "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" + ), + rules: str = typer.Option( + ..., + "--rules", + help=( + f"JSON|@file|- array of {_RULES_SHAPE} objects; visible_columns is the " + "allowlist of columns that principal can read" + ), + ), + target_project: list[str] | None = typer.Option( + None, + "--target-project", + help=( + "Project ID this policy also applies to (repeatable). Omit for plain " + "organization scope -- still only applies where source_project_id / " + "target_project_ids match at read time, never by table-name text alone." + ), + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview the projection without writing"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Create one CLS policy for one table. + + Always authored at ``organization`` scope, or ``targeted`` scope when + ``--target-project`` is given -- never plain ``project`` scope. + """ + formatter = get_formatter(ctx) + parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) + _check_dialect(formatter, dialect) + service = get_service(ctx, "cls_service") + kwargs = { + "alias": project, + "table": table, + "dialect": dialect, + "rules": parsed_rules, + "target_project_ids": target_project, + } + + if not dry_run and not yes and not formatter.json_mode: + # A validation/network error here is reported properly by the real call below. + with contextlib.suppress(ConfigError, KeboolaApiError): + _print_preview(formatter, service.create_policy(**kwargs, dry_run=True)) + _confirm_or_exit(formatter, f"Create CLS policy on {table}?") + + result = _call(formatter, service.create_policy, **kwargs, dry_run=dry_run) + + if formatter.json_mode: + formatter.output(result) + return + if not dry_run: + formatter.success(f"Created CLS policy {result.get('id', '')} on {table}") + _print_preview(formatter, result) + + +@cls_app.command("update") +def cls_update( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), + table: str | None = typer.Option(None, "--table", help="New table key (unset = unchanged)"), + dialect: str | None = typer.Option( + None, "--dialect", help=f"New dialect: {' | '.join(RLS_DIALECTS)} (unset = unchanged)" + ), + rules: str | None = typer.Option( + None, "--rules", help="New JSON|@file|- rules array (unset = unchanged)" + ), + target_project: list[str] | None = typer.Option( + None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" + ), + dry_run: bool = typer.Option(False, "--dry-run", help="Preview the projection without writing"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Update an existing CLS policy. + + Fetch-then-merge: only the flags you pass are changed -- an omitted flag + keeps the policy's current value, it is never silently blanked. + """ + formatter = get_formatter(ctx) + parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) if rules is not None else None + _check_dialect(formatter, dialect) + service = get_service(ctx, "cls_service") + + if not dry_run and not yes and not formatter.json_mode: + _confirm_or_exit(formatter, f"Update CLS policy {policy_id}?") + + result = _call( + formatter, + service.update_policy, + alias=project, + policy_id=policy_id, + table=table, + dialect=dialect, + rules=parsed_rules, + target_project_ids=target_project, + dry_run=dry_run, + ) + + if formatter.json_mode: + formatter.output(result) + return + if not dry_run: + formatter.success(f"Updated CLS policy {policy_id}") + _print_preview(formatter, result) + + +@cls_app.command("delete") +def cls_delete( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), +) -> None: + """Delete a CLS policy.""" + formatter = get_formatter(ctx) + service = get_service(ctx, "cls_service") + + if not yes and not formatter.json_mode: + _confirm_or_exit( + formatter, f"Delete CLS policy {policy_id}? Its table's columns become unrestricted." + ) + + result = _call(formatter, service.delete_policy, alias=project, policy_id=policy_id) + + if formatter.json_mode: + formatter.output(result) + else: + formatter.success(f"Deleted CLS policy {policy_id}") diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 6f76b434..66b98e36 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -14,15 +14,11 @@ **Every policy this group writes is authored at ``organization`` or ``targeted`` scope, never ``project``** -- there is deliberately no -``--scope`` flag that could select ``project``, matching the metastore -backend's own restriction (an RLS policy is centrally governed, never -authored by a project's own admin for its own tables -- see the RFC in +``--scope`` flag that could select ``project`` (see the RFC in ``keboola-mcp-server``'s ``feature_spec/rls_query_tool/RFC.md``). -The ``rls-policy`` metastore object type does not exist on any deployed -backend yet -- see ``gotchas.md``'s "RLS backend not yet available" entry. -Every read/write command here will answer with a clean, classified error -against a real project until that backend work (tracked separately) lands. +On a stack whose metastore predates the ``rls-policy`` schema every command here +answers with a clean, classified error -- see ``gotchas.md``'s RLS/CLS entry. ``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``. ``create``/``update``/``setup`` are gated as ``admin`` (organization-level, @@ -56,7 +52,7 @@ rls_app = typer.Typer( help=( - "Manage row-level security policies (metastore-backed, org-admin-only). " + "Manage row-level security policies (metastore-backed). " "'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' " "are gated as admin -- see CONTRIBUTING.md." ) @@ -129,7 +125,10 @@ def _print_preview(formatter: Any, result: dict[str, Any]) -> None: formatter.console.print(f" {entry.get('principal')}: WHERE {entry.get('condition')}") -def _parse_rules_arg(formatter: Any, raw: str) -> list[dict[str, Any]]: +def _parse_rules_arg( + formatter: Any, raw: str, shape: str = "{principal|principals, condition}" +) -> list[dict[str, Any]]: + """Parse ``--rules``; ``shape`` is only the hint in the error (``cls`` passes its own).""" try: parsed = parse_json_arg(raw, label="--rules") except ValueError as exc: @@ -137,7 +136,7 @@ def _parse_rules_arg(formatter: Any, raw: str) -> list[dict[str, Any]]: raise typer.Exit(code=2) from None if not isinstance(parsed, list): formatter.error( - message="--rules must be a JSON array of {principal|principals, condition} objects", + message=f"--rules must be a JSON array of {shape} objects", error_code=ErrorCode.INVALID_ARGUMENT, ) raise typer.Exit(code=2) from None @@ -224,9 +223,8 @@ def rls_schema( """Print the live ``rls-policy`` JSON Schema fetched from the metastore. Unlike ``flow schema``, there is no offline bundled snapshot -- the - schema is authoritative only from the live metastore, and (until the - go-monorepo backend work lands) fetching it will fail with a clean, - classified error rather than returning a schema. + schema is authoritative only from the live metastore, and on a stack whose + metastore predates it, fetching fails with a clean, classified error. """ formatter = get_formatter(ctx) service = get_service(ctx, "rls_service") diff --git a/src/keboola_agent_cli/errors.py b/src/keboola_agent_cli/errors.py index f5962dbc..b9d3fdb3 100644 --- a/src/keboola_agent_cli/errors.py +++ b/src/keboola_agent_cli/errors.py @@ -183,6 +183,9 @@ class ErrorCode(StrEnum): # for conditional flows: a candidate `rls-policy` body (table/dialect/ # rules/condition) that fails op-name or JSON-Schema structural checks. INVALID_RLS_POLICY = "INVALID_RLS_POLICY" + # Column-level security: same role for a candidate `cls-policy` body + # (table/dialect/rules/visible_columns). + INVALID_CLS_POLICY = "INVALID_CLS_POLICY" def mask_token(token: str) -> str: @@ -493,6 +496,7 @@ class ErrorClass: # Flow ErrorCode.SCHEDULE_DELETE_FAILED: ErrorClass("api", 1), ErrorCode.INVALID_FLOW_DEFINITION: ErrorClass("api", 1), + ErrorCode.INVALID_CLS_POLICY: ErrorClass("api", 1), ErrorCode.INVALID_RLS_POLICY: ErrorClass("api", 1), # Data apps ErrorCode.DATA_APP_BUILD_FAILED: ErrorClass("api", 1), diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 9cce5ffa..1a6fd81c 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -93,6 +93,7 @@ "semantic-glossary", "semantic-reference-data", "rls-policy", + "cls-policy", ] diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 61735e53..62b51d94 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -16,6 +16,7 @@ ("DELETE", "/agents/{task_id}"): "agent delete", ("DELETE", "/branches/{project}/metadata/{metadata_id}"): "branch metadata-delete", ("DELETE", "/branches/{project}/{branch_id}"): "branch delete", + ("DELETE", "/cls/{project}/{policy_id}"): "cls delete", ("DELETE", "/configs/{project}/{component_id}/{config_id}"): "config delete", ( "DELETE", @@ -60,6 +61,9 @@ ("GET", "/branches/{project}/metadata"): "branch metadata-list", ("GET", "/branches/{project}/metadata/{key}"): "branch metadata-get", ("GET", "/changelog"): "changelog", + ("GET", "/cls/{project}"): "cls list", + ("GET", "/cls/{project}/schema"): "cls schema", + ("GET", "/cls/{project}/{policy_id}"): "cls detail", ("GET", "/components"): "component list", ("GET", "/components/{component_id}"): "component detail", ("GET", "/configs"): "config list", @@ -190,6 +194,7 @@ ("POST", "/branches/{project}"): "branch create", ("POST", "/branches/{project}/reset"): "branch reset", ("POST", "/branches/{project}/use"): "branch use", + ("POST", "/cls/{project}"): "cls create", ("POST", "/components/{component_id}/actions/{action}"): "component sync-action", ("POST", "/components/{component_id}/scaffold"): "config new", ("POST", "/configs/{project}/{component_id}"): "config new", @@ -299,6 +304,7 @@ ("POST", "/workspaces/{project}/{workspace_id}/password"): "workspace password", ("POST", "/workspaces/{project}/{workspace_id}/query"): "workspace query", ("PUT", "/branches/{project}/metadata/{key}"): "branch metadata-set", + ("PUT", "/cls/{project}/{policy_id}"): "cls update", ("PUT", "/configs/{project}/{component_id}/{config_id}/metadata/{key}"): "config set-metadata", ("PUT", "/merge-requests/{project}/{merge_request_id}"): "merge-request update", ("PUT", "/merge-requests/{project}/{merge_request_id}/auto-merge"): "merge-request auto-merge", diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index f0bd0973..1d316465 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -60,6 +60,7 @@ auth, billing, branches, + cls, components, configs, data_apps, @@ -367,6 +368,17 @@ "`auth register-projects`)." ), }, + { + "name": "cls", + "description": ( + "**Development.** " + "Column-level security policies (metastore-backed `cls-policy` " + "objects: a per-principal `visible_columns` allowlist), always " + "authored at `organization`/`targeted` scope -- never `project` " + "scope. Mirrors `kbagent cls list|detail|schema|create|update|" + "delete`." + ), + }, # ---- AI & Tools ---- { "name": "kai", @@ -1030,6 +1042,7 @@ async def _generic_handler(request: Request, exc: Exception): app.include_router(search.router) app.include_router(semantic_layer.router) app.include_router(rls.router) + app.include_router(cls.router) app.include_router(transformation.router) app.include_router(docs.router) app.include_router(org.router) diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index 8462b502..8218683f 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -28,6 +28,7 @@ from ..services.auth_service import AuthService from ..services.billing_service import BillingService from ..services.branch_service import BranchService +from ..services.cls_service import ClsService from ..services.component_service import ComponentService from ..services.config_service import ConfigService from ..services.data_app_git_service import DataAppGitService @@ -127,6 +128,7 @@ class ServiceRegistry: dev_portal: DeveloperPortalService = field(init=False) semantic_layer: SemanticLayerService = field(init=False) rls: RlsService = field(init=False) + cls: ClsService = field(init=False) repo_validate: RepoValidateService = field(init=False) kai: KaiService = field(init=False) encrypt: EncryptService = field(init=False) @@ -174,6 +176,7 @@ def __post_init__(self) -> None: # optional metastore_client_factory; the defaults work for both. self.semantic_layer = SemanticLayerService(config_store=cs) self.rls = RlsService(config_store=cs) + self.cls = ClsService(config_store=cs) self.repo_validate = RepoValidateService(config_store=cs) self.kai = KaiService(config_store=cs) self.encrypt = EncryptService(config_store=cs) diff --git a/src/keboola_agent_cli/server/routers/cls.py b/src/keboola_agent_cli/server/routers/cls.py new file mode 100644 index 00000000..4a40832e --- /dev/null +++ b/src/keboola_agent_cli/server/routers/cls.py @@ -0,0 +1,134 @@ +"""Column-level security endpoints -- the REST mirror of ``kbagent cls *``. + +1:1 with the CLI group. Same shape and rules as the ``rls`` router (every +route permission-gated; ``GET /{project}/schema`` registered before +``GET /{project}/{policy_id}`` so it is never shadowed), over the +``cls-policy`` object type: each rule is ``{principal|principals, +visible_columns}``. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends +from pydantic import BaseModel + +from ...errors import ErrorCode, KeboolaApiError +from ..dependencies import ServiceRegistry, get_registry, require_permission + +router = APIRouter(prefix="/cls", tags=["cls"]) + + +def _perm(operation: str) -> Any: + return Depends(require_permission(f"cls.{operation}")) + + +# -- Bodies -------------------------------------------------------------------------------------- + + +class ClsPolicyCreate(BaseModel): + table: str + dialect: str + rules: list[dict[str, Any]] + target_project_ids: list[str] | None = None + dry_run: bool = False + + +class ClsPolicyUpdate(BaseModel): + table: str | None = None + dialect: str | None = None + rules: list[dict[str, Any]] | None = None + target_project_ids: list[str] | None = None + dry_run: bool = False + + +# -- Reads ----------------------------------------------------------------------------------- + + +@router.get("/{project}", summary="List CLS policies", dependencies=[_perm("list")]) +def list_policies( + project: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """List column-level security policies visible to a project. Mirrors `kbagent cls list`.""" + return registry.cls.list_policies(project) + + +@router.get( + "/{project}/schema", + summary="Fetch the live cls-policy JSON Schema", + dependencies=[_perm("schema")], +) +def get_schema(project: str, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: + """Live schema from the metastore -- no offline bundled snapshot. Mirrors `kbagent cls schema`.""" + fetch = registry.cls.fetch_schema(project) + if fetch.schema is None: + raise KeboolaApiError( + message=f"Could not fetch the cls-policy schema: {fetch.reason}", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + return {"format": "json-schema", "source": "live", "schema": fetch.schema} + + +@router.get("/{project}/{policy_id}", summary="Get one CLS policy", dependencies=[_perm("detail")]) +def get_policy( + project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Full rule set for one policy. Mirrors `kbagent cls detail`.""" + return registry.cls.get_policy(project, policy_id) + + +# -- Writes ---------------------------------------------------------------------------------- + + +@router.post("/{project}", summary="Create a CLS policy", dependencies=[_perm("create")]) +def create_policy( + project: str, body: ClsPolicyCreate, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Create one policy for one table -- always `organization`/`targeted` scope, + + never `project` scope (no such option exists on this body). Mirrors + `kbagent cls create`. + """ + return registry.cls.create_policy( + project, + table=body.table, + dialect=body.dialect, + rules=body.rules, + target_project_ids=body.target_project_ids, + dry_run=body.dry_run, + ) + + +@router.put("/{project}/{policy_id}", summary="Update a CLS policy", dependencies=[_perm("update")]) +def update_policy( + project: str, + policy_id: str, + body: ClsPolicyUpdate, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Fetch-then-merge: an omitted field keeps its current value, it is never + + silently blanked. Mirrors `kbagent cls update`. + """ + return registry.cls.update_policy( + project, + policy_id, + table=body.table, + dialect=body.dialect, + rules=body.rules, + target_project_ids=body.target_project_ids, + dry_run=body.dry_run, + ) + + +@router.delete( + "/{project}/{policy_id}", summary="Delete a CLS policy", dependencies=[_perm("delete")] +) +def delete_policy( + project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) +) -> dict[str, Any]: + """Delete a policy, un-protecting its columns. Mirrors `kbagent cls delete`.""" + return registry.cls.delete_policy(project, policy_id) diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py index 42b7cd17..a9dec675 100644 --- a/src/keboola_agent_cli/server/routers/rls.py +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -9,7 +9,7 @@ ``POST /rls/{project}`` route below. **Every route enforces the permission policy** (``Depends(require_permission)``) --- RLS is security-sensitive enough (org-admin-only authorship) that +-- RLS is security-sensitive enough (admin-class writes) that CLI-gates-but-REST-doesn't would be a real hole, mirroring ``merge_requests.py`` rather than the (ungated) ``notifications.py``. diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 632ddbb8..6158db72 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -95,6 +95,20 @@ def validate_condition_ops(condition: Any) -> list[str]: return errors +def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: + """Each rule names exactly one of ``principal``/``principals`` (the schema's ``oneOf``). + + Shared by the RLS and CLS (``cls_service``) local checks -- both policy + types use the identical principal shape. + """ + principals_value = rule.get("principals") + has_principal = bool(rule.get("principal")) + has_principals = isinstance(principals_value, list) and bool(principals_value) + if has_principal == has_principals: # both or neither + return [f"rules[{index}] must set exactly one of 'principal'/'principals'"] + return [] + + def validate_rules_local(rules: Any) -> list[str]: """Semantic checks on ``rules`` that hold regardless of schema availability. @@ -113,11 +127,7 @@ def validate_rules_local(rules: Any) -> list[str]: if not isinstance(rule, dict): errors.append(f"rules[{index}] must be an object") continue - principals_value = rule.get("principals") - has_principal = bool(rule.get("principal")) - has_principals = isinstance(principals_value, list) and bool(principals_value) - if has_principal == has_principals: # both or neither - errors.append(f"rules[{index}] must set exactly one of 'principal'/'principals'") + errors.extend(validate_principal_fields(rule, index)) condition = rule.get("condition") if condition is None: errors.append(f"rules[{index}] is missing 'condition'") diff --git a/src/keboola_agent_cli/services/cls_service.py b/src/keboola_agent_cli/services/cls_service.py new file mode 100644 index 00000000..61ebd9c0 --- /dev/null +++ b/src/keboola_agent_cli/services/cls_service.py @@ -0,0 +1,69 @@ +"""Column-level security service -- business logic for ``kbagent cls``. + +The ``cls-policy`` metastore object is the column-level sibling of +``rls-policy``: one object per protected table, ``{table, dialect, rules}``, +where each rule names a principal and the columns that principal may see +(``visible_columns`` -- an allowlist projection; masking is not supported). + +Everything except the rule shape is identical to :class:`RlsService` (never +``project`` scope, fetch-then-merge update, live-schema validation that +degrades to a warning), so this class only overrides the policy type and the +two rule hooks. Enforcement happens in ``keboola-mcp-server``'s +``query_data``, not here -- kbagent only authors policies. +""" + +from __future__ import annotations + +import re +from typing import Any + +from ..errors import ErrorCode +from ..metastore_client import SemanticType +from . import _rls_condition +from .rls_service import RlsService + +CLS_ITEM_TYPE: SemanticType = "cls-policy" + +__all__ = ["CLS_ITEM_TYPE", "ClsService"] + +# Same identifier pattern the ``cls-policy`` schema enforces on ``visible_columns`` entries. +_COLUMN_RE = re.compile(r"^[A-Za-z0-9_]+$") + + +class ClsService(RlsService): + """Business logic for the ``cls`` command group.""" + + item_type: SemanticType = CLS_ITEM_TYPE + label = "CLS" + invalid_error_code = ErrorCode.INVALID_CLS_POLICY + + def _local_rule_errors(self, rules: Any) -> list[str]: + """Principal shape + a non-empty identifier allowlist per rule (schema-independent).""" + if not isinstance(rules, list) or not rules: + return ["'rules' must be a non-empty array"] + errors: list[str] = [] + for index, rule in enumerate(rules): + if not isinstance(rule, dict): + errors.append(f"rules[{index}] must be an object") + continue + errors.extend(_rls_condition.validate_principal_fields(rule, index)) + columns = rule.get("visible_columns") + if not isinstance(columns, list) or not columns: + errors.append(f"rules[{index}] needs a non-empty 'visible_columns' array") + continue + errors.extend( + f"rules[{index}].visible_columns: {col!r} is not a valid column name" + for col in columns + if not isinstance(col, str) or not _COLUMN_RE.fullmatch(col) + ) + return errors + + def _preview_rules(self, rules: list[dict[str, Any]], dialect: str) -> list[dict[str, Any]]: + """Show each principal's allowed projection (``dialect`` is unused: no SQL is rendered).""" + return [ + { + "principal": rule.get("principal") or rule.get("principals"), + "visible_columns": rule["visible_columns"], + } + for rule in rules + ] diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 2470f07c..802fdbd0 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -13,13 +13,10 @@ restriction (RFC "Scope is organization or targeted -- never plain project, by design, not merely by convention"). -The ``rls-policy`` metastore object type does not exist on any deployed -backend yet (go-monorepo companion work, not shipped -- see -``keboola-mcp-server``'s ``feature_spec/rls_query_tool/PLAN.md``). Every -method here is unit-tested against a mocked :class:`MetastoreClient`; a real -project will answer with a schema-fetch/list/get/post failure until that -backend work lands (see ``fetch_schema`` and the ``gotchas.md`` entry this -PR adds). +On a stack whose metastore predates the ``rls-policy`` schema, a real project +answers with a schema-fetch/list/get/post failure (see ``fetch_schema`` and the +``gotchas.md`` entry). Every method here is unit-tested against a mocked +:class:`MetastoreClient`. """ from __future__ import annotations @@ -31,7 +28,7 @@ from ..config_store import ConfigStore from ..errors import ErrorCode, KeboolaApiError -from ..metastore_client import MetastoreClient, MetastoreScope +from ..metastore_client import MetastoreClient, MetastoreScope, SemanticType from ..models import ProjectConfig from . import _rls_condition from ._rls_condition import RLS_DIALECTS @@ -39,7 +36,7 @@ logger = logging.getLogger(__name__) -RLS_ITEM_TYPE = "rls-policy" +RLS_ITEM_TYPE: SemanticType = "rls-policy" __all__ = ["RLS_DIALECTS", "RLS_ITEM_TYPE", "RlsSchemaFetch", "RlsService"] @@ -51,8 +48,8 @@ class RlsSchemaFetch: """Result of fetching the live ``rls-policy`` JSON Schema. ``schema`` is ``None`` on any failure (network error, ``KeboolaApiError``, - malformed/empty schema -- most likely today: the object type simply isn't - registered on the backend yet). A ``None`` schema must NOT block a write; + malformed/empty schema -- most likely: the object type isn't registered on + that stack's metastore). A ``None`` schema must NOT block a write; callers degrade to the schema-independent checks in ``_rls_condition`` and surface ``reason`` as a warning -- mirrors ``FlowService._fetch_flow_schema`` / ``FlowSchemaFetch`` exactly. @@ -70,8 +67,18 @@ class RlsService(BaseService): same pattern as :class:`SemanticLayerService`, so command-layer operations can target the metastore without polluting the Storage API client. + + ``item_type`` / ``label`` / ``invalid_error_code`` and the two rule hooks + (:meth:`_local_rule_errors`, :meth:`_preview_rules`) are all that differ + for the column-level sibling, :class:`ClsService`, which subclasses this: + the scope rule, fetch-then-merge update and schema degradation are the + same for both policy types. """ + item_type: SemanticType = RLS_ITEM_TYPE + label = "RLS" + invalid_error_code = ErrorCode.INVALID_RLS_POLICY + def __init__( self, config_store: ConfigStore, @@ -104,18 +111,20 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: """Fetch the live ``rls-policy`` schema. Never raises -- any failure (network error, ``KeboolaApiError``, empty/malformed schema -- most - likely today: the object type isn't registered on the backend yet) + likely: the object type isn't registered on the stack's metastore) degrades to ``schema=None`` + ``reason``, never blocks a caller. """ with self._new_metastore_client(project) as client: try: - schema = client.get_schema(RLS_ITEM_TYPE) + schema = client.get_schema(self.item_type) except KeboolaApiError as exc: return RlsSchemaFetch(schema=None, reason=exc.message) except Exception as exc: # any fetch failure must degrade, never block a write return RlsSchemaFetch(schema=None, reason=str(exc)) if not schema: - return RlsSchemaFetch(schema=None, reason="metastore returned no schema for rls-policy") + return RlsSchemaFetch( + schema=None, reason=f"metastore returned no schema for {self.item_type}" + ) return RlsSchemaFetch(schema=schema, reason=None) @staticmethod @@ -149,7 +158,7 @@ def _validate_policy( errors: list[str] = [] if dialect not in _rls_condition.RLS_DIALECTS: errors.append(f"dialect must be one of {_rls_condition.RLS_DIALECTS}, got {dialect!r}") - errors.extend(_rls_condition.validate_rules_local(rules)) + errors.extend(self._local_rule_errors(rules)) if errors: return errors @@ -159,11 +168,25 @@ def _validate_policy( errors.extend(_rls_condition.validate_policy_structural(policy_body, fetch.schema)) return errors + def _local_rule_errors(self, rules: Any) -> list[str]: + """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" + return _rls_condition.validate_rules_local(rules) + + def _preview_rules(self, rules: list[dict[str, Any]], dialect: str) -> list[dict[str, Any]]: + """Per-rule display preview shown by ``--dry-run`` (hook for :class:`ClsService`).""" + return [ + { + "principal": rule.get("principal") or rule.get("principals"), + "condition": _rls_condition.compile_condition_preview(rule["condition"], dialect), + } + for rule in rules + ] + def _raise_invalid(self, errors: list[str]) -> None: raise KeboolaApiError( - message="RLS policy is invalid: " + "; ".join(errors), + message=f"{self.label} policy is invalid: " + "; ".join(errors), status_code=400, - error_code=ErrorCode.INVALID_RLS_POLICY, + error_code=self.invalid_error_code, retryable=False, ) @@ -175,14 +198,14 @@ def list_policies(self, alias: str) -> dict[str, Any]: """List every ``rls-policy`` object visible to ``alias``'s project.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: - raw = client.list_items(RLS_ITEM_TYPE) + raw = client.list_items(self.item_type) return {"project": alias, "policies": [self._row_from_item(item) for item in raw]} def get_policy(self, alias: str, policy_id: str) -> dict[str, Any]: """Fetch one ``rls-policy`` object's full attributes + meta.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: - item = client.get_item(RLS_ITEM_TYPE, policy_id) + item = client.get_item(self.item_type, policy_id) row = self._row_from_item(item) row["rules"] = (item.get("attributes") or {}).get("rules", []) row["revision"] = (item.get("meta") or {}).get("revision") @@ -215,13 +238,7 @@ def create_policy( self._raise_invalid(errors) scope = self._resolve_scope(target_project_ids) - preview = [ - { - "principal": rule.get("principal") or rule.get("principals"), - "condition": _rls_condition.compile_condition_preview(rule["condition"], dialect), - } - for rule in rules - ] + preview = self._preview_rules(rules, dialect) if dry_run: return { "project": alias, @@ -235,14 +252,16 @@ def create_policy( with self._new_metastore_client(project) as client: created = client.post_item( - RLS_ITEM_TYPE, + self.item_type, name=table, data={"table": table, "dialect": dialect, "rules": rules}, scope=scope, target_project_ids=target_project_ids, ) if scope == "targeted" and target_project_ids: - client.put_target_projects(RLS_ITEM_TYPE, created.get("id", ""), target_project_ids) + client.put_target_projects( + self.item_type, created.get("id", ""), target_project_ids + ) row = self._row_from_item(created) row["preview"] = preview return row @@ -268,7 +287,7 @@ def update_policy( """ project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: - current = client.get_item(RLS_ITEM_TYPE, policy_id) + current = client.get_item(self.item_type, policy_id) attrs = current.get("attributes") or {} meta = current.get("meta") or {} @@ -286,15 +305,7 @@ def update_policy( self._raise_invalid(errors) scope = self._resolve_scope(merged_targets) - preview = [ - { - "principal": rule.get("principal") or rule.get("principals"), - "condition": _rls_condition.compile_condition_preview( - rule["condition"], merged_dialect - ), - } - for rule in merged_rules - ] + preview = self._preview_rules(merged_rules, merged_dialect) if dry_run: return { "project": alias, @@ -309,7 +320,7 @@ def update_policy( with self._new_metastore_client(project) as client: updated = client.put_item( - RLS_ITEM_TYPE, + self.item_type, policy_id, name=merged_table, data={"table": merged_table, "dialect": merged_dialect, "rules": merged_rules}, @@ -317,7 +328,7 @@ def update_policy( target_project_ids=merged_targets, ) if scope == "targeted" and merged_targets: - client.put_target_projects(RLS_ITEM_TYPE, policy_id, merged_targets) + client.put_target_projects(self.item_type, policy_id, merged_targets) row = self._row_from_item(updated) row["preview"] = preview return row @@ -326,5 +337,5 @@ def delete_policy(self, alias: str, policy_id: str) -> dict[str, Any]: """Delete one ``rls-policy`` object by id.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: - client.delete_item(RLS_ITEM_TYPE, policy_id) + client.delete_item(self.item_type, policy_id) return {"project": alias, "policy_id": policy_id, "deleted": True} diff --git a/tests/test_cls_cli.py b/tests/test_cls_cli.py new file mode 100644 index 00000000..0b0562f6 --- /dev/null +++ b/tests/test_cls_cli.py @@ -0,0 +1,428 @@ +"""Tests for `kbagent cls` CLI commands via CliRunner. + +Covers JSON + human output, exit codes (ConfigError 5, API error mapping, +invalid arguments 2), confirmation prompts, and ``--dry-run`` / +``--yes`` / ``--json`` skipping the prompt. Mirrors ``test_rls_cli.py``. +""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig + +runner = CliRunner() +RULES_JSON = json.dumps([{"principal": "a@x.com", "visible_columns": ["id", "region"]}]) +CREATE_ARGS = [ + "cls", + "create", + "--project", + "prod", + "--table", + "in.c-crm.invoices", + "--dialect", + "snowflake", + "--rules", + RULES_JSON, +] + + +def _store(tmp_path: Path) -> ConfigStore: + store = ConfigStore(config_dir=tmp_path / "cfg") + if "prod" not in store.load().projects: # a test may invoke `_run` more than once + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token="999-token-abc", + project_name="prod", + project_id=1234, + ), + ) + return store + + +def _run(args: list[str], tmp_path: Path, service: MagicMock, input: str | None = None) -> Any: + with ( + patch("keboola_agent_cli.cli.ConfigStore") as MockStore, + patch("keboola_agent_cli.cli.ClsService") as MockCls, + ): + MockStore.return_value = _store(tmp_path) + MockCls.return_value = service + return runner.invoke(app, args, input=input) + + +def _row(**overrides: Any) -> dict[str, Any]: + return { + "id": "p-1", + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "rule_count": 1, + "scope": "organization", + "source_project_id": "5725", + "target_project_ids": [], + **overrides, + } + + +def _preview(**overrides: Any) -> dict[str, Any]: + return { + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "scope": "organization", + "preview": [{"principal": "a@x.com", "visible_columns": ["id", "region"]}], + **overrides, + } + + +class TestList: + def test_json(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_policies.return_value = {"project": "prod", "policies": [_row()]} + + result = _run(["--json", "cls", "list", "--project", "prod"], tmp_path, service) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"]["policies"][0]["table"] == "in.c-crm.invoices" + service.list_policies.assert_called_once_with(alias="prod") + + def test_human_table_and_empty(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_policies.return_value = {"project": "prod", "policies": [_row()]} + assert ( + "in.c-crm.invoices" + in _run(["cls", "list", "--project", "prod"], tmp_path, service).output + ) + + service.list_policies.return_value = {"project": "prod", "policies": []} + assert ( + "No CLS policies found" + in _run(["cls", "list", "--project", "prod"], tmp_path, service).output + ) + + def test_config_error_exits_5(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_policies.side_effect = ConfigError("unknown alias") + + assert _run(["cls", "list", "--project", "prod"], tmp_path, service).exit_code == 5 + + def test_api_error_is_mapped(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_policies.side_effect = KeboolaApiError( + message="nope", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + + result = _run(["--json", "cls", "list", "--project", "prod"], tmp_path, service) + + assert result.exit_code != 0 + assert "nope" in result.output + + +class TestDetail: + def test_json_and_human(self, tmp_path: Path) -> None: + service = MagicMock() + service.get_policy.return_value = _row( + rules=[{"principal": "a@x.com", "visible_columns": ["id", "region"]}], revision=3 + ) + + as_json = _run( + ["--json", "cls", "detail", "--project", "prod", "--policy-id", "p-1"], + tmp_path, + service, + ) + human = _run( + ["cls", "detail", "--project", "prod", "--policy-id", "p-1"], tmp_path, service + ) + + assert json.loads(as_json.output)["data"]["revision"] == 3 + assert "a@x.com: id, region" in human.output + service.get_policy.assert_called_with(alias="prod", policy_id="p-1") + + +class TestSchema: + def test_success(self, tmp_path: Path) -> None: + service = MagicMock() + service.fetch_schema.return_value = MagicMock(schema={"type": "object"}, reason=None) + + result = _run(["--json", "cls", "schema", "--project", "prod"], tmp_path, service) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"]["schema"] == {"type": "object"} + + def test_human_prints_json(self, tmp_path: Path) -> None: + service = MagicMock() + service.fetch_schema.return_value = MagicMock(schema={"title": "cls-policy"}, reason=None) + + assert ( + "cls-policy" in _run(["cls", "schema", "--project", "prod"], tmp_path, service).output + ) + + def test_fetch_failure_exits_4_with_cls_message(self, tmp_path: Path) -> None: + service = MagicMock() + service.fetch_schema.return_value = MagicMock(schema=None, reason="not registered") + + result = _run(["--json", "cls", "schema", "--project", "prod"], tmp_path, service) + + assert result.exit_code == 4 + assert "cls-policy schema" in result.output + + def test_config_error_exits_5(self, tmp_path: Path) -> None: + service = MagicMock() + service.fetch_schema.side_effect = ConfigError("unknown alias") + + assert _run(["cls", "schema", "--project", "prod"], tmp_path, service).exit_code == 5 + + +class TestCreate: + def test_json_skips_prompt_and_writes_once(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run(["--json", *CREATE_ARGS], tmp_path, service) + + assert result.exit_code == 0, result.output + service.create_policy.assert_called_once() + kwargs = service.create_policy.call_args.kwargs + assert kwargs["rules"][0]["visible_columns"] == ["id", "region"] + assert kwargs["dry_run"] is False + + def test_dry_run_prints_projection_without_prompt(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = _preview(dry_run=True) + + result = _run([*CREATE_ARGS, "--dry-run"], tmp_path, service) + + assert result.exit_code == 0, result.output + assert "a@x.com: SELECT id, region" in result.output + assert service.create_policy.call_args.kwargs["dry_run"] is True + + def test_confirmed_prompt_previews_then_writes(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run(CREATE_ARGS, tmp_path, service, input="y\n") + + assert result.exit_code == 0, result.output + assert "Created CLS policy p-1" in result.output + assert [c.kwargs["dry_run"] for c in service.create_policy.call_args_list] == [True, False] + + def test_declined_prompt_aborts_without_writing(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = _preview(dry_run=True) + + result = _run(CREATE_ARGS, tmp_path, service, input="n\n") + + assert result.exit_code == 0 + assert "Aborted" in result.output + assert [c.kwargs["dry_run"] for c in service.create_policy.call_args_list] == [True] + + def test_yes_skips_prompt_and_passes_target_projects(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run( + [*CREATE_ARGS, "--yes", "--target-project", "7", "--target-project", "8"], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + service.create_policy.assert_called_once() + assert service.create_policy.call_args.kwargs["target_project_ids"] == ["7", "8"] + + def test_invalid_dialect_exits_2_before_any_call(self, tmp_path: Path) -> None: + service = MagicMock() + args = [a if a != "snowflake" else "oracle" for a in CREATE_ARGS] + + result = _run(["--json", *args], tmp_path, service) + + assert result.exit_code == 2 + service.create_policy.assert_not_called() + + @pytest.mark.parametrize("bad_rules", ["not json", '{"principal": "a@x.com"}']) + def test_invalid_rules_exit_2(self, tmp_path: Path, bad_rules: str) -> None: + service = MagicMock() + args = [*CREATE_ARGS[:-1], bad_rules] + + result = _run(["--json", *args], tmp_path, service) + + assert result.exit_code == 2 + if bad_rules.startswith("{"): + assert "{principal|principals, visible_columns}" in result.output + service.create_policy.assert_not_called() + + def test_validation_error_surfaces_cls_code(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.side_effect = KeboolaApiError( + message="CLS policy is invalid: x", + status_code=400, + error_code=ErrorCode.INVALID_CLS_POLICY, + ) + + result = _run(["--json", *CREATE_ARGS], tmp_path, service) + + assert result.exit_code != 0 + assert "INVALID_CLS_POLICY" in result.output + + +class TestUpdate: + def test_passes_only_given_flags(self, tmp_path: Path) -> None: + service = MagicMock() + service.update_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run( + [ + "--json", + "cls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--rules", + RULES_JSON, + ], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + kwargs = service.update_policy.call_args.kwargs + assert kwargs["table"] is None + assert kwargs["dialect"] is None + assert kwargs["rules"][0]["principal"] == "a@x.com" + + def test_declined_prompt_aborts(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + ["cls", "update", "--project", "prod", "--policy-id", "p-1"], + tmp_path, + service, + input="n\n", + ) + + assert "Aborted" in result.output + service.update_policy.assert_not_called() + + def test_confirmed_human_output(self, tmp_path: Path) -> None: + service = MagicMock() + service.update_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run( + ["cls", "update", "--project", "prod", "--policy-id", "p-1"], + tmp_path, + service, + input="y\n", + ) + + assert "Updated CLS policy p-1" in result.output + + def test_dry_run_skips_prompt(self, tmp_path: Path) -> None: + service = MagicMock() + service.update_policy.return_value = _preview(dry_run=True) + + result = _run( + [ + "cls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--dialect", + "bigquery", + "--dry-run", + ], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + assert service.update_policy.call_args.kwargs["dry_run"] is True + + def test_invalid_dialect_exits_2(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + [ + "--json", + "cls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--dialect", + "oracle", + ], + tmp_path, + service, + ) + + assert result.exit_code == 2 + service.update_policy.assert_not_called() + + +class TestDelete: + def test_json_skips_prompt(self, tmp_path: Path) -> None: + service = MagicMock() + service.delete_policy.return_value = { + "project": "prod", + "policy_id": "p-1", + "deleted": True, + } + + result = _run( + ["--json", "cls", "delete", "--project", "prod", "--policy-id", "p-1"], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + assert json.loads(result.output)["data"]["deleted"] is True + + def test_declined_prompt_does_not_delete(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + ["cls", "delete", "--project", "prod", "--policy-id", "p-1"], + tmp_path, + service, + input="n\n", + ) + + assert "Aborted" in result.output + service.delete_policy.assert_not_called() + + def test_yes_deletes(self, tmp_path: Path) -> None: + service = MagicMock() + service.delete_policy.return_value = {"deleted": True} + + result = _run( + ["cls", "delete", "--project", "prod", "--policy-id", "p-1", "--yes"], tmp_path, service + ) + + assert "Deleted CLS policy p-1" in result.output + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1") + + +def test_permission_check_denies_admin_writes_but_not_reads(tmp_path: Path) -> None: + """`cls.*` is registered in OPERATION_REGISTRY: create/update/delete=admin, reads=read.""" + from keboola_agent_cli.permissions import OPERATION_REGISTRY + + assert {op: OPERATION_REGISTRY[f"cls.{op}"] for op in ("list", "detail", "schema")} == { + "list": "read", + "detail": "read", + "schema": "read", + } + assert all(OPERATION_REGISTRY[f"cls.{op}"] == "admin" for op in ("create", "update", "delete")) diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py new file mode 100644 index 00000000..2cdc6999 --- /dev/null +++ b/tests/test_cls_service.py @@ -0,0 +1,272 @@ +"""Service-layer tests for ``ClsService`` (column-level security). + +Same approach as ``test_rls_service.py``: a ``MagicMock`` metastore client +factory, so we verify orchestration (item type, scope resolution, validation +order, fetch-then-merge) without HTTP. The shared plumbing is inherited from +``RlsService`` and covered there; these tests pin what ``ClsService`` changes. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.cls_service import ClsService + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +_RULES = [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] + + +def _make_service(tmp_path: Path) -> tuple[ClsService, MagicMock]: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=5725, + ), + ) + mock = MagicMock() + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + mock.get_schema.side_effect = KeboolaApiError( + message="no schema", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + return ClsService(config_store=store, metastore_client_factory=lambda url, token: mock), mock + + +def _item(rules: list[dict[str, Any]] | None = None, **meta: Any) -> dict[str, Any]: + return { + "type": "cls-policy", + "id": "p-1", + "attributes": { + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "rules": _RULES if rules is None else rules, + }, + "meta": {"scope": "organization", "sourceProjectId": "5725", "revision": 2, **meta}, + } + + +class TestLocalRuleValidation: + @pytest.mark.parametrize( + ("rules", "fragment"), + [ + (None, "non-empty array"), + ([], "non-empty array"), + (["x"], "must be an object"), + ([{"visible_columns": ["id"]}], "exactly one of 'principal'/'principals'"), + ( + [{"principal": "a@x.com", "principals": ["b@x.com"], "visible_columns": ["id"]}], + "exactly one of 'principal'/'principals'", + ), + ([{"principal": "a@x.com"}], "non-empty 'visible_columns'"), + ([{"principal": "a@x.com", "visible_columns": []}], "non-empty 'visible_columns'"), + ([{"principal": "a@x.com", "visible_columns": "id"}], "non-empty 'visible_columns'"), + ( + [{"principal": "a@x.com", "visible_columns": ["id; DROP"]}], + "not a valid column name", + ), + ([{"principal": "a@x.com", "visible_columns": [1]}], "not a valid column name"), + ], + ) + def test_rejects(self, tmp_path: Path, rules: Any, fragment: str) -> None: + service, mock = _make_service(tmp_path) + + with pytest.raises(KeboolaApiError) as exc: + service.create_policy("prod", table="in.c-crm.t", dialect="snowflake", rules=rules) + + assert exc.value.error_code == ErrorCode.INVALID_CLS_POLICY + assert fragment in exc.value.message + assert exc.value.message.startswith("CLS policy is invalid") + mock.post_item.assert_not_called() + + def test_accepts_principals_list(self, tmp_path: Path) -> None: + service, _ = _make_service(tmp_path) + rules = [{"principals": ["a@x.com", "b@x.com"], "visible_columns": ["id"]}] + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=rules, dry_run=True + ) + + assert result["preview"] == [ + {"principal": ["a@x.com", "b@x.com"], "visible_columns": ["id"]} + ] + + def test_bad_dialect_is_rejected_before_any_write(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + + with pytest.raises(KeboolaApiError, match="dialect must be one of"): + service.create_policy("prod", table="in.c-crm.t", dialect="oracle", rules=_RULES) + + mock.post_item.assert_not_called() + + +class TestStructuralValidation: + def test_live_schema_violation_is_reported_as_cls_error(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_schema.side_effect = None + mock.get_schema.return_value = { + "type": "object", + "properties": {"table": {"type": "string", "pattern": "^[a-z.]+$"}}, + } + + with pytest.raises(KeboolaApiError) as exc: + service.create_policy("prod", table="BAD TABLE", dialect="snowflake", rules=_RULES) + + mock.get_schema.assert_called_with("cls-policy") + assert exc.value.error_code == ErrorCode.INVALID_CLS_POLICY + assert "Schema error at table" in exc.value.message + + def test_missing_schema_degrades_instead_of_blocking(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.post_item.return_value = _item() + + service.create_policy("prod", table="in.c-crm.invoices", dialect="snowflake", rules=_RULES) + + mock.post_item.assert_called_once() + + def test_fetch_schema_uses_cls_item_type(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_schema.side_effect = None + mock.get_schema.return_value = {"type": "object"} + + fetch = service.fetch_schema("prod") + + mock.get_schema.assert_called_once_with("cls-policy") + assert fetch.schema == {"type": "object"} + + +class TestReads: + def test_list_uses_cls_item_type(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.list_items.return_value = [_item()] + + result = service.list_policies("prod") + + mock.list_items.assert_called_once_with("cls-policy") + assert result["policies"][0]["rule_count"] == 1 + + def test_detail_returns_visible_columns(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_item.return_value = _item() + + result = service.get_policy("prod", "p-1") + + mock.get_item.assert_called_once_with("cls-policy", "p-1") + assert result["rules"] == _RULES + assert result["revision"] == 2 + + +class TestCreate: + def test_writes_cls_policy_at_organization_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.post_item.return_value = _item() + + result = service.create_policy( + "prod", table="in.c-crm.invoices", dialect="snowflake", rules=_RULES + ) + + mock.post_item.assert_called_once_with( + "cls-policy", + name="in.c-crm.invoices", + data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": _RULES}, + scope="organization", + target_project_ids=None, + ) + mock.put_target_projects.assert_not_called() + assert result["preview"] == [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] + + def test_target_projects_make_it_targeted_and_grant(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.post_item.return_value = _item(scope="targeted") + + service.create_policy( + "prod", + table="in.c-crm.invoices", + dialect="bigquery", + rules=_RULES, + target_project_ids=["7", "8"], + ) + + assert mock.post_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", ["7", "8"]) + + def test_dry_run_never_writes(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES, dry_run=True + ) + + assert result["dry_run"] is True + assert result["scope"] == "organization" + mock.post_item.assert_not_called() + + +class TestUpdate: + def test_merges_only_the_given_fields(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_item.return_value = _item() + mock.put_item.return_value = _item() + new_rules = [{"principal": "b@x.com", "visible_columns": ["id"]}] + + service.update_policy("prod", "p-1", rules=new_rules) + + mock.put_item.assert_called_once_with( + "cls-policy", + "p-1", + name="in.c-crm.invoices", + data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": new_rules}, + scope="organization", + target_project_ids=None, + ) + + def test_keeps_existing_targets_and_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_item.return_value = _item(scope="targeted", targetProjectIds=["7"]) + mock.put_item.return_value = _item(scope="targeted") + + service.update_policy("prod", "p-1", table="in.c-crm.other") + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", ["7"]) + + def test_dry_run_never_writes(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_item.return_value = _item() + + result = service.update_policy("prod", "p-1", dialect="bigquery", dry_run=True) + + assert result["dry_run"] is True + assert result["dialect"] == "bigquery" + mock.put_item.assert_not_called() + + def test_invalid_merged_rules_are_rejected(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + mock.get_item.return_value = _item() + + with pytest.raises(KeboolaApiError) as exc: + service.update_policy("prod", "p-1", rules=[{"principal": "a@x.com"}]) + + assert exc.value.error_code == ErrorCode.INVALID_CLS_POLICY + mock.put_item.assert_not_called() + + +def test_delete_uses_cls_item_type(tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + + result = service.delete_policy("prod", "p-1") + + mock.delete_item.assert_called_once_with("cls-policy", "p-1") + assert result == {"project": "prod", "policy_id": "p-1", "deleted": True} diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 324cec87..0a46d386 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -683,8 +683,8 @@ def test_full_cli_e2e(self) -> None: _step( "40.5", - "rls schema", - "the rls-policy metastore object type isn't registered on any real stack yet", + "rls/cls schema + list", + "the rls-policy / cls-policy metastore object types may not be registered on a stack", ) self._test_rls_commands() @@ -3722,10 +3722,9 @@ def _test_mcp_parity_commands(self) -> None: print(f" WARN: failed to delete transformation {tf_config_id}: {exc}") def _test_rls_commands(self) -> None: - """`rls schema` -- the `rls-policy` metastore object type isn't - - registered on any real stack yet (companion go-monorepo work, CLI-17), - so a clean, classified failure here is the expected outcome, not a + """`rls`/`cls` `schema` + `list` -- the `rls-policy` / `cls-policy` metastore + object types are only registered on stacks with a recent metastore, so + a clean, classified failure here is the expected outcome, not a kbagent bug -- same SKIP-on-failure shape as `_test_mcp_parity_commands`'s `semantic-layer schema` case above. Full CRUD coverage (create/update/ delete against a real `rls-policy` object) is exercised against a @@ -3733,22 +3732,25 @@ def _test_rls_commands(self) -> None: `tests/test_rls_cli.py` and will get a live E2E round-trip once the backend registers the type. """ - result = self._run("rls", "schema", "--project", self.alias) - if result.exit_code != 0: - print(f" {_YELLOW}SKIP: rls schema (rls-policy backend not registered yet){_RESET}") - else: - data = _json_ok(result) - assert isinstance(data["data"]["schema"], dict) and data["data"]["schema"] - assert data["data"]["source"] == "live" - - # `rls list` must degrade the same way -- never a traceback -- on a - # stack without the object type registered. - result = self._run("rls", "list", "--project", self.alias) - if result.exit_code != 0: - print(f" {_YELLOW}SKIP: rls list (rls-policy backend not registered yet){_RESET}") - else: - data = _json_ok(result) - assert isinstance(data["data"]["policies"], list) + for group in ("rls", "cls"): + result = self._run(group, "schema", "--project", self.alias) + if result.exit_code != 0: + print( + f" {_YELLOW}SKIP: {group} schema ({group}-policy not registered here){_RESET}" + ) + else: + data = _json_ok(result) + assert isinstance(data["data"]["schema"], dict) and data["data"]["schema"] + assert data["data"]["source"] == "live" + + # `list` must degrade the same way -- never a traceback -- on a stack + # whose metastore lacks the object type. + result = self._run(group, "list", "--project", self.alias) + if result.exit_code != 0: + print(f" {_YELLOW}SKIP: {group} list ({group}-policy not registered here){_RESET}") + else: + data = _json_ok(result) + assert isinstance(data["data"]["policies"], list) def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" diff --git a/tests/test_server_cls.py b/tests/test_server_cls.py new file mode 100644 index 00000000..506ff252 --- /dev/null +++ b/tests/test_server_cls.py @@ -0,0 +1,172 @@ +"""Tests for `server/routers/cls.py` -- router -> service call parity + permission gating. + +Mirrors ``test_server_rls.py``: kwarg parity via a mocked ``ServiceRegistry`` +and persisted-policy -> 403. CLS writes are ``admin``, like RLS. +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +if importlib.util.find_spec("fastapi") is None: # pragma: no cover + pytest.skip( + "FastAPI not installed; run `uv pip install -e '.[server]'`", allow_module_level=True + ) + +from fastapi.testclient import TestClient + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import PermissionPolicy +from keboola_agent_cli.server import create_app +from keboola_agent_cli.server.dependencies import ServiceRegistry, get_registry + +AUTH = {"Authorization": "Bearer test-token"} +PROJECT = "prod" +POLICY_ID = "p-1" +RULES = [{"principal": "a@x.com", "visible_columns": ["id"]}] + + +def _app(tmp_path: Path, cls_svc: MagicMock) -> Any: + app = create_app(config_dir=str(tmp_path), auth_token="test-token") + registry = ServiceRegistry.__new__(ServiceRegistry) + registry.cls = cls_svc + app.dependency_overrides[get_registry] = lambda: registry + return app + + +def _persist_policy(config_dir: Path, policy: PermissionPolicy) -> None: + store = ConfigStore(config_dir=config_dir) + config = store.load() + config.permissions = policy + store.save(config) + + +def test_list_and_detail_call_service(tmp_path: Path) -> None: + svc = MagicMock() + svc.list_policies.return_value = {"project": PROJECT, "policies": []} + svc.get_policy.return_value = {"id": POLICY_ID} + + with TestClient(_app(tmp_path, svc)) as client: + assert client.get(f"/cls/{PROJECT}", headers=AUTH).status_code == 200 + assert client.get(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH).status_code == 200 + + svc.list_policies.assert_called_once_with(PROJECT) + svc.get_policy.assert_called_once_with(PROJECT, POLICY_ID) + + +def test_schema_success_and_not_shadowed_by_policy_id_route(tmp_path: Path) -> None: + svc = MagicMock() + svc.fetch_schema.return_value = MagicMock(schema={"type": "object"}, reason=None) + + with TestClient(_app(tmp_path, svc)) as client: + res = client.get(f"/cls/{PROJECT}/schema", headers=AUTH) + + assert res.status_code == 200, res.text + assert res.json()["schema"] == {"type": "object"} + svc.fetch_schema.assert_called_once_with(PROJECT) + svc.get_policy.assert_not_called() + + +def test_schema_fetch_failure_is_404(tmp_path: Path) -> None: + svc = MagicMock() + svc.fetch_schema.return_value = MagicMock(schema=None, reason="not registered") + + with TestClient(_app(tmp_path, svc)) as client: + res = client.get(f"/cls/{PROJECT}/schema", headers=AUTH) + + assert res.status_code == 404, res.text + assert res.json()["error"]["code"] == "NOT_FOUND" + assert "cls-policy" in res.json()["error"]["message"] + + +def test_create_and_update_pass_kwargs(tmp_path: Path) -> None: + svc = MagicMock() + svc.create_policy.return_value = {"id": POLICY_ID} + svc.update_policy.return_value = {"id": POLICY_ID} + body = { + "table": "in.c-crm.invoices", + "dialect": "snowflake", + "rules": RULES, + "target_project_ids": ["999"], + } + + with TestClient(_app(tmp_path, svc)) as client: + assert client.post(f"/cls/{PROJECT}", headers=AUTH, json=body).status_code == 200 + res = client.put(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"rules": RULES}) + assert res.status_code == 200, res.text + + create_kwargs = svc.create_policy.call_args.kwargs + assert create_kwargs["rules"] == RULES + assert create_kwargs["target_project_ids"] == ["999"] + assert create_kwargs["dry_run"] is False + args, update_kwargs = svc.update_policy.call_args + assert args == (PROJECT, POLICY_ID) + assert update_kwargs["rules"] == RULES + assert update_kwargs["table"] is None + + +def test_delete_calls_service(tmp_path: Path) -> None: + svc = MagicMock() + svc.delete_policy.return_value = {"deleted": True} + + with TestClient(_app(tmp_path, svc)) as client: + res = client.delete(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 200, res.text + svc.delete_policy.assert_called_once_with(PROJECT, POLICY_ID) + + +def test_service_error_maps_to_http_status(tmp_path: Path) -> None: + svc = MagicMock() + svc.get_policy.side_effect = KeboolaApiError( + message="not found", status_code=404, error_code=ErrorCode.NOT_FOUND + ) + + with TestClient(_app(tmp_path, svc)) as client: + res = client.get(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH) + + assert res.status_code == 404, res.text + + +class TestPermissionGating: + @pytest.mark.parametrize("deny", ["cli:admin", "cli:write"]) + def test_denied_policy_blocks_every_write(self, tmp_path: Path, deny: str) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=[deny])) + svc = MagicMock() + body = {"table": "t", "dialect": "snowflake", "rules": RULES} + + with TestClient(_app(tmp_path, svc)) as client: + responses = [ + client.post(f"/cls/{PROJECT}", headers=AUTH, json=body), + client.put(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table": "t"}), + client.delete(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH), + ] + + for res in responses: + assert res.status_code == 403, res.text + assert res.json()["error"]["code"] == "PERMISSION_DENIED" + svc.create_policy.assert_not_called() + svc.update_policy.assert_not_called() + svc.delete_policy.assert_not_called() + + def test_denying_cli_admin_still_allows_reads(self, tmp_path: Path) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) + svc = MagicMock() + svc.list_policies.return_value = {"project": PROJECT, "policies": []} + svc.get_policy.return_value = {"id": POLICY_ID} + svc.fetch_schema.return_value = MagicMock(schema={"type": "object"}, reason=None) + + with TestClient(_app(tmp_path, svc)) as client: + statuses = [ + client.get(f"/cls/{PROJECT}", headers=AUTH).status_code, + client.get(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH).status_code, + client.get(f"/cls/{PROJECT}/schema", headers=AUTH).status_code, + ] + + assert statuses == [200, 200, 200] From 77413ad6d6b8cd93d5eb17ff5e19aac4f0af9521 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Fri, 2 Oct 2026 17:34:45 +0200 Subject: [PATCH 08/35] fix(CLI-17): resolve the default schema version for rls/cls schema and validation The metastore's bare /api/v1/schema/ endpoint returns only a {"versions": [...]} listing; the JSON Schema itself lives at /. rls/cls schema therefore printed version metadata, and the live structural validation in create/update checked nothing (a listing has no constraints, so any body passed). Move the existing default-version resolution of SemanticLayerService into a shared fetch_resolved_schema() in metastore_client.py and use it from the semantic-layer, rls and cls services. Tests: resolver unit tests (default version, first-version fallback, pass-through) and a cls regression test that validation really runs against the resolved schema. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/services/rls_service.py | 9 +++++-- .../services/semantic_layer_service.py | 24 ++++--------------- tests/test_cls_service.py | 24 +++++++++++++++++++ tests/test_metastore_client.py | 2 ++ 4 files changed, 38 insertions(+), 21 deletions(-) diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 802fdbd0..f8669fee 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -28,7 +28,12 @@ from ..config_store import ConfigStore from ..errors import ErrorCode, KeboolaApiError -from ..metastore_client import MetastoreClient, MetastoreScope, SemanticType +from ..metastore_client import ( + MetastoreClient, + MetastoreScope, + SemanticType, + fetch_resolved_schema, +) from ..models import ProjectConfig from . import _rls_condition from ._rls_condition import RLS_DIALECTS @@ -116,7 +121,7 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: """ with self._new_metastore_client(project) as client: try: - schema = client.get_schema(self.item_type) + schema, _version = fetch_resolved_schema(client, self.item_type) except KeboolaApiError as exc: return RlsSchemaFetch(schema=None, reason=exc.message) except Exception as exc: # any fetch failure must degrade, never block a write diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 13ccab8e..8a7f0596 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -359,26 +359,12 @@ def get_schema(self, alias: str, types: list[str]) -> dict[str, Any]: def _fetch_resolved_schema(client: MetastoreClient, wire_type: SemanticType) -> dict[str, Any]: """Fetch the actual JSON Schema for a type, resolving the default version. - Live metastore behavior (2026-07): the bare ``/api/v1/schema/{type}`` - endpoint returns only a ``{"versions": [...]}`` listing (metadata, no - schema body); the real JSON Schema lives at ``/{version}``. The - upstream MCP tool passes the bare listing through -- an upstream gap - we deliberately do NOT mirror: this resolves ``isDefault`` (falling - back to the first entry) and fetches the versioned document. If the - server someday returns the schema directly (no ``versions`` key), - it is passed through unchanged. + See :func:`fetch_resolved_schema` -- the bare endpoint is only a version + listing. (The upstream MCP tool passes that bare listing through; we + deliberately do not mirror that gap.) """ - body = client.get_schema(wire_type) - versions = body.get("versions") - if not isinstance(versions, list) or not versions: - return {"schema": body, "schema_version": None} - default = next( - (v for v in versions if v.get("isDefault")), - versions[0], - ) - version_id = str(default.get("version", "")) - resolved = client.get_schema(wire_type, version=version_id) if version_id else body - return {"schema": resolved, "schema_version": version_id or None} + schema, version = fetch_resolved_schema(client, wire_type) + return {"schema": schema, "schema_version": version} # Internal helpers (model-scoped fetches). diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index 2cdc6999..417a90c2 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -128,6 +128,30 @@ def test_live_schema_violation_is_reported_as_cls_error(self, tmp_path: Path) -> assert exc.value.error_code == ErrorCode.INVALID_CLS_POLICY assert "Schema error at table" in exc.value.message + def test_version_listing_is_resolved_to_the_real_schema_before_validating( + self, tmp_path: Path + ) -> None: + """The live metastore's bare endpoint returns only `{"versions": [...]}`; validating + against that listing would pass anything, so the default version must be fetched.""" + service, mock = _make_service(tmp_path) + real_schema = { + "type": "object", + "properties": {"table": {"type": "string", "pattern": "^[a-z.]+$"}}, + } + mock.get_schema.side_effect = [ + {"versions": [{"version": "0.9.0"}, {"version": "1.0.0", "isDefault": True}]}, + real_schema, + ] + + with pytest.raises(KeboolaApiError) as exc: + service.create_policy("prod", table="BAD TABLE", dialect="snowflake", rules=_RULES) + + assert [c.args + tuple(c.kwargs.items()) for c in mock.get_schema.call_args_list] == [ + ("cls-policy",), + ("cls-policy", ("version", "1.0.0")), + ] + assert "Schema error at table" in exc.value.message + def test_missing_schema_degrades_instead_of_blocking(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) mock.post_item.return_value = _item() diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index aa88a9b8..d4c015ae 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -9,6 +9,7 @@ import json from typing import ClassVar +from unittest.mock import MagicMock import pytest @@ -17,6 +18,7 @@ from keboola_agent_cli.metastore_client import ( SEMANTIC_TYPES, MetastoreClient, + fetch_resolved_schema, ) STACK_URL_US = "https://connection.keboola.com" From c80ee5d0e341bdbb7f93c7379f125fc971a0d919 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 05:52:02 +0200 Subject: [PATCH 09/35] fix(CLI-17): address Copilot review on rls/cls - rls setup: validate --dialect before the wizard; require an interactive stdin AND stdout before any API call; exit non-zero (and name the tables) when any policy could not be created instead of exiting 0. - validation: the true sentinel must be exactly {"true": true}; the shape of every condition primitive (column, value, values) is checked locally, so a malformed rule fails as INVALID_RLS_POLICY even when the live schema is unavailable instead of crashing the preview. - schema fetch: an authentication/permission failure (INVALID_TOKEN, MISSING_MASTER_TOKEN, access denied, 401/403) is raised instead of being reported as a missing schema; only an unavailable schema degrades. rls/cls schema report it with the right exit code. - create/update add a "warnings" entry (printed in human mode) when live structural validation was skipped. - update: --clear-target-projects revokes every grant (mutually exclusive with --target-project); the policy keeps its current scope. - SEMANTIC_TYPES includes cls-policy; the E2E schema skip now asserts NOT_FOUND. - Docs: new flag, gotchas, rls-workflow wording. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 6 +- .../kbagent/references/commands-reference.md | 6 +- .../skills/kbagent/references/rls-workflow.md | 2 +- src/keboola_agent_cli/commands/cls.py | 20 +- src/keboola_agent_cli/commands/rls.py | 56 +++- src/keboola_agent_cli/metastore_client.py | 1 + .../services/_rls_condition.py | 23 ++ src/keboola_agent_cli/services/rls_service.py | 115 ++++++--- tests/test_cls_cli.py | 63 +++++ tests/test_e2e.py | 3 + tests/test_metastore_client.py | 1 + tests/test_project_ref.py | 1 + tests/test_rls_cli.py | 242 ++++++++++++++++++ tests/test_rls_service.py | 218 ++++++++++++++++ 14 files changed, 705 insertions(+), 52 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9676c7fa..611d0549 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1386,7 +1386,7 @@ kbagent rls list --project P kbagent rls detail --project P --policy-id ID kbagent rls schema --project P kbagent rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] -kbagent rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes] +kbagent rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes] kbagent rls delete --project P --policy-id ID [--yes] kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes] # rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, @@ -1402,7 +1402,7 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # for admin sanity-checking only, drift between the two is acceptable since the preview string # is never executed. `rls setup` is a guided, interactive-terminal-only wizard (checkbox table # picker + guided condition builder, reusing `storage tables` and the same write path `rls -# create` uses) -- it refuses under --json or a non-TTY stdout with a hint to use `rls create` +# create` uses) -- it refuses under --json or without an interactive terminal (stdin or stdout not a TTY) with a hint to use `rls create` # directly, same carve-out as `auth register-projects`'s picker, and has no REST route. # A stack whose metastore predates the `rls-policy`/`cls-policy` schemas answers a clean, # classified error (schema fetch failure, NOT_FOUND); expected there, not a kbagent bug. @@ -1414,7 +1414,7 @@ kbagent cls list --project P kbagent cls detail --project P --policy-id ID kbagent cls schema --project P kbagent cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] -kbagent cls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes] +kbagent cls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes] kbagent cls delete --project P --policy-id ID [--yes] # cls: column-level security policy authoring (`cls-policy` object type, one object per # protected table), sibling of `rls`: same organization/targeted-only scope, fetch-then-merge diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 44eab7bf..3fa44269 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -475,9 +475,9 @@ Author `rls-policy` metastore objects -- one object per protected table, each ho - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals, condition}], revision}`. - `rls schema --project P` -- live `rls-policy` JSON Schema fetched from the metastore. Unlike `flow schema`, there is no offline bundled snapshot -- always live-only, `NOT_FOUND` (exit 4) on fetch failure rather than a stale local copy. - `rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or a `{"true": true}` sentinel) -- never a raw predicate string. Validated locally (op names, `principal`-xor-`principals`) always, plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any network write. `--target-project` (repeatable) shifts scope from `organization` to `targeted` and registers the grant via a follow-up `PUT .../target-projects` call. `--dry-run` runs every validation and returns the same compiled-condition preview a real write would, without writing -- see the preview-vs-enforcement caveat in [gotchas.md](gotchas.md). -- `rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes]` -- fetch-then-merge: `get_item` first, only the flags you pass overwrite the stored `table`/`dialect`/`rules`/target projects, everything else is carried forward untouched (the metastore's `PUT` is a whole-record replace, so this merge step is what stops a partial update from silently blanking fields). +- `rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes]` -- fetch-then-merge: `get_item` first, only the flags you pass overwrite the stored `table`/`dialect`/`rules`/target projects, everything else is carried forward untouched (the metastore's `PUT` is a whole-record replace, so this merge step is what stops a partial update from silently blanking fields). - `rls delete --project P --policy-id ID [--yes]` -- delete a policy, un-protecting its table. Confirmation prompt unless `--yes`; `--json` skips it too. -- `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table (same write path, never a second one). Refuses cleanly under `--json` or a non-TTY stdout with a hint to use `rls create` directly -- there is no non-interactive path through `setup` itself, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. +- `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table (same write path, never a second one). Refuses cleanly under `--json` or without an interactive terminal (stdin or stdout not a TTY) with a hint to use `rls create` directly -- there is no non-interactive path through `setup` itself, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. ## Column-Level Security (CLS) @@ -487,7 +487,7 @@ Author `cls-policy` metastore objects -- one object per protected table, each ho - `cls detail --project P --policy-id ID` -- full rule set: `{..., rules: [{principal|principals, visible_columns}], revision}`. - `cls schema --project P` -- live `cls-policy` JSON Schema from the metastore (`NOT_FOUND`, exit 4, when the stack's metastore does not have the object type). - `cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each principal's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. -- `cls update --project P --policy-id ID [--table ...] [--dialect ...] [--rules JSON|@file|-] [--target-project ID ...] [--dry-run] [--yes]` -- fetch-then-merge; omitted flags keep their current value. +- `cls update --project P --policy-id ID [--table ...] [--dialect ...] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes]` -- fetch-then-merge; omitted flags keep their current value. - `cls delete --project P --policy-id ID [--yes]` -- delete a policy; the table's columns become unrestricted for those principals. ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index ba60fdd3..aa260d3f 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -52,7 +52,7 @@ Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). `rls setup` is interactive-terminal-only -- it needs a real TTY for the checkbox table picker and the condition-builder prompts. It refuses cleanly -under `--json` or a piped/non-TTY stdout, printing a one-line hint instead +under `--json` or without an interactive terminal (stdin or stdout piped), printing a one-line hint instead of hanging or producing malformed JSON output. ```bash diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index b3e94669..ea2c3ad8 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -36,7 +36,7 @@ from ..errors import ConfigError, ErrorCode, KeboolaApiError from ..services.rls_service import RLS_DIALECTS from ._helpers import check_cli_permission, get_formatter, get_service, map_error_to_exit_code -from .rls import _format_policy_table, _parse_rules_arg +from .rls import _format_policy_table, _parse_rules_arg, _print_warnings cls_app = typer.Typer( help=( @@ -166,6 +166,9 @@ def cls_schema( except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) raise typer.Exit(code=5) from None + except KeboolaApiError as exc: # an auth/permission failure is not "schema unavailable" + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None if fetch.schema is None: formatter.error( @@ -240,6 +243,7 @@ def cls_create( if formatter.json_mode: formatter.output(result) return + _print_warnings(formatter, result) if not dry_run: formatter.success(f"Created CLS policy {result.get('id', '')} on {table}") _print_preview(formatter, result) @@ -260,6 +264,11 @@ def cls_update( target_project: list[str] | None = typer.Option( None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" ), + clear_target_projects: bool = typer.Option( + False, + "--clear-target-projects", + help="Revoke every project the policy is shared with (cannot be combined with --target-project)", + ), dry_run: bool = typer.Option(False, "--dry-run", help="Preview the projection without writing"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: @@ -270,6 +279,12 @@ def cls_update( """ formatter = get_formatter(ctx) parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) if rules is not None else None + if clear_target_projects and target_project: + formatter.error( + message="--clear-target-projects cannot be combined with --target-project", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None _check_dialect(formatter, dialect) service = get_service(ctx, "cls_service") @@ -284,13 +299,14 @@ def cls_update( table=table, dialect=dialect, rules=parsed_rules, - target_project_ids=target_project, + target_project_ids=[] if clear_target_projects else target_project, dry_run=dry_run, ) if formatter.json_mode: formatter.output(result) return + _print_warnings(formatter, result) if not dry_run: formatter.success(f"Updated CLS policy {policy_id}") _print_preview(formatter, result) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 66b98e36..f4726076 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -30,7 +30,6 @@ import json import logging -import sys from typing import Any import typer @@ -39,7 +38,7 @@ from ..errors import ConfigError, ErrorCode, KeboolaApiError from ..services.rls_service import RLS_DIALECTS -from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select +from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( check_cli_permission, get_formatter, @@ -64,9 +63,13 @@ def _rls_permission_check(ctx: typer.Context) -> None: check_cli_permission(ctx, "rls") -def _is_stdout_tty() -> bool: - """True when stdout is an interactive terminal (picker eligibility check).""" - return hasattr(sys.stdout, "isatty") and sys.stdout.isatty() +def _is_interactive() -> bool: + """True when BOTH stdin and stdout are terminals -- the same rule the checkbox picker applies. + + Checked up front so a non-interactive `rls setup` refuses before it makes any API call, rather + than listing the project's tables and only then failing in the picker. + """ + return _stdio_is_tty() # --------------------------------------------------------------------------- @@ -125,6 +128,12 @@ def _print_preview(formatter: Any, result: dict[str, Any]) -> None: formatter.console.print(f" {entry.get('principal')}: WHERE {entry.get('condition')}") +def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: + """Human mode only -- in `--json` mode the warnings are already part of the payload.""" + for warning in result.get("warnings", []): + formatter.warning(warning) + + def _parse_rules_arg( formatter: Any, raw: str, shape: str = "{principal|principals, condition}" ) -> list[dict[str, Any]]: @@ -234,6 +243,9 @@ def rls_schema( except ConfigError as exc: formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) raise typer.Exit(code=5) from None + except KeboolaApiError as exc: # an auth/permission failure is not "schema unavailable" + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + raise typer.Exit(code=map_error_to_exit_code(exc)) from None if fetch.schema is None: formatter.error( @@ -343,6 +355,7 @@ def rls_create( formatter.output(result) return + _print_warnings(formatter, result) if dry_run: _print_preview(formatter, result) return @@ -370,6 +383,11 @@ def rls_update( target_project: list[str] | None = typer.Option( None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" ), + clear_target_projects: bool = typer.Option( + False, + "--clear-target-projects", + help="Revoke every project the policy is shared with (cannot be combined with --target-project)", + ), dry_run: bool = typer.Option( False, "--dry-run", help="Preview the compiled condition without writing" ), @@ -382,6 +400,12 @@ def rls_update( """ formatter = get_formatter(ctx) parsed_rules = _parse_rules_arg(formatter, rules) if rules is not None else None + if clear_target_projects and target_project: + formatter.error( + message="--clear-target-projects cannot be combined with --target-project", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None if dialect is not None and dialect not in RLS_DIALECTS: formatter.error( @@ -405,7 +429,7 @@ def rls_update( table=table, dialect=dialect, rules=parsed_rules, - target_project_ids=target_project, + target_project_ids=[] if clear_target_projects else target_project, dry_run=dry_run, ) except ConfigError as exc: @@ -419,6 +443,7 @@ def rls_update( formatter.output(result) return + _print_warnings(formatter, result) if dry_run: _print_preview(formatter, result) return @@ -554,7 +579,13 @@ def rls_setup( create`` uses. """ formatter = get_formatter(ctx) - if formatter.json_mode or not _is_stdout_tty(): + if dialect is not None and dialect not in RLS_DIALECTS: + formatter.error( + message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + if formatter.json_mode or not _is_interactive(): target_console = formatter.err_console if formatter.json_mode else formatter.console target_console.print(_SETUP_HINT) raise typer.Exit(code=2) @@ -622,6 +653,7 @@ def rls_setup( formatter.console.print("[yellow]Aborted.[/yellow]") raise typer.Exit(code=0) + failed: list[str] = [] for table_id in selected_tables: try: result = service.create_policy( @@ -633,5 +665,15 @@ def rls_setup( ) except (ConfigError, KeboolaApiError) as exc: formatter.warning(f"{table_id}: {exc}") + failed.append(table_id) continue + _print_warnings(formatter, result) formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") + if failed: + # Automation must not read a partly (or wholly) failed setup as success. + formatter.error( + message=f"{len(failed)} of {len(selected_tables)} polic{'y' if len(selected_tables) == 1 else 'ies'}" + f" could not be created: {', '.join(failed)}", + error_code=ErrorCode.API_ERROR, + ) + raise typer.Exit(code=1) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 1a6fd81c..0487f9da 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -106,6 +106,7 @@ "semantic-glossary", "semantic-reference-data", "rls-policy", + "cls-policy", ) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 6158db72..003771ef 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -67,6 +67,13 @@ def validate_policy_structural(policy: dict[str, Any], schema: dict[str, Any]) - return errors +def _is_true_sentinel(condition: dict[Any, Any]) -> bool: + """Exactly ``{"true": true}``. ``{"true": false}`` (or ``0``/extra keys) would otherwise read as an + always-true policy -- the opposite of what an author who typed ``false`` meant. ``is True`` because + ``1 == True`` in Python.""" + return len(condition) == 1 and condition["true"] is True + + def validate_condition_ops(condition: Any) -> list[str]: """Recursively check every operator name used in ``condition`` is known. @@ -79,6 +86,8 @@ def validate_condition_ops(condition: Any) -> list[str]: if not isinstance(condition, dict): return [f"condition must be an object, got {type(condition).__name__}"] if "true" in condition: + if not _is_true_sentinel(condition): + errors.append("'true' condition must be exactly {\"true\": true}") return errors if "and" in condition or "or" in condition: clauses = condition.get("and", condition.get("or")) @@ -92,6 +101,18 @@ def validate_condition_ops(condition: Any) -> list[str]: op = condition.get("op") if op not in RLS_CONDITION_OPS: errors.append(f"unknown condition op {op!r} (expected one of {sorted(RLS_CONDITION_OPS)})") + return errors + # The primitive's shape: checked here so a malformed rule fails as INVALID_RLS_POLICY even when the + # live schema is unavailable, instead of reaching `compile_condition_preview` and crashing it. + column = condition.get("column") + if not isinstance(column, str) or not column: + errors.append(f"condition with op {op!r} needs a non-empty string 'column'") + if op in RLS_COMPARISON_OPS and "value" not in condition: + errors.append(f"condition with op {op!r} needs a 'value'") + if op in RLS_MEMBERSHIP_OPS: + values = condition.get("values") + if not isinstance(values, list) or not values: + errors.append(f"condition with op {op!r} needs a non-empty list 'values'") return errors @@ -146,6 +167,8 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: rendering something misleading). """ if "true" in condition: + if not _is_true_sentinel(condition): + raise ValueError(f"'true' condition must be exactly {{\"true\": true}}: {condition!r}") return "TRUE" if "and" in condition: clauses = condition["and"] diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index f8669fee..e683eaae 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -47,6 +47,16 @@ MetastoreClientFactory = Callable[[str, str], MetastoreClient] +# Failures of the caller's own credentials/permissions: never to be mistaken for "schema not available". +_AUTH_ERROR_CODES = frozenset( + { + ErrorCode.INVALID_TOKEN, + ErrorCode.MISSING_MASTER_TOKEN, + ErrorCode.ACCESS_DENIED, + ErrorCode.PERMISSION_DENIED, + } +) + @dataclass(frozen=True) class RlsSchemaFetch: @@ -113,16 +123,20 @@ def fetch_schema(self, alias: str) -> RlsSchemaFetch: return self._fetch_schema_for_project(project) def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: - """Fetch the live ``rls-policy`` schema. Never raises -- any failure + """Fetch the live policy schema, degrading when it is merely *unavailable*. - (network error, ``KeboolaApiError``, empty/malformed schema -- most - likely: the object type isn't registered on the stack's metastore) - degrades to ``schema=None`` + ``reason``, never blocks a caller. + A network error, an empty/malformed schema or a non-auth ``KeboolaApiError`` (most likely: the + object type isn't registered on the stack's metastore) degrades to ``schema=None`` + ``reason`` + and never blocks a caller. An authentication/authorization failure (``INVALID_TOKEN``, + ``MISSING_MASTER_TOKEN``, access denied) is a real problem the user must see, so it is + re-raised instead of being reported as a missing schema. """ with self._new_metastore_client(project) as client: try: schema, _version = fetch_resolved_schema(client, self.item_type) except KeboolaApiError as exc: + if exc.error_code in _AUTH_ERROR_CODES or exc.status_code in (401, 403): + raise return RlsSchemaFetch(schema=None, reason=exc.message) except Exception as exc: # any fetch failure must degrade, never block a write return RlsSchemaFetch(schema=None, reason=str(exc)) @@ -152,26 +166,29 @@ def _resolve_scope(self, target_project_ids: list[str] | None) -> MetastoreScope def _validate_policy( self, *, table: str, dialect: str, rules: list[dict[str, Any]], project: ProjectConfig - ) -> list[str]: - """Run every validation this module can, local checks first. + ) -> tuple[list[str], list[str]]: + """Run every validation this module can, local checks first; returns ``(errors, warnings)``. Local (schema-independent) checks always run. Structural (Draft7, against the live-fetched schema) checks run too when the fetch - succeeds -- when it doesn't, the caller gets a warning, not a block - (see :class:`RlsSchemaFetch`). + succeeds -- when it doesn't, the caller gets a warning (the reduced + validation is reported, not hidden), not a block (see :class:`RlsSchemaFetch`). """ errors: list[str] = [] if dialect not in _rls_condition.RLS_DIALECTS: errors.append(f"dialect must be one of {_rls_condition.RLS_DIALECTS}, got {dialect!r}") errors.extend(self._local_rule_errors(rules)) if errors: - return errors + return errors, [] + warnings: list[str] = [] fetch = self._fetch_schema_for_project(project) if fetch.schema: policy_body = {"table": table, "dialect": dialect, "rules": rules} errors.extend(_rls_condition.validate_policy_structural(policy_body, fetch.schema)) - return errors + else: + warnings.append(f"Live schema validation was skipped: {fetch.reason}") + return errors, warnings def _local_rule_errors(self, rules: Any) -> list[str]: """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" @@ -187,6 +204,13 @@ def _preview_rules(self, rules: list[dict[str, Any]], dialect: str) -> list[dict for rule in rules ] + @staticmethod + def _with_warnings(result: dict[str, Any], warnings: list[str]) -> dict[str, Any]: + """Attach validation warnings (e.g. the live schema was unavailable) only when there are any.""" + if warnings: + result["warnings"] = warnings + return result + def _raise_invalid(self, errors: list[str]) -> None: raise KeboolaApiError( message=f"{self.label} policy is invalid: " + "; ".join(errors), @@ -238,22 +262,27 @@ def create_policy( fetch used for validation happens. """ project = self._resolve_one_project(alias) - errors = self._validate_policy(table=table, dialect=dialect, rules=rules, project=project) + errors, warnings = self._validate_policy( + table=table, dialect=dialect, rules=rules, project=project + ) if errors: self._raise_invalid(errors) scope = self._resolve_scope(target_project_ids) preview = self._preview_rules(rules, dialect) if dry_run: - return { - "project": alias, - "table": table, - "dialect": dialect, - "scope": scope, - "target_project_ids": target_project_ids or [], - "preview": preview, - "dry_run": True, - } + return self._with_warnings( + { + "project": alias, + "table": table, + "dialect": dialect, + "scope": scope, + "target_project_ids": target_project_ids or [], + "preview": preview, + "dry_run": True, + }, + warnings, + ) with self._new_metastore_client(project) as client: created = client.post_item( @@ -269,7 +298,7 @@ def create_policy( ) row = self._row_from_item(created) row["preview"] = preview - return row + return self._with_warnings(row, warnings) def update_policy( self, @@ -282,13 +311,17 @@ def update_policy( target_project_ids: list[str] | None = None, dry_run: bool = False, ) -> dict[str, Any]: - """Update one ``rls-policy`` object. + """Update one policy object. ``put_item`` is a whole-record replace -- fetch the current item first and merge only the given overrides onto it, so an ``update`` call that only changes ``rules`` never silently wipes ``table`` or ``dialect`` (the failure mode this repo's own ``merge-request resolve`` docs warn about for exactly this kind of PUT-based API). + + ``target_project_ids``: ``None`` keeps the current grants, a non-empty list replaces them, and + ``[]`` revokes them all (through the dedicated grants call). A policy keeps its current scope + when its grants are revoked -- scope changes are not done through ``PUT``. """ project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: @@ -303,25 +336,33 @@ def update_policy( target_project_ids if target_project_ids is not None else meta.get("targetProjectIds") ) - errors = self._validate_policy( + errors, warnings = self._validate_policy( table=merged_table, dialect=merged_dialect, rules=merged_rules, project=project ) if errors: self._raise_invalid(errors) - scope = self._resolve_scope(merged_targets) + revoking = target_project_ids == [] and bool(meta.get("targetProjectIds")) + scope = ( + meta.get("scope") or "organization" + if target_project_ids == [] + else self._resolve_scope(merged_targets) + ) preview = self._preview_rules(merged_rules, merged_dialect) if dry_run: - return { - "project": alias, - "policy_id": policy_id, - "table": merged_table, - "dialect": merged_dialect, - "scope": scope, - "target_project_ids": merged_targets or [], - "preview": preview, - "dry_run": True, - } + return self._with_warnings( + { + "project": alias, + "policy_id": policy_id, + "table": merged_table, + "dialect": merged_dialect, + "scope": scope, + "target_project_ids": merged_targets or [], + "preview": preview, + "dry_run": True, + }, + warnings, + ) with self._new_metastore_client(project) as client: updated = client.put_item( @@ -332,11 +373,13 @@ def update_policy( scope=scope, target_project_ids=merged_targets, ) - if scope == "targeted" and merged_targets: + if revoking: + client.put_target_projects(self.item_type, policy_id, []) + elif scope == "targeted" and merged_targets: client.put_target_projects(self.item_type, policy_id, merged_targets) row = self._row_from_item(updated) row["preview"] = preview - return row + return self._with_warnings(row, warnings) def delete_policy(self, alias: str, policy_id: str) -> dict[str, Any]: """Delete one ``rls-policy`` object by id.""" diff --git a/tests/test_cls_cli.py b/tests/test_cls_cli.py index 0b0562f6..5de54030 100644 --- a/tests/test_cls_cli.py +++ b/tests/test_cls_cli.py @@ -426,3 +426,66 @@ def test_permission_check_denies_admin_writes_but_not_reads(tmp_path: Path) -> N "schema": "read", } assert all(OPERATION_REGISTRY[f"cls.{op}"] == "admin" for op in ("create", "update", "delete")) + + +class TestClsReviewFixes: + def test_schema_auth_failure_is_an_error_not_a_missing_schema(self, tmp_path: Path) -> None: + service = MagicMock() + service.fetch_schema.side_effect = KeboolaApiError( + message="bad token", status_code=401, error_code=ErrorCode.INVALID_TOKEN + ) + + result = _run(["--json", "cls", "schema", "--project", "prod"], tmp_path, service) + + assert result.exit_code == 3 + assert "INVALID_TOKEN" in result.output + + def test_update_clear_target_projects_passes_an_empty_list(self, tmp_path: Path) -> None: + service = MagicMock() + service.update_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run( + [ + "--json", + "cls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--clear-target-projects", + ], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + assert service.update_policy.call_args.kwargs["target_project_ids"] == [] + + def test_update_clear_and_target_project_are_mutually_exclusive(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + [ + "--json", "cls", "update", "--project", "prod", "--policy-id", "p-1", + "--clear-target-projects", "--target-project", "7", + ], + tmp_path, + service, + ) # fmt: skip + + assert result.exit_code == 2 + service.update_policy.assert_not_called() + + def test_create_prints_the_validation_warning_in_human_mode(self, tmp_path: Path) -> None: + service = MagicMock() + service.create_policy.return_value = { + **_row(), + "preview": _preview()["preview"], + "warnings": ["Live schema validation was skipped: no schema"], + } + + result = _run([*CREATE_ARGS, "--yes"], tmp_path, service) + + assert result.exit_code == 0, result.output + assert "Live schema validation was skipped" in result.output diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 0a46d386..c31878b1 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -3735,6 +3735,9 @@ def _test_rls_commands(self) -> None: for group in ("rls", "cls"): result = self._run(group, "schema", "--project", self.alias) if result.exit_code != 0: + # Only a missing object type is an expected skip. An auth failure or any other error + # must fail the suite instead of being skipped along with it. + assert "NOT_FOUND" in result.output, result.output print( f" {_YELLOW}SKIP: {group} schema ({group}-policy not registered here){_RESET}" ) diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index d4c015ae..5545c42c 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -358,6 +358,7 @@ def test_semantic_types_complete(self) -> None: "semantic-glossary", "semantic-reference-data", "rls-policy", + "cls-policy", } diff --git a/tests/test_project_ref.py b/tests/test_project_ref.py index 299dc877..07f228a2 100644 --- a/tests/test_project_ref.py +++ b/tests/test_project_ref.py @@ -153,6 +153,7 @@ def test_no_shadow_notice_for_aliases_of_the_same_project(self) -> None: "be registered); config clone's --target-project is in ALIAS_OPTIONS" ), "--all-projects": "a boolean flag", + "--clear-target-projects": "a boolean flag (rls/cls update): revoke every grant, takes no project", "--register-projects": "a boolean flag", "--remove-projects": "a boolean flag", } diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index aca4a70e..5853641e 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -13,9 +13,11 @@ from typing import Any from unittest.mock import MagicMock, patch +import pytest from typer.testing import CliRunner from keboola_agent_cli.cli import app +from keboola_agent_cli.commands.rls import _is_interactive from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError from keboola_agent_cli.models import ProjectConfig @@ -478,3 +480,243 @@ def test_json_mode_refuses_without_launching_picker(self, tmp_path: Path) -> Non assert result.exit_code == 2 storage_service.list_tables.assert_not_called() service.create_policy.assert_not_called() + + +class _Stream: + def __init__(self, tty: bool) -> None: + self._tty = tty + + def isatty(self) -> bool: + return self._tty + + +@pytest.mark.parametrize( + ("stdin_tty", "stdout_tty", "expected"), + [(True, True, True), (True, False, False), (False, True, False), (False, False, False)], +) +def test_setup_is_interactive_only_when_stdin_and_stdout_are_terminals( + monkeypatch: pytest.MonkeyPatch, stdin_tty: bool, stdout_tty: bool, expected: bool +) -> None: + """A TTY stdout with a piped stdin used to pass, so `rls setup` hit the API before the picker failed.""" + monkeypatch.setattr("sys.stdin", _Stream(stdin_tty)) + monkeypatch.setattr("sys.stdout", _Stream(stdout_tty)) + + assert _is_interactive() is expected + + +class TestRlsReviewFixes: + def test_schema_auth_failure_is_an_error_not_a_missing_schema(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.fetch_schema.side_effect = KeboolaApiError( + message="bad token", status_code=401, error_code=ErrorCode.INVALID_TOKEN + ) + + result = _run(["--json", "rls", "schema", "--project", "prod"], store, service) + + assert result.exit_code == 3 # authentication, not 4 (NOT_FOUND) + assert "INVALID_TOKEN" in result.output + + def test_update_clear_target_projects_passes_an_empty_list(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.update_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + [ + "--json", + "rls", + "update", + "--project", + "prod", + "--policy-id", + "p-1", + "--clear-target-projects", + ], + store, + service, + ) + + assert result.exit_code == 0, result.output + assert service.update_policy.call_args.kwargs["target_project_ids"] == [] + + def test_update_clear_and_target_project_are_mutually_exclusive(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + [ + "--json", "rls", "update", "--project", "prod", "--policy-id", "p-1", + "--clear-target-projects", "--target-project", "7", + ], + store, + service, + ) # fmt: skip + + assert result.exit_code == 2 + service.update_policy.assert_not_called() + + def test_update_without_the_flag_keeps_grants_unchanged(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.update_policy.return_value = {**_policy_row(), "preview": []} + + _run( + ["--json", "rls", "update", "--project", "prod", "--policy-id", "p-1", "--yes"], + store, + service, + ) + + assert service.update_policy.call_args.kwargs["target_project_ids"] is None + + def test_create_prints_the_validation_warning_in_human_mode(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = { + **_policy_row(), + "preview": [], + "warnings": ["Live schema validation was skipped: no schema"], + } + + result = _run( + [ + "rls", "create", "--project", "prod", "--table", "in.c-crm.invoices", "--dialect", "snowflake", + "--rules", '[{"principal":"a@x.com","condition":{"true":true}}]', "--yes", + ], + store, + service, + ) # fmt: skip + + assert result.exit_code == 0, result.output + assert "Live schema validation was skipped" in result.output + + +class TestRlsSetupReviewFixes: + """`rls setup` needs a terminal, so the interactive gate and the picker are patched out.""" + + @staticmethod + def _setup_run( + args: list[str], store: ConfigStore, service: MagicMock, storage: MagicMock + ) -> Any: + with ( + patch("keboola_agent_cli.commands.rls._is_interactive", return_value=True), + patch("keboola_agent_cli.commands.rls.checkbox_select", return_value=[0, 1]), + ): + return _run(args, store, service, storage_service=storage) + + @staticmethod + def _storage() -> MagicMock: + storage = MagicMock() + storage.list_tables.return_value = { + "tables": [{"id": "in.c-crm.a", "rows_count": 1}, {"id": "in.c-crm.b", "rows_count": 2}] + } + return storage + + RULES = '[{"principal":"a@x.com","condition":{"true":true}}]' + + def test_an_invalid_dialect_is_rejected_before_any_api_call(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + storage = self._storage() + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "postgres", + "--rules", + self.RULES, + "--yes", + ], + store, + MagicMock(), + storage, + ) + + assert result.exit_code == 2 + storage.list_tables.assert_not_called() + + def test_every_table_failing_is_a_non_zero_exit(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.side_effect = KeboolaApiError( + message="needs a master token", + status_code=403, + error_code=ErrorCode.MISSING_MASTER_TOKEN, + ) + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + self._storage(), + ) + + assert result.exit_code == 1 + assert "2 of 2 policies could not be created" in result.output + + def test_a_partial_failure_is_a_non_zero_exit_and_names_the_failed_table( + self, tmp_path: Path + ) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + ok = {**_policy_row(), "preview": []} + failure = KeboolaApiError(message="boom", status_code=500, error_code=ErrorCode.API_ERROR) + # The wizard previews each table (dry-run) first, then writes: only the second write fails. + service.create_policy.side_effect = [ok, ok, ok, failure] + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + self._storage(), + ) + + assert result.exit_code == 1 + assert "1 of 2 policies could not be created: in.c-crm.b" in result.output + + def test_all_tables_created_exits_zero(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + self._storage(), + ) + + assert result.exit_code == 0, result.output diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index a0853262..c3e28da9 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -88,6 +88,16 @@ def _policy_item( _RULES = [{"principal": "a@x.com", "condition": {"column": "region", "op": "eq", "value": "EU"}}] +# `1 == True` and any non-True value would otherwise read as "true": only `{"true": true}` is the sentinel. +_BAD_TRUE_SENTINELS = [ + {"true": False}, + {"true": 0}, + {"true": 1}, + {"true": None}, + {"true": "yes"}, + {"true": True, "extra": 1}, +] + # --------------------------------------------------------------------------- # Read @@ -145,6 +155,44 @@ def test_api_error_degrades(self, tmp_path: Path) -> None: assert fetch.schema is None assert fetch.reason == "not found" + @pytest.mark.parametrize( + ("error_code", "status"), + [ + (ErrorCode.INVALID_TOKEN, 401), + (ErrorCode.MISSING_MASTER_TOKEN, 403), + (ErrorCode.ACCESS_DENIED, 403), + (ErrorCode.PERMISSION_DENIED, 403), + (ErrorCode.API_ERROR, 401), # by status alone + (ErrorCode.API_ERROR, 403), + ], + ) + def test_auth_and_permission_errors_are_not_reported_as_a_missing_schema( + self, tmp_path: Path, error_code: ErrorCode, status: int + ) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError( + message="nope", status_code=status, error_code=error_code + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.fetch_schema("prod") + + assert excinfo.value.error_code == error_code + + @pytest.mark.parametrize("status", [404, 500, 502]) + def test_other_api_errors_still_degrade(self, tmp_path: Path, status: int) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError( + message="unavailable", status_code=status, error_code=ErrorCode.API_ERROR + ) + + fetch = service.fetch_schema("prod") + + assert fetch.schema is None + assert fetch.reason == "unavailable" + def test_unexpected_error_degrades(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) @@ -348,6 +396,11 @@ class TestCompileConditionPreview: def test_true_sentinel(self) -> None: assert compile_condition_preview({"true": True}, "snowflake") == "TRUE" + @pytest.mark.parametrize("bad", _BAD_TRUE_SENTINELS) + def test_non_true_sentinel_is_never_rendered_as_always_true(self, bad: dict) -> None: + with pytest.raises(ValueError, match="exactly"): + compile_condition_preview(bad, "snowflake") + @pytest.mark.parametrize( "op,sql", [ @@ -421,6 +474,23 @@ def test_unknown_op_reported(self) -> None: def test_true_sentinel_always_valid(self) -> None: assert validate_condition_ops({"true": True}) == [] + @pytest.mark.parametrize("bad", _BAD_TRUE_SENTINELS) + def test_true_sentinel_must_be_exactly_true(self, bad: dict) -> None: + """`{"true": false}` must not pass: it would silently author an always-true policy.""" + errors = validate_condition_ops(bad) + assert errors and "exactly" in errors[0] + + def test_a_false_sentinel_is_rejected_before_any_write(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + service, mock = _make_service(store) + rules = [{"principal": "a@x.com", "condition": {"true": False}}] + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t", dialect="snowflake", rules=rules) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() + def test_and_requires_at_least_two(self) -> None: errors = validate_condition_ops({"and": [{"true": True}]}) assert errors @@ -481,3 +551,151 @@ def test_missing_required_field_reported(self) -> None: schema = {"type": "object", "required": ["table", "dialect", "rules"]} errors = validate_policy_structural({"table": "t"}, schema) assert errors + + +class TestValidationWarnings: + def test_a_skipped_live_validation_is_reported_not_hidden(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + mock.post_item.return_value = _policy_item() + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES + ) + + assert result["warnings"] == ["Live schema validation was skipped: no schema"] + + def test_dry_run_reports_it_too(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES, dry_run=True + ) + + assert result["warnings"] == ["Live schema validation was skipped: no schema"] + + def test_no_warning_when_the_schema_was_available(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = {"type": "object"} + mock.post_item.return_value = _policy_item() + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES + ) + + assert "warnings" not in result + + def test_update_reports_it_as_well(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + mock.get_item.return_value = _policy_item() + mock.put_item.return_value = _policy_item() + + result = service.update_policy("prod", "p-1", dialect="bigquery") + + assert result["warnings"] == ["Live schema validation was skipped: no schema"] + + +class TestClearingGrants: + """`target_project_ids`: None keeps the grants, a list replaces them, [] revokes them all.""" + + def _targeted(self) -> dict[str, Any]: + return _policy_item(scope="targeted", target_project_ids=["7", "8"]) + + def test_empty_list_revokes_every_grant_and_keeps_the_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = self._targeted() + mock.put_item.return_value = self._targeted() + + result = service.update_policy("prod", "p-1", target_project_ids=[]) + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + assert mock.put_item.call_args.kwargs["target_project_ids"] == [] + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) + assert result["id"] == "p-1" + + def test_none_keeps_the_existing_grants(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = self._targeted() + mock.put_item.return_value = self._targeted() + + service.update_policy("prod", "p-1", dialect="bigquery") + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["7", "8"]) + + def test_clearing_a_policy_that_has_no_grants_makes_no_grant_call(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = _policy_item() # organization scope, no targets + mock.put_item.return_value = _policy_item() + + service.update_policy("prod", "p-1", target_project_ids=[]) + + assert mock.put_item.call_args.kwargs["scope"] == "organization" + mock.put_target_projects.assert_not_called() + + def test_dry_run_previews_the_cleared_state_without_writing(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = self._targeted() + + result = service.update_policy("prod", "p-1", target_project_ids=[], dry_run=True) + + assert result["target_project_ids"] == [] + mock.put_item.assert_not_called() + mock.put_target_projects.assert_not_called() + + +class TestPrimitiveShape: + """The shape of a leaf condition is checked locally, so it fails as INVALID_RLS_POLICY even when the + live schema is unavailable (the documented state on stacks without the object type).""" + + @pytest.mark.parametrize( + ("condition", "fragment"), + [ + ({"column": "", "op": "eq", "value": 1}, "non-empty string 'column'"), + ({"op": "eq", "value": 1}, "non-empty string 'column'"), + ({"column": 5, "op": "is_null"}, "non-empty string 'column'"), + ({"column": "a", "op": "eq"}, "needs a 'value'"), + ({"column": "a", "op": "gt"}, "needs a 'value'"), + ({"column": "a", "op": "in"}, "non-empty list 'values'"), + ({"column": "a", "op": "not_in", "values": []}, "non-empty list 'values'"), + ({"column": "a", "op": "in", "values": "abc"}, "non-empty list 'values'"), + ], + ) + def test_malformed_primitives_are_rejected(self, condition: dict, fragment: str) -> None: + errors = validate_condition_ops(condition) + assert errors and fragment in errors[0] + + @pytest.mark.parametrize( + "condition", + [ + {"column": "a", "op": "eq", "value": 0}, + {"column": "a", "op": "eq", "value": None}, # an explicit null value is still a value + {"column": "a", "op": "in", "values": [1]}, + {"column": "a", "op": "is_null"}, + { + "and": [ + {"column": "a", "op": "eq", "value": 1}, + {"column": "b", "op": "is_not_null"}, + ] + }, + ], + ) + def test_well_formed_primitives_pass(self, condition: dict) -> None: + assert validate_condition_ops(condition) == [] + + def test_a_malformed_rule_never_reaches_the_preview_or_the_write(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + rules = [{"principal": "a@x.com", "condition": {"column": "", "op": "eq"}}] + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t", dialect="snowflake", rules=rules, dry_run=True) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() From b64ac1ebad4f03c2362ef51ffff6f7c86f18d1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 06:19:27 +0200 Subject: [PATCH 10/35] fix(CLI-17): send target project ids the way the metastore expects, address review Found while answering a Copilot comment on a stale docstring: the metastore decodes targetProjectIds as []int, and PUT .../target-projects takes {"targetProjectIds": [int]} (an empty list clears every grant). The client sent string ids in the create/update envelope (a 400) and {"projectIds": [...]} to the grants endpoint -- a field the backend ignores, so --target-project would have cleared the grants instead of setting them. It also parsed the 204 No Content reply as JSON. - Convert target project ids to positive integers (INVALID_ARGUMENT before any request otherwise); send the right field; treat 204 as an empty result. - Replace the "speculative" docstring with the deployed contract. - fetch_resolved_schema returns a named ResolvedSchema (schema, version) instead of a tuple. - A non-empty but malformed schema from the metastore degrades to a warning instead of raising SchemaError from the structural validation. - E2E: rls/cls list may only be skipped on NOT_FOUND, like schema. - Docs: fix the contradictory cli:write / cli:admin wording; note numeric --target-project ids. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/metastore_client.py | 23 +++++++ src/keboola_agent_cli/services/rls_service.py | 13 +++- .../services/semantic_layer_service.py | 4 +- tests/test_e2e.py | 1 + tests/test_metastore_client.py | 64 +++++++++++++++++-- tests/test_rls_service.py | 21 ++++++ 6 files changed, 116 insertions(+), 10 deletions(-) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 0487f9da..9e1c41d7 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -49,6 +49,7 @@ import logging import re +from dataclasses import dataclass from typing import Any, Literal, Self import httpx @@ -148,6 +149,26 @@ def _name_conflict(item_type: str, name: str, exc: KeboolaApiError) -> KeboolaAp ) +def _project_ids_as_ints(project_ids: list[Any]) -> list[int]: + """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). + + The CLI takes ids as strings; a non-numeric or non-positive one is rejected here, before any request, + instead of reaching the backend as a 400 (or, in the grants call, being ignored). + """ + try: + ids = [int(project_id) for project_id in project_ids] + except (TypeError, ValueError): + ids = [0] + if any(project_id <= 0 for project_id in ids): + raise KeboolaApiError( + message=f"Target project IDs must be positive integers, got {project_ids!r}", + status_code=400, + error_code=ErrorCode.INVALID_ARGUMENT, + retryable=False, + ) + return ids + + class MetastoreClient(BaseHttpClient): """HTTP client for the Keboola Metastore (semantic layer repository). @@ -435,6 +456,8 @@ def elevate_to_organization(self, item_type: SemanticType, item_id: str) -> dict f"/api/v1/repository/{item_type}/{item_id}", json={"scope": "organization"}, ) + if not response.content: + return {} body = response.json() return body.get("data", body) if isinstance(body, dict) else body diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index e683eaae..27e86d51 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -26,6 +26,8 @@ from dataclasses import dataclass from typing import Any +import jsonschema + from ..config_store import ConfigStore from ..errors import ErrorCode, KeboolaApiError from ..metastore_client import ( @@ -133,7 +135,7 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: """ with self._new_metastore_client(project) as client: try: - schema, _version = fetch_resolved_schema(client, self.item_type) + schema = fetch_resolved_schema(client, self.item_type).schema except KeboolaApiError as exc: if exc.error_code in _AUTH_ERROR_CODES or exc.status_code in (401, 403): raise @@ -144,6 +146,15 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: return RlsSchemaFetch( schema=None, reason=f"metastore returned no schema for {self.item_type}" ) + try: + jsonschema.Draft7Validator.check_schema(schema) + except jsonschema.SchemaError as exc: + # A non-empty but malformed schema would make the structural validation raise a traceback; + # report it as unavailable instead (same as an empty one). + return RlsSchemaFetch( + schema=None, + reason=f"the {self.item_type} schema from the metastore is malformed: {exc.message}", + ) return RlsSchemaFetch(schema=schema, reason=None) @staticmethod diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index 8a7f0596..ed16f72a 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -363,8 +363,8 @@ def _fetch_resolved_schema(client: MetastoreClient, wire_type: SemanticType) -> listing. (The upstream MCP tool passes that bare listing through; we deliberately do not mirror that gap.) """ - schema, version = fetch_resolved_schema(client, wire_type) - return {"schema": schema, "schema_version": version} + resolved = fetch_resolved_schema(client, wire_type) + return {"schema": resolved.schema, "schema_version": resolved.version} # Internal helpers (model-scoped fetches). diff --git a/tests/test_e2e.py b/tests/test_e2e.py index c31878b1..e480bfe4 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -3750,6 +3750,7 @@ def _test_rls_commands(self) -> None: # whose metastore lacks the object type. result = self._run(group, "list", "--project", self.alias) if result.exit_code != 0: + assert "NOT_FOUND" in result.output, result.output print(f" {_YELLOW}SKIP: {group} list ({group}-policy not registered here){_RESET}") else: data = _json_ok(result) diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 5545c42c..f5f0c586 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -18,6 +18,7 @@ from keboola_agent_cli.metastore_client import ( SEMANTIC_TYPES, MetastoreClient, + ResolvedSchema, fetch_resolved_schema, ) @@ -412,7 +413,8 @@ def test_targeted_scope_includes_target_project_ids(self, httpx_mock, metastore_ ) body = json.loads(httpx_mock.get_requests()[0].content) assert body["scope"] == "targeted" - assert body["targetProjectIds"] == ["111", "222"] + # The metastore decodes targetProjectIds as []int: integers on the wire, never strings. + assert body["targetProjectIds"] == [111, 222] def test_put_item_accepts_scope_too(self, httpx_mock, metastore_client) -> None: httpx_mock.add_response( @@ -433,20 +435,68 @@ def test_put_item_accepts_scope_too(self, httpx_mock, metastore_client) -> None: class TestPutTargetProjects: - def test_puts_project_ids_body(self, httpx_mock, metastore_client) -> None: + def test_puts_the_target_project_ids_as_integers(self, httpx_mock, metastore_client) -> None: + """The backend body is `{"targetProjectIds": [int]}` and ignores unknown fields: a wrong key + would be read as an EMPTY list, which clears every grant.""" httpx_mock.add_response( method="PUT", url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", - json={"data": {"type": "rls-policy", "id": "rls-1"}}, - status_code=200, + status_code=204, ) + result = metastore_client.put_target_projects("rls-policy", "rls-1", ["111", "222"]) - assert result["id"] == "rls-1" + request = httpx_mock.get_requests()[0] assert request.method == "PUT" assert request.url.path.endswith("/rls-policy/rls-1/target-projects") - body = json.loads(request.content) - assert body["projectIds"] == ["111", "222"] + assert json.loads(request.content) == {"targetProjectIds": [111, 222]} + assert result == {} # 204 No Content: there is no body to parse + + def test_an_empty_list_is_sent_explicitly_to_clear_all_grants( + self, httpx_mock, metastore_client + ) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", + status_code=204, + ) + + metastore_client.put_target_projects("rls-policy", "rls-1", []) + + assert json.loads(httpx_mock.get_requests()[0].content) == {"targetProjectIds": []} + + def test_a_response_body_is_still_returned_when_the_server_sends_one( + self, httpx_mock, metastore_client + ) -> None: + httpx_mock.add_response( + method="PUT", + url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", + json={"data": {"type": "rls-policy", "id": "rls-1"}}, + status_code=200, + ) + + assert metastore_client.put_target_projects("rls-policy", "rls-1", ["111"])["id"] == "rls-1" + + @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-5"], ["1.5"], [None]]) + def test_non_positive_or_non_numeric_ids_are_rejected_before_any_request( + self, httpx_mock, metastore_client, bad: list + ) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.put_target_projects("rls-policy", "rls-1", bad) + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert httpx_mock.get_requests() == [] + + def test_post_item_rejects_a_non_numeric_target_before_any_request( + self, httpx_mock, metastore_client + ) -> None: + with pytest.raises(KeboolaApiError) as excinfo: + metastore_client.post_item( + "rls-policy", name="t", data={}, scope="targeted", target_project_ids=["abc"] + ) + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + assert httpx_mock.get_requests() == [] class TestProjectScope401Reclassification: diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index c3e28da9..88264c54 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -193,6 +193,27 @@ def test_other_api_errors_still_degrade(self, tmp_path: Path, status: int) -> No assert fetch.schema is None assert fetch.reason == "unavailable" + @pytest.mark.parametrize( + "malformed", [{"type": 123}, {"properties": {"table": {"type": "nope"}}}] + ) + def test_a_non_empty_but_malformed_schema_degrades_instead_of_raising( + self, tmp_path: Path, malformed: dict + ) -> None: + """`Draft7Validator(schema)` would raise SchemaError mid-validation: report it as unavailable.""" + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = malformed + mock.post_item.return_value = _policy_item() + + fetch = service.fetch_schema("prod") + assert fetch.schema is None + assert "malformed" in (fetch.reason or "") + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES + ) + assert any("malformed" in w for w in result["warnings"]) + mock.post_item.assert_called_once() + def test_unexpected_error_degrades(self, tmp_path: Path) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) From 5b6910cb462a3b0ec7f94e3ff28fd9b7dc8e3571 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 06:30:31 +0200 Subject: [PATCH 11/35] fix(CLI-17): keep a policy's scope on partial updates; validate target ids in --dry-run - update: only an explicit, non-empty --target-project list derives a new scope. When the option is omitted or the grants are being cleared, the policy keeps its current scope. A targeted policy whose grants were cleared is legitimately "targeted" with an empty list; an unrelated update used to turn it into "organization". - create/update: validate the target project ids before any network call, so --dry-run runs the same validation as the real write (INVALID_ARGUMENT) instead of reporting a successful preview for an id the write would reject. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/metastore_client.py | 2 +- src/keboola_agent_cli/services/rls_service.py | 17 +++- tests/test_rls_service.py | 93 +++++++++++++++++++ 3 files changed, 108 insertions(+), 4 deletions(-) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 9e1c41d7..a757f369 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -149,7 +149,7 @@ def _name_conflict(item_type: str, name: str, exc: KeboolaApiError) -> KeboolaAp ) -def _project_ids_as_ints(project_ids: list[Any]) -> list[int]: +def project_ids_as_ints(project_ids: list[Any]) -> list[int]: """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). The CLI takes ids as strings; a non-numeric or non-positive one is rejected here, before any request, diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 27e86d51..82c18459 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -35,6 +35,7 @@ MetastoreScope, SemanticType, fetch_resolved_schema, + project_ids_as_ints, ) from ..models import ProjectConfig from . import _rls_condition @@ -272,6 +273,9 @@ def create_policy( metastore's write endpoint at all -- only the (read-only) schema fetch used for validation happens. """ + project_ids_as_ints( + target_project_ids or [] + ) # same validation as the write, also for --dry-run project = self._resolve_one_project(alias) errors, warnings = self._validate_policy( table=table, dialect=dialect, rules=rules, project=project @@ -334,6 +338,9 @@ def update_policy( ``[]`` revokes them all (through the dedicated grants call). A policy keeps its current scope when its grants are revoked -- scope changes are not done through ``PUT``. """ + project_ids_as_ints( + target_project_ids or [] + ) # same validation as the write, also for --dry-run project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: current = client.get_item(self.item_type, policy_id) @@ -354,10 +361,14 @@ def update_policy( self._raise_invalid(errors) revoking = target_project_ids == [] and bool(meta.get("targetProjectIds")) + # An explicit, non-empty target list makes the policy `targeted`. When the option is omitted or the + # grants are being cleared, the current scope is kept: a `targeted` policy whose grants were cleared + # is legitimately `targeted` with an empty list, and an unrelated update must not turn it into + # `organization`. scope = ( - meta.get("scope") or "organization" - if target_project_ids == [] - else self._resolve_scope(merged_targets) + self._resolve_scope(target_project_ids) + if target_project_ids + else (meta.get("scope") or self._resolve_scope(merged_targets)) ) preview = self._preview_rules(merged_rules, merged_dialect) if dry_run: diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 88264c54..2d96ba33 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -720,3 +720,96 @@ def test_a_malformed_rule_never_reaches_the_preview_or_the_write(self, tmp_path: assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY mock.post_item.assert_not_called() + + +class TestScopeIsKeptWhenTargetsAreOmitted: + def _targeted_without_grants(self) -> dict[str, Any]: + """What the metastore returns after every grant was revoked: still `targeted`, empty list.""" + return _policy_item(scope="targeted", target_project_ids=[]) + + def test_an_unrelated_update_does_not_turn_a_cleared_targeted_policy_into_organization( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = self._targeted_without_grants() + mock.put_item.return_value = self._targeted_without_grants() + + service.update_policy("prod", "p-1", dialect="bigquery") # no target option at all + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_not_called() + + def test_an_explicit_target_list_makes_the_policy_targeted(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = _policy_item() # organization scope + mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=["7"]) + + service.update_policy("prod", "p-1", target_project_ids=["7"]) + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["7"]) + + def test_clearing_keeps_the_current_scope(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = _policy_item(scope="targeted", target_project_ids=["7"]) + mock.put_item.return_value = self._targeted_without_grants() + + service.update_policy("prod", "p-1", target_project_ids=[]) + + assert mock.put_item.call_args.kwargs["scope"] == "targeted" + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) + + +class TestDryRunValidatesTargetIds: + """A preview must run the same validation as the write -- including the target project ids.""" + + @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-3"], ["1", "x"]]) + def test_create_dry_run_rejects_a_bad_target_before_any_network_call( + self, tmp_path: Path, bad: list + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy( + "prod", + table="t", + dialect="snowflake", + rules=_RULES, + target_project_ids=bad, + dry_run=True, + ) + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.get_schema.assert_not_called() + mock.post_item.assert_not_called() + + def test_update_dry_run_rejects_a_bad_target_before_any_network_call( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(KeboolaApiError) as excinfo: + service.update_policy("prod", "p-1", target_project_ids=["abc"], dry_run=True) + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.get_item.assert_not_called() + mock.put_item.assert_not_called() + + def test_valid_targets_still_preview(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + + result = service.create_policy( + "prod", + table="t", + dialect="snowflake", + rules=_RULES, + target_project_ids=["7", "8"], + dry_run=True, + ) + + assert result["scope"] == "targeted" + assert result["target_project_ids"] == ["7", "8"] From 7029bac97a9cdbbd813a86940eeb883c3ee0ffd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 06:43:02 +0200 Subject: [PATCH 12/35] fix(CLI-17): reject conditions with several composition keys or stray keys (Copilot review) validate_condition_ops handled `{"and": [...], "or": [...]}` by validating only the "and" branch and returning, so with the live schema unavailable the policy no longer matched the submitted tree. An and/or condition must now have exactly one of those keys and no others, and a leaf condition may carry only its op's own keys (value for comparisons, values for in/not_in), as the schema's additionalProperties:false requires. compile_condition_preview is equally strict. Co-Authored-By: Claude Sonnet 5.5 --- .../services/_rls_condition.py | 35 +++++++--- tests/test_rls_service.py | 65 ++++++++++++++++++- 2 files changed, 91 insertions(+), 9 deletions(-) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 003771ef..21507b96 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -90,9 +90,17 @@ def validate_condition_ops(condition: Any) -> list[str]: errors.append("'true' condition must be exactly {\"true\": true}") return errors if "and" in condition or "or" in condition: - clauses = condition.get("and", condition.get("or")) + # Exactly one composition key and nothing else: `{"and": [...], "or": [...]}` would otherwise have + # one branch silently validated and the other dropped, so the policy would no longer be the tree + # the author submitted. + if set(condition) not in ({"and"}, {"or"}): + errors.append( + f"an 'and'/'or' condition must have exactly one of those keys and no others, got {sorted(condition)}" + ) + return errors + key = "and" if "and" in condition else "or" + clauses = condition[key] if not isinstance(clauses, list) or len(clauses) < 2: - key = "and" if "and" in condition else "or" errors.append(f"'{key}' requires at least 2 nested conditions") return errors for clause in clauses: @@ -113,6 +121,13 @@ def validate_condition_ops(condition: Any) -> list[str]: values = condition.get("values") if not isinstance(values, list) or not values: errors.append(f"condition with op {op!r} needs a non-empty list 'values'") + # No keys beyond the op's own shape (the schema forbids additional properties): a stray `values` on an + # `eq`, or a `value` on an `in`, is an authoring mistake that would otherwise be silently ignored. + allowed = {"column", "op"} | ( + {"value"} if op in RLS_COMPARISON_OPS else {"values"} if op in RLS_MEMBERSHIP_OPS else set() + ) + if extra := sorted(set(condition) - allowed): + errors.append(f"condition with op {op!r} has unexpected keys {extra}") return errors @@ -170,12 +185,16 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: if not _is_true_sentinel(condition): raise ValueError(f"'true' condition must be exactly {{\"true\": true}}: {condition!r}") return "TRUE" - if "and" in condition: - clauses = condition["and"] - return " AND ".join(f"({compile_condition_preview(c, dialect)})" for c in clauses) - if "or" in condition: - clauses = condition["or"] - return " OR ".join(f"({compile_condition_preview(c, dialect)})" for c in clauses) + if "and" in condition or "or" in condition: + if len(condition) != 1: + raise ValueError( + f"an 'and'/'or' condition must have exactly one of those keys and no others: {condition!r}" + ) + if "and" in condition: + return " AND ".join( + f"({compile_condition_preview(c, dialect)})" for c in condition["and"] + ) + return " OR ".join(f"({compile_condition_preview(c, dialect)})" for c in condition["or"]) column = condition.get("column") op = condition.get("op") diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 2d96ba33..a7d2e7f9 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -8,7 +8,7 @@ from __future__ import annotations from pathlib import Path -from typing import Any +from typing import Any, ClassVar from unittest.mock import MagicMock import pytest @@ -813,3 +813,66 @@ def test_valid_targets_still_preview(self, tmp_path: Path) -> None: assert result["scope"] == "targeted" assert result["target_project_ids"] == ["7", "8"] + + +class TestCompositionAndKeysAreExact: + """One composition key, and no keys beyond an op's own shape: nothing may be validated and then dropped.""" + + _LEAF: ClassVar[dict[str, Any]] = {"column": "a", "op": "eq", "value": 1} + + @pytest.mark.parametrize( + "bad", + [ + { + "and": [_LEAF, _LEAF], + "or": [_LEAF, _LEAF], + }, # the second branch used to be silently dropped + {"or": [_LEAF, _LEAF], "and": [_LEAF, _LEAF]}, + {"and": [_LEAF, _LEAF], "column": "a"}, + {"or": [_LEAF, _LEAF], "op": "eq"}, + ], + ) + def test_a_condition_with_several_composition_keys_is_rejected(self, bad: dict) -> None: + errors = validate_condition_ops(bad) + assert errors and "exactly one of those keys" in errors[0] + with pytest.raises(ValueError, match="exactly one of those keys"): + compile_condition_preview(bad, "snowflake") + + @pytest.mark.parametrize( + ("condition", "extra"), + [ + ({"column": "a", "op": "eq", "value": 1, "values": [1]}, ["values"]), + ({"column": "a", "op": "in", "values": [1], "value": 1}, ["value"]), + ({"column": "a", "op": "is_null", "value": 1}, ["value"]), + ({"column": "a", "op": "eq", "value": 1, "note": "x"}, ["note"]), + ], + ) + def test_keys_beyond_the_ops_shape_are_rejected(self, condition: dict, extra: list) -> None: + errors = validate_condition_ops(condition) + assert errors and f"unexpected keys {extra}" in errors[0] + + def test_a_valid_nested_tree_still_passes_and_renders(self) -> None: + tree = { + "or": [ + {"and": [self._LEAF, {"column": "b", "op": "in", "values": [1, 2]}]}, + {"column": "c", "op": "is_null"}, + ] + } + assert validate_condition_ops(tree) == [] + assert "OR" in compile_condition_preview(tree, "snowflake") + + def test_a_multi_key_condition_never_reaches_a_write(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + rules = [ + { + "principal": "a@x.com", + "condition": {"and": [self._LEAF, self._LEAF], "or": [self._LEAF, self._LEAF]}, + } + ] + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t", dialect="snowflake", rules=rules) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() From 32f8199b11f4f5dfc0a68793de0527861bb325a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 06:54:18 +0200 Subject: [PATCH 13/35] fix(CLI-17): validate principal fields by value and reject an unresolved version listing (Copilot review) - validate_principal_fields decided "exactly one of principal/principals" by truthiness, so {"principal": "a", "principals": []} or {"principal": 1} passed when the live schema was unavailable and the service wrote a rule that does not satisfy the documented shape. Key presence now decides "exactly one", and the value is checked: principal must be a non-empty string, principals a non-empty list of non-empty strings. (Shared with the cls service.) - fetch_resolved_schema returns the bare {"versions": [...]} listing when it cannot pick a version. That is a valid, constraint-free JSON Schema, so validating against it silently skipped every structural check without the documented warning. It is now treated as unavailable (schema=None with a reason, surfaced as a warning on create/update). Co-Authored-By: Claude Sonnet 5.5 --- .../services/_rls_condition.py | 19 ++++- src/keboola_agent_cli/services/rls_service.py | 11 ++- tests/test_rls_service.py | 74 +++++++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 21507b96..4d658f0b 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -137,11 +137,24 @@ def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: Shared by the RLS and CLS (``cls_service``) local checks -- both policy types use the identical principal shape. """ - principals_value = rule.get("principals") - has_principal = bool(rule.get("principal")) - has_principals = isinstance(principals_value, list) and bool(principals_value) + # Key PRESENCE decides "exactly one" (a truthiness test would let `{"principal": "a", "principals": []}` + # through as if `principals` were absent); the VALUE is then checked on its own. + has_principal = "principal" in rule + has_principals = "principals" in rule if has_principal == has_principals: # both or neither return [f"rules[{index}] must set exactly one of 'principal'/'principals'"] + if has_principal: + principal = rule["principal"] + if not isinstance(principal, str) or not principal: + return [f"rules[{index}].principal must be a non-empty string"] + return [] + principals = rule["principals"] + if ( + not isinstance(principals, list) + or not principals + or not all(isinstance(name, str) and name for name in principals) + ): + return [f"rules[{index}].principals must be a non-empty list of non-empty strings"] return [] diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 82c18459..8a6dd243 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -136,7 +136,8 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: """ with self._new_metastore_client(project) as client: try: - schema = fetch_resolved_schema(client, self.item_type).schema + resolved = fetch_resolved_schema(client, self.item_type) + schema = resolved.schema except KeboolaApiError as exc: if exc.error_code in _AUTH_ERROR_CODES or exc.status_code in (401, 403): raise @@ -147,6 +148,14 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: return RlsSchemaFetch( schema=None, reason=f"metastore returned no schema for {self.item_type}" ) + if resolved.version is None and isinstance(schema.get("versions"), list): + # `fetch_resolved_schema` hands back the bare version listing when it cannot pick a version + # (e.g. an empty listing). It is a valid, constraint-free "schema", so validating against it + # would silently skip every structural check -- treat it as unavailable instead. + return RlsSchemaFetch( + schema=None, + reason=f"the metastore returned only a version listing, no schema, for {self.item_type}", + ) try: jsonschema.Draft7Validator.check_schema(schema) except jsonschema.SchemaError as exc: diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index a7d2e7f9..e658ae59 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -876,3 +876,77 @@ def test_a_multi_key_condition_never_reaches_a_write(self, tmp_path: Path) -> No assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY mock.post_item.assert_not_called() + + +class TestPrincipalFieldsAreValidatedByValueNotTruthiness: + _COND: ClassVar[dict[str, Any]] = {"true": True} + + @pytest.mark.parametrize( + ("rule", "fragment"), + [ + ({"principal": "a", "principals": [], "condition": _COND}, "exactly one of"), + ({"principal": "", "principals": ["b"], "condition": _COND}, "exactly one of"), + ({"principal": 1, "condition": _COND}, "principal must be a non-empty string"), + ({"principal": "", "condition": _COND}, "principal must be a non-empty string"), + ({"principal": None, "condition": _COND}, "principal must be a non-empty string"), + ({"principal": ["a"], "condition": _COND}, "principal must be a non-empty string"), + ({"principals": [], "condition": _COND}, "principals must be a non-empty list"), + ({"principals": "a@x.com", "condition": _COND}, "principals must be a non-empty list"), + ({"principals": ["a", ""], "condition": _COND}, "principals must be a non-empty list"), + ({"principals": ["a", 2], "condition": _COND}, "principals must be a non-empty list"), + ({"condition": _COND}, "exactly one of"), + ], + ) + def test_malformed_principal_fields_are_rejected_locally( + self, rule: dict, fragment: str + ) -> None: + errors = validate_rules_local([rule]) + assert errors and fragment in errors[0] + + @pytest.mark.parametrize( + "rule", + [ + {"principal": "a@x.com", "condition": _COND}, + {"principals": ["a@x.com", "b@x.com"], "condition": _COND}, + ], + ) + def test_well_formed_principal_fields_pass(self, rule: dict) -> None: + assert validate_rules_local([rule]) == [] + + def test_a_malformed_principal_never_reaches_a_write_when_the_schema_is_unavailable( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy( + "prod", + table="t", + dialect="snowflake", + rules=[{"principal": 1, "condition": self._COND}], + ) + + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.post_item.assert_not_called() + + +class TestUnresolvedVersionListing: + @pytest.mark.parametrize("listing", [{"versions": []}, {"versions": [{"isDefault": True}]}]) + def test_a_version_listing_that_cannot_be_resolved_is_unavailable_not_a_schema( + self, tmp_path: Path, listing: dict + ) -> None: + """It is a valid, constraint-free JSON Schema, so validating against it would silently skip every + structural check without the documented warning.""" + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = listing + mock.post_item.return_value = _policy_item() + + fetch = service.fetch_schema("prod") + assert fetch.schema is None + assert "version listing" in (fetch.reason or "") + + result = service.create_policy( + "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES + ) + assert any("version listing" in w for w in result["warnings"]) From 52cdc36a91128b1d978224207ba78298787d9205 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 07:36:48 +0200 Subject: [PATCH 14/35] fix(CLI-17): setup honours rls.create denial and exit-code classes; update re-sends no grants; non-string op (Copilot review) - `rls setup` performs `create`'s writes but was only gated as `rls.setup`; an exact `rls.create` denial now blocks it too (checked before any API call). - When every table of a `setup` fails, the exit code now follows the error class of the failures (auth -> 3, config -> 5, ...) like `rls create`, instead of a synthetic general failure; a mix of classes stays 1. - `update` omitting --target-project re-sent the grant snapshot it had just read through put_target_projects, which could overwrite a concurrent grant change. Only an explicit list (or the clear path) touches grants now. - validate_condition_ops tested `op in ` with the raw value, so an unhashable op such as `[]` raised TypeError. A non-string op is reported as an unknown op. - E2E: create/detail/update/delete round-trip for rls and cls (skips on a stack or token that cannot author policies; the policy is always deleted) and the `rls setup` refusal under --json. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/commands/rls.py | 9 ++- .../services/_rls_condition.py | 2 +- src/keboola_agent_cli/services/rls_service.py | 6 +- tests/test_e2e.py | 49 +++++++++++++ tests/test_rls_cli.py | 70 ++++++++++++++++++- tests/test_rls_service.py | 6 ++ 6 files changed, 137 insertions(+), 5 deletions(-) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index f4726076..4d74cb51 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -40,6 +40,7 @@ from ..services.rls_service import RLS_DIALECTS from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( + check_cli_operation, check_cli_permission, get_formatter, get_service, @@ -578,6 +579,8 @@ def rls_setup( all sharing the same rules, via the exact same write path ``rls create`` uses. """ + # `setup` performs `create`'s writes, so an exact `rls.create` denial must cover it too. + check_cli_operation(ctx, "rls.create") formatter = get_formatter(ctx) if dialect is not None and dialect not in RLS_DIALECTS: formatter.error( @@ -654,6 +657,7 @@ def rls_setup( raise typer.Exit(code=0) failed: list[str] = [] + exit_codes: set[int] = set() for table_id in selected_tables: try: result = service.create_policy( @@ -666,6 +670,7 @@ def rls_setup( except (ConfigError, KeboolaApiError) as exc: formatter.warning(f"{table_id}: {exc}") failed.append(table_id) + exit_codes.add(5 if isinstance(exc, ConfigError) else map_error_to_exit_code(exc)) continue _print_warnings(formatter, result) formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") @@ -676,4 +681,6 @@ def rls_setup( f" could not be created: {', '.join(failed)}", error_code=ErrorCode.API_ERROR, ) - raise typer.Exit(code=1) + # Same exit-code contract as `rls create`: when every failure maps to one code (auth -> 3, + # config -> 5, ...) use it; a mix is a general failure. + raise typer.Exit(code=exit_codes.pop() if len(exit_codes) == 1 else 1) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 4d658f0b..fd4033fe 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -107,7 +107,7 @@ def validate_condition_ops(condition: Any) -> list[str]: errors.extend(validate_condition_ops(clause)) return errors op = condition.get("op") - if op not in RLS_CONDITION_OPS: + if not isinstance(op, str) or op not in RLS_CONDITION_OPS: errors.append(f"unknown condition op {op!r} (expected one of {sorted(RLS_CONDITION_OPS)})") return errors # The primitive's shape: checked here so a malformed rule fails as INVALID_RLS_POLICY even when the diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 8a6dd243..8370d21a 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -406,8 +406,10 @@ def update_policy( ) if revoking: client.put_target_projects(self.item_type, policy_id, []) - elif scope == "targeted" and merged_targets: - client.put_target_projects(self.item_type, policy_id, merged_targets) + elif target_ids: + # Only an explicit list replaces the grants; an update that omits the option must not + # re-send the snapshot read above (it could overwrite a concurrent grant change). + client.put_target_projects(self.item_type, policy_id, target_ids) row = self._row_from_item(updated) row["preview"] = preview return self._with_warnings(row, warnings) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index e480bfe4..eb24381f 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -687,6 +687,7 @@ def test_full_cli_e2e(self) -> None: "the rls-policy / cls-policy metastore object types may not be registered on a stack", ) self._test_rls_commands() + self._test_rls_cls_crud() # ============================================================== # PHASE 15: Cleanup @@ -3756,6 +3757,54 @@ def _test_rls_commands(self) -> None: data = _json_ok(result) assert isinstance(data["data"]["policies"], list) + def _test_rls_cls_crud(self) -> None: + """create -> detail -> update -> delete for both policy types, then the `rls setup` refusal. + + Authoring needs an org-admin-capable token on a stack whose metastore registers the type, so a + classified NOT_FOUND / ACCESS_DENIED / MISSING_MASTER_TOKEN on the create is an expected skip + (the policy is never created then); any other failure fails the suite. The policy is always + deleted again. Targets a table that does not exist, so no real data is ever governed. + """ + table = "in.c-kbagent-e2e-rls.policy_crud" + samples = { + "rls": '[{"principal":"e2e@example.com","condition":{"true":true}}]', + "cls": '[{"principal":"e2e@example.com","visible_columns":["id"]}]', + } + for group, rules in samples.items(): + created = self._run( + group, "create", "--project", self.alias, "--table", table, + "--dialect", "snowflake", "--rules", rules, + ) # fmt: skip + if created.exit_code != 0: + assert any( + code in created.output + for code in ("NOT_FOUND", "ACCESS_DENIED", "MISSING_MASTER_TOKEN") + ), created.output + print(f" {_YELLOW}SKIP: {group} create/update/delete (cannot author here){_RESET}") + continue + policy_id = _json_ok(created)["data"]["id"] + try: + detail = self._run_ok( + group, "detail", "--project", self.alias, "--policy-id", policy_id + ) + assert detail["data"]["table"] == table + updated = self._run_ok( + group, "update", "--project", self.alias, "--policy-id", policy_id, + "--dialect", "bigquery", + ) # fmt: skip + assert updated["data"]["dialect"] == "bigquery" + assert ( + updated["data"]["table"] == table + ) # fetch-then-merge kept the untouched field + finally: + deleted = self._run( + group, "delete", "--project", self.alias, "--policy-id", policy_id + ) + assert deleted.exit_code == 0, deleted.output + # Under --json (no terminal) the guided wizard refuses before any API call. + refused = self._run("rls", "setup", "--project", self.alias) + assert refused.exit_code == 2, refused.output + def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" # job list diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 5853641e..abeb1cff 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -638,6 +638,74 @@ def test_an_invalid_dialect_is_rejected_before_any_api_call(self, tmp_path: Path assert result.exit_code == 2 storage.list_tables.assert_not_called() + def test_an_exact_create_denial_also_blocks_setup(self, tmp_path: Path) -> None: + from keboola_agent_cli.models import PermissionPolicy + + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + cfg = store.load() + cfg.permissions = PermissionPolicy(mode="allow", deny=["rls.create"]) + store.save(cfg) + service = MagicMock() + storage = self._storage() + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + storage, + ) + + assert result.exit_code == 6, result.output + storage.list_tables.assert_not_called() + service.create_policy.assert_not_called() + + @pytest.mark.parametrize( + ("error", "exit_code"), + [ + ( + KeboolaApiError(message="x", status_code=401, error_code=ErrorCode.INVALID_TOKEN), + 3, + ), + (KeboolaApiError(message="x", status_code=500, error_code=ErrorCode.API_ERROR), 1), + (ConfigError("no such project"), 5), + ], + ) + def test_every_table_failing_keeps_the_error_class_in_the_exit_code( + self, tmp_path: Path, error: Exception, exit_code: int + ) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.side_effect = error + + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + self._storage(), + ) + + assert result.exit_code == exit_code + def test_every_table_failing_is_a_non_zero_exit(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) service = MagicMock() @@ -664,7 +732,7 @@ def test_every_table_failing_is_a_non_zero_exit(self, tmp_path: Path) -> None: self._storage(), ) - assert result.exit_code == 1 + assert result.exit_code == 3 # an auth-class failure keeps the authentication exit code assert "2 of 2 policies could not be created" in result.output def test_a_partial_failure_is_a_non_zero_exit_and_names_the_failed_table( diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index e658ae59..7e961a34 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -492,6 +492,12 @@ def test_unknown_op_reported(self) -> None: errors = validate_condition_ops({"column": "x", "op": "bogus", "value": 1}) assert errors and "bogus" in errors[0] + @pytest.mark.parametrize("op", [[], {}, 1, None]) + def test_a_non_string_op_is_reported_not_a_crash(self, op: object) -> None: + errors = validate_condition_ops({"column": "id", "op": op, "value": 1}) + + assert errors and "unknown condition op" in errors[0] + def test_true_sentinel_always_valid(self) -> None: assert validate_condition_ops({"true": True}) == [] From 72bf930e97940984c7c324171f1a6fb536f2e149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 07:50:41 +0200 Subject: [PATCH 15/35] fix(CLI-17): reject bool/non-integral project ids; keep the failures' error code in the setup envelope (Copilot review) - project_ids_as_ints used int(), which truncates 1.5 to 1 and turns True into 1, so a REST or programmatic caller could grant a policy to a different project than the one named. Booleans and non-integral floats are rejected (2.0 is still accepted as 2). - When every table of `rls setup` fails with the same error class, the final error now carries that class's code (INVALID_TOKEN, CONFIG_ERROR, ...) instead of a fixed API_ERROR, matching its exit code; a mix of classes stays API_ERROR. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/commands/rls.py | 6 ++- src/keboola_agent_cli/metastore_client.py | 11 ++++- tests/test_rls_cli.py | 51 ++++++++++++++--------- 3 files changed, 46 insertions(+), 22 deletions(-) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 4d74cb51..6a28a1a9 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -658,6 +658,7 @@ def rls_setup( failed: list[str] = [] exit_codes: set[int] = set() + error_codes: set[str] = set() for table_id in selected_tables: try: result = service.create_policy( @@ -671,6 +672,9 @@ def rls_setup( formatter.warning(f"{table_id}: {exc}") failed.append(table_id) exit_codes.add(5 if isinstance(exc, ConfigError) else map_error_to_exit_code(exc)) + error_codes.add( + ErrorCode.CONFIG_ERROR if isinstance(exc, ConfigError) else exc.error_code + ) continue _print_warnings(formatter, result) formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") @@ -679,7 +683,7 @@ def rls_setup( formatter.error( message=f"{len(failed)} of {len(selected_tables)} polic{'y' if len(selected_tables) == 1 else 'ies'}" f" could not be created: {', '.join(failed)}", - error_code=ErrorCode.API_ERROR, + error_code=error_codes.pop() if len(error_codes) == 1 else ErrorCode.API_ERROR, ) # Same exit-code contract as `rls create`: when every failure maps to one code (auth -> 3, # config -> 5, ...) use it; a mix is a general failure. diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index a757f369..41a0fc73 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -149,6 +149,15 @@ def _name_conflict(item_type: str, name: str, exc: KeboolaApiError) -> KeboolaAp ) +def _exact_int(value: Any) -> int: + """``int(value)`` without its silent conversions: a bool or a non-integral float is rejected.""" + if isinstance(value, bool): + raise TypeError("a bool is not a project id") + if isinstance(value, float) and not value.is_integer(): + raise ValueError("a non-integral float is not a project id") + return int(value) + + def project_ids_as_ints(project_ids: list[Any]) -> list[int]: """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). @@ -156,7 +165,7 @@ def project_ids_as_ints(project_ids: list[Any]) -> list[int]: instead of reaching the backend as a 400 (or, in the grants call, being ignored). """ try: - ids = [int(project_id) for project_id in project_ids] + ids = [_exact_int(project_id) for project_id in project_ids] except (TypeError, ValueError): ids = [0] if any(project_id <= 0 for project_id in ids): diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index abeb1cff..15d9ba57 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -21,6 +21,7 @@ from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.output import OutputFormatter runner = CliRunner() TEST_TOKEN = "999-token-abc" @@ -670,41 +671,51 @@ def test_an_exact_create_denial_also_blocks_setup(self, tmp_path: Path) -> None: service.create_policy.assert_not_called() @pytest.mark.parametrize( - ("error", "exit_code"), + ("error", "exit_code", "error_code"), [ ( KeboolaApiError(message="x", status_code=401, error_code=ErrorCode.INVALID_TOKEN), 3, + "INVALID_TOKEN", ), - (KeboolaApiError(message="x", status_code=500, error_code=ErrorCode.API_ERROR), 1), - (ConfigError("no such project"), 5), + ( + KeboolaApiError(message="x", status_code=500, error_code=ErrorCode.API_ERROR), + 1, + "API_ERROR", + ), + (ConfigError("no such project"), 5, "CONFIG_ERROR"), ], ) def test_every_table_failing_keeps_the_error_class_in_the_exit_code( - self, tmp_path: Path, error: Exception, exit_code: int + self, tmp_path: Path, error: Exception, exit_code: int, error_code: str ) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) service = MagicMock() service.create_policy.side_effect = error - result = self._setup_run( - [ - "rls", - "setup", - "--project", - "prod", - "--dialect", - "snowflake", - "--rules", - self.RULES, - "--yes", - ], - store, - service, - self._storage(), - ) + with patch("keboola_agent_cli.commands.rls.get_formatter") as get_formatter_mock: + real = OutputFormatter(json_mode=False) + get_formatter_mock.return_value = real + with patch.object(real, "error", wraps=real.error) as error_spy: + result = self._setup_run( + [ + "rls", + "setup", + "--project", + "prod", + "--dialect", + "snowflake", + "--rules", + self.RULES, + "--yes", + ], + store, + service, + self._storage(), + ) assert result.exit_code == exit_code + assert error_spy.call_args.kwargs["error_code"] == error_code # the failures' own code def test_every_table_failing_is_a_non_zero_exit(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) From 117d835a3b41c5985f261118a40bb4e9c61bf886 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 5 Oct 2026 08:06:24 +0200 Subject: [PATCH 16/35] fix(CLI-17): make the project-id types covariant so `ty` passes The services took `list[int | str]`, which `list[int]` from the REST bodies is not assignable to (lists are invariant). They now take `Sequence[int | str]`, and project_ids_as_ints takes a Sequence. The test helper's id list is typed loosely. No behaviour change. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/metastore_client.py | 3 ++- src/keboola_agent_cli/services/rls_service.py | 2 +- tests/test_rls_service.py | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 41a0fc73..b63de5ba 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -49,6 +49,7 @@ import logging import re +from collections.abc import Sequence from dataclasses import dataclass from typing import Any, Literal, Self @@ -158,7 +159,7 @@ def _exact_int(value: Any) -> int: return int(value) -def project_ids_as_ints(project_ids: list[Any]) -> list[int]: +def project_ids_as_ints(project_ids: Sequence[Any]) -> list[int]: """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). The CLI takes ids as strings; a non-numeric or non-positive one is rejected here, before any request, diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 8370d21a..30cf5f2a 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -22,7 +22,7 @@ from __future__ import annotations import logging -from collections.abc import Callable +from collections.abc import Callable, Sequence from dataclasses import dataclass from typing import Any diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 7e961a34..7492aa20 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -65,7 +65,7 @@ def _policy_item( rules: list[dict[str, Any]] | None = None, scope: str = "organization", source_project_id: str | None = "5725", - target_project_ids: list[str] | None = None, + target_project_ids: list[Any] | None = None, ) -> dict[str, Any]: return { "type": "rls-policy", From 9b6c79c3e5fbfb4a2030a4662d61391fd44b3aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 17:26:21 +0200 Subject: [PATCH 17/35] fix(CLI-17): adopt main's metastore scope client, accept integer project ids in REST Replaces the merge commit a121636f that the rebase onto main flattened: the rls/cls services use the scope client #715 shipped on main, and the REST bodies accept integer project ids (Copilot review). Co-Authored-By: Claude Opus 5.5 --- plugins/kbagent/agents/keboola-expert.md | 1 + plugins/kbagent/skills/kbagent/SKILL.md | 14 ++ .../skills/kbagent/references/gotchas.md | 79 ++++++++ .../skills/kbagent/references/rls-workflow.md | 13 +- src/keboola_agent_cli/metastore_client.py | 40 +++- src/keboola_agent_cli/permissions.py | 22 +++ src/keboola_agent_cli/server/routers/cls.py | 4 +- src/keboola_agent_cli/server/routers/rls.py | 4 +- src/keboola_agent_cli/services/rls_service.py | 60 +++--- .../services/semantic_layer_service.py | 2 +- tests/test_cls_service.py | 7 +- tests/test_metastore_client.py | 172 +++++------------- tests/test_project_ref.py | 3 +- tests/test_rls_service.py | 45 ++--- tests/test_server_cls.py | 4 +- tests/test_server_rls.py | 4 +- 16 files changed, 262 insertions(+), 212 deletions(-) diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index c59f48bf..89a34e5f 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,6 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P ...` -- ALWAYS `organization`/`targeted` scope (never `project`, no such flag exists); `--rules` is a declarative JSON condition tree, never free-text SQL; `--dry-run` previews (kbagent's OWN preview renderer, NOT the enforcement engine -- that's `keboola-mcp-server`'s `query_data`). `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; refuses under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 6ec60c0d..3b3c66de 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -371,6 +371,19 @@ When working inside a git repository or project directory, run `kbagent init` (o | Flag a project-scoped item as awaiting an org-admin's step-up decision | `kbagent sl scope request-create --project PROJECT --type TYPE- --context-id CONTEXT-ID` | | Withdraw a pending scope-elevation request. | `kbagent sl scope request-delete --project PROJECT --type TYPE- --context-id CONTEXT-ID` | | List items of --type awaiting an org-admin's elevation decision, across the org | `kbagent sl scope request-list --project PROJECT --type TYPE-` | +| List row-level security policies visible to a project | `kbagent rls list --project PROJECT` | +| Show one RLS policy's full rule set | `kbagent rls detail --project PROJECT --policy-id POLICY-ID` | +| Print the live ``rls-policy`` JSON Schema fetched from the metastore | `kbagent rls schema --project PROJECT` | +| Create one RLS policy for one table | `kbagent rls create --project PROJECT --table TABLE --dialect DIALECT --rules RULES` | +| Update an existing RLS policy | `kbagent rls update --project PROJECT --policy-id POLICY-ID` | +| Delete an RLS policy | `kbagent rls delete --project PROJECT --policy-id POLICY-ID` | +| Guided RLS setup: pick tables, build a condition, preview, then write | `kbagent rls setup --project PROJECT` | +| List column-level security policies visible to a project | `kbagent cls list --project PROJECT` | +| Show one CLS policy's full rule set | `kbagent cls detail --project PROJECT --policy-id POLICY-ID` | +| Print the live ``cls-policy`` JSON Schema fetched from the metastore | `kbagent cls schema --project PROJECT` | +| Create one CLS policy for one table | `kbagent cls create --project PROJECT --table TABLE --dialect DIALECT --rules RULES` | +| Update an existing CLS policy | `kbagent cls update --project PROJECT --policy-id POLICY-ID` | +| Delete a CLS policy | `kbagent cls delete --project PROJECT --policy-id POLICY-ID` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | @@ -467,6 +480,7 @@ For detailed response parsing rules and common pitfalls, see [gotchas](reference | SQL migration (input mapping removal) | [sql-migration-workflow](references/sql-migration-workflow.md) | | **Semantic layer (metastore)** -- models, metrics, datasets, constraints, glossary; validate / export / diff / promote / build / token | [semantic-layer-workflow](references/semantic-layer-workflow.md) | | **Share a semantic-layer item across projects / make it org-wide** -- `--scope`, `--target-project`, `scope get\|add\|remove\|set\|request-*` | [metastore-scope-workflow](references/metastore-scope-workflow.md) | +| **Row-level / column-level security (RLS, CLS)** -- author `rls-policy` / `cls-policy` objects, organization/targeted scope, guided RLS setup, condition primitives, column allowlists | [rls-workflow](references/rls-workflow.md) | | **Developer Portal** (identity CRUD, list/get apps, create/patch/upload-icon/publish/deprecate; TTY-confirm on writes) | [dev-portal-workflow](references/dev-portal-workflow.md) | | **Config metadata** (list/get/set/delete arbitrary key-value metadata on a configuration) | [config-metadata-workflow](references/config-metadata-workflow.md) | | **Storage descriptions** (describe bucket / table / column, batch from YAML) | [storage-describe-workflow](references/storage-describe-workflow.md) | diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 6131f13c..3ab6c3ce 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6747,3 +6747,82 @@ The package was named `keboola-agent-cli` up to 0.62.0 and is named `keboola-cli - **`col_map.in_table` matched by substring**, so a mapping could land on the wrong input edge. It must now be the full input table id, its `alias:`-qualified form, or a unique input-mapping destination; anything else is skipped and counted in `ai_col_map_skipped`. - **Stale AI results are not applied** *(since 0.99.1)*: a result whose `_code_hash` differs from the config's current code hash describes code that no longer exists; it is skipped entirely and listed in `ai_status.ai_results_stale` (a fresh task is emitted for it). A result without `_code_hash` is still applied. A `col_map` naming a table the config reads under two aliases now maps onto both edges instead of being skipped as ambiguous. - `ai_status` keys: `ai_edges_added` (reads + writes), `ai_reads_added`, `ai_writes_added`, `ai_columns_merged`, `ai_external_added`, `ai_col_map_skipped`, `ai_outputs_unresolved`, `ai_results_stale`, `ai_tasks_missing_code_file`. + +## Row-Level and Column-Level Security (RLS, CLS): organization/targeted scope only + +*(since vNEXT)* + +`kbagent rls` authors metastore `rls-policy` objects -- one per protected table. It never +enforces anything itself: the actual SQL rewrite happens in `keboola-mcp-server`'s `query_data` +tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full recipes. + +- **Availability depends on the stack's metastore.** The `rls-policy` and `cls-policy` object types are + registered by the metastore's `2026-09-29` schema migrations. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. Check with + `kbagent --json rls schema --project P` / `cls schema`. All commands are covered by + `tests/test_rls_*.py` / `tests/test_cls_*.py` / `tests/test_server_{rls,cls}.py` against a mocked + `MetastoreClient`. +- **`--scope project` does not exist anywhere in this group, on purpose.** Every write + (`create`/`update`/`setup`) is issued at `organization` scope (default) or `targeted` scope + (`--target-project`, repeatable) -- never `project`. This is structural, not merely validated: + the command surface has no flag that could even attempt `project` scope, the metastore ACL reserves + `organization` scope and cross-project grants for organization admins (it additionally lets the + owning project's admin author `targeted` policies for that project). `--project` on every command + names which project's tables/metastore instance you're working against. +- **One `rls-policy` object per protected table**, never one blob per project. `rls setup` + creates one policy per table selected in the checkbox picker, all sharing the same rules -- + if tables need different rules, run `setup`/`create` again per table. +- **`organization` scope is visible org-wide but only APPLIES where it matches.** A fetched policy + is visible to every project in the org once it's `organization` scope, but the enforcement + engine (and `rls list`/`rls detail`'s own `source_project_id`/`target_project_ids` fields) only + ever applies it to the project it names -- never merely because a table name matches. Do not + read a policy in `rls list` as protecting the current project without checking + `source_project_id` (or `target_project_ids` for `targeted` scope). +- **`condition` is a fixed six-shape primitive vocabulary, never a free-text predicate.** + Comparison (`eq`/`ne`/`gt`/`gte`/`lt`/`lte`), membership (`in`/`not_in` with `values`), nullness + (`is_null`/`is_not_null`), `and`/`or` (2+ nested conditions each), or the `{"true": true}` + sentinel. Anything else fails local validation (`INVALID_RLS_POLICY`, exit determined by + `map_error_to_exit_code`) before any network call -- there is no server round-trip needed to + catch a typo'd operator name. +- **`--dry-run`'s compiled-condition preview is NOT the enforcement engine.** It's a small, + local, pure-Python renderer (`services/_rls_condition.py::compile_condition_preview`) meant only + for human sanity-checking before a write -- the actual compiler that decides what a real query + returns is `keboola-mcp-server`'s `rls.py::_compile_primitive` (sqlglot-based, a different repo + entirely). The two are deliberately allowed to render differently; never treat kbagent's preview + string as proof of what will be enforced. +- **`rls update` is fetch-then-merge, not a raw PUT passthrough.** The metastore's `PUT` replaces + the whole record; the service fetches the current item first and only overwrites the fields you + actually passed (`--table`/`--dialect`/`--rules`/`--target-project`), so an update that only + changes `--rules` can never silently blank `table` or `dialect`. +- **`rls setup` is interactive-terminal-only and has no REST route.** Same carve-out as `auth + register-projects`'s picker: it refuses under `--json` or without an interactive terminal (stdin or stdout not a TTY) with a one-line hint + to use `rls create` directly, rather than hanging or emitting malformed JSON. `kbagent serve` + has no `/rls/{project}/setup` endpoint -- the CLI's checkbox picker has no HTTP equivalent, and + its write is the same `POST /rls/{project}` route `rls create` already uses. +- **Writes are classified `admin`, not merely `write`.** `rls.create`/`update`/`delete`/`setup` + all sit in the `admin` risk category in `OPERATION_REGISTRY` (`permissions.py`). Denying + `cli:admin` blocks exactly them (and the other admin operations); denying `cli:write` blocks them + too, because `cli:write` spans write, destructive AND admin. A policy that denies only `cli:destructive` + or only `cli:read` leaves them reachable. Every REST route is gated the same way + `merge-request`'s routes are (`Depends(require_permission(...))`), unlike `notifications.py`'s + routes, which currently enforce nothing over HTTP -- RLS was built gated from the start. +- **`cls` is the column-level sibling of `rls`.** `kbagent cls list|detail|schema|create|update|delete` + author `cls-policy` objects (`{table, dialect, rules: [{principal|principals, visible_columns}]}`) with the + same scope rule (`organization`/`targeted`, never `project`), fetch-then-merge `update`, `admin` + permission class for writes, and `/cls/{project}` REST mirror as `rls`. `ClsService` subclasses + `RlsService` (only the item type, error code and rule hooks differ). Differences: `visible_columns` is a + non-empty allowlist of `[A-Za-z0-9_]+` names (masking is not supported), `--dry-run` prints each + principal's projection instead of a SQL condition, validation failures raise `INVALID_CLS_POLICY`, and + there is no `cls setup` wizard. +- **`rls|cls update` can revoke grants, and `schema`/writes tell you when validation was reduced.** + `--clear-target-projects` (mutually exclusive with `--target-project`) revokes every project the policy is + shared with through the dedicated grants call; the policy keeps its current scope (scope is not changed by + `PUT`). If the live schema could not be fetched, `create`/`update` still run the local checks (including the + shape of every condition primitive) but add a `warnings` entry saying structural validation was skipped. An + authentication/permission failure while fetching the schema (`INVALID_TOKEN`, `MISSING_MASTER_TOKEN`, + access denied) is raised as that error -- only an unavailable schema (e.g. `NOT_FOUND`) is degraded. + `rls setup` exits non-zero when any selected table's policy could not be created. +- **`--target-project` takes numeric project IDs.** The metastore decodes `targetProjectIds` as integers and + the grants call as `{"targetProjectIds": [int]}` (an empty list clears every grant), so a non-numeric or + non-positive ID is rejected up front with `INVALID_ARGUMENT` instead of reaching the backend. Earlier builds + sent string IDs under the wrong key to the grants endpoint, which the backend read as an empty list. + diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index aa260d3f..c12572cd 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -203,10 +203,13 @@ meant," never as proof of what will be enforced. ## Workflow 5 -- Share a policy across sibling projects (targeted scope) -`--target-project` (repeatable, on `create`/`update`/`setup`) shifts scope -from `organization` to `targeted` and registers each listed project as a -grant. Use this when one org-authored policy should apply identically -across specific sibling customer projects, not the whole org. +`--target-project` (repeatable, on `create`/`setup`) creates the policy at +`targeted` scope instead of `organization` and registers each listed project +as a grant. On `update` it replaces the grants of an already-`targeted` +policy; an `organization` policy cannot be narrowed (the metastore has no +downgrade), so delete it and create a new one. Use this when one org-authored +policy should apply identically across specific sibling customer projects, +not the whole org. ```bash kbagent --json rls create \ @@ -217,7 +220,7 @@ kbagent --json rls create \ --target-project 22222 \ --target-project 33333 \ --dry-run -# -> {"scope": "targeted", "target_project_ids": ["22222", "33333"], ...} +# -> {"scope": "targeted", "target_project_ids": [22222, 33333], ...} ``` Omitting `--target-project` on a later `rls update` call leaves the current diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index b63de5ba..375539c4 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -162,8 +162,8 @@ def _exact_int(value: Any) -> int: def project_ids_as_ints(project_ids: Sequence[Any]) -> list[int]: """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). - The CLI takes ids as strings; a non-numeric or non-positive one is rejected here, before any request, - instead of reaching the backend as a 400 (or, in the grants call, being ignored). + Callers may hand over strings (CLI) or ints (REST); a non-numeric or non-positive id is rejected here, + before any request, instead of reaching the backend as a 400 (or, in the grants call, being ignored). """ try: ids = [_exact_int(project_id) for project_id in project_ids] @@ -406,9 +406,6 @@ def put_item( item_id: str, name: str, data: dict[str, Any], - *, - scope: MetastoreScope = _ENVELOPE_SCOPE, - target_project_ids: list[str] | None = None, ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). @@ -466,8 +463,6 @@ def elevate_to_organization(self, item_type: SemanticType, item_id: str) -> dict f"/api/v1/repository/{item_type}/{item_id}", json={"scope": "organization"}, ) - if not response.content: - return {} body = response.json() return body.get("data", body) if isinstance(body, dict) else body @@ -543,3 +538,34 @@ def list_organization_items( ) body = response.json() return body.get("data", []) if isinstance(body, dict) else [] + + +@dataclass(frozen=True) +class ResolvedSchema: + """A type's JSON Schema together with the version it was resolved from (``None`` if none was).""" + + schema: dict[str, Any] + version: str | None + + +def fetch_resolved_schema(client: MetastoreClient, item_type: SemanticType) -> ResolvedSchema: + """Fetch the actual JSON Schema for a type, resolving the default version. + + Live metastore behavior (2026-07): the bare ``/api/v1/schema/{type}`` + endpoint returns only a ``{"versions": [...]}`` listing (metadata, no + schema body); the real JSON Schema lives at ``/{version}``. This resolves + ``isDefault`` (falling back to the first entry) and fetches the versioned + document. If the server someday returns the schema directly (no + ``versions`` key) it is passed through unchanged, with ``version=None``. + """ + body = client.get_schema(item_type) + versions = body.get("versions") + if not isinstance(versions, list) or not versions: + return ResolvedSchema(schema=body, version=None) + default = next((v for v in versions if v.get("isDefault")), versions[0]) + version_id = str(default.get("version", "")) + if not version_id: + return ResolvedSchema(schema=body, version=None) + return ResolvedSchema( + schema=client.get_schema(item_type, version=version_id), version=version_id + ) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 419e8e01..180904ae 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -359,6 +359,28 @@ "semantic-layer.reference-data.get": "read", "semantic-layer.reference-data.set": "write", "semantic-layer.reference-data.delete": "destructive", + # Row-level security (metastore) — CLI-17. Classified `admin` (not + # merely `write`): RLS policy authorship is an org-level concern (see + # keboola-mcp-server's feature_spec/rls_query_tool/RFC.md) -- `admin` + # is the class this repo's own taxonomy reserves for org-level + # operations (see `project.add` etc. above). `setup` performs the exact + # same write as `create`, just via an interactive picker, so it must + # carry the same class or a permission policy that allows `setup` but + # not `create` (or vice versa) would leave a hole. + "rls.list": "read", + "rls.detail": "read", + "rls.schema": "read", + "rls.create": "admin", + "rls.update": "admin", + "rls.delete": "admin", + "rls.setup": "admin", + # Column-level security: same classes as rls.* for the same reason. + "cls.list": "read", + "cls.detail": "read", + "cls.schema": "read", + "cls.create": "admin", + "cls.update": "admin", + "cls.delete": "admin", # `scope` sub-app: visibility scope / target-project grants / elevation # requests (PSGO-140). Parent key at the least-privileged level (read), # same pattern as `reference-data` above. `set --scope organization` is diff --git a/src/keboola_agent_cli/server/routers/cls.py b/src/keboola_agent_cli/server/routers/cls.py index 4a40832e..4eb83350 100644 --- a/src/keboola_agent_cli/server/routers/cls.py +++ b/src/keboola_agent_cli/server/routers/cls.py @@ -31,7 +31,7 @@ class ClsPolicyCreate(BaseModel): table: str dialect: str rules: list[dict[str, Any]] - target_project_ids: list[str] | None = None + target_project_ids: list[int] | None = None dry_run: bool = False @@ -39,7 +39,7 @@ class ClsPolicyUpdate(BaseModel): table: str | None = None dialect: str | None = None rules: list[dict[str, Any]] | None = None - target_project_ids: list[str] | None = None + target_project_ids: list[int] | None = None dry_run: bool = False diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py index a9dec675..1be64e53 100644 --- a/src/keboola_agent_cli/server/routers/rls.py +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -43,7 +43,7 @@ class RlsPolicyCreate(BaseModel): table: str dialect: str rules: list[dict[str, Any]] - target_project_ids: list[str] | None = None + target_project_ids: list[int] | None = None dry_run: bool = False @@ -51,7 +51,7 @@ class RlsPolicyUpdate(BaseModel): table: str | None = None dialect: str | None = None rules: list[dict[str, Any]] | None = None - target_project_ids: list[str] | None = None + target_project_ids: list[int] | None = None dry_run: bool = False diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 30cf5f2a..7c57d4b2 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -32,7 +32,7 @@ from ..errors import ErrorCode, KeboolaApiError from ..metastore_client import ( MetastoreClient, - MetastoreScope, + ObjectScope, SemanticType, fetch_resolved_schema, project_ids_as_ints, @@ -181,7 +181,7 @@ def _row_from_item(item: dict[str, Any]) -> dict[str, Any]: "target_project_ids": meta.get("targetProjectIds") or [], } - def _resolve_scope(self, target_project_ids: list[str] | None) -> MetastoreScope: + def _resolve_scope(self, target_project_ids: list[int] | None) -> ObjectScope: """Never ``"project"`` -- ``targeted`` iff target projects were given.""" return "targeted" if target_project_ids else "organization" @@ -272,7 +272,7 @@ def create_policy( table: str, dialect: str, rules: list[dict[str, Any]], - target_project_ids: list[str] | None = None, + target_project_ids: Sequence[int | str] | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Create one ``rls-policy`` object -- always at ``organization``/``targeted`` scope. @@ -282,9 +282,7 @@ def create_policy( metastore's write endpoint at all -- only the (read-only) schema fetch used for validation happens. """ - project_ids_as_ints( - target_project_ids or [] - ) # same validation as the write, also for --dry-run + target_ids = project_ids_as_ints(target_project_ids or []) # also validates for --dry-run project = self._resolve_one_project(alias) errors, warnings = self._validate_policy( table=table, dialect=dialect, rules=rules, project=project @@ -292,7 +290,7 @@ def create_policy( if errors: self._raise_invalid(errors) - scope = self._resolve_scope(target_project_ids) + scope = self._resolve_scope(target_ids) preview = self._preview_rules(rules, dialect) if dry_run: return self._with_warnings( @@ -301,7 +299,7 @@ def create_policy( "table": table, "dialect": dialect, "scope": scope, - "target_project_ids": target_project_ids or [], + "target_project_ids": target_ids, "preview": preview, "dry_run": True, }, @@ -314,12 +312,10 @@ def create_policy( name=table, data={"table": table, "dialect": dialect, "rules": rules}, scope=scope, - target_project_ids=target_project_ids, + target_project_ids=target_ids or None, ) - if scope == "targeted" and target_project_ids: - client.put_target_projects( - self.item_type, created.get("id", ""), target_project_ids - ) + if target_ids: + client.put_target_projects(self.item_type, created.get("id", ""), target_ids) row = self._row_from_item(created) row["preview"] = preview return self._with_warnings(row, warnings) @@ -332,7 +328,7 @@ def update_policy( table: str | None = None, dialect: str | None = None, rules: list[dict[str, Any]] | None = None, - target_project_ids: list[str] | None = None, + target_project_ids: Sequence[int | str] | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Update one policy object. @@ -347,9 +343,9 @@ def update_policy( ``[]`` revokes them all (through the dedicated grants call). A policy keeps its current scope when its grants are revoked -- scope changes are not done through ``PUT``. """ - project_ids_as_ints( - target_project_ids or [] - ) # same validation as the write, also for --dry-run + target_ids = ( + project_ids_as_ints(target_project_ids) if target_project_ids is not None else None + ) # also validates for --dry-run project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: current = client.get_item(self.item_type, policy_id) @@ -360,7 +356,9 @@ def update_policy( merged_dialect = dialect if dialect is not None else attrs.get("dialect", "") merged_rules = rules if rules is not None else attrs.get("rules", []) merged_targets = ( - target_project_ids if target_project_ids is not None else meta.get("targetProjectIds") + target_ids + if target_ids is not None + else project_ids_as_ints(meta.get("targetProjectIds") or []) ) errors, warnings = self._validate_policy( @@ -369,16 +367,20 @@ def update_policy( if errors: self._raise_invalid(errors) - revoking = target_project_ids == [] and bool(meta.get("targetProjectIds")) - # An explicit, non-empty target list makes the policy `targeted`. When the option is omitted or the - # grants are being cleared, the current scope is kept: a `targeted` policy whose grants were cleared - # is legitimately `targeted` with an empty list, and an unrelated update must not turn it into - # `organization`. - scope = ( - self._resolve_scope(target_project_ids) - if target_project_ids - else (meta.get("scope") or self._resolve_scope(merged_targets)) - ) + revoking = target_ids == [] and bool(meta.get("targetProjectIds")) + # A plain PUT never changes scope, and grants only exist on `targeted` policies, so the scope is + # always the current one: a `targeted` policy whose grants were cleared stays `targeted` with an + # empty list, and an `organization` policy cannot be narrowed to specific projects. + scope = meta.get("scope") or self._resolve_scope(merged_targets) + if target_ids and scope != "targeted": + self._raise_invalid( + [ + ( + f"the policy has {scope!r} scope, which cannot be narrowed to target projects; " + "delete it and create a new one with --target-project" + ) + ] + ) preview = self._preview_rules(merged_rules, merged_dialect) if dry_run: return self._with_warnings( @@ -401,8 +403,6 @@ def update_policy( policy_id, name=merged_table, data={"table": merged_table, "dialect": merged_dialect, "rules": merged_rules}, - scope=scope, - target_project_ids=merged_targets, ) if revoking: client.put_target_projects(self.item_type, policy_id, []) diff --git a/src/keboola_agent_cli/services/semantic_layer_service.py b/src/keboola_agent_cli/services/semantic_layer_service.py index ed16f72a..b04cf472 100644 --- a/src/keboola_agent_cli/services/semantic_layer_service.py +++ b/src/keboola_agent_cli/services/semantic_layer_service.py @@ -24,7 +24,7 @@ from ..auth.sentinel import require_static_token from ..config_store import ConfigStore from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..metastore_client import MetastoreClient, ObjectScope, SemanticType +from ..metastore_client import MetastoreClient, ObjectScope, SemanticType, fetch_resolved_schema from ..models import ProjectConfig from . import _semantic_layer_reference_data as _refdata from . import _semantic_layer_scope as _scope diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index 417a90c2..35d4d605 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -224,7 +224,7 @@ def test_target_projects_make_it_targeted_and_grant(self, tmp_path: Path) -> Non ) assert mock.post_item.call_args.kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", ["7", "8"]) + mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", [7, 8]) def test_dry_run_never_writes(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) @@ -252,8 +252,6 @@ def test_merges_only_the_given_fields(self, tmp_path: Path) -> None: "p-1", name="in.c-crm.invoices", data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": new_rules}, - scope="organization", - target_project_ids=None, ) def test_keeps_existing_targets_and_scope(self, tmp_path: Path) -> None: @@ -263,8 +261,7 @@ def test_keeps_existing_targets_and_scope(self, tmp_path: Path) -> None: service.update_policy("prod", "p-1", table="in.c-crm.other") - assert mock.put_item.call_args.kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", ["7"]) + mock.put_target_projects.assert_not_called() # the omitted option re-sends no grants def test_dry_run_never_writes(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index f5f0c586..6d37e9f6 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -20,6 +20,7 @@ MetastoreClient, ResolvedSchema, fetch_resolved_schema, + project_ids_as_ints, ) STACK_URL_US = "https://connection.keboola.com" @@ -363,140 +364,20 @@ def test_semantic_types_complete(self) -> None: } -class TestPostItemScope: - """post_item/put_item default to project scope but accept organization/ +class TestProjectIdsAsInts: + """`project_ids_as_ints`: the metastore decodes ``targetProjectIds`` as ``[]int``.""" - targeted -- CLI-17: rls-policy is the first type that must never use - plain project scope. - """ - - def test_default_scope_is_still_project(self, httpx_mock, metastore_client) -> None: - """Every existing caller (semantic-layer) keeps its current behavior.""" - httpx_mock.add_response( - url=f"{METASTORE_URL_US}/api/v1/repository/semantic-metric", - json={"data": {"type": "semantic-metric", "id": "new-id", "attributes": {}}}, - status_code=201, - ) - metastore_client.post_item("semantic-metric", name="rev", data={"name": "rev"}) - body = json.loads(httpx_mock.get_requests()[0].content) - assert body["scope"] == "project" - assert "targetProjectIds" not in body - - def test_organization_scope(self, httpx_mock, metastore_client) -> None: - httpx_mock.add_response( - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy", - json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, - status_code=201, - ) - metastore_client.post_item( - "rls-policy", - name="in.c-crm.invoices", - data={"table": "in.c-crm.invoices"}, - scope="organization", - ) - body = json.loads(httpx_mock.get_requests()[0].content) - assert body["scope"] == "organization" - assert "targetProjectIds" not in body - - def test_targeted_scope_includes_target_project_ids(self, httpx_mock, metastore_client) -> None: - httpx_mock.add_response( - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy", - json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, - status_code=201, - ) - metastore_client.post_item( - "rls-policy", - name="in.c-crm.invoices", - data={"table": "in.c-crm.invoices"}, - scope="targeted", - target_project_ids=["111", "222"], - ) - body = json.loads(httpx_mock.get_requests()[0].content) - assert body["scope"] == "targeted" - # The metastore decodes targetProjectIds as []int: integers on the wire, never strings. - assert body["targetProjectIds"] == [111, 222] - - def test_put_item_accepts_scope_too(self, httpx_mock, metastore_client) -> None: - httpx_mock.add_response( - method="PUT", - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1", - json={"data": {"type": "rls-policy", "id": "rls-1", "attributes": {}}}, - status_code=200, - ) - metastore_client.put_item( - "rls-policy", - "rls-1", - name="in.c-crm.invoices", - data={"table": "in.c-crm.invoices"}, - scope="organization", - ) - body = json.loads(httpx_mock.get_requests()[0].content) - assert body["scope"] == "organization" - - -class TestPutTargetProjects: - def test_puts_the_target_project_ids_as_integers(self, httpx_mock, metastore_client) -> None: - """The backend body is `{"targetProjectIds": [int]}` and ignores unknown fields: a wrong key - would be read as an EMPTY list, which clears every grant.""" - httpx_mock.add_response( - method="PUT", - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", - status_code=204, - ) - - result = metastore_client.put_target_projects("rls-policy", "rls-1", ["111", "222"]) - - request = httpx_mock.get_requests()[0] - assert request.method == "PUT" - assert request.url.path.endswith("/rls-policy/rls-1/target-projects") - assert json.loads(request.content) == {"targetProjectIds": [111, 222]} - assert result == {} # 204 No Content: there is no body to parse - - def test_an_empty_list_is_sent_explicitly_to_clear_all_grants( - self, httpx_mock, metastore_client - ) -> None: - httpx_mock.add_response( - method="PUT", - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", - status_code=204, - ) + def test_strings_and_ints_become_ints(self) -> None: + assert project_ids_as_ints(["111", 222]) == [111, 222] + assert project_ids_as_ints([2.0]) == [2] + assert project_ids_as_ints([]) == [] - metastore_client.put_target_projects("rls-policy", "rls-1", []) - - assert json.loads(httpx_mock.get_requests()[0].content) == {"targetProjectIds": []} - - def test_a_response_body_is_still_returned_when_the_server_sends_one( - self, httpx_mock, metastore_client - ) -> None: - httpx_mock.add_response( - method="PUT", - url=f"{METASTORE_URL_US}/api/v1/repository/rls-policy/rls-1/target-projects", - json={"data": {"type": "rls-policy", "id": "rls-1"}}, - status_code=200, - ) - - assert metastore_client.put_target_projects("rls-policy", "rls-1", ["111"])["id"] == "rls-1" - - @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-5"], ["1.5"], [None]]) - def test_non_positive_or_non_numeric_ids_are_rejected_before_any_request( - self, httpx_mock, metastore_client, bad: list - ) -> None: + @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-5"], ["1.5"], [None], [True], [1.5]]) + def test_non_positive_or_non_numeric_ids_are_rejected(self, bad: list) -> None: with pytest.raises(KeboolaApiError) as excinfo: - metastore_client.put_target_projects("rls-policy", "rls-1", bad) - - assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - assert httpx_mock.get_requests() == [] - - def test_post_item_rejects_a_non_numeric_target_before_any_request( - self, httpx_mock, metastore_client - ) -> None: - with pytest.raises(KeboolaApiError) as excinfo: - metastore_client.post_item( - "rls-policy", name="t", data={}, scope="targeted", target_project_ids=["abc"] - ) + project_ids_as_ints(bad) assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - assert httpx_mock.get_requests() == [] class TestProjectScope401Reclassification: @@ -883,3 +764,36 @@ def test_no_filters_sends_bare_request(self, httpx_mock, metastore_client) -> No status_code=200, ) assert metastore_client.list_organization_items("semantic-model") == [] + + +class TestFetchResolvedSchema: + """`fetch_resolved_schema`: the bare schema endpoint is only a version listing.""" + + @staticmethod + def _client(*responses: dict) -> MagicMock: + client = MagicMock() + client.get_schema.side_effect = list(responses) + return client + + def test_fetches_the_default_version(self) -> None: + listing = {"versions": [{"version": "0.9.0"}, {"version": "1.0.0", "isDefault": True}]} + client = self._client(listing, {"title": "real"}) + + resolved = fetch_resolved_schema(client, "rls-policy") + + assert (resolved.schema, resolved.version) == ({"title": "real"}, "1.0.0") + assert client.get_schema.call_args_list[1].kwargs == {"version": "1.0.0"} + + def test_falls_back_to_the_first_version_without_a_default(self) -> None: + client = self._client({"versions": [{"version": "2.0.0"}, {"version": "1.0.0"}]}, {"t": 1}) + + assert fetch_resolved_schema(client, "cls-policy") == ResolvedSchema({"t": 1}, "2.0.0") + + @pytest.mark.parametrize( + "body", [{"title": "direct"}, {"versions": []}, {"versions": [{"isDefault": True}]}] + ) + def test_passes_through_when_nothing_to_resolve(self, body: dict) -> None: + client = self._client(body) + + assert fetch_resolved_schema(client, "rls-policy") == ResolvedSchema(body, None) + client.get_schema.assert_called_once_with("rls-policy") diff --git a/tests/test_project_ref.py b/tests/test_project_ref.py index 07f228a2..2cd64527 100644 --- a/tests/test_project_ref.py +++ b/tests/test_project_ref.py @@ -150,7 +150,8 @@ def test_no_shadow_notice_for_aliases_of_the_same_project(self) -> None: "--target-project-ids": "already project IDs (sharing share)", "--target-project": ( "semantic-layer: alias OR numeric ID resolved by the service itself (a target need not " - "be registered); config clone's --target-project is in ALIAS_OPTIONS" + "be registered); config clone's --target-project is in ALIAS_OPTIONS; " + "rls/cls create|update: a numeric project ID the policy is granted to, not an alias" ), "--all-projects": "a boolean flag", "--clear-target-projects": "a boolean flag (rls/cls update): revoke every grant, takes no project", diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 7492aa20..eb989bb4 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -270,8 +270,8 @@ def test_target_projects_use_targeted_scope_and_manage_grants(self, tmp_path: Pa _, kwargs = mock.post_item.call_args assert kwargs["scope"] == "targeted" - assert kwargs["target_project_ids"] == ["999"] - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["999"]) + assert kwargs["target_project_ids"] == [999] + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", [999]) def test_invalid_dialect_rejected_before_any_write(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -357,18 +357,20 @@ def test_partial_override_preserves_untouched_fields(self, tmp_path: Path) -> No assert kwargs["data"]["dialect"] == "snowflake" # unchanged, not wiped assert kwargs["data"]["rules"] == _RULES # the one field we changed - def test_targeted_scope_derived_from_merged_targets(self, tmp_path: Path) -> None: + def test_an_organization_policy_cannot_be_narrowed_to_target_projects( + self, tmp_path: Path + ) -> None: store = _make_store(tmp_path) service, mock = _make_service(store) mock.get_item.return_value = _policy_item(scope="organization", target_project_ids=[]) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=["42"]) - service.update_policy("prod", "p-1", target_project_ids=["42"]) + with pytest.raises(KeboolaApiError) as excinfo: + service.update_policy("prod", "p-1", target_project_ids=["42"]) - _, kwargs = mock.put_item.call_args - assert kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["42"]) + assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY + mock.put_item.assert_not_called() + mock.put_target_projects.assert_not_called() def test_invalid_override_rejected_before_write(self, tmp_path: Path) -> None: store = _make_store(tmp_path) @@ -637,9 +639,6 @@ def test_empty_list_revokes_every_grant_and_keeps_the_scope(self, tmp_path: Path mock.put_item.return_value = self._targeted() result = service.update_policy("prod", "p-1", target_project_ids=[]) - - assert mock.put_item.call_args.kwargs["scope"] == "targeted" - assert mock.put_item.call_args.kwargs["target_project_ids"] == [] mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) assert result["id"] == "p-1" @@ -650,9 +649,7 @@ def test_none_keeps_the_existing_grants(self, tmp_path: Path) -> None: mock.put_item.return_value = self._targeted() service.update_policy("prod", "p-1", dialect="bigquery") - - assert mock.put_item.call_args.kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["7", "8"]) + mock.put_target_projects.assert_not_called() # the omitted option re-sends nothing def test_clearing_a_policy_that_has_no_grants_makes_no_grant_call(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) @@ -661,8 +658,6 @@ def test_clearing_a_policy_that_has_no_grants_makes_no_grant_call(self, tmp_path mock.put_item.return_value = _policy_item() service.update_policy("prod", "p-1", target_project_ids=[]) - - assert mock.put_item.call_args.kwargs["scope"] == "organization" mock.put_target_projects.assert_not_called() def test_dry_run_previews_the_cleared_state_without_writing(self, tmp_path: Path) -> None: @@ -742,20 +737,20 @@ def test_an_unrelated_update_does_not_turn_a_cleared_targeted_policy_into_organi mock.put_item.return_value = self._targeted_without_grants() service.update_policy("prod", "p-1", dialect="bigquery") # no target option at all - - assert mock.put_item.call_args.kwargs["scope"] == "targeted" mock.put_target_projects.assert_not_called() - def test_an_explicit_target_list_makes_the_policy_targeted(self, tmp_path: Path) -> None: + def test_an_explicit_target_list_replaces_the_grants_of_a_targeted_policy( + self, tmp_path: Path + ) -> None: service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = _policy_item() # organization scope - mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=["7"]) + mock.get_item.return_value = self._targeted_without_grants() + mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=[7]) service.update_policy("prod", "p-1", target_project_ids=["7"]) - assert mock.put_item.call_args.kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", ["7"]) + assert "scope" not in mock.put_item.call_args.kwargs # a PUT never carries scope + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", [7]) def test_clearing_keeps_the_current_scope(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) @@ -764,8 +759,6 @@ def test_clearing_keeps_the_current_scope(self, tmp_path: Path) -> None: mock.put_item.return_value = self._targeted_without_grants() service.update_policy("prod", "p-1", target_project_ids=[]) - - assert mock.put_item.call_args.kwargs["scope"] == "targeted" mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) @@ -818,7 +811,7 @@ def test_valid_targets_still_preview(self, tmp_path: Path) -> None: ) assert result["scope"] == "targeted" - assert result["target_project_ids"] == ["7", "8"] + assert result["target_project_ids"] == [7, 8] class TestCompositionAndKeysAreExact: diff --git a/tests/test_server_cls.py b/tests/test_server_cls.py index 506ff252..b8ba7a9d 100644 --- a/tests/test_server_cls.py +++ b/tests/test_server_cls.py @@ -93,7 +93,7 @@ def test_create_and_update_pass_kwargs(tmp_path: Path) -> None: "table": "in.c-crm.invoices", "dialect": "snowflake", "rules": RULES, - "target_project_ids": ["999"], + "target_project_ids": [999], } with TestClient(_app(tmp_path, svc)) as client: @@ -103,7 +103,7 @@ def test_create_and_update_pass_kwargs(tmp_path: Path) -> None: create_kwargs = svc.create_policy.call_args.kwargs assert create_kwargs["rules"] == RULES - assert create_kwargs["target_project_ids"] == ["999"] + assert create_kwargs["target_project_ids"] == [999] assert create_kwargs["dry_run"] is False args, update_kwargs = svc.update_policy.call_args assert args == (PROJECT, POLICY_ID) diff --git a/tests/test_server_rls.py b/tests/test_server_rls.py index 07df51ff..385417d6 100644 --- a/tests/test_server_rls.py +++ b/tests/test_server_rls.py @@ -132,7 +132,7 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: "table": "in.c-crm.invoices", "dialect": "snowflake", "rules": [{"principal": "a@x.com", "condition": {"true": True}}], - "target_project_ids": ["999"], + "target_project_ids": [999], "dry_run": False, } @@ -144,7 +144,7 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: assert kwargs["table"] == "in.c-crm.invoices" assert kwargs["dialect"] == "snowflake" assert kwargs["rules"] == body["rules"] - assert kwargs["target_project_ids"] == ["999"] + assert kwargs["target_project_ids"] == [999] assert kwargs["dry_run"] is False From 7f5ccfc9ae95d2310d0bb7ea2694b4ab65295556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:09:33 +0200 Subject: [PATCH 18/35] feat(CLI-17): add MetastoreClient.patch_item for partial updates PATCH merges only the given top-level data keys under a row lock (go-monorepo simpleJSONMerge), so unknown keys and concurrent changes to other keys survive a policy update. The server validates only the patched keys, so callers validate the merged record themselves. Co-Authored-By: Claude Opus 5.5 --- src/keboola_agent_cli/metastore_client.py | 44 +++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 375539c4..b79eba03 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -438,6 +438,50 @@ def put_item( body = response.json() return body.get("data", body) if isinstance(body, dict) else body + def patch_item( + self, + item_type: SemanticType, + item_id: str, + *, + name: str | None = None, + data: dict[str, Any] | None = None, + ) -> dict[str, Any]: + """Partially update an item via ``PATCH`` -- only the given fields change. + + The server merges ``data`` into the stored record one TOP-LEVEL key at + a time (go-monorepo ``simpleJSONMerge``): a key in ``data`` replaces the + stored value wholesale, every key it does not name is kept, all inside + one transaction under a row lock. So unknown keys survive and a + concurrent change to another key is not overwritten -- unlike + :meth:`put_item`. Caveat: the server validates only the patched keys, + never the merged record, so the caller must validate the merged result + itself. Never sends ``scope`` (a scope step-up cannot be combined with + name/data -- :meth:`elevate_to_organization`). + + Raises ``NOT_FOUND`` on 404 and ``ALREADY_EXISTS`` on 409 (a rename to + a taken name). + """ + body: dict[str, Any] = {} + if name is not None: + body["name"] = name + if data is not None: + body["data"] = data + try: + response = self._do_request( + "PATCH", f"/api/v1/repository/{item_type}/{item_id}", json=body + ) + except KeboolaApiError as exc: + if exc.status_code == 409: + raise KeboolaApiError( + message=f"{item_type} with name {name!r} already exists. Pick another name.", + status_code=409, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) from exc + raise + body = response.json() + return body.get("data", body) if isinstance(body, dict) else body + def delete_item(self, item_type: SemanticType, item_id: str) -> None: """Delete an item by its UUID. Returns silently on 204. From 2d567318880aa8ed2faca0f45343314be865e8d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:09:33 +0200 Subject: [PATCH 19/35] fix(CLI-17): refuse policies that block every query; preview like the enforcement The enforcement refuses all queries of a project when one policy names a principal twice on a table, or has a principal with whitespace; a null comparison renders col = NULL and matches nothing. Refuse all three locally. The preview quotes columns per dialect and renders booleans as TRUE/FALSE, as the enforcement does. Target project 0 is rejected (the metastore requires positive IDs). Co-Authored-By: Claude Opus 5.5 --- .../services/_rls_condition.py | 100 ++++++++++++++---- .../services/_semantic_layer_scope.py | 4 +- tests/test_semantic_layer_scope.py | 3 +- 3 files changed, 83 insertions(+), 24 deletions(-) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index fd4033fe..98801300 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -9,20 +9,19 @@ what the enforcement engine (that repo's ``rls.py::_compile_primitive``, sqlglot-based) expects to read back from the metastore. -``compile_condition_preview`` below is a **preview convenience only**: a -small recursive pure-Python string renderer used for ``--dry-run`` / -confirmation display before a write. It is deliberately NOT the enforcement -engine and never will be -- the actual compiler that decides what SQL a -query gets rewritten to lives in ``keboola-mcp-server`` (a different repo, -sqlglot-based). Drift between the two renderings is acceptable here because -this string is only ever shown to an admin for sanity-checking, never -executed. Adding sqlglot as a kbagent dependency just to produce this -confirmation string would be a heavier dependency than the six condition -shapes below warrant. +``compile_condition_preview`` below is a **preview** for ``--dry-run`` / +confirmation display, not the enforcement engine -- that lives in +``keboola-mcp-server`` (sqlglot-based, a different repo). It follows the +enforcement's rendering where an admin could be misled otherwise: columns +quoted per dialect (the enforcement matches them case-exactly), booleans as +``TRUE``/``FALSE``. ``null`` comparisons never reach it: the enforcement would +render ``col = NULL`` (matches nothing), so validation refuses them. Adding +sqlglot as a kbagent dependency just for this string is not warranted. """ from __future__ import annotations +import re from typing import Any import jsonschema @@ -40,6 +39,14 @@ # between (predicates are pinned to one workspace dialect per policy). RLS_DIALECTS: tuple[str, ...] = ("snowflake", "bigquery") +# The enforcement's principal pattern (mcp-server ``rls.py``): no whitespace or control characters. A +# principal that fails it is not ignored there -- it refuses EVERY query in the project. +_PRINCIPAL_RE = re.compile(r"^[^\s\x00-\x1f\x7f]+$") + +# How each dialect quotes an identifier. The enforcement compiles columns quoted (case-exact), so the +# preview must too -- an unquoted preview would hide a case mismatch the real filter does not forgive. +_QUOTE = {"snowflake": '"', "bigquery": "`"} + _COMPARISON_SQL = { "eq": "=", "ne": "!=", @@ -115,12 +122,22 @@ def validate_condition_ops(condition: Any) -> list[str]: column = condition.get("column") if not isinstance(column, str) or not column: errors.append(f"condition with op {op!r} needs a non-empty string 'column'") - if op in RLS_COMPARISON_OPS and "value" not in condition: - errors.append(f"condition with op {op!r} needs a 'value'") + if op in RLS_COMPARISON_OPS: + if "value" not in condition: + errors.append(f"condition with op {op!r} needs a 'value'") + elif condition["value"] is None: + # The enforcement renders `col = NULL`, which matches no row: the principal silently sees nothing. + errors.append( + f"condition with op {op!r} cannot compare to null; use op 'is_null' / 'is_not_null'" + ) if op in RLS_MEMBERSHIP_OPS: values = condition.get("values") if not isinstance(values, list) or not values: errors.append(f"condition with op {op!r} needs a non-empty list 'values'") + elif any(value is None for value in values): + errors.append( + f"condition with op {op!r} cannot list null in 'values' (it never matches); use 'is_null'" + ) # No keys beyond the op's own shape (the schema forbids additional properties): a stray `values` on an # `eq`, or a `value` on an `in`, is an authoring mistake that would otherwise be silently ignored. allowed = {"column", "op"} | ( @@ -147,15 +164,52 @@ def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: principal = rule["principal"] if not isinstance(principal, str) or not principal: return [f"rules[{index}].principal must be a non-empty string"] - return [] - principals = rule["principals"] - if ( - not isinstance(principals, list) - or not principals - or not all(isinstance(name, str) and name for name in principals) - ): - return [f"rules[{index}].principals must be a non-empty list of non-empty strings"] - return [] + names = [principal] + else: + names = rule["principals"] + if ( + not isinstance(names, list) + or not names + or not all(isinstance(name, str) and name for name in names) + ): + return [f"rules[{index}].principals must be a non-empty list of non-empty strings"] + return [ + f"rules[{index}]: principal {name!r} contains whitespace or control characters" + for name in names + if not _PRINCIPAL_RE.fullmatch(name) + ] + + +def rule_principals(rule: dict[str, Any]) -> list[str]: + """The principals one rule names (``principal`` or ``principals``), as written.""" + return [rule["principal"]] if "principal" in rule else list(rule.get("principals") or []) + + +def duplicate_principals( + rules: list[dict[str, Any]], taken: dict[str, str] | None = None +) -> list[str]: + """Principals named twice on one table, case-folded -- the enforcement refuses every query then. + + ``taken`` maps an already-used case-folded principal to where it is used (another policy on the + same table), so the same check covers duplicates inside ``rules`` and across policies. + """ + seen = dict(taken or {}) + errors: list[str] = [] + for index, rule in enumerate(rules): + for name in rule_principals(rule): + folded = name.casefold() + if folded in seen: + errors.append( + f"rules[{index}]: principal {name!r} already has a rule in {seen[folded]}" + ) + else: + seen[folded] = f"rules[{index}]" + return errors + + +def table_key(table: str, dialect: str) -> str: + """How the enforcement compares table keys: case-insensitive on Snowflake, exact on BigQuery.""" + return table.lower() if dialect == "snowflake" else table def validate_rules_local(rules: Any) -> list[str]: @@ -214,6 +268,8 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: if not column or op is None: raise ValueError(f"unrecognized condition shape: {condition!r}") + quote = _QUOTE.get(dialect, '"') + column = f"{quote}{column}{quote}" if op in _COMPARISON_SQL: return f"{column} {_COMPARISON_SQL[op]} {_preview_literal(condition.get('value'))}" if op in RLS_MEMBERSHIP_OPS: @@ -236,6 +292,8 @@ def _preview_literal(value: Any) -> str: """ if isinstance(value, str): return "'" + value.replace("'", "''") + "'" + if isinstance(value, bool): # before the generic str(): Python's `True` is SQL's TRUE + return "TRUE" if value else "FALSE" if value is None: return "NULL" return str(value) diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py index a200b9ca..c19b4b4b 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -53,11 +53,11 @@ def resolve_target_project_ids( for target in parse_target_projects(targets): project = config_store.get_project(target) if project is None: - if not target.isdigit(): + if not target.isdigit() or int(target) == 0: raise KeboolaApiError( message=( f"--target-project {target!r} is neither a registered project alias " - "nor a numeric project ID. See `kbagent project list`." + "nor a positive numeric project ID. See `kbagent project list`." ), error_code=ErrorCode.INVALID_ARGUMENT, ) diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index a16a6072..461cdc7c 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -97,7 +97,8 @@ def test_aliases_ids_and_comma_lists_resolve_and_dedupe(self, tmp_path: Path) -> @pytest.mark.parametrize( ("target", "needle"), [ - ("ghost", "neither a registered project alias nor a numeric project ID"), + ("ghost", "neither a registered project alias nor a positive numeric project ID"), + ("0", "neither a registered project alias nor a positive numeric project ID"), ("eu-project", "different stack"), ], ) From e06378084ffa2d186ac6a736a3742d7bcb16f4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:09:33 +0200 Subject: [PATCH 20/35] fix(CLI-17): targeted scope by default, backend dialect, PATCH updates, destructive deletes Address the review of #778: - default to targeted scope (the schema default); --scope organization is explicit and destructive-class (FLAG_ESCALATIONS, also over REST) - --dialect defaults to and must match the project backend; duplicate principals are refused across the visible policies on the table - --target-project takes alias or ID (#715 resolver); grants go in the create request only; on update they change before the rules - update sends a PATCH of the changed keys and re-reads the result - rls/cls delete are destructive and have --dry-run - own 409 message, owner_project_id in list/detail - command shape per #791: --table-id, --dialect choice, REST table_id / target_projects and PATCH; one router builder for rls and cls - rls setup --json returns a JSON error envelope; typed wizard values Co-Authored-By: Claude Opus 5.5 --- src/keboola_agent_cli/commands/cls.py | 184 ++--- src/keboola_agent_cli/commands/rls.py | 503 +++++++------- src/keboola_agent_cli/permissions.py | 13 +- .../server/_serve_command_map.py | 4 +- src/keboola_agent_cli/server/app.py | 10 +- src/keboola_agent_cli/server/routers/cls.py | 132 +--- src/keboola_agent_cli/server/routers/rls.py | 228 ++++--- src/keboola_agent_cli/services/rls_service.py | 396 +++++++---- tests/test_cls_cli.py | 66 +- tests/test_cls_service.py | 75 +- tests/test_rls_cli.py | 138 +++- tests/test_rls_service.py | 643 ++++++++++-------- tests/test_server_cls.py | 51 +- tests/test_server_rls.py | 84 ++- 14 files changed, 1372 insertions(+), 1155 deletions(-) diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index ea2c3ad8..4c2d19cb 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -14,29 +14,42 @@ Each rule is ``{principal|principals, visible_columns: [...]}``: the listed columns are the only ones that principal can read (allowlist projection; masking is not supported). As with ``rls``, every write is authored at -``organization`` or ``targeted`` scope, never ``project`` -- there is no -``--scope`` flag. Enforcement happens in ``keboola-mcp-server``'s +``targeted`` scope unless ``--scope organization`` is passed -- never +``project``. Enforcement happens in ``keboola-mcp-server``'s ``query_data``; kbagent only authors policies. ``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``; -``create``/``update``/``delete`` are gated as ``admin`` (``cls.*`` in -``OPERATION_REGISTRY``). +``create``/``update`` are gated as ``admin``, ``delete`` and ``--scope +organization`` as ``destructive`` (``cls.*`` in ``OPERATION_REGISTRY`` / +``FLAG_ESCALATIONS``). """ from __future__ import annotations import contextlib -import json -from collections.abc import Callable from typing import Any import typer -from rich.syntax import Syntax -from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..services.rls_service import RLS_DIALECTS -from ._helpers import check_cli_permission, get_formatter, get_service, map_error_to_exit_code -from .rls import _format_policy_table, _parse_rules_arg, _print_warnings +from ..errors import ConfigError, KeboolaApiError +from ._helpers import check_cli_permission, get_formatter, get_service +from .rls import ( + DIALECT_HELP, + SCOPE_HELP, + TABLE_HELP, + TARGET_HELP, + Dialect, + PolicyScope, + _call, + _confirm_or_exit, + _format_policy_table, + _parse_rules_arg, + _print_warnings, + delete_policy, + gate_scope, + print_schema, + reject_target_conflict, +) cls_app = typer.Typer( help=( @@ -54,25 +67,15 @@ def _cls_permission_check(ctx: typer.Context) -> None: check_cli_permission(ctx, "cls") -def _call(formatter: Any, fn: Callable[..., dict[str, Any]], **kwargs: Any) -> dict[str, Any]: - """Run a service call, turning its errors into a formatted message + exit code.""" - try: - return fn(**kwargs) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None - - def _print_policy(formatter: Any, row: dict[str, Any]) -> None: formatter.console.print( f"\n[bold]{row.get('table', '')}[/bold] [dim](id {row.get('id', '')})[/dim]" ) formatter.console.print(f" Dialect: {row.get('dialect', '')}") formatter.console.print(f" Scope: {row.get('scope', '')}") - formatter.console.print(f" Source project: {row.get('source_project_id') or '(none)'}") + formatter.console.print( + f" Owner project: {row.get('owner_project_id') or '(organization)'}" + ) targets = row.get("target_project_ids") or [] if targets: formatter.console.print(f" Target projects: {', '.join(str(t) for t in targets)}") @@ -96,21 +99,6 @@ def _print_preview(formatter: Any, result: dict[str, Any]) -> None: ) -def _check_dialect(formatter: Any, dialect: str | None) -> None: - if dialect is not None and dialect not in RLS_DIALECTS: - formatter.error( - message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - - -def _confirm_or_exit(formatter: Any, question: str) -> None: - if not typer.confirm(question): - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) - - @cls_app.command("list") def cls_list( ctx: typer.Context, @@ -118,8 +106,7 @@ def cls_list( ) -> None: """List column-level security policies visible to a project.""" formatter = get_formatter(ctx) - service = get_service(ctx, "cls_service") - result = _call(formatter, service.list_policies, alias=project) + result = _call(formatter, get_service(ctx, "cls_service").list_policies, alias=project) if formatter.json_mode: formatter.output(result) @@ -159,42 +146,16 @@ def cls_schema( have the ``cls-policy`` object type registered, this fails with a clean, classified ``NOT_FOUND`` error. """ - formatter = get_formatter(ctx) - service = get_service(ctx, "cls_service") - try: - fetch = service.fetch_schema(project) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: # an auth/permission failure is not "schema unavailable" - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None - - if fetch.schema is None: - formatter.error( - message=f"Could not fetch the cls-policy schema: {fetch.reason}", - error_code=ErrorCode.NOT_FOUND, - ) - raise typer.Exit(code=4) - - if formatter.json_mode: - formatter.output({"format": "json-schema", "source": "live", "schema": fetch.schema}) - return - formatter.console.print( - Syntax(json.dumps(fetch.schema, indent=2), "json", theme="monokai", line_numbers=False) - ) + print_schema(ctx, "cls_service", "cls-policy", project) @cls_app.command("create") def cls_create( ctx: typer.Context, project: str = typer.Option( - ..., "--project", help="Project alias -- the project this policy protects" - ), - table: str = typer.Option(..., "--table", help="Table key, e.g. in.c-crm.invoices"), - dialect: str = typer.Option( - ..., "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" + ..., "--project", help="Project alias -- the project that owns the policy" ), + table_id: str = typer.Option(..., "--table-id", help=TABLE_HELP), rules: str = typer.Option( ..., "--rules", @@ -203,40 +164,30 @@ def cls_create( "allowlist of columns that principal can read" ), ), - target_project: list[str] | None = typer.Option( - None, - "--target-project", - help=( - "Project ID this policy also applies to (repeatable). Omit for plain " - "organization scope -- still only applies where source_project_id / " - "target_project_ids match at read time, never by table-name text alone." - ), - ), + dialect: Dialect | None = typer.Option(None, "--dialect", help=DIALECT_HELP), + scope: PolicyScope = typer.Option(PolicyScope.TARGETED, "--scope", help=SCOPE_HELP), + target_project: list[str] | None = typer.Option(None, "--target-project", help=TARGET_HELP), dry_run: bool = typer.Option(False, "--dry-run", help="Preview the projection without writing"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: - """Create one CLS policy for one table. - - Always authored at ``organization`` scope, or ``targeted`` scope when - ``--target-project`` is given -- never plain ``project`` scope. - """ + """Create one CLS policy for one table (``targeted`` scope unless ``--scope organization``).""" + gate_scope(ctx, "cls", "create", scope) formatter = get_formatter(ctx) - parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) - _check_dialect(formatter, dialect) service = get_service(ctx, "cls_service") kwargs = { "alias": project, - "table": table, + "table": table_id, "dialect": dialect, - "rules": parsed_rules, - "target_project_ids": target_project, + "rules": _parse_rules_arg(formatter, rules, _RULES_SHAPE), + "scope": scope.value, + "target_projects": target_project, } if not dry_run and not yes and not formatter.json_mode: # A validation/network error here is reported properly by the real call below. with contextlib.suppress(ConfigError, KeboolaApiError): _print_preview(formatter, service.create_policy(**kwargs, dry_run=True)) - _confirm_or_exit(formatter, f"Create CLS policy on {table}?") + _confirm_or_exit(formatter, f"Create CLS policy on {table_id}?") result = _call(formatter, service.create_policy, **kwargs, dry_run=dry_run) @@ -245,7 +196,7 @@ def cls_create( return _print_warnings(formatter, result) if not dry_run: - formatter.success(f"Created CLS policy {result.get('id', '')} on {table}") + formatter.success(f"Created CLS policy {result.get('id', '')} on {table_id}") _print_preview(formatter, result) @@ -254,15 +205,15 @@ def cls_update( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), - table: str | None = typer.Option(None, "--table", help="New table key (unset = unchanged)"), - dialect: str | None = typer.Option( - None, "--dialect", help=f"New dialect: {' | '.join(RLS_DIALECTS)} (unset = unchanged)" + table_id: str | None = typer.Option(None, "--table-id", help="New table (unset = unchanged)"), + dialect: Dialect | None = typer.Option( + None, "--dialect", help="New dialect, must match the project backend (unset = unchanged)" ), rules: str | None = typer.Option( None, "--rules", help="New JSON|@file|- rules array (unset = unchanged)" ), target_project: list[str] | None = typer.Option( - None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" + None, "--target-project", help=f"{TARGET_HELP}; replaces the list (unset = unchanged)" ), clear_target_projects: bool = typer.Option( False, @@ -274,18 +225,12 @@ def cls_update( ) -> None: """Update an existing CLS policy. - Fetch-then-merge: only the flags you pass are changed -- an omitted flag - keeps the policy's current value, it is never silently blanked. + Only the flags you pass change -- an omitted flag keeps the policy's + current value. The write is a partial update (PATCH) of just those keys. """ formatter = get_formatter(ctx) parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) if rules is not None else None - if clear_target_projects and target_project: - formatter.error( - message="--clear-target-projects cannot be combined with --target-project", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - _check_dialect(formatter, dialect) + reject_target_conflict(formatter, target_project, clear_target_projects) service = get_service(ctx, "cls_service") if not dry_run and not yes and not formatter.json_mode: @@ -296,10 +241,10 @@ def cls_update( service.update_policy, alias=project, policy_id=policy_id, - table=table, + table=table_id, dialect=dialect, rules=parsed_rules, - target_project_ids=[] if clear_target_projects else target_project, + target_projects=[] if clear_target_projects else target_project, dry_run=dry_run, ) @@ -317,20 +262,17 @@ def cls_delete( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), + dry_run: bool = typer.Option(False, "--dry-run", help="Show the policy without deleting it"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: - """Delete a CLS policy.""" - formatter = get_formatter(ctx) - service = get_service(ctx, "cls_service") - - if not yes and not formatter.json_mode: - _confirm_or_exit( - formatter, f"Delete CLS policy {policy_id}? Its table's columns become unrestricted." - ) - - result = _call(formatter, service.delete_policy, alias=project, policy_id=policy_id) - - if formatter.json_mode: - formatter.output(result) - else: - formatter.success(f"Deleted CLS policy {policy_id}") + """Delete a CLS policy -- its table's columns become unrestricted.""" + delete_policy( + ctx, + "cls_service", + "CLS", + "Its table's columns become unrestricted.", + alias=project, + policy_id=policy_id, + dry_run=dry_run, + yes=yes, + ) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 6a28a1a9..68fadd86 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -12,9 +12,11 @@ a checkbox picker, build conditions, preview, then create one policy per table via the same ``create_policy`` path ``create`` uses. -**Every policy this group writes is authored at ``organization`` or -``targeted`` scope, never ``project``** -- there is deliberately no -``--scope`` flag that could select ``project`` (see the RFC in +**Every policy this group writes is authored at ``targeted`` scope (the +default: the owning project plus the ``--target-project`` grants) or, only +when asked for with ``--scope organization``, at ``organization`` scope** +(every project in the organization). ``--scope`` offers no ``project``: the +metastore schema does not support it for policies (see the RFC in ``keboola-mcp-server``'s ``feature_spec/rls_query_tool/RFC.md``). On a stack whose metastore predates the ``rls-policy`` schema every command here @@ -22,14 +24,18 @@ ``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``. ``create``/``update``/``setup`` are gated as ``admin`` (organization-level, -not merely ``write``); ``delete`` is gated as ``admin`` too -- see -``rls.*`` in ``OPERATION_REGISTRY`` (``permissions.py``). +not merely ``write``); ``delete`` and ``--scope organization`` are +``destructive`` -- see ``rls.*`` in ``OPERATION_REGISTRY`` / ``FLAG_ESCALATIONS`` +(``permissions.py``). """ from __future__ import annotations +import contextlib import json import logging +from collections.abc import Callable +from enum import StrEnum from typing import Any import typer @@ -37,7 +43,6 @@ from rich.table import Table from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..services.rls_service import RLS_DIALECTS from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( check_cli_operation, @@ -59,6 +64,72 @@ ) +class Dialect(StrEnum): + """``--dialect`` values (the policy schema's enum); omitted = the project backend.""" + + SNOWFLAKE = "snowflake" + BIGQUERY = "bigquery" + + +class PolicyScope(StrEnum): + """``--scope`` values: the schema supports no ``project`` scope for policies.""" + + TARGETED = "targeted" + ORGANIZATION = "organization" + + +DIALECT_HELP = "Workspace SQL dialect; must match the project backend (default: the backend)" +SCOPE_HELP = ( + "targeted (default): the owning project plus --target-project grants. organization: " + "EVERY project of the organization that has the row-level-security feature -- an explicit, " + "organization-admin-only choice, gated as destructive." +) +TARGET_HELP = ( + "Project alias or ID the policy also applies to (repeatable or comma-separated; " + "--scope targeted only; granting needs the organization-admin role)" +) +TABLE_HELP = "Storage table ID, e.g. in.c-crm.invoices" + + +def _call[T](formatter: Any, fn: Callable[..., T], **kwargs: Any) -> T: + """Run a service call, turning its errors into a formatted message + exit code. + + A bad option value (``INVALID_ARGUMENT``) is a usage error, exit 2. + """ + try: + return fn(**kwargs) + except ConfigError as exc: + formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) + raise typer.Exit(code=5) from None + except KeboolaApiError as exc: + formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) + code = 2 if exc.error_code == ErrorCode.INVALID_ARGUMENT else map_error_to_exit_code(exc) + raise typer.Exit(code=code) from None + + +def _confirm_or_exit(formatter: Any, question: str) -> None: + if not typer.confirm(question): + formatter.console.print("[yellow]Aborted.[/yellow]") + raise typer.Exit(code=0) + + +def gate_scope(ctx: typer.Context, group: str, operation: str, scope: PolicyScope) -> None: + """``--scope organization`` governs the table in every project: a destructive-class flag.""" + if scope == PolicyScope.ORGANIZATION: + check_cli_operation(ctx, f"{group}.{operation} --scope organization") + + +def reject_target_conflict( + formatter: Any, target_project: list[str] | None, clear_target_projects: bool +) -> None: + if clear_target_projects and target_project: + formatter.error( + message="--clear-target-projects cannot be combined with --target-project", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + + @rls_app.callback(invoke_without_command=True) def _rls_permission_check(ctx: typer.Context) -> None: check_cli_permission(ctx, "rls") @@ -85,7 +156,7 @@ def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None "Dialect", "Rules", "Scope", - "Source Project", + "Owner Project", "Target Projects", show_header=True, header_style="bold cyan", @@ -97,7 +168,7 @@ def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None str(policy.get("dialect", "")), str(policy.get("rule_count", 0)), str(policy.get("scope", "")), - str(policy.get("source_project_id") or ""), + str(policy.get("owner_project_id") or "(organization)"), ", ".join(str(p) for p in policy.get("target_project_ids", [])), ) formatter.console.print(tbl) @@ -109,7 +180,9 @@ def _print_policy(formatter: Any, row: dict[str, Any]) -> None: ) formatter.console.print(f" Dialect: {row.get('dialect', '')}") formatter.console.print(f" Scope: {row.get('scope', '')}") - formatter.console.print(f" Source project: {row.get('source_project_id') or '(none)'}") + formatter.console.print( + f" Owner project: {row.get('owner_project_id') or '(organization)'}" + ) targets = row.get("target_project_ids") or [] if targets: formatter.console.print(f" Target projects: {', '.join(str(t) for t in targets)}") @@ -154,7 +227,7 @@ def _parse_rules_arg( # --------------------------------------------------------------------------- -# rls list +# rls list / detail / schema # --------------------------------------------------------------------------- @@ -165,31 +238,14 @@ def rls_list( ) -> None: """List row-level security policies visible to a project.""" formatter = get_formatter(ctx) - service = get_service(ctx, "rls_service") - - try: - result = service.list_policies(alias=project) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + result = _call(formatter, get_service(ctx, "rls_service").list_policies, alias=project) if formatter.json_mode: formatter.output(result) - return - - policies = result.get("policies", []) - if not policies: + elif not result.get("policies"): formatter.console.print("[dim]No RLS policies found.[/dim]") else: - _format_policy_table(formatter, policies) - - -# --------------------------------------------------------------------------- -# rls detail -# --------------------------------------------------------------------------- + _format_policy_table(formatter, result["policies"]) @rls_app.command("detail") @@ -201,26 +257,32 @@ def rls_detail( """Show one RLS policy's full rule set.""" formatter = get_formatter(ctx) service = get_service(ctx, "rls_service") - - try: - result = service.get_policy(alias=project, policy_id=policy_id) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + result = _call(formatter, service.get_policy, alias=project, policy_id=policy_id) if formatter.json_mode: formatter.output(result) - return + else: + _print_policy(formatter, result) - _print_policy(formatter, result) +def print_schema(ctx: typer.Context, service_name: str, item_type: str, project: str) -> None: + """``rls schema`` / ``cls schema``: the live schema, no offline bundled snapshot.""" + formatter = get_formatter(ctx) + # An auth/permission failure is raised (not "schema unavailable") and reported by `_call`. + fetch = _call(formatter, get_service(ctx, service_name).fetch_schema, alias=project) + if fetch.schema is None: + formatter.error( + message=f"Could not fetch the {item_type} schema: {fetch.reason}", + error_code=ErrorCode.NOT_FOUND, + ) + raise typer.Exit(code=4) -# --------------------------------------------------------------------------- -# rls schema -# --------------------------------------------------------------------------- + if formatter.json_mode: + formatter.output({"format": "json-schema", "source": "live", "schema": fetch.schema}) + return + formatter.console.print( + Syntax(json.dumps(fetch.schema, indent=2), "json", theme="monokai", line_numbers=False) + ) @rls_app.command("schema") @@ -236,35 +298,11 @@ def rls_schema( schema is authoritative only from the live metastore, and on a stack whose metastore predates it, fetching fails with a clean, classified error. """ - formatter = get_formatter(ctx) - service = get_service(ctx, "rls_service") - - try: - fetch = service.fetch_schema(project) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: # an auth/permission failure is not "schema unavailable" - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None - - if fetch.schema is None: - formatter.error( - message=f"Could not fetch the rls-policy schema: {fetch.reason}", - error_code=ErrorCode.NOT_FOUND, - ) - raise typer.Exit(code=4) - - if formatter.json_mode: - formatter.output({"format": "json-schema", "source": "live", "schema": fetch.schema}) - return - formatter.console.print( - Syntax(json.dumps(fetch.schema, indent=2), "json", theme="monokai", line_numbers=False) - ) + print_schema(ctx, "rls_service", "rls-policy", project) # --------------------------------------------------------------------------- -# rls create +# rls create / update / delete # --------------------------------------------------------------------------- @@ -272,12 +310,9 @@ def rls_schema( def rls_create( ctx: typer.Context, project: str = typer.Option( - ..., "--project", help="Project alias -- the project this policy protects" - ), - table: str = typer.Option(..., "--table", help="Table key, e.g. in.c-crm.invoices"), - dialect: str = typer.Option( - ..., "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" + ..., "--project", help="Project alias -- the project that owns the policy" ), + table_id: str = typer.Option(..., "--table-id", help=TABLE_HELP), rules: str = typer.Option( ..., "--rules", @@ -286,103 +321,58 @@ def rls_create( "(see `rls schema` / rls-workflow.md for the condition shape)" ), ), - target_project: list[str] | None = typer.Option( - None, - "--target-project", - help=( - "Project ID this policy also applies to (repeatable). Omit for plain " - "organization scope -- still only applies where source_project_id / " - "target_project_ids match at read time, never by table-name text alone." - ), - ), + dialect: Dialect | None = typer.Option(None, "--dialect", help=DIALECT_HELP), + scope: PolicyScope = typer.Option(PolicyScope.TARGETED, "--scope", help=SCOPE_HELP), + target_project: list[str] | None = typer.Option(None, "--target-project", help=TARGET_HELP), dry_run: bool = typer.Option( False, "--dry-run", help="Preview the compiled condition without writing" ), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: - """Create one RLS policy for one table. - - Always authored at ``organization`` scope, or ``targeted`` scope when - ``--target-project`` is given -- never plain ``project`` scope. - """ + """Create one RLS policy for one table (``targeted`` scope unless ``--scope organization``).""" + gate_scope(ctx, "rls", "create", scope) formatter = get_formatter(ctx) - parsed_rules = _parse_rules_arg(formatter, rules) - - if dialect not in RLS_DIALECTS: - formatter.error( - message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - service = get_service(ctx, "rls_service") + kwargs = { + "alias": project, + "table": table_id, + "dialect": dialect, + "rules": _parse_rules_arg(formatter, rules), + "scope": scope.value, + "target_projects": target_project, + } if not dry_run and not yes and not formatter.json_mode: - try: - preview = service.create_policy( - alias=project, - table=table, - dialect=dialect, - rules=parsed_rules, - target_project_ids=target_project, - dry_run=True, - ) - except (ConfigError, KeboolaApiError): - pass # validation/network error -- let the real call below report it properly - else: - _print_preview(formatter, preview) - confirmed = typer.confirm(f"Create RLS policy on {table}?") - if not confirmed: - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) + # A validation/network error here is reported properly by the real call below. + with contextlib.suppress(ConfigError, KeboolaApiError): + _print_preview(formatter, service.create_policy(**kwargs, dry_run=True)) + _confirm_or_exit(formatter, f"Create RLS policy on {table_id}?") - try: - result = service.create_policy( - alias=project, - table=table, - dialect=dialect, - rules=parsed_rules, - target_project_ids=target_project, - dry_run=dry_run, - ) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + result = _call(formatter, service.create_policy, **kwargs, dry_run=dry_run) if formatter.json_mode: formatter.output(result) return - _print_warnings(formatter, result) - if dry_run: - _print_preview(formatter, result) - return - formatter.success(f"Created RLS policy {result.get('id', '')} on {table}") + if not dry_run: + formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") _print_preview(formatter, result) -# --------------------------------------------------------------------------- -# rls update -# --------------------------------------------------------------------------- - - @rls_app.command("update") def rls_update( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), - table: str | None = typer.Option(None, "--table", help="New table key (unset = unchanged)"), - dialect: str | None = typer.Option( - None, "--dialect", help=f"New dialect: {' | '.join(RLS_DIALECTS)} (unset = unchanged)" + table_id: str | None = typer.Option(None, "--table-id", help="New table (unset = unchanged)"), + dialect: Dialect | None = typer.Option( + None, "--dialect", help="New dialect, must match the project backend (unset = unchanged)" ), rules: str | None = typer.Option( None, "--rules", help="New JSON|@file|- rules array (unset = unchanged)" ), target_project: list[str] | None = typer.Option( - None, "--target-project", help="New target-project list (repeatable; unset = unchanged)" + None, "--target-project", help=f"{TARGET_HELP}; replaces the list (unset = unchanged)" ), clear_target_projects: bool = typer.Option( False, @@ -396,65 +386,58 @@ def rls_update( ) -> None: """Update an existing RLS policy. - Fetch-then-merge: only the flags you pass are changed -- an omitted flag - keeps the policy's current value, it is never silently blanked. + Only the flags you pass change -- an omitted flag keeps the policy's + current value. The write is a partial update (PATCH) of just those keys. """ formatter = get_formatter(ctx) parsed_rules = _parse_rules_arg(formatter, rules) if rules is not None else None - if clear_target_projects and target_project: - formatter.error( - message="--clear-target-projects cannot be combined with --target-project", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - - if dialect is not None and dialect not in RLS_DIALECTS: - formatter.error( - message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - + reject_target_conflict(formatter, target_project, clear_target_projects) service = get_service(ctx, "rls_service") if not dry_run and not yes and not formatter.json_mode: - confirmed = typer.confirm(f"Update RLS policy {policy_id}?") - if not confirmed: - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) - - try: - result = service.update_policy( - alias=project, - policy_id=policy_id, - table=table, - dialect=dialect, - rules=parsed_rules, - target_project_ids=[] if clear_target_projects else target_project, - dry_run=dry_run, - ) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None + _confirm_or_exit(formatter, f"Update RLS policy {policy_id}?") + + result = _call( + formatter, + service.update_policy, + alias=project, + policy_id=policy_id, + table=table_id, + dialect=dialect, + rules=parsed_rules, + target_projects=[] if clear_target_projects else target_project, + dry_run=dry_run, + ) if formatter.json_mode: formatter.output(result) return - _print_warnings(formatter, result) - if dry_run: - _print_preview(formatter, result) - return - formatter.success(f"Updated RLS policy {policy_id}") + if not dry_run: + formatter.success(f"Updated RLS policy {policy_id}") _print_preview(formatter, result) -# --------------------------------------------------------------------------- -# rls delete -# --------------------------------------------------------------------------- +def delete_policy( + ctx: typer.Context, service_name: str, label: str, effect: str, **kwargs: Any +) -> None: + """``rls delete`` / ``cls delete``; ``--dry-run`` shows the policy that would be deleted.""" + formatter = get_formatter(ctx) + service = get_service(ctx, service_name) + policy_id, dry_run, yes = kwargs["policy_id"], kwargs["dry_run"], kwargs.pop("yes") + + if not dry_run and not yes and not formatter.json_mode: + _confirm_or_exit(formatter, f"Delete {label} policy {policy_id}? {effect}") + + result = _call(formatter, service.delete_policy, **kwargs) + + if formatter.json_mode: + formatter.output(result) + elif dry_run: + formatter.console.print(f"[bold]Would delete[/bold] {label} policy {policy_id}:") + formatter.console.print_json(data=result["policy"]) + else: + formatter.success(f"Deleted {label} policy {policy_id}") @rls_app.command("delete") @@ -462,31 +445,20 @@ def rls_delete( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), + dry_run: bool = typer.Option(False, "--dry-run", help="Show the policy without deleting it"), yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: - """Delete an RLS policy.""" - formatter = get_formatter(ctx) - service = get_service(ctx, "rls_service") - - if not yes and not formatter.json_mode: - confirmed = typer.confirm(f"Delete RLS policy {policy_id}? This un-protects its table.") - if not confirmed: - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) - - try: - result = service.delete_policy(alias=project, policy_id=policy_id) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None - - if formatter.json_mode: - formatter.output(result) - else: - formatter.success(f"Deleted RLS policy {policy_id}") + """Delete an RLS policy -- its table's rows are no longer filtered.""" + delete_policy( + ctx, + "rls_service", + "RLS", + "This un-protects its table.", + alias=project, + policy_id=policy_id, + dry_run=dry_run, + yes=yes, + ) # --------------------------------------------------------------------------- @@ -496,7 +468,7 @@ def rls_delete( _SETUP_HINT = ( "rls setup is an interactive picker and needs a real terminal. " - "Use `rls create --project P --table T --dialect D --rules '[...]'` " + "Use `rls create --project P --table-id T --rules '[...]'` " "directly instead (see rls-workflow.md)." ) @@ -508,6 +480,7 @@ def rls_delete( ) _COMPARISON_OPS = ("eq", "ne", "gt", "gte", "lt", "lte") +_VALUE_HINT = 'parsed as JSON when it is one (42, 4.5, true); quote it ("42") to force a string' def _prompt_choice(text: str, choices: tuple[str, ...], default: str) -> str: @@ -518,6 +491,19 @@ def _prompt_choice(text: str, choices: tuple[str, ...], default: str) -> str: typer.echo(f"Enter one of: {', '.join(choices)}") +def _typed_value(raw: str) -> Any: + """A prompted value as the JSON scalar it spells (``42``, ``true``), else the text itself. + + Without this every value would be stored as a string, and the filter would compare a numeric or + boolean column to a string literal. ``null`` is not a value here: it never matches (use option 3). + """ + try: + value = json.loads(raw) + except ValueError: + return raw + return value if isinstance(value, str | int | float | bool) else raw + + def _prompt_condition() -> dict[str, Any]: typer.echo(_CONDITION_MENU) choice = _prompt_choice("Choice", ("1", "2", "3", "4"), default="1") @@ -526,12 +512,12 @@ def _prompt_condition() -> dict[str, Any]: column = typer.prompt("Column name") if choice == "1": op = _prompt_choice("Operator", _COMPARISON_OPS, default="eq") - value = typer.prompt("Value") + value = _typed_value(typer.prompt(f"Value ({_VALUE_HINT})")) return {"column": column, "op": op, "value": value} if choice == "2": op = _prompt_choice("Operator", ("in", "not_in"), default="in") - raw_values = typer.prompt("Values (comma-separated)") - values = [v.strip() for v in raw_values.split(",") if v.strip()] + raw_values = typer.prompt(f"Values (comma-separated; each {_VALUE_HINT})") + values = [_typed_value(v.strip()) for v in raw_values.split(",") if v.strip()] return {"column": column, "op": op, "values": values} op = _prompt_choice("Operator", ("is_null", "is_not_null"), default="is_null") return {"column": column, "op": op} @@ -559,51 +545,34 @@ def rls_setup( project: str = typer.Option( ..., "--project", help="Project alias whose tables you're protecting" ), - dialect: str | None = typer.Option( - None, "--dialect", help=f"Workspace SQL dialect: {' | '.join(RLS_DIALECTS)}" - ), + dialect: Dialect | None = typer.Option(None, "--dialect", help=DIALECT_HELP), rules: str | None = typer.Option( None, "--rules", help="JSON|@file|- rules array -- skips the interactive condition builder", ), - target_project: list[str] | None = typer.Option( - None, "--target-project", help="Project ID to also share the policy with (repeatable)" - ), + scope: PolicyScope = typer.Option(PolicyScope.TARGETED, "--scope", help=SCOPE_HELP), + target_project: list[str] | None = typer.Option(None, "--target-project", help=TARGET_HELP), yes: bool = typer.Option(False, "--yes", "-y", help="Skip the final confirmation"), ) -> None: """Guided RLS setup: pick tables, build a condition, preview, then write. - Interactive-terminal-only (no ``--json`` / non-TTY path -- use ``rls - create`` directly there). Creates one RLS policy per selected table, - all sharing the same rules, via the exact same write path ``rls - create`` uses. + Interactive-terminal-only (with ``--json`` or without a terminal it exits 2 + with a usage error -- use ``rls create`` there). Creates one RLS policy per + selected table, all sharing the same rules, via the exact same write path + ``rls create`` uses. """ # `setup` performs `create`'s writes, so an exact `rls.create` denial must cover it too. check_cli_operation(ctx, "rls.create") + gate_scope(ctx, "rls", "create", scope) formatter = get_formatter(ctx) - if dialect is not None and dialect not in RLS_DIALECTS: - formatter.error( - message=f"--dialect must be one of {RLS_DIALECTS}, got {dialect!r}", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None if formatter.json_mode or not _is_interactive(): - target_console = formatter.err_console if formatter.json_mode else formatter.console - target_console.print(_SETUP_HINT) + formatter.error(message=_SETUP_HINT, error_code=ErrorCode.INVALID_ARGUMENT) raise typer.Exit(code=2) - storage_service = get_service(ctx, "storage_service") - try: - tables_result = storage_service.list_tables(aliases=[project]) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None - - tables = tables_result.get("tables", []) + tables = _call( + formatter, get_service(ctx, "storage_service").list_tables, aliases=[project] + ).get("tables", []) if not tables: formatter.console.print(f"[dim]No tables found in project '{project}'.[/dim]") raise typer.Exit(code=0) @@ -615,7 +584,7 @@ def rls_setup( try: indices = checkbox_select(items, title=f"Select tables to protect with RLS in '{project}'") except CheckboxUnavailable: - formatter.console.print(f"[yellow]{_SETUP_HINT}[/yellow]") + formatter.error(message=_SETUP_HINT, error_code=ErrorCode.INVALID_ARGUMENT) raise typer.Exit(code=2) from None selected = indices or [] @@ -623,51 +592,39 @@ def rls_setup( formatter.console.print("No tables selected.") raise typer.Exit(code=0) - resolved_dialect = dialect or _prompt_choice("Dialect", RLS_DIALECTS, default="snowflake") - - if rules is not None: - parsed_rules = _parse_rules_arg(formatter, rules) - else: - parsed_rules = _build_rules_interactively() - + parsed_rules = ( + _parse_rules_arg(formatter, rules) if rules is not None else _build_rules_interactively() + ) service = get_service(ctx, "rls_service") selected_tables = [str(tables[i]["id"]) for i in selected] + kwargs = { + "alias": project, + "dialect": dialect, # None = the project backend, resolved by the service + "rules": parsed_rules, + "scope": scope.value, + "target_projects": target_project, + } for table_id in selected_tables: try: - preview = service.create_policy( - alias=project, - table=table_id, - dialect=resolved_dialect, - rules=parsed_rules, - target_project_ids=target_project, - dry_run=True, - ) + preview = service.create_policy(table=table_id, **kwargs, dry_run=True) except (ConfigError, KeboolaApiError) as exc: formatter.console.print(f"[yellow]{table_id}: preview failed ({exc}).[/yellow]") continue _print_preview(formatter, preview) if not yes: - confirmed = typer.confirm( - f"Create {len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}?" + _confirm_or_exit( + formatter, + f"Create {len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}?", ) - if not confirmed: - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) failed: list[str] = [] exit_codes: set[int] = set() error_codes: set[str] = set() for table_id in selected_tables: try: - result = service.create_policy( - alias=project, - table=table_id, - dialect=resolved_dialect, - rules=parsed_rules, - target_project_ids=target_project, - ) + result = service.create_policy(table=table_id, **kwargs) except (ConfigError, KeboolaApiError) as exc: formatter.warning(f"{table_id}: {exc}") failed.append(table_id) diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 180904ae..40815fe2 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -366,13 +366,16 @@ # operations (see `project.add` etc. above). `setup` performs the exact # same write as `create`, just via an interactive picker, so it must # carry the same class or a permission policy that allows `setup` but - # not `create` (or vice versa) would leave a hole. + # not `create` (or vice versa) would leave a hole. `delete` is + # `destructive`, like every other remote delete: removing a policy removes + # the row filter (or column allowlist) it enforced. `create --scope + # organization` is escalated to `destructive` too (FLAG_ESCALATIONS). "rls.list": "read", "rls.detail": "read", "rls.schema": "read", "rls.create": "admin", "rls.update": "admin", - "rls.delete": "admin", + "rls.delete": "destructive", "rls.setup": "admin", # Column-level security: same classes as rls.* for the same reason. "cls.list": "read", @@ -380,7 +383,7 @@ "cls.schema": "read", "cls.create": "admin", "cls.update": "admin", - "cls.delete": "admin", + "cls.delete": "destructive", # `scope` sub-app: visibility scope / target-project grants / elevation # requests (PSGO-140). Parent key at the least-privileged level (read), # same pattern as `reference-data` above. `set --scope organization` is @@ -502,6 +505,10 @@ "semantic-layer.import --scope organization": "destructive", "semantic-layer.promote --scope organization": "destructive", "semantic-layer.build --scope organization": "destructive", + # An organization-scope policy governs its table in EVERY project of the + # organization (one invalid policy refuses all of their queries). + "rls.create --scope organization": "destructive", + "cls.create --scope organization": "destructive", } # Operations that exist ONLY on the `kbagent serve` REST surface. They are real diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 62b51d94..4941e8b6 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -173,6 +173,7 @@ ("GET", "/workspaces"): "workspace list", ("GET", "/workspaces/{project}/{workspace_id}"): "workspace detail", ("PATCH", "/agents/{task_id}"): "agent update", + ("PATCH", "/cls/{project}/{policy_id}"): "cls update", ("PATCH", "/configs/{project}/{component_id}/{config_id}"): "config update", ("PATCH", "/configs/{project}/{component_id}/{config_id}/variables"): "config variables-set", ("PATCH", "/data-apps/{project}/{app_id}/secrets"): "data-app secrets-set", @@ -181,6 +182,7 @@ ("PATCH", "/flows/{project}/{config_id}"): "flow update", ("PATCH", "/projects/{alias}"): "project edit", ("PATCH", "/semantic-layer/items/{kind}/{name}"): "", + ("PATCH", "/rls/{project}/{policy_id}"): "rls update", ("PATCH", "/transformations/{project}/{config_id}"): "transformation edit", ("POST", "/agents"): "agent create", ("POST", "/agents/prompt/improve"): "agent prompt-improve", @@ -304,7 +306,6 @@ ("POST", "/workspaces/{project}/{workspace_id}/password"): "workspace password", ("POST", "/workspaces/{project}/{workspace_id}/query"): "workspace query", ("PUT", "/branches/{project}/metadata/{key}"): "branch metadata-set", - ("PUT", "/cls/{project}/{policy_id}"): "cls update", ("PUT", "/configs/{project}/{component_id}/{config_id}/metadata/{key}"): "config set-metadata", ("PUT", "/merge-requests/{project}/{merge_request_id}"): "merge-request update", ("PUT", "/merge-requests/{project}/{merge_request_id}/auto-merge"): "merge-request auto-merge", @@ -312,7 +313,6 @@ ("PUT", "/configs/{project}/{component_id}/{config_id}/variables"): "config variables-set", ("PUT", "/data-apps/{project}/{app_id}/secrets"): "data-app secrets-set", ("PUT", "/projects/{alias}/description"): "project description-set", - ("PUT", "/rls/{project}/{policy_id}"): "rls update", ("PUT", "/semantic-layer/items/{kind}/{name}"): "", ("PUT", "/semantic-layer/reference-data"): "semantic-layer reference-data set", ("PUT", "/semantic-layer/scope/{context_id}"): "semantic-layer scope set", diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index 1d316465..fd63f0da 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -361,8 +361,8 @@ "description": ( "**Development.** " "Row-level security policies (metastore-backed `rls-policy` " - "objects), always authored at `organization`/`targeted` scope -- " - "never `project` scope. Mirrors `kbagent rls list|detail|schema|" + "objects), authored at `targeted` scope unless `organization` is " + "asked for -- never `project` scope. Mirrors `kbagent rls list|detail|schema|" "create|update|delete` (`rls setup`'s interactive picker has no " "endpoint of its own, same carve-out as " "`auth register-projects`)." @@ -373,9 +373,9 @@ "description": ( "**Development.** " "Column-level security policies (metastore-backed `cls-policy` " - "objects: a per-principal `visible_columns` allowlist), always " - "authored at `organization`/`targeted` scope -- never `project` " - "scope. Mirrors `kbagent cls list|detail|schema|create|update|" + "objects: a per-principal `visible_columns` allowlist), authored " + "at `targeted` scope unless `organization` is asked for -- never " + "`project` scope. Mirrors `kbagent cls list|detail|schema|create|update|" "delete`." ), }, diff --git a/src/keboola_agent_cli/server/routers/cls.py b/src/keboola_agent_cli/server/routers/cls.py index 4eb83350..5fb2d2fa 100644 --- a/src/keboola_agent_cli/server/routers/cls.py +++ b/src/keboola_agent_cli/server/routers/cls.py @@ -1,134 +1,12 @@ """Column-level security endpoints -- the REST mirror of ``kbagent cls *``. -1:1 with the CLI group. Same shape and rules as the ``rls`` router (every -route permission-gated; ``GET /{project}/schema`` registered before -``GET /{project}/{policy_id}`` so it is never shadowed), over the -``cls-policy`` object type: each rule is ``{principal|principals, -visible_columns}``. +Built by :func:`.rls.build_policy_router`: same routes, permission gates and +bodies as the ``rls`` router, over the ``cls-policy`` object type (each rule +is ``{principal|principals, visible_columns}``, validated by the service). """ from __future__ import annotations -from typing import Any +from .rls import build_policy_router -from fastapi import APIRouter, Depends -from pydantic import BaseModel - -from ...errors import ErrorCode, KeboolaApiError -from ..dependencies import ServiceRegistry, get_registry, require_permission - -router = APIRouter(prefix="/cls", tags=["cls"]) - - -def _perm(operation: str) -> Any: - return Depends(require_permission(f"cls.{operation}")) - - -# -- Bodies -------------------------------------------------------------------------------------- - - -class ClsPolicyCreate(BaseModel): - table: str - dialect: str - rules: list[dict[str, Any]] - target_project_ids: list[int] | None = None - dry_run: bool = False - - -class ClsPolicyUpdate(BaseModel): - table: str | None = None - dialect: str | None = None - rules: list[dict[str, Any]] | None = None - target_project_ids: list[int] | None = None - dry_run: bool = False - - -# -- Reads ----------------------------------------------------------------------------------- - - -@router.get("/{project}", summary="List CLS policies", dependencies=[_perm("list")]) -def list_policies( - project: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """List column-level security policies visible to a project. Mirrors `kbagent cls list`.""" - return registry.cls.list_policies(project) - - -@router.get( - "/{project}/schema", - summary="Fetch the live cls-policy JSON Schema", - dependencies=[_perm("schema")], -) -def get_schema(project: str, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: - """Live schema from the metastore -- no offline bundled snapshot. Mirrors `kbagent cls schema`.""" - fetch = registry.cls.fetch_schema(project) - if fetch.schema is None: - raise KeboolaApiError( - message=f"Could not fetch the cls-policy schema: {fetch.reason}", - status_code=404, - error_code=ErrorCode.NOT_FOUND, - retryable=False, - ) - return {"format": "json-schema", "source": "live", "schema": fetch.schema} - - -@router.get("/{project}/{policy_id}", summary="Get one CLS policy", dependencies=[_perm("detail")]) -def get_policy( - project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Full rule set for one policy. Mirrors `kbagent cls detail`.""" - return registry.cls.get_policy(project, policy_id) - - -# -- Writes ---------------------------------------------------------------------------------- - - -@router.post("/{project}", summary="Create a CLS policy", dependencies=[_perm("create")]) -def create_policy( - project: str, body: ClsPolicyCreate, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Create one policy for one table -- always `organization`/`targeted` scope, - - never `project` scope (no such option exists on this body). Mirrors - `kbagent cls create`. - """ - return registry.cls.create_policy( - project, - table=body.table, - dialect=body.dialect, - rules=body.rules, - target_project_ids=body.target_project_ids, - dry_run=body.dry_run, - ) - - -@router.put("/{project}/{policy_id}", summary="Update a CLS policy", dependencies=[_perm("update")]) -def update_policy( - project: str, - policy_id: str, - body: ClsPolicyUpdate, - registry: ServiceRegistry = Depends(get_registry), -) -> dict[str, Any]: - """Fetch-then-merge: an omitted field keeps its current value, it is never - - silently blanked. Mirrors `kbagent cls update`. - """ - return registry.cls.update_policy( - project, - policy_id, - table=body.table, - dialect=body.dialect, - rules=body.rules, - target_project_ids=body.target_project_ids, - dry_run=body.dry_run, - ) - - -@router.delete( - "/{project}/{policy_id}", summary="Delete a CLS policy", dependencies=[_perm("delete")] -) -def delete_policy( - project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Delete a policy, un-protecting its columns. Mirrors `kbagent cls delete`.""" - return registry.cls.delete_policy(project, policy_id) +router = build_policy_router("cls", "CLS") diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py index 1be64e53..f2f4914e 100644 --- a/src/keboola_agent_cli/server/routers/rls.py +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -8,10 +8,15 @@ its write (create one policy per selected table) is the same ``POST /rls/{project}`` route below. +:func:`build_policy_router` also builds the ``cls`` router: the two groups +differ only in the service and the rule shape, which the service validates. + **Every route enforces the permission policy** (``Depends(require_permission)``) -- RLS is security-sensitive enough (admin-class writes) that CLI-gates-but-REST-doesn't would be a real hole, mirroring -``merge_requests.py`` rather than the (ungated) ``notifications.py``. +``merge_requests.py`` rather than the (ungated) ``notifications.py``. A +create at ``organization`` scope is also checked as +``.create --scope organization`` (destructive), like the CLI flag. The ``GET /{project}/schema`` route is registered before ``GET /{project}/{policy_id}`` so ``schema`` is never shadowed by the @@ -21,128 +26,153 @@ from __future__ import annotations -from typing import Any +from typing import Any, Literal from fastapi import APIRouter, Depends -from pydantic import BaseModel +from pydantic import BaseModel, model_validator from ...errors import ErrorCode, KeboolaApiError -from ..dependencies import ServiceRegistry, get_registry, require_permission - -router = APIRouter(prefix="/rls", tags=["rls"]) - - -def _perm(operation: str) -> Any: - return Depends(require_permission(f"rls.{operation}")) - +from ...permissions import PermissionEngine +from ..dependencies import ( + ServiceRegistry, + get_permission_engine, + get_registry, + require_permission, +) # -- Bodies -------------------------------------------------------------------------------------- -class RlsPolicyCreate(BaseModel): - table: str - dialect: str +class PolicyCreate(BaseModel): + """``target_projects`` takes aliases or project IDs, like the semantic-layer bodies.""" + + table_id: str rules: list[dict[str, Any]] - target_project_ids: list[int] | None = None + dialect: Literal["snowflake", "bigquery"] | None = None # None = the project backend + scope: Literal["targeted", "organization"] = "targeted" + target_projects: list[str | int] | None = None dry_run: bool = False + @model_validator(mode="after") + def _targets_need_targeted_scope(self) -> PolicyCreate: + if self.target_projects and self.scope != "targeted": + raise ValueError("target_projects requires scope='targeted'.") + return self + + +class PolicyUpdate(BaseModel): + """Partial update: an omitted field is unchanged; ``target_projects: []`` revokes every grant.""" -class RlsPolicyUpdate(BaseModel): - table: str | None = None - dialect: str | None = None + table_id: str | None = None + dialect: Literal["snowflake", "bigquery"] | None = None rules: list[dict[str, Any]] | None = None - target_project_ids: list[int] | None = None + target_projects: list[str | int] | None = None dry_run: bool = False -# -- Reads ----------------------------------------------------------------------------------- +def build_policy_router(group: Literal["rls", "cls"], label: str) -> APIRouter: + """The ``/rls`` or ``/cls`` router; ``label`` is "RLS"/"CLS" in summaries.""" + router = APIRouter(prefix=f"/{group}", tags=[group]) + def perm(operation: str) -> Any: + return Depends(require_permission(f"{group}.{operation}")) -@router.get("/{project}", summary="List RLS policies", dependencies=[_perm("list")]) -def list_policies( - project: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """List row-level security policies visible to a project. Mirrors `kbagent rls list`.""" - return registry.rls.list_policies(project) + def service(registry: ServiceRegistry) -> Any: + return getattr(registry, group) + @router.get("/{project}", summary=f"List {label} policies", dependencies=[perm("list")]) + def list_policies( + project: str, registry: ServiceRegistry = Depends(get_registry) + ) -> dict[str, Any]: + """Policies visible to a project. Mirrors `kbagent list`.""" + return service(registry).list_policies(project) -@router.get( - "/{project}/schema", - summary="Fetch the live rls-policy JSON Schema", - dependencies=[_perm("schema")], -) -def get_schema(project: str, registry: ServiceRegistry = Depends(get_registry)) -> dict[str, Any]: - """Live schema from the metastore -- no offline bundled snapshot. Mirrors `kbagent rls schema`.""" - fetch = registry.rls.fetch_schema(project) - if fetch.schema is None: - raise KeboolaApiError( - message=f"Could not fetch the rls-policy schema: {fetch.reason}", - status_code=404, - error_code=ErrorCode.NOT_FOUND, - retryable=False, + @router.get( + "/{project}/schema", + summary=f"Fetch the live {group}-policy JSON Schema", + dependencies=[perm("schema")], + ) + def get_schema( + project: str, registry: ServiceRegistry = Depends(get_registry) + ) -> dict[str, Any]: + """Live schema from the metastore -- no offline bundled snapshot.""" + fetch = service(registry).fetch_schema(project) + if fetch.schema is None: + raise KeboolaApiError( + message=f"Could not fetch the {group}-policy schema: {fetch.reason}", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + return {"format": "json-schema", "source": "live", "schema": fetch.schema} + + @router.get( + "/{project}/{policy_id}", summary=f"Get one {label} policy", dependencies=[perm("detail")] + ) + def get_policy( + project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) + ) -> dict[str, Any]: + """Full rule set for one policy. Mirrors `kbagent detail`.""" + return service(registry).get_policy(project, policy_id) + + @router.post("/{project}", summary=f"Create a {label} policy", dependencies=[perm("create")]) + def create_policy( + project: str, + body: PolicyCreate, + registry: ServiceRegistry = Depends(get_registry), + engine: PermissionEngine = Depends(get_permission_engine), + ) -> dict[str, Any]: + """Create one policy for one table at `targeted` (default) or `organization` scope. + + Never `project` scope (the schema does not support it). Mirrors `kbagent create`. + """ + if body.scope == "organization": + engine.check_or_raise(f"{group}.create --scope organization") + return service(registry).create_policy( + project, + table=body.table_id, + rules=body.rules, + dialect=body.dialect, + scope=body.scope, + target_projects=body.target_projects, + dry_run=body.dry_run, ) - return {"format": "json-schema", "source": "live", "schema": fetch.schema} - -@router.get("/{project}/{policy_id}", summary="Get one RLS policy", dependencies=[_perm("detail")]) -def get_policy( - project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Full rule set for one policy. Mirrors `kbagent rls detail`.""" - return registry.rls.get_policy(project, policy_id) - - -# -- Writes ---------------------------------------------------------------------------------- - - -@router.post("/{project}", summary="Create an RLS policy", dependencies=[_perm("create")]) -def create_policy( - project: str, body: RlsPolicyCreate, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Create one policy for one table -- always `organization`/`targeted` scope, - - never `project` scope (no such option exists on this body). Mirrors - `kbagent rls create`. - """ - return registry.rls.create_policy( - project, - table=body.table, - dialect=body.dialect, - rules=body.rules, - target_project_ids=body.target_project_ids, - dry_run=body.dry_run, + @router.patch( + "/{project}/{policy_id}", summary=f"Update a {label} policy", dependencies=[perm("update")] ) + def update_policy( + project: str, + policy_id: str, + body: PolicyUpdate, + registry: ServiceRegistry = Depends(get_registry), + ) -> dict[str, Any]: + """Partial update: an omitted field keeps its current value. Mirrors `kbagent update`.""" + return service(registry).update_policy( + project, + policy_id, + table=body.table_id, + dialect=body.dialect, + rules=body.rules, + target_projects=body.target_projects, + dry_run=body.dry_run, + ) - -@router.put( - "/{project}/{policy_id}", summary="Update an RLS policy", dependencies=[_perm("update")] -) -def update_policy( - project: str, - policy_id: str, - body: RlsPolicyUpdate, - registry: ServiceRegistry = Depends(get_registry), -) -> dict[str, Any]: - """Fetch-then-merge: an omitted field keeps its current value, it is never - - silently blanked. Mirrors `kbagent rls update`. - """ - return registry.rls.update_policy( - project, - policy_id, - table=body.table, - dialect=body.dialect, - rules=body.rules, - target_project_ids=body.target_project_ids, - dry_run=body.dry_run, + @router.delete( + "/{project}/{policy_id}", + summary=f"Delete a {label} policy", + dependencies=[perm("delete")], ) + def delete_policy( + project: str, + policy_id: str, + dry_run: bool = False, + registry: ServiceRegistry = Depends(get_registry), + ) -> dict[str, Any]: + """Delete a policy (``?dry_run=true`` only shows it). Mirrors `kbagent delete`.""" + return service(registry).delete_policy(project, policy_id, dry_run=dry_run) + return router -@router.delete( - "/{project}/{policy_id}", summary="Delete an RLS policy", dependencies=[_perm("delete")] -) -def delete_policy( - project: str, policy_id: str, registry: ServiceRegistry = Depends(get_registry) -) -> dict[str, Any]: - """Delete a policy, un-protecting its table. Mirrors `kbagent rls delete`.""" - return registry.rls.delete_policy(project, policy_id) + +router = build_policy_router("rls", "RLS") diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 7c57d4b2..86e25274 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -6,12 +6,23 @@ to one object type instead of seven. The one structural rule every write path in this module enforces: an -``rls-policy`` object is **never** created or updated at plain ``project`` -scope. Authorship is centralized at ``organization`` (default) or -``targeted`` scope only -- this command surface does not even have a -``--scope`` flag that could select ``project``, matching the backend's own -restriction (RFC "Scope is organization or targeted -- never plain -project, by design, not merely by convention"). +``rls-policy`` object is **never** created at plain ``project`` scope (the +metastore schema supports only ``organization`` and ``targeted``). The +default is ``targeted`` -- the schema's own default: the policy governs the +owning project plus the target projects it is granted to. ``organization`` +governs the table in EVERY project of the organization, so it is only ever +an explicit choice (RFC: "org-wide scope is a deliberate choice an admin +makes, not a default"). + +Who may write (metastore schema ACL): a project admin (master token with the +admin role) may create/update/delete ``targeted`` policies of its own project +WITHOUT target projects; granting target projects and ``organization`` scope +need the organization-admin role. + +The enforcement (keboola-mcp-server ``query_data``) refuses EVERY query of a +project when one policy it loads is invalid -- a dialect other than the +project backend, or one principal with two rules on one table. Both are +refused here before any write. On a stack whose metastore predates the ``rls-policy`` schema, a real project answers with a schema-fetch/list/get/post failure (see ``fetch_schema`` and the @@ -35,18 +46,18 @@ ObjectScope, SemanticType, fetch_resolved_schema, - project_ids_as_ints, ) from ..models import ProjectConfig from . import _rls_condition from ._rls_condition import RLS_DIALECTS +from ._semantic_layer_scope import resolve_target_project_ids from .base import BaseService, ClientFactory, make_session_aware_client_factory logger = logging.getLogger(__name__) RLS_ITEM_TYPE: SemanticType = "rls-policy" -__all__ = ["RLS_DIALECTS", "RLS_ITEM_TYPE", "RlsSchemaFetch", "RlsService"] +__all__ = ["POLICY_SCOPES", "RLS_DIALECTS", "RLS_ITEM_TYPE", "RlsSchemaFetch", "RlsService"] MetastoreClientFactory = Callable[[str, str], MetastoreClient] @@ -61,6 +72,18 @@ ) +# The scopes a policy may be authored at (the schema's `x-metastore.scope.supported`). +POLICY_SCOPES: tuple[ObjectScope, ...] = ("targeted", "organization") + + +@dataclass(frozen=True) +class PolicyValidation: + """Outcome of :meth:`RlsService._validate_policy`: blocking ``errors`` and non-blocking ``warnings``.""" + + errors: list[str] + warnings: list[str] + + @dataclass(frozen=True) class RlsSchemaFetch: """Result of fetching the live ``rls-policy`` JSON Schema. @@ -177,39 +200,92 @@ def _row_from_item(item: dict[str, Any]) -> dict[str, Any]: "dialect": attrs.get("dialect", ""), "rule_count": len(attrs.get("rules") or []), "scope": meta.get("scope", ""), + # The owning project; the metastore omits it for an organization-scope policy. + "owner_project_id": meta.get("projectId"), "source_project_id": meta.get("sourceProjectId"), "target_project_ids": meta.get("targetProjectIds") or [], } - def _resolve_scope(self, target_project_ids: list[int] | None) -> ObjectScope: - """Never ``"project"`` -- ``targeted`` iff target projects were given.""" - return "targeted" if target_project_ids else "organization" + def _project_dialect(self, project: ProjectConfig) -> str: + """The project's backend -- the only dialect the enforcement accepts for its policies.""" + with self._client_factory(project.stack_url, project.token) as client: + backend = (client.verify_token().default_backend or "").lower() + if backend not in RLS_DIALECTS: + self._raise_invalid( + [ + f"the project backend is {backend or 'unknown'!r}; policies support {RLS_DIALECTS} only" + ] + ) + return backend + + def _resolve_targets(self, alias: str, targets: Sequence[int | str] | None) -> list[int]: + """``--target-project`` values (alias or project ID, comma lists) -> unique project IDs.""" + return resolve_target_project_ids( + self._config_store, alias, [str(t) for t in targets or []] + ) + + def _taken_principals( + self, client: MetastoreClient, table: str, dialect: str, exclude_id: str | None + ) -> dict[str, str]: + """Principals other policies on the same table already have a rule for (case-folded). + + Best effort: the metastore lists only the policies this token may read -- for a project admin, + those its project owns -- so a clash with a policy owned elsewhere is not visible here. + """ + key = _rls_condition.table_key(table, dialect) + taken: dict[str, str] = {} + for item in client.list_items(self.item_type): + attrs = item.get("attributes") or {} + if item.get("id") == exclude_id or not isinstance(attrs.get("table"), str): + continue + if _rls_condition.table_key(attrs["table"], dialect) != key: + continue + for rule in attrs.get("rules") or []: + if isinstance(rule, dict): + for name in _rls_condition.rule_principals(rule): + taken.setdefault(str(name).casefold(), f"policy {item.get('id')}") + return taken def _validate_policy( - self, *, table: str, dialect: str, rules: list[dict[str, Any]], project: ProjectConfig - ) -> tuple[list[str], list[str]]: - """Run every validation this module can, local checks first; returns ``(errors, warnings)``. - - Local (schema-independent) checks always run. Structural (Draft7, - against the live-fetched schema) checks run too when the fetch - succeeds -- when it doesn't, the caller gets a warning (the reduced - validation is reported, not hidden), not a block (see :class:`RlsSchemaFetch`). + self, + client: MetastoreClient, + project: ProjectConfig, + *, + policy: dict[str, Any], + backend: str, + exclude_id: str | None = None, + ) -> PolicyValidation: + """Run every check this module can on the WHOLE policy, local checks first. + + Local (schema-independent) checks always run: the dialect matches the project backend, the + rules are well-formed and no principal has two rules on the table (within the policy or across + the other visible policies). Structural (Draft7, against the live schema) checks run when the + fetch succeeds -- when it doesn't, the caller gets a warning, not a block (see + :class:`RlsSchemaFetch`). The whole policy is validated even for a partial update, because the + metastore's PATCH validates only the keys it receives. """ + table, dialect, rules = policy["table"], policy["dialect"], policy["rules"] errors: list[str] = [] - if dialect not in _rls_condition.RLS_DIALECTS: - errors.append(f"dialect must be one of {_rls_condition.RLS_DIALECTS}, got {dialect!r}") + if dialect != backend: + errors.append( + f"dialect {dialect!r} does not match the project backend {backend!r} " + "(the enforcement refuses every query of a project with such a policy)" + ) errors.extend(self._local_rule_errors(rules)) if errors: - return errors, [] + return PolicyValidation(errors, []) + errors.extend( + _rls_condition.duplicate_principals( + rules, self._taken_principals(client, table, dialect, exclude_id) + ) + ) + if errors: + return PolicyValidation(errors, []) - warnings: list[str] = [] fetch = self._fetch_schema_for_project(project) - if fetch.schema: - policy_body = {"table": table, "dialect": dialect, "rules": rules} - errors.extend(_rls_condition.validate_policy_structural(policy_body, fetch.schema)) - else: - warnings.append(f"Live schema validation was skipped: {fetch.reason}") - return errors, warnings + if not fetch.schema: + return PolicyValidation([], [f"Live schema validation was skipped: {fetch.reason}"]) + return PolicyValidation(_rls_condition.validate_policy_structural(policy, fetch.schema), []) def _local_rule_errors(self, rules: Any) -> list[str]: """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" @@ -240,22 +316,43 @@ def _raise_invalid(self, errors: list[str]) -> None: retryable=False, ) + @staticmethod + def _raise_usage(message: str) -> None: + raise KeboolaApiError( + message=message, status_code=400, error_code=ErrorCode.INVALID_ARGUMENT, retryable=False + ) + + def _already_exists(self, table: str, exc: KeboolaApiError) -> KeboolaApiError: + """The client's generic 409 message talks about semantic models; policies are named by table.""" + return KeboolaApiError( + message=( + f"An {self.label} policy for table {table!r} already exists in this project. " + f"Change it with `{self.label.lower()} update --policy-id ...`, " + f"or `{self.label.lower()} delete` it first." + ), + status_code=exc.status_code, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) + # ------------------------------------------------------------------ # Read # ------------------------------------------------------------------ def list_policies(self, alias: str) -> dict[str, Any]: - """List every ``rls-policy`` object visible to ``alias``'s project.""" + """List every policy object visible to ``alias``'s project.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: raw = client.list_items(self.item_type) return {"project": alias, "policies": [self._row_from_item(item) for item in raw]} def get_policy(self, alias: str, policy_id: str) -> dict[str, Any]: - """Fetch one ``rls-policy`` object's full attributes + meta.""" + """Fetch one policy object's full attributes + meta.""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: - item = client.get_item(self.item_type, policy_id) + return self._detail_row(client.get_item(self.item_type, policy_id)) + + def _detail_row(self, item: dict[str, Any]) -> dict[str, Any]: row = self._row_from_item(item) row["rules"] = (item.get("attributes") or {}).get("rules", []) row["revision"] = (item.get("meta") or {}).get("revision") @@ -270,55 +367,60 @@ def create_policy( alias: str, *, table: str, - dialect: str, rules: list[dict[str, Any]], - target_project_ids: Sequence[int | str] | None = None, + dialect: str | None = None, + scope: str = "targeted", + target_projects: Sequence[int | str] | None = None, dry_run: bool = False, ) -> dict[str, Any]: - """Create one ``rls-policy`` object -- always at ``organization``/``targeted`` scope. + """Create one policy -- at ``targeted`` scope (default) or, explicitly, ``organization``. - ``dry_run=True`` returns the same preview shape a real write would - produce (compiled condition previews per rule) without calling the - metastore's write endpoint at all -- only the (read-only) schema - fetch used for validation happens. + ``dialect`` defaults to the project backend and must equal it. ``target_projects`` (aliases or + project IDs) only with ``targeted``; they go in the create request itself, which stores the + grants in the same transaction (and needs the organization-admin role). ``dry_run=True`` + returns the preview without calling the write endpoint. """ - target_ids = project_ids_as_ints(target_project_ids or []) # also validates for --dry-run + if scope not in POLICY_SCOPES: + self._raise_usage(f"scope must be one of {POLICY_SCOPES}, got {scope!r}") + if target_projects and scope != "targeted": + self._raise_usage("target projects require scope 'targeted'") + target_ids = self._resolve_targets(alias, target_projects) project = self._resolve_one_project(alias) - errors, warnings = self._validate_policy( - table=table, dialect=dialect, rules=rules, project=project - ) - if errors: - self._raise_invalid(errors) - - scope = self._resolve_scope(target_ids) - preview = self._preview_rules(rules, dialect) - if dry_run: - return self._with_warnings( - { - "project": alias, - "table": table, - "dialect": dialect, - "scope": scope, - "target_project_ids": target_ids, - "preview": preview, - "dry_run": True, - }, - warnings, - ) + backend = self._project_dialect(project) + policy = {"table": table, "dialect": dialect or backend, "rules": rules} with self._new_metastore_client(project) as client: - created = client.post_item( - self.item_type, - name=table, - data={"table": table, "dialect": dialect, "rules": rules}, - scope=scope, - target_project_ids=target_ids or None, - ) - if target_ids: - client.put_target_projects(self.item_type, created.get("id", ""), target_ids) + check = self._validate_policy(client, project, policy=policy, backend=backend) + if check.errors: + self._raise_invalid(check.errors) + preview = self._preview_rules(rules, policy["dialect"]) + if dry_run: + return self._with_warnings( + { + "project": alias, + **policy, + "scope": scope, + "target_project_ids": target_ids, + "preview": preview, + "dry_run": True, + }, + check.warnings, + ) + try: + created = client.post_item( + self.item_type, + name=table, + data=policy, + scope="organization" if scope == "organization" else "targeted", + target_project_ids=target_ids or None, + ) + except KeboolaApiError as exc: + if exc.error_code == ErrorCode.ALREADY_EXISTS: + raise self._already_exists(table, exc) from exc + raise row = self._row_from_item(created) row["preview"] = preview - return self._with_warnings(row, warnings) + return self._with_warnings(row, check.warnings) def update_policy( self, @@ -328,95 +430,97 @@ def update_policy( table: str | None = None, dialect: str | None = None, rules: list[dict[str, Any]] | None = None, - target_project_ids: Sequence[int | str] | None = None, + target_projects: Sequence[int | str] | None = None, dry_run: bool = False, ) -> dict[str, Any]: - """Update one policy object. + """Update one policy: only the given fields change. - ``put_item`` is a whole-record replace -- fetch the current item - first and merge only the given overrides onto it, so an ``update`` - call that only changes ``rules`` never silently wipes ``table`` or - ``dialect`` (the failure mode this repo's own ``merge-request - resolve`` docs warn about for exactly this kind of PUT-based API). + Reads the policy to validate the MERGED result (the metastore's PATCH validates only the keys + it receives) and for the ``--dry-run`` preview, then sends a ``PATCH`` with only the changed + keys -- unknown keys and concurrent changes to other keys survive. - ``target_project_ids``: ``None`` keeps the current grants, a non-empty list replaces them, and - ``[]`` revokes them all (through the dedicated grants call). A policy keeps its current scope - when its grants are revoked -- scope changes are not done through ``PUT``. + ``target_projects``: ``None`` keeps the grants, a list replaces them, ``[]`` revokes them all. + Grants change first (an organization-admin-only call): when that is refused, nothing is written. + The result is re-read after the write, so it shows the grants as they now are. """ + changes = { + key: value + for key, value in (("table", table), ("dialect", dialect), ("rules", rules)) + if value is not None + } + if not changes and target_projects is None: + self._raise_usage("nothing to update: pass a table, dialect, rules or target projects") target_ids = ( - project_ids_as_ints(target_project_ids) if target_project_ids is not None else None - ) # also validates for --dry-run - project = self._resolve_one_project(alias) - with self._new_metastore_client(project) as client: - current = client.get_item(self.item_type, policy_id) - attrs = current.get("attributes") or {} - meta = current.get("meta") or {} - - merged_table = table if table is not None else attrs.get("table", "") - merged_dialect = dialect if dialect is not None else attrs.get("dialect", "") - merged_rules = rules if rules is not None else attrs.get("rules", []) - merged_targets = ( - target_ids - if target_ids is not None - else project_ids_as_ints(meta.get("targetProjectIds") or []) - ) - - errors, warnings = self._validate_policy( - table=merged_table, dialect=merged_dialect, rules=merged_rules, project=project + self._resolve_targets(alias, target_projects) if target_projects is not None else None ) - if errors: - self._raise_invalid(errors) - - revoking = target_ids == [] and bool(meta.get("targetProjectIds")) - # A plain PUT never changes scope, and grants only exist on `targeted` policies, so the scope is - # always the current one: a `targeted` policy whose grants were cleared stays `targeted` with an - # empty list, and an `organization` policy cannot be narrowed to specific projects. - scope = meta.get("scope") or self._resolve_scope(merged_targets) - if target_ids and scope != "targeted": - self._raise_invalid( - [ - ( - f"the policy has {scope!r} scope, which cannot be narrowed to target projects; " - "delete it and create a new one with --target-project" - ) - ] - ) - preview = self._preview_rules(merged_rules, merged_dialect) - if dry_run: - return self._with_warnings( - { - "project": alias, - "policy_id": policy_id, - "table": merged_table, - "dialect": merged_dialect, - "scope": scope, - "target_project_ids": merged_targets or [], - "preview": preview, - "dry_run": True, - }, - warnings, - ) + project = self._resolve_one_project(alias) + backend = self._project_dialect(project) with self._new_metastore_client(project) as client: - updated = client.put_item( - self.item_type, - policy_id, - name=merged_table, - data={"table": merged_table, "dialect": merged_dialect, "rules": merged_rules}, + current = client.get_item(self.item_type, policy_id) + attrs = current.get("attributes") or {} + meta = current.get("meta") or {} + scope = meta.get("scope", "") + if target_ids is not None and scope != "targeted": + self._raise_invalid( + [ + ( + f"the policy has {scope!r} scope, which has no target projects " + "(organization scope already applies everywhere and cannot be narrowed)" + ) + ] + ) + policy = { + "table": attrs.get("table", ""), + "dialect": attrs.get("dialect", ""), + "rules": attrs.get("rules", []), + **changes, + } + check = self._validate_policy( + client, project, policy=policy, backend=backend, exclude_id=policy_id ) - if revoking: - client.put_target_projects(self.item_type, policy_id, []) - elif target_ids: - # Only an explicit list replaces the grants; an update that omits the option must not - # re-send the snapshot read above (it could overwrite a concurrent grant change). + if check.errors: + self._raise_invalid(check.errors) + preview = self._preview_rules(policy["rules"], policy["dialect"]) + if dry_run: + return self._with_warnings( + { + "project": alias, + "policy_id": policy_id, + **policy, + "scope": scope, + "target_project_ids": ( + target_ids + if target_ids is not None + else meta.get("targetProjectIds") or [] + ), + "preview": preview, + "dry_run": True, + }, + check.warnings, + ) + if target_ids is not None: client.put_target_projects(self.item_type, policy_id, target_ids) - row = self._row_from_item(updated) + if changes: + try: + client.patch_item(self.item_type, policy_id, name=table, data=changes) + except KeboolaApiError as exc: + if exc.error_code == ErrorCode.ALREADY_EXISTS: + raise self._already_exists(policy["table"], exc) from exc + raise + row = self._detail_row(client.get_item(self.item_type, policy_id)) row["preview"] = preview - return self._with_warnings(row, warnings) + return self._with_warnings(row, check.warnings) - def delete_policy(self, alias: str, policy_id: str) -> dict[str, Any]: - """Delete one ``rls-policy`` object by id.""" + def delete_policy(self, alias: str, policy_id: str, *, dry_run: bool = False) -> dict[str, Any]: + """Delete one policy by id; ``dry_run`` shows what would be deleted (one read, no write).""" project = self._resolve_one_project(alias) with self._new_metastore_client(project) as client: + if dry_run: + return { + "project": alias, + "policy": self._detail_row(client.get_item(self.item_type, policy_id)), + "dry_run": True, + } client.delete_item(self.item_type, policy_id) return {"project": alias, "policy_id": policy_id, "deleted": True} diff --git a/tests/test_cls_cli.py b/tests/test_cls_cli.py index 5de54030..1aaf272c 100644 --- a/tests/test_cls_cli.py +++ b/tests/test_cls_cli.py @@ -27,7 +27,7 @@ "create", "--project", "prod", - "--table", + "--table-id", "in.c-crm.invoices", "--dialect", "snowflake", @@ -237,7 +237,7 @@ def test_yes_skips_prompt_and_passes_target_projects(self, tmp_path: Path) -> No assert result.exit_code == 0, result.output service.create_policy.assert_called_once() - assert service.create_policy.call_args.kwargs["target_project_ids"] == ["7", "8"] + assert service.create_policy.call_args.kwargs["target_projects"] == ["7", "8"] def test_invalid_dialect_exits_2_before_any_call(self, tmp_path: Path) -> None: service = MagicMock() @@ -413,11 +413,11 @@ def test_yes_deletes(self, tmp_path: Path) -> None: ) assert "Deleted CLS policy p-1" in result.output - service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1") + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1", dry_run=False) -def test_permission_check_denies_admin_writes_but_not_reads(tmp_path: Path) -> None: - """`cls.*` is registered in OPERATION_REGISTRY: create/update/delete=admin, reads=read.""" +def test_permission_classes_of_the_cls_operations(tmp_path: Path) -> None: + """`cls.*` in OPERATION_REGISTRY: create/update=admin, delete=destructive, reads=read.""" from keboola_agent_cli.permissions import OPERATION_REGISTRY assert {op: OPERATION_REGISTRY[f"cls.{op}"] for op in ("list", "detail", "schema")} == { @@ -425,7 +425,8 @@ def test_permission_check_denies_admin_writes_but_not_reads(tmp_path: Path) -> N "detail": "read", "schema": "read", } - assert all(OPERATION_REGISTRY[f"cls.{op}"] == "admin" for op in ("create", "update", "delete")) + assert all(OPERATION_REGISTRY[f"cls.{op}"] == "admin" for op in ("create", "update")) + assert OPERATION_REGISTRY["cls.delete"] == "destructive" class TestClsReviewFixes: @@ -460,7 +461,7 @@ def test_update_clear_target_projects_passes_an_empty_list(self, tmp_path: Path) ) assert result.exit_code == 0, result.output - assert service.update_policy.call_args.kwargs["target_project_ids"] == [] + assert service.update_policy.call_args.kwargs["target_projects"] == [] def test_update_clear_and_target_project_are_mutually_exclusive(self, tmp_path: Path) -> None: service = MagicMock() @@ -489,3 +490,54 @@ def test_create_prints_the_validation_warning_in_human_mode(self, tmp_path: Path assert result.exit_code == 0, result.output assert "Live schema validation was skipped" in result.output + + +class TestDestructiveGates: + """`--scope organization` and `delete` are destructive-class: `--deny-destructive` blocks them.""" + + @pytest.mark.parametrize(("extra", "exit_code"), [([], 0), (["--scope", "organization"], 6)]) + def test_deny_destructive_blocks_only_organization_scope( + self, tmp_path: Path, extra: list[str], exit_code: int + ) -> None: + service = MagicMock() + service.create_policy.return_value = {**_row(), "preview": _preview()["preview"]} + + result = _run(["--json", "--deny-destructive", *CREATE_ARGS, *extra], tmp_path, service) + + assert result.exit_code == exit_code, result.output + assert service.create_policy.called is (exit_code == 0) + + def test_deny_destructive_blocks_delete(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + [ + "--json", + "--deny-destructive", + "cls", + "delete", + "--project", + "prod", + "--policy-id", + "p-1", + ], + tmp_path, + service, + ) + + assert result.exit_code == 6 + service.delete_policy.assert_not_called() + + def test_delete_dry_run_needs_no_confirm(self, tmp_path: Path) -> None: + service = MagicMock() + service.delete_policy.return_value = {"project": "prod", "policy": _row(), "dry_run": True} + + result = _run( + ["cls", "delete", "--project", "prod", "--policy-id", "p-1", "--dry-run"], + tmp_path, + service, + ) + + assert result.exit_code == 0, result.output + assert "Would delete" in result.output + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1", dry_run=True) diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index 35d4d605..f347af9a 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -19,6 +19,8 @@ from keboola_agent_cli.models import ProjectConfig from keboola_agent_cli.services.cls_service import ClsService +from .test_rls_service import storage_client_factory + TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" _RULES = [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] @@ -42,7 +44,13 @@ def _make_service(tmp_path: Path) -> tuple[ClsService, MagicMock]: mock.get_schema.side_effect = KeboolaApiError( message="no schema", status_code=404, error_code=ErrorCode.NOT_FOUND ) - return ClsService(config_store=store, metastore_client_factory=lambda url, token: mock), mock + mock.list_items.return_value = [] + service = ClsService( + config_store=store, + client_factory=storage_client_factory(), + metastore_client_factory=lambda url, token: mock, + ) + return service, mock def _item(rules: list[dict[str, Any]] | None = None, **meta: Any) -> dict[str, Any]: @@ -103,11 +111,13 @@ def test_accepts_principals_list(self, tmp_path: Path) -> None: {"principal": ["a@x.com", "b@x.com"], "visible_columns": ["id"]} ] - def test_bad_dialect_is_rejected_before_any_write(self, tmp_path: Path) -> None: + def test_a_dialect_other_than_the_backend_is_rejected_before_any_write( + self, tmp_path: Path + ) -> None: service, mock = _make_service(tmp_path) - with pytest.raises(KeboolaApiError, match="dialect must be one of"): - service.create_policy("prod", table="in.c-crm.t", dialect="oracle", rules=_RULES) + with pytest.raises(KeboolaApiError, match="does not match the project backend"): + service.create_policy("prod", table="in.c-crm.t", dialect="bigquery", rules=_RULES) mock.post_item.assert_not_called() @@ -193,71 +203,66 @@ def test_detail_returns_visible_columns(self, tmp_path: Path) -> None: class TestCreate: - def test_writes_cls_policy_at_organization_scope(self, tmp_path: Path) -> None: + def test_writes_cls_policy_at_targeted_scope_by_default(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) - mock.post_item.return_value = _item() + mock.post_item.return_value = _item(scope="targeted") - result = service.create_policy( - "prod", table="in.c-crm.invoices", dialect="snowflake", rules=_RULES - ) + result = service.create_policy("prod", table="in.c-crm.invoices", rules=_RULES) mock.post_item.assert_called_once_with( "cls-policy", name="in.c-crm.invoices", data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": _RULES}, - scope="organization", + scope="targeted", target_project_ids=None, ) - mock.put_target_projects.assert_not_called() assert result["preview"] == [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] - def test_target_projects_make_it_targeted_and_grant(self, tmp_path: Path) -> None: + def test_target_projects_are_granted_in_the_create_request(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) mock.post_item.return_value = _item(scope="targeted") service.create_policy( - "prod", - table="in.c-crm.invoices", - dialect="bigquery", - rules=_RULES, - target_project_ids=["7", "8"], + "prod", table="in.c-crm.invoices", rules=_RULES, target_projects=["7,8"] ) - assert mock.post_item.call_args.kwargs["scope"] == "targeted" - mock.put_target_projects.assert_called_once_with("cls-policy", "p-1", [7, 8]) + assert mock.post_item.call_args.kwargs["target_project_ids"] == [7, 8] + mock.put_target_projects.assert_not_called() + + def test_duplicate_principals_are_rejected(self, tmp_path: Path) -> None: + service, mock = _make_service(tmp_path) + rules = [*_RULES, {"principal": "A@x.com", "visible_columns": ["id"]}] + + with pytest.raises(KeboolaApiError, match="already has a rule"): + service.create_policy("prod", table="in.c-crm.t", rules=rules) + + mock.post_item.assert_not_called() def test_dry_run_never_writes(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES, dry_run=True - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES, dry_run=True) assert result["dry_run"] is True - assert result["scope"] == "organization" + assert result["scope"] == "targeted" mock.post_item.assert_not_called() class TestUpdate: - def test_merges_only_the_given_fields(self, tmp_path: Path) -> None: + def test_patches_only_the_given_fields(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) mock.get_item.return_value = _item() - mock.put_item.return_value = _item() new_rules = [{"principal": "b@x.com", "visible_columns": ["id"]}] service.update_policy("prod", "p-1", rules=new_rules) - mock.put_item.assert_called_once_with( - "cls-policy", - "p-1", - name="in.c-crm.invoices", - data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": new_rules}, + mock.patch_item.assert_called_once_with( + "cls-policy", "p-1", name=None, data={"rules": new_rules} ) def test_keeps_existing_targets_and_scope(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) mock.get_item.return_value = _item(scope="targeted", targetProjectIds=["7"]) - mock.put_item.return_value = _item(scope="targeted") service.update_policy("prod", "p-1", table="in.c-crm.other") @@ -267,11 +272,11 @@ def test_dry_run_never_writes(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) mock.get_item.return_value = _item() - result = service.update_policy("prod", "p-1", dialect="bigquery", dry_run=True) + result = service.update_policy("prod", "p-1", table="in.c-crm.other", dry_run=True) assert result["dry_run"] is True - assert result["dialect"] == "bigquery" - mock.put_item.assert_not_called() + assert result["table"] == "in.c-crm.other" + mock.patch_item.assert_not_called() def test_invalid_merged_rules_are_rejected(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) @@ -281,7 +286,7 @@ def test_invalid_merged_rules_are_rejected(self, tmp_path: Path) -> None: service.update_policy("prod", "p-1", rules=[{"principal": "a@x.com"}]) assert exc.value.error_code == ErrorCode.INVALID_CLS_POLICY - mock.put_item.assert_not_called() + mock.patch_item.assert_not_called() def test_delete_uses_cls_item_type(tmp_path: Path) -> None: diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 15d9ba57..3b364635 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -207,7 +207,7 @@ def test_json_output_no_confirm_needed(self, tmp_path: Path) -> None: "create", "--project", "prod", - "--table", + "--table-id", "in.c-crm.invoices", "--dialect", "snowflake", @@ -232,7 +232,7 @@ def test_invalid_dialect_exit_2_before_service_call(self, tmp_path: Path) -> Non "create", "--project", "prod", - "--table", + "--table-id", "t", "--dialect", "postgres", @@ -257,7 +257,7 @@ def test_invalid_rules_json_exit_2(self, tmp_path: Path) -> None: "create", "--project", "prod", - "--table", + "--table-id", "t", "--dialect", "snowflake", @@ -282,7 +282,7 @@ def test_dry_run_uses_dry_run_flag_and_needs_no_confirm(self, tmp_path: Path) -> "create", "--project", "prod", - "--table", + "--table-id", "t", "--dialect", "snowflake", @@ -309,7 +309,7 @@ def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: "create", "--project", "prod", - "--table", + "--table-id", "t", "--dialect", "snowflake", @@ -340,7 +340,7 @@ def test_config_error_exit_5(self, tmp_path: Path) -> None: "create", "--project", "prod", - "--table", + "--table-id", "t", "--dialect", "snowflake", @@ -374,7 +374,7 @@ def test_json_output(self, tmp_path: Path) -> None: "prod", "--policy-id", "p-1", - "--table", + "--table-id", "in.c-crm.invoices", ], store, @@ -407,7 +407,7 @@ def test_yes_flag_skips_confirm(self, tmp_path: Path) -> None: ) assert result.exit_code == 0, result.output - service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1") + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1", dry_run=False) def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) @@ -539,7 +539,7 @@ def test_update_clear_target_projects_passes_an_empty_list(self, tmp_path: Path) ) assert result.exit_code == 0, result.output - assert service.update_policy.call_args.kwargs["target_project_ids"] == [] + assert service.update_policy.call_args.kwargs["target_projects"] == [] def test_update_clear_and_target_project_are_mutually_exclusive(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) @@ -568,7 +568,7 @@ def test_update_without_the_flag_keeps_grants_unchanged(self, tmp_path: Path) -> service, ) - assert service.update_policy.call_args.kwargs["target_project_ids"] is None + assert service.update_policy.call_args.kwargs["target_projects"] is None def test_create_prints_the_validation_warning_in_human_mode(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) @@ -581,7 +581,7 @@ def test_create_prints_the_validation_warning_in_human_mode(self, tmp_path: Path result = _run( [ - "rls", "create", "--project", "prod", "--table", "in.c-crm.invoices", "--dialect", "snowflake", + "rls", "create", "--project", "prod", "--table-id", "in.c-crm.invoices", "--dialect", "snowflake", "--rules", '[{"principal":"a@x.com","condition":{"true":true}}]', "--yes", ], store, @@ -799,3 +799,119 @@ def test_all_tables_created_exits_zero(self, tmp_path: Path) -> None: ) assert result.exit_code == 0, result.output + + +class TestScopeAndDestructiveGates: + """`--scope organization` and `delete` are destructive-class: `--deny-destructive` blocks them.""" + + RULES = '[{"principal":"a@x.com","condition":{"true":true}}]' + + @pytest.mark.parametrize(("extra", "exit_code"), [([], 0), (["--scope", "organization"], 6)]) + def test_deny_destructive_blocks_only_organization_scope( + self, tmp_path: Path, extra: list[str], exit_code: int + ) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + [ + "--json", "--deny-destructive", "rls", "create", "--project", "prod", + "--table-id", "in.c-crm.t", "--rules", self.RULES, *extra, + ], + store, + service, + ) # fmt: skip + + assert result.exit_code == exit_code, result.output + assert service.create_policy.called is (exit_code == 0) + + def test_scope_is_passed_to_the_service(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + + _run( + [ + "--json", "rls", "create", "--project", "prod", "--table-id", "in.c-crm.t", + "--rules", self.RULES, "--scope", "organization", + ], + store, + service, + ) # fmt: skip + + kwargs = service.create_policy.call_args.kwargs + assert kwargs["scope"] == "organization" + assert kwargs["dialect"] is None # omitted: the service uses the project backend + + def test_deny_destructive_blocks_delete(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + [ + "--json", + "--deny-destructive", + "rls", + "delete", + "--project", + "prod", + "--policy-id", + "p-1", + ], + store, + service, + ) + + assert result.exit_code == 6 + service.delete_policy.assert_not_called() + + def test_delete_dry_run_needs_no_confirm_and_deletes_nothing(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.delete_policy.return_value = { + "project": "prod", + "policy": _policy_row(), + "dry_run": True, + } + + result = _run( + ["rls", "delete", "--project", "prod", "--policy-id", "p-1", "--dry-run"], + store, + service, + ) + + assert result.exit_code == 0, result.output + assert "Would delete" in result.output + service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1", dry_run=True) + + def test_a_service_usage_error_exits_2(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.side_effect = KeboolaApiError( + message="--target-project 'ghost' is neither ...", + status_code=400, + error_code=ErrorCode.INVALID_ARGUMENT, + ) + + result = _run( + [ + "--json", "rls", "create", "--project", "prod", "--table-id", "t.x", + "--rules", self.RULES, "--target-project", "ghost", + ], + store, + service, + ) # fmt: skip + + assert result.exit_code == 2 + + def test_setup_json_prints_a_json_error_envelope(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + + result = _run(["--json", "rls", "setup", "--project", "prod"], store, MagicMock()) + + assert result.exit_code == 2 + envelope = json.loads(result.output) + assert envelope["status"] == "error" + assert envelope["error"]["code"] == "INVALID_ARGUMENT" + assert "rls create" in envelope["error"]["message"] diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index eb989bb4..03cdc835 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -54,10 +54,24 @@ def _make_service( mock = metastore_mock or MagicMock() mock.__enter__ = MagicMock(return_value=mock) mock.__exit__ = MagicMock(return_value=False) - service = RlsService(config_store=store, metastore_client_factory=lambda url, token: mock) + service = RlsService( + config_store=store, + client_factory=storage_client_factory(), + metastore_client_factory=lambda url, token: mock, + ) + mock.list_items.return_value = [] return service, mock +def storage_client_factory(backend: str = "snowflake") -> Any: + """A Storage client factory whose token belongs to a project on ``backend``.""" + storage = MagicMock() + storage.__enter__ = MagicMock(return_value=storage) + storage.__exit__ = MagicMock(return_value=False) + storage.verify_token.return_value.default_backend = backend + return lambda url, token: storage + + def _policy_item( item_id: str = "p-1", table: str = "in.c-crm.invoices", @@ -208,9 +222,7 @@ def test_a_non_empty_but_malformed_schema_degrades_instead_of_raising( assert fetch.schema is None assert "malformed" in (fetch.reason or "") - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES) assert any("malformed" in w for w in result["warnings"]) mock.post_item.assert_called_once() @@ -241,76 +253,119 @@ def test_empty_schema_degrades(self, tmp_path: Path) -> None: class TestCreatePolicy: - def test_default_scope_is_organization_never_project(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + def test_default_scope_is_targeted_with_no_grants(self, tmp_path: Path) -> None: + """The schema's own default: only the owning project is governed until grants are added.""" + service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.post_item.return_value = _policy_item() + mock.post_item.return_value = _policy_item(scope="targeted") - service.create_policy("prod", table="in.c-crm.invoices", dialect="snowflake", rules=_RULES) + service.create_policy("prod", table="in.c-crm.invoices", rules=_RULES) - _, kwargs = mock.post_item.call_args - assert kwargs["scope"] == "organization" - assert kwargs["scope"] != "project" + kwargs = mock.post_item.call_args.kwargs + assert kwargs["scope"] == "targeted" + assert kwargs["target_project_ids"] is None + assert kwargs["data"]["dialect"] == "snowflake" # defaulted from the project backend mock.put_target_projects.assert_not_called() - def test_target_projects_use_targeted_scope_and_manage_grants(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + def test_organization_scope_only_when_asked_for(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.post_item.return_value = _policy_item(scope="targeted", target_project_ids=["999"]) - - service.create_policy( - "prod", - table="in.c-crm.invoices", - dialect="snowflake", - rules=_RULES, - target_project_ids=["999"], - ) + mock.post_item.return_value = _policy_item() - _, kwargs = mock.post_item.call_args - assert kwargs["scope"] == "targeted" - assert kwargs["target_project_ids"] == [999] - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", [999]) + service.create_policy("prod", table="t.x", rules=_RULES, scope="organization") - def test_invalid_dialect_rejected_before_any_write(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + assert mock.post_item.call_args.kwargs["scope"] == "organization" - with pytest.raises(KeboolaApiError) as excinfo: - service.create_policy("prod", table="t", dialect="postgres", rules=_RULES) + @pytest.mark.parametrize( + ("kwargs", "fragment"), + [ + ({"scope": "project"}, "scope must be one of"), + ({"scope": "organization", "target_projects": ["7"]}, "require scope 'targeted'"), + ], + ) + def test_bad_scope_combinations_are_usage_errors( + self, tmp_path: Path, kwargs: dict, fragment: str + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(KeboolaApiError, match=fragment) as excinfo: + service.create_policy("prod", table="t.x", rules=_RULES, **kwargs) + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.post_item.assert_not_called() + + def test_target_projects_go_in_the_create_request_only(self, tmp_path: Path) -> None: + """The POST stores the grants; a second grants call could fail after the policy exists.""" + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.return_value = _policy_item(scope="targeted", target_project_ids=[999]) + + service.create_policy("prod", table="t.x", rules=_RULES, target_projects=["999", "999"]) + + assert mock.post_item.call_args.kwargs["target_project_ids"] == [999] # de-duplicated + mock.put_target_projects.assert_not_called() + + @pytest.mark.parametrize( + ("backend", "dialect"), [("snowflake", "bigquery"), ("bigquery", "snowflake")] + ) + def test_a_dialect_other_than_the_backend_is_refused( + self, tmp_path: Path, backend: str, dialect: str + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + service._client_factory = storage_client_factory(backend) + + with pytest.raises(KeboolaApiError, match="does not match the project backend") as excinfo: + service.create_policy("prod", table="t.x", dialect=dialect, rules=_RULES) assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY mock.post_item.assert_not_called() - mock.get_schema.assert_not_called() - def test_unknown_condition_op_rejected_before_any_write(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - bad_rules = [ - {"principal": "a@x.com", "condition": {"column": "x", "op": "bogus", "value": 1}} - ] + def test_a_backend_without_policy_support_is_refused(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + service._client_factory = storage_client_factory("synapse") - with pytest.raises(KeboolaApiError): - service.create_policy("prod", table="t", dialect="snowflake", rules=bad_rules) + with pytest.raises(KeboolaApiError, match="synapse"): + service.create_policy("prod", table="t.x", rules=_RULES) mock.post_item.assert_not_called() - def test_rule_with_both_principal_and_principals_rejected(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - bad_rules = [ - {"principal": "a@x.com", "principals": ["b@x.com"], "condition": {"true": True}} - ] + def test_a_duplicate_name_gets_a_policy_message_not_the_semantic_one( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.side_effect = KeboolaApiError( + message="already exists in the target model", + status_code=409, + error_code=ErrorCode.ALREADY_EXISTS, + ) + + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t.x", rules=_RULES) + + assert excinfo.value.error_code == ErrorCode.ALREADY_EXISTS + assert "rls update --policy-id" in excinfo.value.message + assert "model" not in excinfo.value.message + + @pytest.mark.parametrize( + "bad_rules", + [ + [{"principal": "a@x.com", "condition": {"column": "x", "op": "bogus", "value": 1}}], + [{"principal": "a@x.com", "principals": ["b@x.com"], "condition": {"true": True}}], + ], + ) + def test_malformed_rules_are_rejected_before_any_write( + self, tmp_path: Path, bad_rules: list + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) with pytest.raises(KeboolaApiError): - service.create_policy("prod", table="t", dialect="snowflake", rules=bad_rules) + service.create_policy("prod", table="t.x", rules=bad_rules) mock.post_item.assert_not_called() def test_structural_validation_runs_when_schema_available(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + service, mock = _make_service(_make_store(tmp_path)) # A schema that requires a `description` field the candidate body lacks. mock.get_schema.return_value = { "type": "object", @@ -318,23 +373,62 @@ def test_structural_validation_runs_when_schema_available(self, tmp_path: Path) } with pytest.raises(KeboolaApiError) as excinfo: - service.create_policy("prod", table="t", dialect="snowflake", rules=_RULES) + service.create_policy("prod", table="t.x", rules=_RULES) assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY mock.post_item.assert_not_called() def test_dry_run_never_calls_post_item(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - result = service.create_policy( - "prod", table="t", dialect="snowflake", rules=_RULES, dry_run=True - ) + result = service.create_policy("prod", table="t.x", rules=_RULES, dry_run=True) mock.post_item.assert_not_called() assert result["dry_run"] is True - assert result["preview"][0]["condition"] == "region = 'EU'" + assert result["scope"] == "targeted" + assert result["preview"][0]["condition"] == "\"region\" = 'EU'" + + +class TestDuplicatePrincipals: + """One principal with two rules on one table makes the enforcement refuse every query.""" + + def test_within_one_policy_case_insensitively(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + rules = [ + {"principal": "a@x.com", "condition": {"true": True}}, + {"principals": ["b@x.com", "A@X.com"], "condition": {"true": True}}, + ] + + with pytest.raises(KeboolaApiError, match=r"'A@X.com' already has a rule in rules\[0\]"): + service.create_policy("prod", table="t.x", rules=rules, dry_run=True) + + mock.post_item.assert_not_called() + + def test_across_policies_on_the_same_table(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + # Snowflake table keys compare case-insensitively. + mock.list_items.return_value = [_policy_item(item_id="other", table="IN.C-CRM.T")] + + with pytest.raises(KeboolaApiError, match="already has a rule in policy other"): + service.create_policy("prod", table="in.c-crm.t", rules=_RULES) + + mock.post_item.assert_not_called() + + def test_policies_on_other_tables_and_the_policy_itself_do_not_count( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.list_items.return_value = [ + _policy_item(item_id="other", table="in.c-crm.other"), + _policy_item(item_id="p-1", table="in.c-crm.invoices"), + ] + mock.get_item.return_value = _policy_item(item_id="p-1") + + service.update_policy("prod", "p-1", rules=_RULES) + + mock.patch_item.assert_called_once() # --------------------------------------------------------------------------- @@ -343,55 +437,156 @@ def test_dry_run_never_calls_post_item(self, tmp_path: Path) -> None: class TestUpdatePolicy: - def test_partial_override_preserves_untouched_fields(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + def test_patches_only_the_changed_keys_and_validates_the_merged_policy( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) mock.get_item.return_value = _policy_item(table="in.c-crm.invoices", dialect="snowflake") - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.put_item.return_value = _policy_item() + mock.get_schema.return_value = {"type": "object", "required": ["table", "dialect", "rules"]} service.update_policy("prod", "p-1", rules=_RULES) - _, kwargs = mock.put_item.call_args - assert kwargs["data"]["table"] == "in.c-crm.invoices" # unchanged, not wiped - assert kwargs["data"]["dialect"] == "snowflake" # unchanged, not wiped - assert kwargs["data"]["rules"] == _RULES # the one field we changed + mock.patch_item.assert_called_once_with( + "rls-policy", "p-1", name=None, data={"rules": _RULES} + ) + mock.put_item.assert_not_called() + + def test_a_new_table_renames_the_policy(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _policy_item() + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + + service.update_policy("prod", "p-1", table="in.c-crm.new") + + mock.patch_item.assert_called_once_with( + "rls-policy", "p-1", name="in.c-crm.new", data={"table": "in.c-crm.new"} + ) + + def test_nothing_to_update_is_a_usage_error(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + with pytest.raises(KeboolaApiError) as excinfo: + service.update_policy("prod", "p-1") + + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.get_item.assert_not_called() + + def test_the_result_is_re_read_after_the_write(self, tmp_path: Path) -> None: + """The grants change after the PATCH response was built -- report what is stored now.""" + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + before = _policy_item(scope="targeted", target_project_ids=[7]) + after = _policy_item(scope="targeted", target_project_ids=[8]) + mock.get_item.side_effect = [before, after] + + result = service.update_policy("prod", "p-1", target_projects=["8"]) + + assert result["target_project_ids"] == [8] + + def test_grants_change_before_the_rules(self, tmp_path: Path) -> None: + """Granting is organization-admin only: when it is refused, the rules stay unwritten.""" + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = _policy_item(scope="targeted") + mock.put_target_projects.side_effect = KeboolaApiError( + message="Insufficient permissions", status_code=403, error_code=ErrorCode.ACCESS_DENIED + ) + + with pytest.raises(KeboolaApiError): + service.update_policy("prod", "p-1", rules=_RULES, target_projects=["8"]) + + mock.patch_item.assert_not_called() def test_an_organization_policy_cannot_be_narrowed_to_target_projects( self, tmp_path: Path ) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _policy_item(scope="organization", target_project_ids=[]) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _policy_item(scope="organization") with pytest.raises(KeboolaApiError) as excinfo: - service.update_policy("prod", "p-1", target_project_ids=["42"]) + service.update_policy("prod", "p-1", target_projects=["42"]) assert excinfo.value.error_code == ErrorCode.INVALID_RLS_POLICY - mock.put_item.assert_not_called() + mock.patch_item.assert_not_called() mock.put_target_projects.assert_not_called() - def test_invalid_override_rejected_before_write(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + def test_a_dialect_other_than_the_backend_is_refused(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) mock.get_item.return_value = _policy_item() - with pytest.raises(KeboolaApiError): - service.update_policy("prod", "p-1", dialect="postgres") + with pytest.raises(KeboolaApiError, match="does not match the project backend"): + service.update_policy("prod", "p-1", dialect="bigquery") - mock.put_item.assert_not_called() + mock.patch_item.assert_not_called() - def test_dry_run_never_calls_put_item(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) - mock.get_item.return_value = _policy_item() + def test_dry_run_never_writes(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _policy_item(scope="targeted", target_project_ids=[7]) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) result = service.update_policy("prod", "p-1", rules=_RULES, dry_run=True) - mock.put_item.assert_not_called() + mock.patch_item.assert_not_called() + mock.put_target_projects.assert_not_called() assert result["dry_run"] is True + assert result["target_project_ids"] == [7] + + +class TestGrants: + """`target_projects`: None keeps the grants, a list replaces them, [] revokes them all.""" + + @pytest.mark.parametrize( + ("targets", "expected_call"), + [(None, None), ([], []), (["7", "8,9"], [7, 8, 9])], + ) + def test_target_projects_semantics( + self, tmp_path: Path, targets: list | None, expected_call: list | None + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.get_item.return_value = _policy_item(scope="targeted", target_project_ids=[7]) + + service.update_policy("prod", "p-1", rules=_RULES, target_projects=targets) + + if expected_call is None: + mock.put_target_projects.assert_not_called() + else: + mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", expected_call) + + def test_a_registered_alias_resolves_to_its_project_id(self, tmp_path: Path) -> None: + store = _make_store(tmp_path) + store.add_project( + "shared", + ProjectConfig( + stack_url="https://connection.keboola.com", token=TEST_TOKEN, project_id=77 + ), + ) + service, mock = _make_service(store) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + + result = service.create_policy( + "prod", table="t.x", rules=_RULES, target_projects=["shared"], dry_run=True + ) + + assert result["target_project_ids"] == [77] + + @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-3"], ["1", "x"]]) + def test_a_bad_target_is_rejected_before_any_network_call( + self, tmp_path: Path, bad: list + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + + for call in ( + lambda: service.create_policy( + "prod", table="t.x", rules=_RULES, target_projects=bad, dry_run=True + ), + lambda: service.update_policy("prod", "p-1", target_projects=bad, dry_run=True), + ): + with pytest.raises(KeboolaApiError) as excinfo: + call() + assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT + mock.get_item.assert_not_called() + mock.post_item.assert_not_called() # --------------------------------------------------------------------------- @@ -401,14 +596,23 @@ def test_dry_run_never_calls_put_item(self, tmp_path: Path) -> None: class TestDeletePolicy: def test_calls_delete_item(self, tmp_path: Path) -> None: - store = _make_store(tmp_path) - service, mock = _make_service(store) + service, mock = _make_service(_make_store(tmp_path)) result = service.delete_policy("prod", "p-1") mock.delete_item.assert_called_once_with("rls-policy", "p-1") assert result == {"project": "prod", "policy_id": "p-1", "deleted": True} + def test_dry_run_shows_the_policy_and_deletes_nothing(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_item.return_value = _policy_item() + + result = service.delete_policy("prod", "p-1", dry_run=True) + + mock.delete_item.assert_not_called() + assert result["dry_run"] is True + assert result["policy"]["id"] == "p-1" + # --------------------------------------------------------------------------- # Pure condition helpers @@ -425,61 +629,64 @@ def test_non_true_sentinel_is_never_rendered_as_always_true(self, bad: dict) -> compile_condition_preview(bad, "snowflake") @pytest.mark.parametrize( - "op,sql", + ("condition", "dialect", "sql"), [ - ("eq", "region = 'EU'"), - ("ne", "region != 'EU'"), - ("gt", "region > 'EU'"), - ("gte", "region >= 'EU'"), - ("lt", "region < 'EU'"), - ("lte", "region <= 'EU'"), + *( + ( + {"column": "region", "op": op, "value": "EU"}, + "snowflake", + f"\"region\" {sql} 'EU'", + ) + for op, sql in [ + ("eq", "="), + ("ne", "!="), + ("gt", ">"), + ("gte", ">="), + ("lt", "<"), + ("lte", "<="), + ] + ), + # Quoted per dialect, like the enforcement (so a column-case mismatch is visible). + ({"column": "Region", "op": "eq", "value": "EU"}, "bigquery", "`Region` = 'EU'"), + ( + {"column": "region", "op": "in", "values": ["EU", "US"]}, + "snowflake", + "\"region\" IN ('EU', 'US')", + ), + ( + {"column": "region", "op": "not_in", "values": ["EU"]}, + "snowflake", + "\"region\" NOT IN ('EU')", + ), + ({"column": "d", "op": "is_null"}, "snowflake", '"d" IS NULL'), + ({"column": "d", "op": "is_not_null"}, "snowflake", '"d" IS NOT NULL'), + ({"column": "amount", "op": "gt", "value": 100}, "snowflake", '"amount" > 100'), + ({"column": "active", "op": "eq", "value": True}, "snowflake", '"active" = TRUE'), + ({"column": "o", "op": "eq", "value": "O'Brien"}, "snowflake", "\"o\" = 'O''Brien'"), + ( + { + "and": [ + {"column": "r", "op": "eq", "value": "EU"}, + {"column": "s", "op": "ne", "value": "x"}, + ] + }, + "snowflake", + "(\"r\" = 'EU') AND (\"s\" != 'x')", + ), + ( + { + "or": [ + {"column": "r", "op": "eq", "value": "EU"}, + {"column": "r", "op": "eq", "value": "US"}, + ] + }, + "snowflake", + "(\"r\" = 'EU') OR (\"r\" = 'US')", + ), ], ) - def test_comparison_ops(self, op: str, sql: str) -> None: - condition = {"column": "region", "op": op, "value": "EU"} - assert compile_condition_preview(condition, "snowflake") == sql - - def test_in_op(self) -> None: - condition = {"column": "region", "op": "in", "values": ["EU", "US"]} - assert compile_condition_preview(condition, "snowflake") == "region IN ('EU', 'US')" - - def test_not_in_op(self) -> None: - condition = {"column": "region", "op": "not_in", "values": ["EU"]} - assert compile_condition_preview(condition, "snowflake") == "region NOT IN ('EU')" - - def test_is_null(self) -> None: - condition = {"column": "deleted_at", "op": "is_null"} - assert compile_condition_preview(condition, "snowflake") == "deleted_at IS NULL" - - def test_is_not_null(self) -> None: - condition = {"column": "deleted_at", "op": "is_not_null"} - assert compile_condition_preview(condition, "snowflake") == "deleted_at IS NOT NULL" - - def test_and_nesting(self) -> None: - condition = { - "and": [ - {"column": "region", "op": "eq", "value": "EU"}, - {"column": "status", "op": "ne", "value": "draft"}, - ] - } - assert compile_condition_preview(condition, "snowflake") == ( - "(region = 'EU') AND (status != 'draft')" - ) - - def test_or_nesting(self) -> None: - condition = { - "or": [ - {"column": "region", "op": "eq", "value": "EU"}, - {"column": "region", "op": "eq", "value": "US"}, - ] - } - assert compile_condition_preview(condition, "snowflake") == ( - "(region = 'EU') OR (region = 'US')" - ) - - def test_numeric_and_null_literals_not_quoted(self) -> None: - condition = {"column": "amount", "op": "gt", "value": 100} - assert compile_condition_preview(condition, "snowflake") == "amount > 100" + def test_renders_like_the_enforcement(self, condition: dict, dialect: str, sql: str) -> None: + assert compile_condition_preview(condition, dialect) == sql def test_unrecognized_shape_raises(self) -> None: with pytest.raises(ValueError): @@ -588,9 +795,7 @@ def test_a_skipped_live_validation_is_reported_not_hidden(self, tmp_path: Path) mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) mock.post_item.return_value = _policy_item() - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES) assert result["warnings"] == ["Live schema validation was skipped: no schema"] @@ -598,9 +803,7 @@ def test_dry_run_reports_it_too(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES, dry_run=True - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES, dry_run=True) assert result["warnings"] == ["Live schema validation was skipped: no schema"] @@ -609,9 +812,7 @@ def test_no_warning_when_the_schema_was_available(self, tmp_path: Path) -> None: mock.get_schema.return_value = {"type": "object"} mock.post_item.return_value = _policy_item() - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES) assert "warnings" not in result @@ -619,59 +820,12 @@ def test_update_reports_it_as_well(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="no schema", status_code=404) mock.get_item.return_value = _policy_item() - mock.put_item.return_value = _policy_item() - result = service.update_policy("prod", "p-1", dialect="bigquery") + result = service.update_policy("prod", "p-1", rules=_RULES) assert result["warnings"] == ["Live schema validation was skipped: no schema"] -class TestClearingGrants: - """`target_project_ids`: None keeps the grants, a list replaces them, [] revokes them all.""" - - def _targeted(self) -> dict[str, Any]: - return _policy_item(scope="targeted", target_project_ids=["7", "8"]) - - def test_empty_list_revokes_every_grant_and_keeps_the_scope(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = self._targeted() - mock.put_item.return_value = self._targeted() - - result = service.update_policy("prod", "p-1", target_project_ids=[]) - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) - assert result["id"] == "p-1" - - def test_none_keeps_the_existing_grants(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = self._targeted() - mock.put_item.return_value = self._targeted() - - service.update_policy("prod", "p-1", dialect="bigquery") - mock.put_target_projects.assert_not_called() # the omitted option re-sends nothing - - def test_clearing_a_policy_that_has_no_grants_makes_no_grant_call(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = _policy_item() # organization scope, no targets - mock.put_item.return_value = _policy_item() - - service.update_policy("prod", "p-1", target_project_ids=[]) - mock.put_target_projects.assert_not_called() - - def test_dry_run_previews_the_cleared_state_without_writing(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = self._targeted() - - result = service.update_policy("prod", "p-1", target_project_ids=[], dry_run=True) - - assert result["target_project_ids"] == [] - mock.put_item.assert_not_called() - mock.put_target_projects.assert_not_called() - - class TestPrimitiveShape: """The shape of a leaf condition is checked locally, so it fails as INVALID_RLS_POLICY even when the live schema is unavailable (the documented state on stacks without the object type).""" @@ -687,6 +841,9 @@ class TestPrimitiveShape: ({"column": "a", "op": "in"}, "non-empty list 'values'"), ({"column": "a", "op": "not_in", "values": []}, "non-empty list 'values'"), ({"column": "a", "op": "in", "values": "abc"}, "non-empty list 'values'"), + # `col = NULL` / `IN (NULL)` match no row: the principal would silently see nothing. + ({"column": "a", "op": "eq", "value": None}, "use op 'is_null'"), + ({"column": "a", "op": "in", "values": [1, None]}, "cannot list null"), ], ) def test_malformed_primitives_are_rejected(self, condition: dict, fragment: str) -> None: @@ -697,7 +854,6 @@ def test_malformed_primitives_are_rejected(self, condition: dict, fragment: str) "condition", [ {"column": "a", "op": "eq", "value": 0}, - {"column": "a", "op": "eq", "value": None}, # an explicit null value is still a value {"column": "a", "op": "in", "values": [1]}, {"column": "a", "op": "is_null"}, { @@ -723,97 +879,6 @@ def test_a_malformed_rule_never_reaches_the_preview_or_the_write(self, tmp_path: mock.post_item.assert_not_called() -class TestScopeIsKeptWhenTargetsAreOmitted: - def _targeted_without_grants(self) -> dict[str, Any]: - """What the metastore returns after every grant was revoked: still `targeted`, empty list.""" - return _policy_item(scope="targeted", target_project_ids=[]) - - def test_an_unrelated_update_does_not_turn_a_cleared_targeted_policy_into_organization( - self, tmp_path: Path - ) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = self._targeted_without_grants() - mock.put_item.return_value = self._targeted_without_grants() - - service.update_policy("prod", "p-1", dialect="bigquery") # no target option at all - mock.put_target_projects.assert_not_called() - - def test_an_explicit_target_list_replaces_the_grants_of_a_targeted_policy( - self, tmp_path: Path - ) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = self._targeted_without_grants() - mock.put_item.return_value = _policy_item(scope="targeted", target_project_ids=[7]) - - service.update_policy("prod", "p-1", target_project_ids=["7"]) - - assert "scope" not in mock.put_item.call_args.kwargs # a PUT never carries scope - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", [7]) - - def test_clearing_keeps_the_current_scope(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.get_item.return_value = _policy_item(scope="targeted", target_project_ids=["7"]) - mock.put_item.return_value = self._targeted_without_grants() - - service.update_policy("prod", "p-1", target_project_ids=[]) - mock.put_target_projects.assert_called_once_with("rls-policy", "p-1", []) - - -class TestDryRunValidatesTargetIds: - """A preview must run the same validation as the write -- including the target project ids.""" - - @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-3"], ["1", "x"]]) - def test_create_dry_run_rejects_a_bad_target_before_any_network_call( - self, tmp_path: Path, bad: list - ) -> None: - service, mock = _make_service(_make_store(tmp_path)) - - with pytest.raises(KeboolaApiError) as excinfo: - service.create_policy( - "prod", - table="t", - dialect="snowflake", - rules=_RULES, - target_project_ids=bad, - dry_run=True, - ) - - assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - mock.get_schema.assert_not_called() - mock.post_item.assert_not_called() - - def test_update_dry_run_rejects_a_bad_target_before_any_network_call( - self, tmp_path: Path - ) -> None: - service, mock = _make_service(_make_store(tmp_path)) - - with pytest.raises(KeboolaApiError) as excinfo: - service.update_policy("prod", "p-1", target_project_ids=["abc"], dry_run=True) - - assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - mock.get_item.assert_not_called() - mock.put_item.assert_not_called() - - def test_valid_targets_still_preview(self, tmp_path: Path) -> None: - service, mock = _make_service(_make_store(tmp_path)) - mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - - result = service.create_policy( - "prod", - table="t", - dialect="snowflake", - rules=_RULES, - target_project_ids=["7", "8"], - dry_run=True, - ) - - assert result["scope"] == "targeted" - assert result["target_project_ids"] == [7, 8] - - class TestCompositionAndKeysAreExact: """One composition key, and no keys beyond an op's own shape: nothing may be validated and then dropped.""" @@ -945,7 +1010,5 @@ def test_a_version_listing_that_cannot_be_resolved_is_unavailable_not_a_schema( assert fetch.schema is None assert "version listing" in (fetch.reason or "") - result = service.create_policy( - "prod", table="in.c-crm.t", dialect="snowflake", rules=_RULES - ) + result = service.create_policy("prod", table="in.c-crm.t", rules=_RULES) assert any("version listing" in w for w in result["warnings"]) diff --git a/tests/test_server_cls.py b/tests/test_server_cls.py index b8ba7a9d..86233028 100644 --- a/tests/test_server_cls.py +++ b/tests/test_server_cls.py @@ -1,7 +1,8 @@ """Tests for `server/routers/cls.py` -- router -> service call parity + permission gating. Mirrors ``test_server_rls.py``: kwarg parity via a mocked ``ServiceRegistry`` -and persisted-policy -> 403. CLS writes are ``admin``, like RLS. +and persisted-policy -> 403. Same classes as RLS: create/update ``admin``, +delete ``destructive``. """ from __future__ import annotations @@ -90,20 +91,20 @@ def test_create_and_update_pass_kwargs(tmp_path: Path) -> None: svc.create_policy.return_value = {"id": POLICY_ID} svc.update_policy.return_value = {"id": POLICY_ID} body = { - "table": "in.c-crm.invoices", + "table_id": "in.c-crm.invoices", "dialect": "snowflake", "rules": RULES, - "target_project_ids": [999], + "target_projects": [999], } with TestClient(_app(tmp_path, svc)) as client: assert client.post(f"/cls/{PROJECT}", headers=AUTH, json=body).status_code == 200 - res = client.put(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"rules": RULES}) + res = client.patch(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"rules": RULES}) assert res.status_code == 200, res.text create_kwargs = svc.create_policy.call_args.kwargs assert create_kwargs["rules"] == RULES - assert create_kwargs["target_project_ids"] == [999] + assert create_kwargs["target_projects"] == [999] assert create_kwargs["dry_run"] is False args, update_kwargs = svc.update_policy.call_args assert args == (PROJECT, POLICY_ID) @@ -119,7 +120,7 @@ def test_delete_calls_service(tmp_path: Path) -> None: res = client.delete(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH) assert res.status_code == 200, res.text - svc.delete_policy.assert_called_once_with(PROJECT, POLICY_ID) + svc.delete_policy.assert_called_once_with(PROJECT, POLICY_ID, dry_run=False) def test_service_error_maps_to_http_status(tmp_path: Path) -> None: @@ -135,25 +136,33 @@ def test_service_error_maps_to_http_status(tmp_path: Path) -> None: class TestPermissionGating: - @pytest.mark.parametrize("deny", ["cli:admin", "cli:write"]) - def test_denied_policy_blocks_every_write(self, tmp_path: Path, deny: str) -> None: + @pytest.mark.parametrize( + ("deny", "blocked"), + [ + ("cli:admin", {"create", "update"}), + ("cli:destructive", {"delete"}), + ("cli:write", {"create", "update", "delete"}), # spans write+destructive+admin + ], + ) + def test_each_write_is_gated_by_its_class( + self, tmp_path: Path, deny: str, blocked: set[str] + ) -> None: _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=[deny])) svc = MagicMock() - body = {"table": "t", "dialect": "snowflake", "rules": RULES} + for method in ("create_policy", "update_policy", "delete_policy"): + getattr(svc, method).return_value = {"id": POLICY_ID} + body = {"table_id": "t.x", "rules": RULES} with TestClient(_app(tmp_path, svc)) as client: - responses = [ - client.post(f"/cls/{PROJECT}", headers=AUTH, json=body), - client.put(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table": "t"}), - client.delete(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH), - ] - - for res in responses: - assert res.status_code == 403, res.text - assert res.json()["error"]["code"] == "PERMISSION_DENIED" - svc.create_policy.assert_not_called() - svc.update_policy.assert_not_called() - svc.delete_policy.assert_not_called() + statuses = { + "create": client.post(f"/cls/{PROJECT}", headers=AUTH, json=body).status_code, + "update": client.patch( + f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table_id": "t.y"} + ).status_code, + "delete": client.delete(f"/cls/{PROJECT}/{POLICY_ID}", headers=AUTH).status_code, + } + + assert statuses == {op: 403 if op in blocked else 200 for op in statuses} def test_denying_cli_admin_still_allows_reads(self, tmp_path: Path) -> None: _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) diff --git a/tests/test_server_rls.py b/tests/test_server_rls.py index 385417d6..3d0b72e8 100644 --- a/tests/test_server_rls.py +++ b/tests/test_server_rls.py @@ -2,10 +2,9 @@ Two families, mirroring ``test_server_router_calls.py`` (kwarg parity via a mocked ``ServiceRegistry``) and ``test_server_permissions.py`` (persisted -policy -> 403). RLS is classified ``admin`` for every write -- these tests -pin that ``cli:admin`` (not merely ``cli:write``) is what has to be denied -to block ``create``/``update``/``delete``, and that ``list``/``detail``/ -``schema`` stay reachable under that same denial. +policy -> 403). RLS ``create``/``update`` are ``admin``, ``delete`` and a +create at ``organization`` scope are ``destructive``; ``list``/``detail``/ +``schema`` stay reachable under either denial. """ from __future__ import annotations @@ -129,10 +128,10 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: rls_svc.create_policy.return_value = {"id": POLICY_ID} app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) body = { - "table": "in.c-crm.invoices", + "table_id": "in.c-crm.invoices", "dialect": "snowflake", "rules": [{"principal": "a@x.com", "condition": {"true": True}}], - "target_project_ids": [999], + "target_projects": [999, "analytics"], "dry_run": False, } @@ -144,7 +143,8 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: assert kwargs["table"] == "in.c-crm.invoices" assert kwargs["dialect"] == "snowflake" assert kwargs["rules"] == body["rules"] - assert kwargs["target_project_ids"] == [999] + assert kwargs["scope"] == "targeted" # the default + assert kwargs["target_projects"] == [999, "analytics"] assert kwargs["dry_run"] is False @@ -154,13 +154,42 @@ def test_update_policy_passes_kwargs(tmp_path: Path) -> None: app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) with TestClient(app) as client: - res = client.put(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table": "new.table"}) + res = client.patch( + f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table_id": "new.table"} + ) assert res.status_code == 200, res.text args, kwargs = rls_svc.update_policy.call_args assert args == (PROJECT, POLICY_ID) assert kwargs["table"] == "new.table" assert kwargs["rules"] is None + assert kwargs["target_projects"] is None + + +def test_update_is_patch_not_put(tmp_path: Path) -> None: + app = _make_app_with_registry(tmp_path, _mock_registry(rls=MagicMock())) + + with TestClient(app) as client: + res = client.put(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH, json={"table_id": "x"}) + + assert res.status_code == 405, res.text + + +def test_target_projects_with_organization_scope_is_422(tmp_path: Path) -> None: + rls_svc = MagicMock() + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + body = { + "table_id": "t.x", + "rules": [{"principal": "a@x.com", "condition": {"true": True}}], + "scope": "organization", + "target_projects": [7], + } + + with TestClient(app) as client: + res = client.post(f"/rls/{PROJECT}", headers=AUTH, json=body) + + assert res.status_code == 422, res.text + rls_svc.create_policy.assert_not_called() def test_delete_policy_calls_service(tmp_path: Path) -> None: @@ -170,9 +199,12 @@ def test_delete_policy_calls_service(tmp_path: Path) -> None: with TestClient(app) as client: res = client.delete(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) + dry = client.delete(f"/rls/{PROJECT}/{POLICY_ID}?dry_run=true", headers=AUTH) assert res.status_code == 200, res.text - rls_svc.delete_policy.assert_called_once_with(PROJECT, POLICY_ID) + assert dry.status_code == 200, dry.text + assert [c.kwargs["dry_run"] for c in rls_svc.delete_policy.call_args_list] == [False, True] + assert rls_svc.delete_policy.call_args.args == (PROJECT, POLICY_ID) def test_keboola_api_error_from_service_maps_to_http_status(tmp_path: Path) -> None: @@ -189,7 +221,7 @@ def test_keboola_api_error_from_service_maps_to_http_status(tmp_path: Path) -> N # --------------------------------------------------------------------------- -# Permission gating -- RLS writes are `admin`, not merely `write` +# Permission gating -- create/update `admin`; delete and organization scope `destructive` # --------------------------------------------------------------------------- @@ -204,7 +236,7 @@ def test_denying_cli_admin_blocks_create(self, tmp_path: Path) -> None: f"/rls/{PROJECT}", headers=AUTH, json={ - "table": "t", + "table_id": "t", "dialect": "snowflake", "rules": [{"principal": "a@x.com", "condition": {"true": True}}], }, @@ -214,16 +246,38 @@ def test_denying_cli_admin_blocks_create(self, tmp_path: Path) -> None: assert res.json()["error"]["code"] == "PERMISSION_DENIED" rls_svc.create_policy.assert_not_called() - def test_denying_cli_admin_blocks_delete(self, tmp_path: Path) -> None: - _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) + @pytest.mark.parametrize(("deny", "status"), [("cli:destructive", 403), ("cli:admin", 200)]) + def test_delete_is_destructive_not_admin(self, tmp_path: Path, deny: str, status: int) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=[deny])) rls_svc = MagicMock() + rls_svc.delete_policy.return_value = {"deleted": True} app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) with TestClient(app) as client: res = client.delete(f"/rls/{PROJECT}/{POLICY_ID}", headers=AUTH) - assert res.status_code == 403, res.text - rls_svc.delete_policy.assert_not_called() + assert res.status_code == status, res.text + assert rls_svc.delete_policy.called is (status == 200) + + @pytest.mark.parametrize(("scope", "status"), [("targeted", 200), ("organization", 403)]) + def test_denying_cli_destructive_blocks_only_organization_scope( + self, tmp_path: Path, scope: str, status: int + ) -> None: + _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:destructive"])) + rls_svc = MagicMock() + rls_svc.create_policy.return_value = {"id": POLICY_ID} + app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) + body = { + "table_id": "t.x", + "rules": [{"principal": "a@x.com", "condition": {"true": True}}], + "scope": scope, + } + + with TestClient(app) as client: + res = client.post(f"/rls/{PROJECT}", headers=AUTH, json=body) + + assert res.status_code == status, res.text + assert rls_svc.create_policy.called is (status == 200) def test_denying_cli_admin_still_allows_reads(self, tmp_path: Path) -> None: _persist_policy(tmp_path, PermissionPolicy(mode="allow", deny=["cli:admin"])) From 52854a238a4d9ed67cc1f7126c8ac93b5e5fc14e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:09:33 +0200 Subject: [PATCH 21/35] docs(CLI-17): correct rls/cls scope, prerequisites and command shape Every listed policy applies to the project; organization scope reaches every project of the organization. Document the schema ACL (project admin vs organization admin), the row-level-security feature, the checks that keep one policy from blocking all queries, and the new flags. Restore the SKILL.md trigger keywords; regenerate the endpoint reference. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 63 +++-- docs/web-server-endpoints.md | 12 +- plugins/kbagent/agents/keboola-expert.md | 2 +- plugins/kbagent/skills/kbagent/SKILL.md | 12 +- .../kbagent/references/commands-reference.md | 20 +- .../skills/kbagent/references/gotchas.md | 124 ++++---- .../skills/kbagent/references/rls-workflow.md | 264 ++++++++++-------- 7 files changed, 262 insertions(+), 235 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 611d0549..8c6b73a8 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1385,43 +1385,46 @@ kbagent semantic-layer reference-data delete --project P --id ID [--yes] kbagent rls list --project P kbagent rls detail --project P --policy-id ID kbagent rls schema --project P -kbagent rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] -kbagent rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes] -kbagent rls delete --project P --policy-id ID [--yes] -kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes] +kbagent rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes] +kbagent rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] +kbagent rls delete --project P --policy-id ID [--dry-run] [--yes] +kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes] # rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, -# one object per protected table). SCOPE: every write (create/update/ -# setup) is authored at organization scope (default) or targeted scope (--target-project, -# repeatable), never project scope -- there is no --scope flag offering project at all (the metastore ACL -# reserves organization scope and cross-project grants for organization admins). --rules is a JSON array of {principal|principals, condition} objects; condition is a +# one object per protected table, named by it). SCOPE: `targeted` by default (the schema's own +# default: the owning project plus --target-project grants); `--scope organization` governs the +# table in EVERY project of the org (the enforcement applies every listed policy) and is +# destructive-class (FLAG_ESCALATIONS); there is no project scope. WHO: a project admin may +# write `targeted` policies of its own project without grants; grants and organization scope +# need the organization-admin role (403 otherwise). Enforced only where the project has the +# `row-level-security` feature. One invalid policy refuses EVERY query of a project in the +# enforcement, so writes refuse a --dialect other than the project backend (it defaults to the +# backend) and a principal with two rules on one table (case-insensitive, also across visible +# policies). --rules is a JSON array of {principal|principals, condition}; condition is a # declarative primitive tree (column/op/value comparisons, in/not_in, is_null/is_not_null, -# and/or nesting, or a {"true": true} sentinel) -- never a free-text predicate string, closing -# the injection surface a hand-written-SQL model would have. `--dry-run` previews the compiled -# condition via kbagent's OWN preview renderer, which is explicitly NOT the enforcement engine -# (that lives in keboola-mcp-server's `query_data`, a different repo/runtime) -- the preview is -# for admin sanity-checking only, drift between the two is acceptable since the preview string -# is never executed. `rls setup` is a guided, interactive-terminal-only wizard (checkbox table -# picker + guided condition builder, reusing `storage tables` and the same write path `rls -# create` uses) -- it refuses under --json or without an interactive terminal (stdin or stdout not a TTY) with a hint to use `rls create` -# directly, same carve-out as `auth register-projects`'s picker, and has no REST route. -# A stack whose metastore predates the `rls-policy`/`cls-policy` schemas answers a clean, -# classified error (schema fetch failure, NOT_FOUND); expected there, not a kbagent bug. -# New error code: INVALID_RLS_POLICY. Version gate for this whole -# entry lives in gotchas.md -- a `(since vNEXT)` tag cannot be written on these `# ` comment -# lines (check_version_gates.py parses them as ATX markdown headings). +# and/or nesting, or {"true": true}) -- never a free-text predicate; a null comparison is +# refused (use is_null). `update` reads the policy, validates the MERGED result, then PATCHes +# only the changed keys (grants first); the result is re-read. `delete` is destructive-class. +# `--dry-run` previews with columns quoted like the enforcement (keboola-mcp-server +# `query_data`), but it is not the enforcement. `rls setup` is interactive-terminal-only +# (under --json / no TTY: INVALID_ARGUMENT, exit 2) and has no REST route. REST: POST +# /rls/{project}, PATCH /rls/{project}/{policy_id}, DELETE ...?dry_run=; body field +# `target_projects` (alias or ID). A stack whose metastore predates the schemas answers +# NOT_FOUND. New error code: INVALID_RLS_POLICY. Version gate for this whole entry lives in +# gotchas.md -- a `(since vNEXT)` tag cannot be written on these `# ` comment lines +# (check_version_gates.py parses them as ATX markdown headings). kbagent cls list --project P kbagent cls detail --project P --policy-id ID kbagent cls schema --project P -kbagent cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes] -kbagent cls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes] -kbagent cls delete --project P --policy-id ID [--yes] +kbagent cls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes] +kbagent cls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] +kbagent cls delete --project P --policy-id ID [--dry-run] [--yes] # cls: column-level security policy authoring (`cls-policy` object type, one object per -# protected table), sibling of `rls`: same organization/targeted-only scope, fetch-then-merge -# update, master-token requirement and permission class (`admin` for writes). --rules is a -# JSON array of {principal|principals, visible_columns: [col, ...]} objects -- an allowlist -# projection (masking is not supported). `--dry-run` prints each principal's projection. -# No `setup` wizard; the REST routes under /cls/{project} mirror /rls. +# protected table), sibling of `rls`: same scope default, write permissions, dialect and +# duplicate-principal checks, partial update and permission classes. --rules is a JSON array +# of {principal|principals, visible_columns: [col, ...]} objects -- an allowlist projection +# (masking is not supported). `--dry-run` prints each principal's projection. No `setup` +# wizard; the REST routes under /cls/{project} mirror /rls. # New error code: INVALID_CLS_POLICY. Version gate lives in gotchas.md. kbagent http get PATH [--timeout SECONDS] diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 81c8fab4..26ebbc30 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -430,20 +430,20 @@ Model, validate, import/export, diff, promote, and build semantic layer artifact ### `rls` (6 operations) -Row-level security policies (metastore-backed `rls-policy` objects), always authored at `organization`/`targeted` scope -- never `project` scope. Mirrors `kbagent rls list|detail|schema|create|update|delete` (`rls setup`'s interactive picker has no endpoint of its own, same carve-out as `auth register-projects`). +Row-level security policies (metastore-backed `rls-policy` objects), authored at `targeted` scope unless `organization` is asked for -- never `project` scope. Mirrors `kbagent rls list|detail|schema|create|update|delete` (`rls setup`'s interactive picker has no endpoint of its own, same carve-out as `auth register-projects`). | Method | Path | Summary | |---|---|---| | `GET` | `/rls/{project}` | List RLS policies | -| `POST` | `/rls/{project}` | Create an RLS policy | +| `POST` | `/rls/{project}` | Create a RLS policy | | `GET` | `/rls/{project}/schema` | Fetch the live rls-policy JSON Schema | | `GET` | `/rls/{project}/{policy_id}` | Get one RLS policy | -| `PUT` | `/rls/{project}/{policy_id}` | Update an RLS policy | -| `DELETE` | `/rls/{project}/{policy_id}` | Delete an RLS policy | +| `PATCH` | `/rls/{project}/{policy_id}` | Update a RLS policy | +| `DELETE` | `/rls/{project}/{policy_id}` | Delete a RLS policy | ### `cls` (6 operations) -Column-level security policies (metastore-backed `cls-policy` objects: a per-principal `visible_columns` allowlist), always authored at `organization`/`targeted` scope -- never `project` scope. Mirrors `kbagent cls list|detail|schema|create|update|delete`. +Column-level security policies (metastore-backed `cls-policy` objects: a per-principal `visible_columns` allowlist), authored at `targeted` scope unless `organization` is asked for -- never `project` scope. Mirrors `kbagent cls list|detail|schema|create|update|delete`. | Method | Path | Summary | |---|---|---| @@ -451,7 +451,7 @@ Column-level security policies (metastore-backed `cls-policy` objects: a per-pri | `POST` | `/cls/{project}` | Create a CLS policy | | `GET` | `/cls/{project}/schema` | Fetch the live cls-policy JSON Schema | | `GET` | `/cls/{project}/{policy_id}` | Get one CLS policy | -| `PUT` | `/cls/{project}/{policy_id}` | Update a CLS policy | +| `PATCH` | `/cls/{project}/{policy_id}` | Update a CLS policy | | `DELETE` | `/cls/{project}/{policy_id}` | Delete a CLS policy | ## AI & Tools diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 89a34e5f..8b50cb78 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P ...` -- ALWAYS `organization`/`targeted` scope (never `project`, no such flag exists); `--rules` is a declarative JSON condition tree, never free-text SQL; `--dry-run` previews (kbagent's OWN preview renderer, NOT the enforcement engine -- that's `keboola-mcp-server`'s `query_data`). `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; refuses under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch, or one principal with two rules on a table, is refused: it would block every query of the project). `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 3b3c66de..cd2d41b6 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -8,7 +8,7 @@ description: > Storage tables/files/snapshots, large (200 GB) file uploads, data apps, flows/schedules, invitations, feature flags, OTLP data streams, scoped Storage tokens, semantic - layer, row-level security, Developer Portal, + layer, row/column-level security, Developer Portal, first-time setup and logout. Triggers: kbagent, Keboola, keboola config, keboola job, keboola lineage, keboola sync, gitops, dev branch, @@ -18,7 +18,7 @@ description: > feature flag, flow schedule, invite member, SQL transformation edit, sync action, keboola docs, table snapshot, auth, login, sign in, PAYG credits, flow notifications, alert recipients, config trash, - restore config, rls, cls, zero-copy clone, workspace load type, + restore config, zero-copy clone, workspace load type, rls, cls, multipart, import job, job-detail, upload-table no-wait, set up keboola, connect project, logout. --- @@ -374,16 +374,16 @@ When working inside a git repository or project directory, run `kbagent init` (o | List row-level security policies visible to a project | `kbagent rls list --project PROJECT` | | Show one RLS policy's full rule set | `kbagent rls detail --project PROJECT --policy-id POLICY-ID` | | Print the live ``rls-policy`` JSON Schema fetched from the metastore | `kbagent rls schema --project PROJECT` | -| Create one RLS policy for one table | `kbagent rls create --project PROJECT --table TABLE --dialect DIALECT --rules RULES` | +| Create one RLS policy for one table (``targeted`` scope unless ``--scope organization``) | `kbagent rls create --project PROJECT --table-id TABLE-ID --rules RULES` | | Update an existing RLS policy | `kbagent rls update --project PROJECT --policy-id POLICY-ID` | -| Delete an RLS policy | `kbagent rls delete --project PROJECT --policy-id POLICY-ID` | +| Delete an RLS policy -- its table's rows are no longer filtered | `kbagent rls delete --project PROJECT --policy-id POLICY-ID` | | Guided RLS setup: pick tables, build a condition, preview, then write | `kbagent rls setup --project PROJECT` | | List column-level security policies visible to a project | `kbagent cls list --project PROJECT` | | Show one CLS policy's full rule set | `kbagent cls detail --project PROJECT --policy-id POLICY-ID` | | Print the live ``cls-policy`` JSON Schema fetched from the metastore | `kbagent cls schema --project PROJECT` | -| Create one CLS policy for one table | `kbagent cls create --project PROJECT --table TABLE --dialect DIALECT --rules RULES` | +| Create one CLS policy for one table (``targeted`` scope unless ``--scope organization``) | `kbagent cls create --project PROJECT --table-id TABLE-ID --rules RULES` | | Update an existing CLS policy | `kbagent cls update --project PROJECT --policy-id POLICY-ID` | -| Delete a CLS policy | `kbagent cls delete --project PROJECT --policy-id POLICY-ID` | +| Delete a CLS policy -- its table's columns become unrestricted | `kbagent cls delete --project PROJECT --policy-id POLICY-ID` | | GET an endpoint on the running kbagent serve | `kbagent http get ` | | POST to an endpoint on the running kbagent serve | `kbagent http post ` | | PATCH an endpoint on the running kbagent serve | `kbagent http patch ` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 3fa44269..c5d81b02 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,26 +469,26 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo). Same metastore/master-token requirements as Semantic Layer above (see [gotchas.md](gotchas.md)). **Every write is `organization`/`targeted` scope, never `project`** -- no `--scope project` option exists anywhere in this group. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. See [rls-workflow.md](rls-workflow.md) Workflow 7 for how to tell that apart from a real problem. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend and a principal with two rules on one table (both would make the enforcement refuse every query of the project). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. -- `rls list --project P` -- list policies visible to a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, source_project_id, target_project_ids}, ...]}`. An `organization`-scope row's `source_project_id` may not match the current project -- see `rls detail`/the workflow doc before assuming it applies here. +- `rls list --project P` -- list policies visible to (and applied in) a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, owner_project_id, source_project_id, target_project_ids}, ...]}`. `owner_project_id` is null for an `organization` policy; `target_project_ids` is shown to the owner only. - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals, condition}], revision}`. - `rls schema --project P` -- live `rls-policy` JSON Schema fetched from the metastore. Unlike `flow schema`, there is no offline bundled snapshot -- always live-only, `NOT_FOUND` (exit 4) on fetch failure rather than a stale local copy. -- `rls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or a `{"true": true}` sentinel) -- never a raw predicate string. Validated locally (op names, `principal`-xor-`principals`) always, plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any network write. `--target-project` (repeatable) shifts scope from `organization` to `targeted` and registers the grant via a follow-up `PUT .../target-projects` call. `--dry-run` runs every validation and returns the same compiled-condition preview a real write would, without writing -- see the preview-vs-enforcement caveat in [gotchas.md](gotchas.md). -- `rls update --project P --policy-id ID [--table BUCKET.TABLE] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes]` -- fetch-then-merge: `get_item` first, only the flags you pass overwrite the stored `table`/`dialect`/`rules`/target projects, everything else is carried forward untouched (the metastore's `PUT` is a whole-record replace, so this merge step is what stops a partial update from silently blanking fields). -- `rls delete --project P --policy-id ID [--yes]` -- delete a policy, un-protecting its table. Confirmation prompt unless `--yes`; `--json` skips it too. -- `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table (same write path, never a second one). Refuses cleanly under `--json` or without an interactive terminal (stdin or stdout not a TTY) with a hint to use `rls create` directly -- there is no non-interactive path through `setup` itself, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. +- `rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or a `{"true": true}` sentinel) -- never a raw predicate string; a `null` comparison value is refused (use `is_null`). `--dialect` defaults to the project backend and must equal it. `--target-project` (alias or ID, repeatable or comma-separated, `targeted` only) is sent in the create request itself. Validated locally always (rule shape, dialect, duplicate principals across the policies the token can read), plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any write; a policy already on that table is `ALREADY_EXISTS`. `--dry-run` returns the same compiled-condition preview (columns quoted per dialect) a real write would, without writing. +- `rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- reads the policy, validates the MERGED result, then sends a partial `PATCH` of only the changed keys (unknown keys and concurrent changes to other keys survive). `--target-project` replaces the grants of a `targeted` policy, `--clear-target-projects` revokes them; grants change before the rules. The result is re-read after the write. Nothing to change is `INVALID_ARGUMENT` (exit 2). +- `rls delete --project P --policy-id ID [--dry-run] [--yes]` -- delete a policy, un-protecting its table. Destructive-class (`--deny-destructive` blocks it). `--dry-run` shows the policy without deleting it. Confirmation prompt unless `--yes`; `--json` skips it too. +- `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder (a value is read as JSON when it is one: `42`, `true`) or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table. Under `--json` or without an interactive terminal it exits 2 with an `INVALID_ARGUMENT` error, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. ## Column-Level Security (CLS) -Author `cls-policy` metastore objects -- one object per protected table, each holding per-principal `visible_columns` allowlists (columns not listed are omitted from that principal's result; masking is not supported). Sibling of RLS above: same scope rule (**`organization`/`targeted` only, never `project`**), same fetch-then-merge `update`, same master-token requirement, no interactive `setup`. Enforcement (and RLS+CLS composition) happens in `keboola-mcp-server`'s `query_data`. See [rls-workflow.md](rls-workflow.md#column-level-security-cls). +Author `cls-policy` metastore objects -- one object per protected table, each holding per-principal `visible_columns` allowlists (columns not listed are omitted from that principal's result; masking is not supported). Sibling of RLS above: same scope default and options, write permissions, dialect and duplicate-principal checks, partial `update` and `delete --dry-run`; no interactive `setup`. Enforcement (and RLS+CLS composition) happens in `keboola-mcp-server`'s `query_data`. See [rls-workflow.md](rls-workflow.md#column-level-security-cls). - `cls list --project P` -- list policies visible to a project (same row shape as `rls list`). - `cls detail --project P --policy-id ID` -- full rule set: `{..., rules: [{principal|principals, visible_columns}], revision}`. - `cls schema --project P` -- live `cls-policy` JSON Schema from the metastore (`NOT_FOUND`, exit 4, when the stack's metastore does not have the object type). -- `cls create --project P --table BUCKET.TABLE --dialect snowflake|bigquery --rules JSON|@file|- [--target-project ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each principal's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. -- `cls update --project P --policy-id ID [--table ...] [--dialect ...] [--rules JSON|@file|-] [--target-project ID ... | --clear-target-projects] [--dry-run] [--yes]` -- fetch-then-merge; omitted flags keep their current value. -- `cls delete --project P --policy-id ID [--yes]` -- delete a policy; the table's columns become unrestricted for those principals. +- `cls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each principal's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. +- `cls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect ...] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- partial update, same as `rls update`. +- `cls delete --project P --policy-id ID [--dry-run] [--yes]` -- delete a policy; the table's columns become unrestricted for those principals. Destructive-class. ## Self-Call HTTP (inside `kbagent serve` subprocesses; since v0.40.0) - `http get PATH [--timeout SECONDS]` -- GET an endpoint on the running `kbagent serve` diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 3ab6c3ce..72648d63 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6748,7 +6748,7 @@ The package was named `keboola-agent-cli` up to 0.62.0 and is named `keboola-cli - **Stale AI results are not applied** *(since 0.99.1)*: a result whose `_code_hash` differs from the config's current code hash describes code that no longer exists; it is skipped entirely and listed in `ai_status.ai_results_stale` (a fresh task is emitted for it). A result without `_code_hash` is still applied. A `col_map` naming a table the config reads under two aliases now maps onto both edges instead of being skipped as ambiguous. - `ai_status` keys: `ai_edges_added` (reads + writes), `ai_reads_added`, `ai_writes_added`, `ai_columns_merged`, `ai_external_added`, `ai_col_map_skipped`, `ai_outputs_unresolved`, `ai_results_stale`, `ai_tasks_missing_code_file`. -## Row-Level and Column-Level Security (RLS, CLS): organization/targeted scope only +## Row-Level and Column-Level Security (RLS, CLS): targeted by default, never project scope *(since vNEXT)* @@ -6757,72 +6757,68 @@ enforces anything itself: the actual SQL rewrite happens in `keboola-mcp-server` tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full recipes. - **Availability depends on the stack's metastore.** The `rls-policy` and `cls-policy` object types are - registered by the metastore's `2026-09-29` schema migrations. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. Check with - `kbagent --json rls schema --project P` / `cls schema`. All commands are covered by - `tests/test_rls_*.py` / `tests/test_cls_*.py` / `tests/test_server_{rls,cls}.py` against a mocked - `MetastoreClient`. -- **`--scope project` does not exist anywhere in this group, on purpose.** Every write - (`create`/`update`/`setup`) is issued at `organization` scope (default) or `targeted` scope - (`--target-project`, repeatable) -- never `project`. This is structural, not merely validated: - the command surface has no flag that could even attempt `project` scope, the metastore ACL reserves - `organization` scope and cross-project grants for organization admins (it additionally lets the - owning project's admin author `targeted` policies for that project). `--project` on every command - names which project's tables/metastore instance you're working against. -- **One `rls-policy` object per protected table**, never one blob per project. `rls setup` - creates one policy per table selected in the checkbox picker, all sharing the same rules -- - if tables need different rules, run `setup`/`create` again per table. -- **`organization` scope is visible org-wide but only APPLIES where it matches.** A fetched policy - is visible to every project in the org once it's `organization` scope, but the enforcement - engine (and `rls list`/`rls detail`'s own `source_project_id`/`target_project_ids` fields) only - ever applies it to the project it names -- never merely because a table name matches. Do not - read a policy in `rls list` as protecting the current project without checking - `source_project_id` (or `target_project_ids` for `targeted` scope). + registered by the metastore's `2026-09-29` schema migrations. A stack whose metastore predates them + answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected + there, not a kbagent bug. Check with `kbagent --json rls schema --project P` / `cls schema`. +- **Scope is `targeted` unless you pass `--scope organization`; there is no `project` scope.** `targeted` + (the metastore schema's own default) governs the owning project plus the `--target-project` grants. + `organization` governs the table in EVERY project of the organization with the `row-level-security` + feature: the enforcement applies every policy the metastore lists for a project and does not re-check + the owner, so an organization policy is never "only where it matches". Every listed policy applies to + the listing project. `--scope organization` is destructive-class (`rls.create --scope organization` / + `cls.create --scope organization` in `FLAG_ESCALATIONS`). +- **Who may write.** A project admin (master token, admin role) may create/update/delete `targeted` + policies of its own project without grants. Grants (`--target-project`, `--clear-target-projects`) and + `--scope organization` need the organization-admin role -- a master token alone gets a 403 + (`ACCESS_DENIED`). A project admin reads only the policies its project owns. Policies are enforced only + where the consuming project has the `row-level-security` feature. +- **One invalid policy refuses every query of a project.** The enforcement refuses ALL `query_data` + calls of a project when a policy it loads has a `dialect` other than the project backend, or names one + principal (case-insensitive) twice on one table -- in one policy or across policies. The metastore + accepts both, so kbagent refuses them before writing (`INVALID_RLS_POLICY` / `INVALID_CLS_POLICY`). + `--dialect` defaults to the project backend (`verify_token` -> `defaultBackend`). The cross-policy check + only sees policies the token can read. An `organization` policy also reaches projects on the OTHER + backend, where it blocks every query -- prefer `targeted` in a mixed-backend organization. +- **One `rls-policy` object per protected table**, never one blob per project, named by its table: a + second policy on the same table in the same project answers `ALREADY_EXISTS` (update or delete the + first). `rls setup` creates one policy per selected table, all sharing the same rules. - **`condition` is a fixed six-shape primitive vocabulary, never a free-text predicate.** Comparison (`eq`/`ne`/`gt`/`gte`/`lt`/`lte`), membership (`in`/`not_in` with `values`), nullness - (`is_null`/`is_not_null`), `and`/`or` (2+ nested conditions each), or the `{"true": true}` - sentinel. Anything else fails local validation (`INVALID_RLS_POLICY`, exit determined by - `map_error_to_exit_code`) before any network call -- there is no server round-trip needed to - catch a typo'd operator name. -- **`--dry-run`'s compiled-condition preview is NOT the enforcement engine.** It's a small, - local, pure-Python renderer (`services/_rls_condition.py::compile_condition_preview`) meant only - for human sanity-checking before a write -- the actual compiler that decides what a real query - returns is `keboola-mcp-server`'s `rls.py::_compile_primitive` (sqlglot-based, a different repo - entirely). The two are deliberately allowed to render differently; never treat kbagent's preview - string as proof of what will be enforced. -- **`rls update` is fetch-then-merge, not a raw PUT passthrough.** The metastore's `PUT` replaces - the whole record; the service fetches the current item first and only overwrites the fields you - actually passed (`--table`/`--dialect`/`--rules`/`--target-project`), so an update that only - changes `--rules` can never silently blank `table` or `dialect`. -- **`rls setup` is interactive-terminal-only and has no REST route.** Same carve-out as `auth - register-projects`'s picker: it refuses under `--json` or without an interactive terminal (stdin or stdout not a TTY) with a one-line hint - to use `rls create` directly, rather than hanging or emitting malformed JSON. `kbagent serve` - has no `/rls/{project}/setup` endpoint -- the CLI's checkbox picker has no HTTP equivalent, and - its write is the same `POST /rls/{project}` route `rls create` already uses. -- **Writes are classified `admin`, not merely `write`.** `rls.create`/`update`/`delete`/`setup` - all sit in the `admin` risk category in `OPERATION_REGISTRY` (`permissions.py`). Denying - `cli:admin` blocks exactly them (and the other admin operations); denying `cli:write` blocks them - too, because `cli:write` spans write, destructive AND admin. A policy that denies only `cli:destructive` - or only `cli:read` leaves them reachable. Every REST route is gated the same way - `merge-request`'s routes are (`Depends(require_permission(...))`), unlike `notifications.py`'s - routes, which currently enforce nothing over HTTP -- RLS was built gated from the start. + (`is_null`/`is_not_null`), `and`/`or` (2+ nested conditions each, no `not`), or the `{"true": true}` + sentinel. A `null` comparison value (or `null` in `values`) is refused: the enforcement renders + `col = NULL`, which matches nothing -- use `is_null`. A principal with whitespace or control + characters is refused (the enforcement would reject the whole policy). +- **The `--dry-run` preview follows the enforcement's rendering but is not the enforcement.** Columns + are quoted per dialect (`"col"` / `` `col` ``, so matching is case-exact), booleans render + `TRUE`/`FALSE`. The compiler that decides what a query returns is `keboola-mcp-server`'s + `rls.py::_compile_primitive` (sqlglot). +- **`rls update` sends a partial update (`PATCH`) of only the changed keys.** It still reads the policy + first: the metastore's PATCH validates only the keys it receives, so kbagent validates the MERGED + policy (and builds the `--dry-run` preview from it). Unknown keys and concurrent changes to other keys + survive. Grants change before the rules, so a refused grant writes nothing, and the result is re-read + after the write. An `update` with nothing to change is `INVALID_ARGUMENT` (exit 2). +- **`--target-project` takes an alias or a project ID**, repeatable or comma-separated, de-duplicated + (the metastore requires unique, positive IDs). An alias must be on the owner's stack. On `create` the + grants go in the create request itself (one transaction); on `update` the list replaces the grants of a + `targeted` policy and `--clear-target-projects` revokes them. An `organization` policy has no grants + and cannot be narrowed. +- **`delete` is destructive-class and has `--dry-run`.** `rls.delete` / `cls.delete` are `destructive` + (removing a policy removes the filter it enforced), so `--deny-destructive` blocks them. + `create`/`update`/`setup` stay `admin`; `cli:write` spans write, destructive AND admin. +- **`rls setup` is interactive-terminal-only and has no REST route.** Under `--json` or without an + interactive terminal it exits 2 with an `INVALID_ARGUMENT` error (a JSON envelope under `--json`) + pointing at `rls create`. A prompted value is read as JSON when it is one (`42`, `true`); quote it to + keep a string. Exits non-zero when any selected table's policy could not be created. +- **REST mirror.** `POST /rls/{project}` (`table_id`, `rules`, `dialect?`, `scope`, `target_projects`), + `PATCH /rls/{project}/{policy_id}` (partial; `target_projects: []` revokes), `DELETE ...?dry_run=true`; + same for `/cls`. Every route is permission-gated; `scope: "organization"` is also checked as + `.create --scope organization`. - **`cls` is the column-level sibling of `rls`.** `kbagent cls list|detail|schema|create|update|delete` author `cls-policy` objects (`{table, dialect, rules: [{principal|principals, visible_columns}]}`) with the - same scope rule (`organization`/`targeted`, never `project`), fetch-then-merge `update`, `admin` - permission class for writes, and `/cls/{project}` REST mirror as `rls`. `ClsService` subclasses - `RlsService` (only the item type, error code and rule hooks differ). Differences: `visible_columns` is a + same scope, permissions, checks and partial `update` as `rls`. Differences: `visible_columns` is a non-empty allowlist of `[A-Za-z0-9_]+` names (masking is not supported), `--dry-run` prints each - principal's projection instead of a SQL condition, validation failures raise `INVALID_CLS_POLICY`, and - there is no `cls setup` wizard. -- **`rls|cls update` can revoke grants, and `schema`/writes tell you when validation was reduced.** - `--clear-target-projects` (mutually exclusive with `--target-project`) revokes every project the policy is - shared with through the dedicated grants call; the policy keeps its current scope (scope is not changed by - `PUT`). If the live schema could not be fetched, `create`/`update` still run the local checks (including the - shape of every condition primitive) but add a `warnings` entry saying structural validation was skipped. An - authentication/permission failure while fetching the schema (`INVALID_TOKEN`, `MISSING_MASTER_TOKEN`, - access denied) is raised as that error -- only an unavailable schema (e.g. `NOT_FOUND`) is degraded. - `rls setup` exits non-zero when any selected table's policy could not be created. -- **`--target-project` takes numeric project IDs.** The metastore decodes `targetProjectIds` as integers and - the grants call as `{"targetProjectIds": [int]}` (an empty list clears every grant), so a non-numeric or - non-positive ID is rejected up front with `INVALID_ARGUMENT` instead of reaching the backend. Earlier builds - sent string IDs under the wrong key to the grants endpoint, which the backend read as an empty list. + principal's projection, validation failures raise `INVALID_CLS_POLICY`, and there is no `cls setup`. +- **Reduced validation is reported.** If the live schema could not be fetched, writes still run every + local check but add a `warnings` entry; an authentication/permission failure while fetching the schema + is raised as that error. diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index c12572cd..39806991 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -9,22 +9,53 @@ different runtime. `kbagent rls` never executes a query and never decides whether a real query gets filtered -- it only creates, reads, updates and deletes the policy objects the enforcement engine reads. -**Scope is `organization` or `targeted`, never `project`.** Every write (`create`, -`update`, `setup`) is issued at `organization` scope (default) or `targeted` scope -(`--target-project`, repeatable) -- there is no `--scope project` option anywhere in this -command group. The metastore ACL reserves `organization` scope and cross-project grants -for organization admins; it additionally lets the owning project's admin author -`targeted` policies for that project. `--project` on every command names the project -whose tables you're working with (and which metastore instance to query). +**Scope: `targeted` by default, `organization` only when asked for, never `project`.** + +- `targeted` (the default, and the metastore schema's own default) governs the + table in the owning project (`--project`) plus every project granted with + `--target-project` (alias or project ID, repeatable or comma-separated). +- `--scope organization` governs the table in **every project of the + organization** that has the `row-level-security` project feature. The + enforcement applies every policy the metastore lists for a project -- it does + not re-check the owning project -- so an organization policy is never "only + for my project". It is gated as `destructive` (`rls.create --scope + organization` in `FLAG_ESCALATIONS`), so `--deny-destructive` blocks it. +- There is no `--scope project`: the schema does not support it for policies. + +**Who may write** (the metastore schema ACL, not kbagent): + +| Operation | Project admin (master token, admin role) | Organization admin | +|---|---|---| +| `create` / `update` / `delete` a `targeted` policy of its own project, no grants | yes | yes | +| `--target-project` (grants) on `create` or `update`, `--clear-target-projects` | no (403) | yes | +| `--scope organization` | no (403) | yes | +| Read (`list` / `detail`) | only the policies its project owns | everything visible to the project | + +**Enforcement prerequisite.** Policies do nothing until the consuming project has +the `row-level-security` project feature (it gates CLS too). Without it, queries +are unfiltered -- not an error. + +**One bad policy blocks every query.** When the enforcement loads an invalid +policy it refuses **every** `query_data` call of the project, not just queries +of that table. Two such cases pass the metastore's own validation, so kbagent +refuses them before any write: +- a `dialect` other than the project backend (`--dialect` defaults to the + backend; a different value is `INVALID_RLS_POLICY`); +- one principal (case-insensitive) with two rules on one table, inside one + policy or across policies. The cross-policy check sees only the policies the + token can read -- for a project admin, those its project owns. + +An `organization` policy is loaded by every project in the organization, +whatever their backend -- in an organization with both Snowflake and BigQuery +projects, an organization policy blocks the queries of every project on the +other backend. Prefer `targeted` there. Same metastore requirements as [semantic-layer-workflow.md](semantic-layer-workflow.md): a MASTER (project admin) Storage token (`kbagent --json project info --project P` -> `is_master_token`), same `MISSING_MASTER_TOKEN` reclassification -on a non-master token. Pre-flight before anything else in this file: - -```bash -kbagent --json project info --project P | jq '.is_master_token' -``` +on a non-master token. A master token is NOT an organization admin: grants and +organization scope also need the organization-admin role, which the metastore +checks (403 otherwise). **Backend availability.** The `rls-policy` and `cls-policy` object types are registered by the metastore's `2026-09-29` schema migrations. A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` (schema-fetch failure) -- expected there, not a kbagent bug. See Workflow 7. @@ -39,21 +70,22 @@ Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). | See what RLS policies already protect a project | `rls list --project P` | | Inspect one policy's full rule set | `rls detail --project P --policy-id ID` | | Check the live condition/rules shape before hand-writing JSON | `rls schema --project P` | -| Walk an org admin through authoring a new policy interactively | `rls setup --project P` | -| Script or automate policy creation (CI, batch onboarding) | `rls create --project P --table T --dialect D --rules '[...]'` | -| Change one field on an existing policy without touching the rest | `rls update --project P --policy-id ID --table/--dialect/--rules ...` | -| Share one policy across sibling projects | `rls create`/`rls update ... --target-project ID` | +| Walk an admin through authoring a new policy interactively | `rls setup --project P` | +| Script or automate policy creation (CI, batch onboarding) | `rls create --project P --table-id T --rules '[...]'` | +| Change one field on an existing policy without touching the rest | `rls update --project P --policy-id ID --table-id/--dialect/--rules ...` | +| Share one policy with sibling projects (organization admin) | `rls create`/`rls update ... --target-project ALIAS_OR_ID` | +| Govern the table in every project of the organization (organization admin) | `rls create ... --scope organization` | | Preview the compiled condition before writing anything | any write command + `--dry-run` | -| Remove a policy (un-protect a table) | `rls delete --project P --policy-id ID` | +| Remove a policy (un-protect a table) | `rls delete --project P --policy-id ID` (`--dry-run` shows it first) | --- -## Workflow 1 -- Guided setup (the default path for a human org admin) +## Workflow 1 -- Guided setup (the default path for a human admin) `rls setup` is interactive-terminal-only -- it needs a real TTY for the -checkbox table picker and the condition-builder prompts. It refuses cleanly -under `--json` or without an interactive terminal (stdin or stdout piped), printing a one-line hint instead -of hanging or producing malformed JSON output. +checkbox table picker and the condition-builder prompts. Under `--json` or +without an interactive terminal (stdin or stdout piped) it exits 2 with an +`INVALID_ARGUMENT` error (a JSON envelope under `--json`) pointing at `rls create`. ```bash # 1. Run it directly in an interactive terminal -- NOT through kbagent http, @@ -67,21 +99,18 @@ kbagent rls setup --project prod # 3. For the rules, either: # (a) build interactively -- it prompts for principal(s), then a small # menu: comparison / IN-NOT_IN / IS NULL-IS NOT NULL / always-true, -# with an "add another rule?" loop, or +# with an "add another rule?" loop. A typed value is read as JSON when +# it is one (42, 4.5, true); quote it ("42") to keep a string. Or # (b) skip the builder entirely with a prepared --rules file for # anything beyond a single simple comparison per principal: kbagent rls setup --project prod --rules @rls_rules.json # 4. It prints a compiled-condition preview per selected table, then a # single confirm (skip with --yes) before creating one `rls-policy` -# object per table -- via the exact same write path `rls create` uses, -# never a second, parallel one. +# object per table -- via the exact same write path `rls create` uses. +# The dialect is the project backend unless --dialect says otherwise. ``` -If `rls setup` is not usable (no TTY -- a CI job, a scheduled agent task, a -piped invocation), it tells you so and points at `rls create` -- go to -Workflow 2. - ## Workflow 2 -- Scripted / non-interactive policy creation The path for CI, batch onboarding, or any AI-agent-driven invocation. @@ -104,63 +133,54 @@ cat > rls_rules.json <<'EOF' ] EOF -# 2. ALWAYS preview first -- --dry-run runs every validation (dialect, -# rule shape, live schema check when available) and compiles a -# human-readable condition preview, but writes nothing. +# 2. ALWAYS preview first -- --dry-run runs every validation (dialect vs +# backend, rule shape, duplicate principals, live schema check when +# available) and compiles a condition preview, but writes nothing. kbagent --json rls create \ --project prod \ - --table in.c-crm.invoices \ - --dialect snowflake \ + --table-id in.c-crm.invoices \ --rules @rls_rules.json \ --dry-run # -> {"table": "in.c-crm.invoices", "dialect": "snowflake", -# "scope": "organization", "preview": [ -# {"principal": "eu-analyst@example.com", "condition": "region = 'EU'"}, -# {"principal": "us-analyst@example.com, us-lead@example.com", -# "condition": "region = 'US'"} +# "scope": "targeted", "target_project_ids": [], "preview": [ +# {"principal": "eu-analyst@example.com", "condition": "\"region\" = 'EU'"}, +# {"principal": ["us-analyst@example.com", "us-lead@example.com"], +# "condition": "\"region\" = 'US'"} # ], "dry_run": true} # 3. Then the real write (still --json for a clean, single-document # response -- no confirmation prompt in --json mode): -kbagent --json rls create \ - --project prod \ - --table in.c-crm.invoices \ - --dialect snowflake \ - --rules @rls_rules.json +kbagent --json rls create --project prod --table-id in.c-crm.invoices --rules @rls_rules.json ``` -A table with no rows matching ANY rule for a given caller means the query -is refused, not silently returned empty -- fail-closed is the enforcement -engine's whole design point (see the RFC in `keboola-mcp-server`). Before -protecting a table, make sure every principal who legitimately needs access -has a matching rule, including yourself if you plan to verify the result -via `query_data`. +A caller with no rule on a governed table is refused, not silently returned +empty -- fail-closed is the enforcement engine's whole design point (see the +RFC in `keboola-mcp-server`). Before protecting a table, make sure every +principal who legitimately needs access has a matching rule, including +yourself if you plan to verify the result via `query_data`. A principal is the +user's email, matched case-insensitively. ## Workflow 3 -- Inspect existing policies before making a change -Always read before you write -- `rls update` merges onto the CURRENT stored -state, so knowing what's there first avoids surprises. - ```bash -# 1. What's protected in this project already? +# 1. What's visible to this project? kbagent --json rls list --project prod # -> {"project": "prod", "policies": [ # {"id": "...", "table": "in.c-crm.invoices", "dialect": "snowflake", -# "rule_count": 2, "scope": "organization", -# "source_project_id": "12345", "target_project_ids": []} +# "rule_count": 2, "scope": "targeted", "owner_project_id": 12345, +# "source_project_id": null, "target_project_ids": [22222]} # ]} # 2. Full rule set for one policy (needed before a partial `rls update`): kbagent --json rls detail --project prod --policy-id ``` -`scope`/`source_project_id`/`target_project_ids` tell you how a policy -became visible here: `organization` scope is visible to every project in -the org, but only APPLIES where `source_project_id` matches the querying -project (or that project is in `target_project_ids` for a `targeted` -policy) -- a policy authored for a different project, with the same table -name, never applies here. Don't assume a listed policy protects the current -project just because it's in the listing; check `source_project_id`. +Every policy in the listing **applies** to the project: the metastore lists +what the project owns, what is granted to it (`targeted`) and every +`organization` policy, and the enforcement applies all of them. +`owner_project_id` is the project that owns (and alone may update or delete) +a `targeted` policy; it is `null` for an `organization` policy. +`target_project_ids` is shown to the owning project only. ## Workflow 4 -- Condition primitive cookbook @@ -170,16 +190,16 @@ hand-written-SQL model would have). `rls schema --project P` returns the live, authoritative JSON Schema; this is the quick-reference version. ```jsonc -// Comparison: eq | ne | gt | gte | lt | lte +// Comparison: eq | ne | gt | gte | lt | lte (value: string, number or boolean) {"column": "status", "op": "eq", "value": "active"} // Membership: in | not_in {"column": "region", "op": "in", "values": ["EU", "US"]} -// Nullness: is_null | is_not_null +// Nullness: is_null | is_not_null -- the ONLY way to match null {"column": "deleted_at", "op": "is_null"} -// Boolean composition (2+ nested conditions each) +// Boolean composition (2+ nested conditions each; there is no "not") {"and": [ {"column": "region", "op": "eq", "value": "EU"}, {"column": "status", "op": "ne", "value": "draft"} @@ -194,48 +214,53 @@ live, authoritative JSON Schema; this is the quick-reference version. {"true": true} ``` -`rls create --dry-run` (or `rls setup`'s preview step) renders any of these -as a `WHERE`-clause-shaped string for human sanity-checking. **That preview -is kbagent's own renderer, not the enforcement engine** -- the engine that -actually decides what a query returns lives in `keboola-mcp-server`'s +- `"value": null` (or `null` inside `values`) is refused: the enforcement would + render `col = NULL`, which matches no row. Use `is_null` / `is_not_null`. +- Column names are matched **case-exactly**: the enforcement quotes them + (`"region"` on Snowflake, `` `region` `` on BigQuery), and so does the preview. + +`rls create --dry-run` (or `rls setup`'s preview step) renders a condition +as a `WHERE`-clause-shaped string. It follows the enforcement's rendering +(quoted columns, `TRUE`/`FALSE`), but it is kbagent's own renderer -- the +engine that decides what a query returns lives in `keboola-mcp-server`'s `rls.py` (sqlglot-based). Treat the preview as "does this look like what I -meant," never as proof of what will be enforced. +meant," not as proof of what will be enforced. + +## Workflow 5 -- Share a policy with sibling projects (targeted grants) -## Workflow 5 -- Share a policy across sibling projects (targeted scope) +`--target-project` (on `create`/`setup`/`update`) takes a registered alias or a +project ID, repeatable or comma-separated, de-duplicated. On `create` the grants +travel in the create request itself (one transaction). On `update` the list +REPLACES the grants of a `targeted` policy, and `--clear-target-projects` revokes +them all. Granting needs the organization-admin role; on `update` the grants +change before the rules, so a refused grant writes nothing. -`--target-project` (repeatable, on `create`/`setup`) creates the policy at -`targeted` scope instead of `organization` and registers each listed project -as a grant. On `update` it replaces the grants of an already-`targeted` -policy; an `organization` policy cannot be narrowed (the metastore has no -downgrade), so delete it and create a new one. Use this when one org-authored -policy should apply identically across specific sibling customer projects, -not the whole org. +An `organization` policy has no grants and cannot be narrowed -- delete it and +create a `targeted` one instead. ```bash kbagent --json rls create \ --project prod \ - --table in.c-crm.invoices \ - --dialect snowflake \ + --table-id in.c-crm.invoices \ --rules @rls_rules.json \ - --target-project 22222 \ - --target-project 33333 \ + --target-project customer-a,33333 \ --dry-run # -> {"scope": "targeted", "target_project_ids": [22222, 33333], ...} ``` -Omitting `--target-project` on a later `rls update` call leaves the current -target list untouched (fetch-then-merge, see Workflow 6) -- pass the FULL -desired list to change it, not just an addition. +Omitting `--target-project` on a later `rls update` leaves the grants +untouched -- pass the FULL desired list to change them, not just an addition. ## Workflow 6 -- Update or revoke a policy safely -`rls update` is fetch-then-merge: only the flags you pass change, every -omitted flag keeps its current stored value -- a partial update can never -silently blank `table`/`dialect`/`rules`/target projects you didn't mean to -touch. +`rls update` changes only the flags you pass. It reads the policy, validates the +MERGED result (the metastore's partial update validates only the keys it +receives), then sends a partial update (`PATCH`) with just the changed keys -- +keys kbagent does not know and concurrent changes to other keys survive. The +result is read back after the write. ```bash -# Change only the rules, leave table/dialect/scope exactly as they are: +# Change only the rules, leave table/dialect/scope/grants exactly as they are: kbagent --json rls update \ --project prod \ --policy-id \ @@ -244,14 +269,14 @@ kbagent --json rls update \ kbagent --json rls update --project prod --policy-id --rules @updated_rls_rules.json -# Revoke entirely (un-protects the table -- confirm this is what you want, -# a project without the feature flag or without an applicable policy is -# simply unfiltered, not an error state on the enforcement side): +# Revoke entirely (un-protects the table). Show it first, then delete: +kbagent --json rls delete --project prod --policy-id --dry-run kbagent rls delete --project prod --policy-id # --yes to skip the confirmation prompt in a script; --json also skips it. +# `delete` is destructive-class: `--deny-destructive` blocks it. ``` -## Workflow 7 -- Troubleshooting "backend not available" errors +## Workflow 7 -- Troubleshooting errors On a stack whose metastore predates the `rls-policy`/`cls-policy` schemas, EVERY command in this group fails with a classified `NOT_FOUND` -- expected, @@ -264,14 +289,22 @@ kbagent --json rls schema --project prod - **`NOT_FOUND` / "Could not fetch the rls-policy schema: ..."** -- the object type isn't registered on this stack's metastore (same for `cls schema`). Nothing here is broken; use a stack with a newer metastore. -- **`MISSING_MASTER_TOKEN`** -- an actual, fixable problem: the registered - token for this project isn't a master token. `kbagent project info - --project P` -> `is_master_token`, then `project edit --project P - --token ...` with a master token. -- **`INVALID_RLS_POLICY`** -- your `--rules`/`--dialect`/`--table` failed - validation (either the schema-independent checks kbagent always runs, or - the live JSON-Schema check when the backend IS reachable). The error - message lists every violation found; nothing was written. + `detail`/`update`/`delete` also answer `NOT_FOUND` for a policy this token + may not read (owned by another project). +- **`MISSING_MASTER_TOKEN`** -- the registered token for this project isn't a + master token. `kbagent project info --project P` -> `is_master_token`, then + `project edit --project P --token ...` with a master token. +- **`ACCESS_DENIED` (403)** on `--target-project`, `--clear-target-projects` or + `--scope organization` -- the token's user is not an organization admin. +- **`INVALID_RLS_POLICY`** -- the policy failed validation: a dialect other than + the project backend, a malformed rule, a null comparison, a principal with two + rules on the table, or the live JSON-Schema check. The message lists every + violation; nothing was written. +- **`ALREADY_EXISTS`** -- the project already has a policy for this table + (policies are named by table). `rls update` or `rls delete` it. +- **`INVALID_ARGUMENT` / exit 2** -- a bad option value: an unknown + `--target-project`, `--target-project` with `--scope organization`, or an + update with nothing to change. - **`CONFIG_ERROR` / exit 5** -- the `--project` alias isn't registered locally. `kbagent project list`. @@ -281,18 +314,13 @@ kbagent --json rls schema --project prod - [gotchas.md](gotchas.md) -- search "rls" for the full list, version-tagged. - One `rls-policy` object per protected table, never one blob per project. -- `organization` scope is visible org-wide but only APPLIES where - `source_project_id` (or `target_project_ids` for `targeted` scope) - matches the querying project -- never by table-name text alone. +- Every policy a project can list applies to it -- an `organization` policy + governs the table in every project of the organization. - `--scope project` does not exist anywhere in this command group. This is intentional, not a missing feature -- see the RFC in `keboola-mcp-server`'s `feature_spec/rls_query_tool/RFC.md`. - `rls setup` has no REST route (`kbagent serve`) -- it is genuinely terminal-only, same carve-out as `auth register-projects`'s picker. -- kbagent's `--dry-run` preview and the real enforcement engine - (`keboola-mcp-server`'s `rls.py`) are two separate implementations by - design -- the preview is a convenience for human review, never the - source of truth for what a query actually returns. --- @@ -301,27 +329,27 @@ kbagent --json rls schema --project prod `kbagent cls` authors `cls-policy` objects: one per protected table, each rule naming a principal and the `visible_columns` that principal may read (an allowlist -- unlisted columns are omitted from the result; masking is not supported). It mirrors `rls` -(`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same scope rule, -master-token requirement, `--dry-run`/`--yes` flags and fetch-then-merge `update`; -there is no `cls setup` wizard. Enforcement and RLS+CLS composition happen in -`keboola-mcp-server`'s `query_data`. +(`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same scope rule and +default, the same write permissions, dialect check, duplicate-principal check, +`--dry-run`/`--yes` flags and partial `update`; there is no `cls setup` wizard. +Enforcement and RLS+CLS composition happen in `keboola-mcp-server`'s `query_data`, +gated by the same `row-level-security` feature. ```bash kbagent --json cls schema --project prod # live shape kbagent --json cls list --project prod # expect [] on a fresh project # Dry-run first: prints each principal's projection, writes nothing. -kbagent cls create --project prod --table in.c-crm.customers --dialect snowflake \ +kbagent cls create --project prod --table-id in.c-crm.customers \ --rules '[{"principal":"analyst@example.com","visible_columns":["id","region","amount"]}, {"principals":["a@example.com","b@example.com"],"visible_columns":["id"]}]' --dry-run -kbagent cls create --project prod --table in.c-crm.customers --dialect snowflake \ - --rules @cls_rules.json --yes +kbagent cls create --project prod --table-id in.c-crm.customers --rules @cls_rules.json --yes kbagent --json cls update --project prod --policy-id --rules @cls_rules_v2.json kbagent cls delete --project prod --policy-id --yes ``` - A table can carry both an `rls-policy` and a `cls-policy`; author them separately. -- `INVALID_CLS_POLICY` -- `--rules` failed validation (exactly one of `principal`/`principals` - per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, plus the live JSON Schema). - Nothing was written. +- `INVALID_CLS_POLICY` -- the policy failed validation (exactly one of `principal`/`principals` + per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, the dialect check, duplicate + principals, plus the live JSON Schema). Nothing was written. From f5b56f3f6d23ac20fe6f856015aa61834fa67891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:09:33 +0200 Subject: [PATCH 22/35] test(CLI-17): E2E authors a targeted policy and changes its rules An organization-scope policy in the shared E2E organization would reach every project there, and switching its dialect would block their queries. Use the default targeted scope and the project's own dialect. Co-Authored-By: Claude Opus 5.5 --- tests/test_e2e.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index eb24381f..9d4cd9b6 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -3760,20 +3760,28 @@ def _test_rls_commands(self) -> None: def _test_rls_cls_crud(self) -> None: """create -> detail -> update -> delete for both policy types, then the `rls setup` refusal. - Authoring needs an org-admin-capable token on a stack whose metastore registers the type, so a - classified NOT_FOUND / ACCESS_DENIED / MISSING_MASTER_TOKEN on the create is an expected skip - (the policy is never created then); any other failure fails the suite. The policy is always - deleted again. Targets a table that does not exist, so no real data is ever governed. + The policy is `targeted` (the default) with no grants and the project's own dialect (the + default), so it only ever governs this project -- never the shared organization, where an + organization-scope policy would reach every project with the feature. Authoring needs a + master token on a stack whose metastore registers the type, so a classified NOT_FOUND / + ACCESS_DENIED / MISSING_MASTER_TOKEN on the create is an expected skip (the policy is never + created then); any other failure fails the suite. The policy is always deleted again. + Targets a table that does not exist, so no real data is ever governed. """ table = "in.c-kbagent-e2e-rls.policy_crud" samples = { - "rls": '[{"principal":"e2e@example.com","condition":{"true":true}}]', - "cls": '[{"principal":"e2e@example.com","visible_columns":["id"]}]', + "rls": ( + '[{"principal":"e2e@example.com","condition":{"column":"id","op":"eq","value":"x"}}]', + '[{"principal":"e2e-2@example.com","condition":{"column":"id","op":"eq","value":"y"}}]', + ), + "cls": ( + '[{"principal":"e2e@example.com","visible_columns":["id"]}]', + '[{"principal":"e2e-2@example.com","visible_columns":["id"]}]', + ), } - for group, rules in samples.items(): + for group, (rules, new_rules) in samples.items(): created = self._run( - group, "create", "--project", self.alias, "--table", table, - "--dialect", "snowflake", "--rules", rules, + group, "create", "--project", self.alias, "--table-id", table, "--rules", rules, ) # fmt: skip if created.exit_code != 0: assert any( @@ -3784,18 +3792,21 @@ def _test_rls_cls_crud(self) -> None: continue policy_id = _json_ok(created)["data"]["id"] try: + assert _json_ok(created)["data"]["scope"] == "targeted" detail = self._run_ok( group, "detail", "--project", self.alias, "--policy-id", policy_id ) assert detail["data"]["table"] == table updated = self._run_ok( group, "update", "--project", self.alias, "--policy-id", policy_id, - "--dialect", "bigquery", + "--rules", new_rules, ) # fmt: skip - assert updated["data"]["dialect"] == "bigquery" - assert ( - updated["data"]["table"] == table - ) # fetch-then-merge kept the untouched field + assert updated["data"]["rules"][0]["principal"] == "e2e-2@example.com" + assert updated["data"]["table"] == table # the partial update kept the table + previewed = self._run_ok( + group, "delete", "--project", self.alias, "--policy-id", policy_id, "--dry-run" + ) + assert previewed["data"]["dry_run"] is True finally: deleted = self._run( group, "delete", "--project", self.alias, "--policy-id", policy_id @@ -3804,6 +3815,7 @@ def _test_rls_cls_crud(self) -> None: # Under --json (no terminal) the guided wizard refuses before any API call. refused = self._run("rls", "setup", "--project", self.alias) assert refused.exit_code == 2, refused.output + assert "INVALID_ARGUMENT" in refused.output def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" From 33280564bcba61be41a5763e19c8ff58fb0741d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:35:26 +0200 Subject: [PATCH 23/35] fix(CLI-17): reject organization scope with target projects up front; scalar operands only (Copilot review) rls setup listed tables and prompted before the service refused --scope organization with --target-project, and a declined prompt then exited 0; the conflict is now a usage error before any call (create too). Comparison and membership operands must be strings, numbers or booleans, so an object or array is refused even when the live schema is unavailable. Co-Authored-By: Claude Opus 5.5 --- src/keboola_agent_cli/commands/cls.py | 2 +- src/keboola_agent_cli/commands/rls.py | 29 +++++++++++++++---- .../services/_rls_condition.py | 9 ++++++ tests/test_rls_cli.py | 18 ++++++++++++ tests/test_rls_service.py | 4 +++ 5 files changed, 55 insertions(+), 7 deletions(-) diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index 4c2d19cb..e9723409 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -171,7 +171,7 @@ def cls_create( yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: """Create one CLS policy for one table (``targeted`` scope unless ``--scope organization``).""" - gate_scope(ctx, "cls", "create", scope) + gate_scope(ctx, "cls", "create", scope, target_project) formatter = get_formatter(ctx) service = get_service(ctx, "cls_service") kwargs = { diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 68fadd86..492774f2 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -113,10 +113,27 @@ def _confirm_or_exit(formatter: Any, question: str) -> None: raise typer.Exit(code=0) -def gate_scope(ctx: typer.Context, group: str, operation: str, scope: PolicyScope) -> None: - """``--scope organization`` governs the table in every project: a destructive-class flag.""" - if scope == PolicyScope.ORGANIZATION: - check_cli_operation(ctx, f"{group}.{operation} --scope organization") +def gate_scope( + ctx: typer.Context, + group: str, + operation: str, + scope: PolicyScope, + target_project: list[str] | None, +) -> None: + """Validate ``--scope`` before any API call or prompt. + + ``--scope organization`` governs the table in every project: a destructive-class flag. It has + no grants, so ``--target-project`` with it is a usage error (exit 2). + """ + if scope != PolicyScope.ORGANIZATION: + return + check_cli_operation(ctx, f"{group}.{operation} --scope organization") + if target_project: + get_formatter(ctx).error( + message="--target-project requires --scope targeted (organization scope has no grants)", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) def reject_target_conflict( @@ -330,7 +347,7 @@ def rls_create( yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: """Create one RLS policy for one table (``targeted`` scope unless ``--scope organization``).""" - gate_scope(ctx, "rls", "create", scope) + gate_scope(ctx, "rls", "create", scope, target_project) formatter = get_formatter(ctx) service = get_service(ctx, "rls_service") kwargs = { @@ -564,7 +581,7 @@ def rls_setup( """ # `setup` performs `create`'s writes, so an exact `rls.create` denial must cover it too. check_cli_operation(ctx, "rls.create") - gate_scope(ctx, "rls", "create", scope) + gate_scope(ctx, "rls", "create", scope, target_project) formatter = get_formatter(ctx) if formatter.json_mode or not _is_interactive(): formatter.error(message=_SETUP_HINT, error_code=ErrorCode.INVALID_ARGUMENT) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 98801300..73b12caf 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -81,6 +81,11 @@ def _is_true_sentinel(condition: dict[Any, Any]) -> bool: return len(condition) == 1 and condition["true"] is True +def _is_scalar(value: Any) -> bool: + """A literal the primitives allow: string, number or boolean -- never an object or array.""" + return isinstance(value, str | int | float | bool) + + def validate_condition_ops(condition: Any) -> list[str]: """Recursively check every operator name used in ``condition`` is known. @@ -130,6 +135,8 @@ def validate_condition_ops(condition: Any) -> list[str]: errors.append( f"condition with op {op!r} cannot compare to null; use op 'is_null' / 'is_not_null'" ) + elif not _is_scalar(condition["value"]): + errors.append(f"condition with op {op!r} needs a string, number or boolean 'value'") if op in RLS_MEMBERSHIP_OPS: values = condition.get("values") if not isinstance(values, list) or not values: @@ -138,6 +145,8 @@ def validate_condition_ops(condition: Any) -> list[str]: errors.append( f"condition with op {op!r} cannot list null in 'values' (it never matches); use 'is_null'" ) + elif not all(_is_scalar(value) for value in values): + errors.append(f"condition with op {op!r} needs string, number or boolean 'values'") # No keys beyond the op's own shape (the schema forbids additional properties): a stray `values` on an # `eq`, or a `value` on an `in`, is an authoring mistake that would otherwise be silently ignored. allowed = {"column", "op"} | ( diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 3b364635..045fcf76 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -905,6 +905,24 @@ def test_a_service_usage_error_exits_2(self, tmp_path: Path) -> None: assert result.exit_code == 2 + @pytest.mark.parametrize( + "command", [["rls", "setup"], ["rls", "create", "--table-id", "t.x", "--rules", "[]"]] + ) + def test_organization_scope_with_target_project_fails_before_any_call( + self, tmp_path: Path, command: list[str] + ) -> None: + """No grants on organization scope: a usage error before the table listing or any write.""" + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service, storage = MagicMock(), MagicMock() + args = [*command, "--project", "prod", "--scope", "organization", "--target-project", "7"] + + result = _run(args, store, service, storage_service=storage) + + assert result.exit_code == 2, result.output + assert "--target-project requires --scope targeted" in result.output + storage.list_tables.assert_not_called() + service.create_policy.assert_not_called() + def test_setup_json_prints_a_json_error_envelope(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 03cdc835..f7e137af 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -844,6 +844,10 @@ class TestPrimitiveShape: # `col = NULL` / `IN (NULL)` match no row: the principal would silently see nothing. ({"column": "a", "op": "eq", "value": None}, "use op 'is_null'"), ({"column": "a", "op": "in", "values": [1, None]}, "cannot list null"), + # Only scalar literals: an object/array must not slip through when the live schema is unavailable. + ({"column": "a", "op": "eq", "value": {"x": 1}}, "string, number or boolean 'value'"), + ({"column": "a", "op": "gt", "value": [1]}, "string, number or boolean 'value'"), + ({"column": "a", "op": "in", "values": [1, [2]]}, "string, number or boolean 'values'"), ], ) def test_malformed_primitives_are_rejected(self, condition: dict, fragment: str) -> None: From d90dc998c17107244b684092f6cb3d07dec6e74c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 18:48:36 +0200 Subject: [PATCH 24/35] fix(CLI-17): ASCII-only numeric target project IDs; patch_item tests; unique E2E policy table (Copilot review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit str.isdigit() is also true for characters like '²', which int() rejects, so --target-project ² escaped INVALID_ARGUMENT with a traceback (rls/cls and semantic-layer scope). Cover MetastoreClient.patch_item directly, and give the E2E policy a unique table: policies are named by table, so concurrent runs would collide with a 409. Co-Authored-By: Claude Opus 5.5 --- .../services/_semantic_layer_scope.py | 3 +- tests/test_e2e.py | 4 +- tests/test_metastore_client.py | 42 +++++++++++++++++++ tests/test_semantic_layer_scope.py | 1 + 4 files changed, 48 insertions(+), 2 deletions(-) diff --git a/src/keboola_agent_cli/services/_semantic_layer_scope.py b/src/keboola_agent_cli/services/_semantic_layer_scope.py index c19b4b4b..a30d7743 100644 --- a/src/keboola_agent_cli/services/_semantic_layer_scope.py +++ b/src/keboola_agent_cli/services/_semantic_layer_scope.py @@ -53,7 +53,8 @@ def resolve_target_project_ids( for target in parse_target_projects(targets): project = config_store.get_project(target) if project is None: - if not target.isdigit() or int(target) == 0: + # isascii(): `str.isdigit()` is also true for e.g. '²', which `int()` then rejects. + if not (target.isascii() and target.isdigit()) or int(target) == 0: raise KeboolaApiError( message=( f"--target-project {target!r} is neither a registered project alias " diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 9d4cd9b6..104782df 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -52,6 +52,7 @@ import shutil import subprocess import time +import uuid from collections.abc import Generator from datetime import UTC, datetime from pathlib import Path @@ -3768,7 +3769,8 @@ def _test_rls_cls_crud(self) -> None: created then); any other failure fails the suite. The policy is always deleted again. Targets a table that does not exist, so no real data is ever governed. """ - table = "in.c-kbagent-e2e-rls.policy_crud" + # Policies are named by table: a unique table keeps concurrent runs from colliding (409). + table = f"in.c-kbagent-e2e-rls.policy_crud_{uuid.uuid4().hex[:8]}" samples = { "rls": ( '[{"principal":"e2e@example.com","condition":{"column":"id","op":"eq","value":"x"}}]', diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 6d37e9f6..1f9fc6cd 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -289,6 +289,48 @@ def test_delete_404(self, httpx_mock) -> None: assert excinfo.value.error_code == ErrorCode.NOT_FOUND +class TestPatchItem: + """patch_item sends only the given fields -- the server merges them into the stored record.""" + + _URL = f"{METASTORE_URL_US}/api/v1/repository/rls-policy/p-1" + + @pytest.mark.parametrize( + ("kwargs", "body"), + [ + ({"data": {"rules": []}}, {"data": {"rules": []}}), + ({"name": "t.x", "data": {"table": "t.x"}}, {"name": "t.x", "data": {"table": "t.x"}}), + ], + ) + def test_sends_only_the_given_fields(self, httpx_mock, kwargs: dict, body: dict) -> None: + httpx_mock.add_response( + method="PATCH", url=self._URL, json={"data": {"id": "p-1"}}, status_code=200 + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + stored = client.patch_item("rls-policy", "p-1", **kwargs) + finally: + client.close() + assert stored == {"id": "p-1"} + request = httpx_mock.get_requests()[0] + assert request.method == "PATCH" + assert json.loads(request.content) == body # never scope, branch or unchanged keys + + @pytest.mark.parametrize( + ("status", "code"), [(409, ErrorCode.ALREADY_EXISTS), (404, ErrorCode.NOT_FOUND)] + ) + def test_errors_are_classified(self, httpx_mock, status: int, code: ErrorCode) -> None: + httpx_mock.add_response( + method="PATCH", url=self._URL, json={"error": "x"}, status_code=status + ) + client = MetastoreClient(stack_url=STACK_URL_US, token=TOKEN) + try: + with pytest.raises(KeboolaApiError) as excinfo: + client.patch_item("rls-policy", "p-1", name="t.x", data={"table": "t.x"}) + finally: + client.close() + assert excinfo.value.error_code == code + + class TestPutItem: """put_item wraps the same envelope as post_item but targets PUT /{type}/{id}.""" diff --git a/tests/test_semantic_layer_scope.py b/tests/test_semantic_layer_scope.py index 461cdc7c..bda4a5ea 100644 --- a/tests/test_semantic_layer_scope.py +++ b/tests/test_semantic_layer_scope.py @@ -99,6 +99,7 @@ def test_aliases_ids_and_comma_lists_resolve_and_dedupe(self, tmp_path: Path) -> [ ("ghost", "neither a registered project alias nor a positive numeric project ID"), ("0", "neither a registered project alias nor a positive numeric project ID"), + ("²", "neither a registered project alias nor a positive numeric project ID"), ("eu-project", "different stack"), ], ) From 842b18a00f5f8a6d41c6993a68ded2abbc7fc111 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 19:18:50 +0200 Subject: [PATCH 25/35] refactor(CLI-17): share rls/cls command bodies; one 409 path; reuse existing helpers - rls/cls commands share list/detail/schema/create/update/delete and the printers through a PolicyGroup descriptor; cls.py keeps only its options - errors go through the existing _handle_service_call (JSON errors keep details); _is_interactive and the unused project_ids_as_ints are gone - MetastoreClient builds ALREADY_EXISTS in one place with a caller hint; the service no longer re-wraps it - json_utils.draft7_errors replaces the copied Draft7 formatting in flow and rls validation; one comparison-operator map - the schema fetch reuses the open metastore client and the project backend is looked up once per project, not on every write Co-Authored-By: Claude Opus 5.5 --- src/keboola_agent_cli/commands/cls.py | 151 ++----- src/keboola_agent_cli/commands/rls.py | 399 ++++++++++-------- src/keboola_agent_cli/json_utils.py | 13 +- src/keboola_agent_cli/metastore_client.py | 48 +-- src/keboola_agent_cli/permissions.py | 15 +- .../services/_rls_condition.py | 33 +- .../services/flow_validation.py | 10 +- src/keboola_agent_cli/services/rls_service.py | 125 +++--- tests/test_cls_service.py | 5 +- tests/test_metastore_client.py | 17 - tests/test_rls_cli.py | 10 +- tests/test_rls_service.py | 28 +- 12 files changed, 383 insertions(+), 471 deletions(-) diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index e9723409..e3b20371 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -26,29 +26,23 @@ from __future__ import annotations -import contextlib -from typing import Any - import typer -from ..errors import ConfigError, KeboolaApiError -from ._helpers import check_cli_permission, get_formatter, get_service +from ._helpers import check_cli_permission from .rls import ( DIALECT_HELP, SCOPE_HELP, TABLE_HELP, TARGET_HELP, Dialect, + PolicyGroup, PolicyScope, - _call, - _confirm_or_exit, - _format_policy_table, - _parse_rules_arg, - _print_warnings, + create_policy, delete_policy, - gate_scope, + list_policies, print_schema, - reject_target_conflict, + show_policy, + update_policy, ) cls_app = typer.Typer( @@ -59,7 +53,13 @@ ) ) -_RULES_SHAPE = "{principal|principals, visible_columns}" +CLS = PolicyGroup( + name="cls", + label="CLS", + rules_shape="{principal|principals, visible_columns}", + rule_text=lambda rule: ", ".join(rule.get("visible_columns") or []), + preview_text=lambda entry: f"SELECT {', '.join(entry.get('visible_columns') or [])}", +) @cls_app.callback(invoke_without_command=True) @@ -67,53 +67,13 @@ def _cls_permission_check(ctx: typer.Context) -> None: check_cli_permission(ctx, "cls") -def _print_policy(formatter: Any, row: dict[str, Any]) -> None: - formatter.console.print( - f"\n[bold]{row.get('table', '')}[/bold] [dim](id {row.get('id', '')})[/dim]" - ) - formatter.console.print(f" Dialect: {row.get('dialect', '')}") - formatter.console.print(f" Scope: {row.get('scope', '')}") - formatter.console.print( - f" Owner project: {row.get('owner_project_id') or '(organization)'}" - ) - targets = row.get("target_project_ids") or [] - if targets: - formatter.console.print(f" Target projects: {', '.join(str(t) for t in targets)}") - rules = row.get("rules") or [] - formatter.console.print(f" Rules ({len(rules)}):") - for rule in rules: - principal = rule.get("principal") or ", ".join(rule.get("principals") or []) - formatter.console.print( - f" - {principal}: {', '.join(rule.get('visible_columns') or [])}" - ) - - -def _print_preview(formatter: Any, result: dict[str, Any]) -> None: - formatter.console.print( - f"\n[bold]Preview[/bold] -- {result.get('table', '')} " - f"[dim]({result.get('dialect', '')}, scope={result.get('scope', '')})[/dim]" - ) - for entry in result.get("preview", []): - formatter.console.print( - f" {entry.get('principal')}: SELECT {', '.join(entry.get('visible_columns') or [])}" - ) - - @cls_app.command("list") def cls_list( ctx: typer.Context, project: str = typer.Option(..., "--project", help="Project alias"), ) -> None: """List column-level security policies visible to a project.""" - formatter = get_formatter(ctx) - result = _call(formatter, get_service(ctx, "cls_service").list_policies, alias=project) - - if formatter.json_mode: - formatter.output(result) - elif not result.get("policies"): - formatter.console.print("[dim]No CLS policies found.[/dim]") - else: - _format_policy_table(formatter, result["policies"]) + list_policies(ctx, CLS, project) @cls_app.command("detail") @@ -123,14 +83,7 @@ def cls_detail( policy_id: str = typer.Option(..., "--policy-id", help="CLS policy ID"), ) -> None: """Show one CLS policy's full rule set.""" - formatter = get_formatter(ctx) - service = get_service(ctx, "cls_service") - result = _call(formatter, service.get_policy, alias=project, policy_id=policy_id) - - if formatter.json_mode: - formatter.output(result) - else: - _print_policy(formatter, result) + show_policy(ctx, CLS, project, policy_id) @cls_app.command("schema") @@ -146,7 +99,7 @@ def cls_schema( have the ``cls-policy`` object type registered, this fails with a clean, classified ``NOT_FOUND`` error. """ - print_schema(ctx, "cls_service", "cls-policy", project) + print_schema(ctx, CLS, project) @cls_app.command("create") @@ -160,7 +113,7 @@ def cls_create( ..., "--rules", help=( - f"JSON|@file|- array of {_RULES_SHAPE} objects; visible_columns is the " + f"JSON|@file|- array of {CLS.rules_shape} objects; visible_columns is the " "allowlist of columns that principal can read" ), ), @@ -171,33 +124,18 @@ def cls_create( yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: """Create one CLS policy for one table (``targeted`` scope unless ``--scope organization``).""" - gate_scope(ctx, "cls", "create", scope, target_project) - formatter = get_formatter(ctx) - service = get_service(ctx, "cls_service") - kwargs = { - "alias": project, - "table": table_id, - "dialect": dialect, - "rules": _parse_rules_arg(formatter, rules, _RULES_SHAPE), - "scope": scope.value, - "target_projects": target_project, - } - - if not dry_run and not yes and not formatter.json_mode: - # A validation/network error here is reported properly by the real call below. - with contextlib.suppress(ConfigError, KeboolaApiError): - _print_preview(formatter, service.create_policy(**kwargs, dry_run=True)) - _confirm_or_exit(formatter, f"Create CLS policy on {table_id}?") - - result = _call(formatter, service.create_policy, **kwargs, dry_run=dry_run) - - if formatter.json_mode: - formatter.output(result) - return - _print_warnings(formatter, result) - if not dry_run: - formatter.success(f"Created CLS policy {result.get('id', '')} on {table_id}") - _print_preview(formatter, result) + create_policy( + ctx, + CLS, + alias=project, + table=table_id, + dialect=dialect, + rules=rules, + scope=scope, + target_project=target_project, + dry_run=dry_run, + yes=yes, + ) @cls_app.command("update") @@ -228,34 +166,20 @@ def cls_update( Only the flags you pass change -- an omitted flag keeps the policy's current value. The write is a partial update (PATCH) of just those keys. """ - formatter = get_formatter(ctx) - parsed_rules = _parse_rules_arg(formatter, rules, _RULES_SHAPE) if rules is not None else None - reject_target_conflict(formatter, target_project, clear_target_projects) - service = get_service(ctx, "cls_service") - - if not dry_run and not yes and not formatter.json_mode: - _confirm_or_exit(formatter, f"Update CLS policy {policy_id}?") - - result = _call( - formatter, - service.update_policy, + update_policy( + ctx, + CLS, alias=project, policy_id=policy_id, table=table_id, dialect=dialect, - rules=parsed_rules, - target_projects=[] if clear_target_projects else target_project, + rules=rules, + target_project=target_project, + clear_target_projects=clear_target_projects, dry_run=dry_run, + yes=yes, ) - if formatter.json_mode: - formatter.output(result) - return - _print_warnings(formatter, result) - if not dry_run: - formatter.success(f"Updated CLS policy {policy_id}") - _print_preview(formatter, result) - @cls_app.command("delete") def cls_delete( @@ -268,8 +192,7 @@ def cls_delete( """Delete a CLS policy -- its table's columns become unrestricted.""" delete_policy( ctx, - "cls_service", - "CLS", + CLS, "Its table's columns become unrestricted.", alias=project, policy_id=policy_id, diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 492774f2..86475a3a 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -35,6 +35,7 @@ import json import logging from collections.abc import Callable +from dataclasses import dataclass from enum import StrEnum from typing import Any @@ -43,6 +44,7 @@ from rich.table import Table from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..services._rls_condition import COMPARISON_SQL from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( check_cli_operation, @@ -52,6 +54,7 @@ map_error_to_exit_code, parse_json_arg, ) +from ._semantic_layer_helpers import _handle_service_call logger = logging.getLogger(__name__) @@ -91,20 +94,38 @@ class PolicyScope(StrEnum): TABLE_HELP = "Storage table ID, e.g. in.c-crm.invoices" -def _call[T](formatter: Any, fn: Callable[..., T], **kwargs: Any) -> T: - """Run a service call, turning its errors into a formatted message + exit code. +@dataclass(frozen=True) +class PolicyGroup: + """What differs between the ``rls`` and ``cls`` command groups; everything else is shared.""" - A bad option value (``INVALID_ARGUMENT``) is a usage error, exit 2. - """ - try: - return fn(**kwargs) - except ConfigError as exc: - formatter.error(message=exc.message, error_code=ErrorCode.CONFIG_ERROR) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - formatter.error(message=exc.message, error_code=exc.error_code, retryable=exc.retryable) - code = 2 if exc.error_code == ErrorCode.INVALID_ARGUMENT else map_error_to_exit_code(exc) - raise typer.Exit(code=code) from None + name: str # "rls" | "cls": the permission prefix, `_service`, `-policy` + label: str # "RLS" | "CLS" + rules_shape: str # shown in the --rules parse error + rule_text: Callable[[dict[str, Any]], str] # one stored rule's restriction + preview_text: Callable[[dict[str, Any]], str] # one --dry-run preview entry's restriction + + @property + def service(self) -> str: + return f"{self.name}_service" + + +RLS = PolicyGroup( + name="rls", + label="RLS", + rules_shape="{principal|principals, condition}", + rule_text=lambda rule: str(rule.get("condition")), + preview_text=lambda entry: f"WHERE {entry.get('condition')}", +) + + +@rls_app.callback(invoke_without_command=True) +def _rls_permission_check(ctx: typer.Context) -> None: + check_cli_permission(ctx, "rls") + + +# --------------------------------------------------------------------------- +# Shared helpers (also used by `cls`) +# --------------------------------------------------------------------------- def _confirm_or_exit(formatter: Any, question: str) -> None: @@ -114,11 +135,7 @@ def _confirm_or_exit(formatter: Any, question: str) -> None: def gate_scope( - ctx: typer.Context, - group: str, - operation: str, - scope: PolicyScope, - target_project: list[str] | None, + ctx: typer.Context, group: PolicyGroup, scope: PolicyScope, target_project: list[str] | None ) -> None: """Validate ``--scope`` before any API call or prompt. @@ -127,7 +144,7 @@ def gate_scope( """ if scope != PolicyScope.ORGANIZATION: return - check_cli_operation(ctx, f"{group}.{operation} --scope organization") + check_cli_operation(ctx, f"{group.name}.create --scope organization") if target_project: get_formatter(ctx).error( message="--target-project requires --scope targeted (organization scope has no grants)", @@ -136,34 +153,9 @@ def gate_scope( raise typer.Exit(code=2) -def reject_target_conflict( - formatter: Any, target_project: list[str] | None, clear_target_projects: bool -) -> None: - if clear_target_projects and target_project: - formatter.error( - message="--clear-target-projects cannot be combined with --target-project", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) - - -@rls_app.callback(invoke_without_command=True) -def _rls_permission_check(ctx: typer.Context) -> None: - check_cli_permission(ctx, "rls") - - -def _is_interactive() -> bool: - """True when BOTH stdin and stdout are terminals -- the same rule the checkbox picker applies. - - Checked up front so a non-interactive `rls setup` refuses before it makes any API call, rather - than listing the project's tables and only then failing in the picker. - """ - return _stdio_is_tty() - - -# --------------------------------------------------------------------------- -# Shared helpers -# --------------------------------------------------------------------------- +def _principal_text(item: dict[str, Any]) -> str: + principal = item.get("principal") or item.get("principals") or [] + return principal if isinstance(principal, str) else ", ".join(principal) def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None: @@ -191,7 +183,7 @@ def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None formatter.console.print(tbl) -def _print_policy(formatter: Any, row: dict[str, Any]) -> None: +def _print_policy(formatter: Any, group: PolicyGroup, row: dict[str, Any]) -> None: formatter.console.print( f"\n[bold]{row.get('table', '')}[/bold] [dim](id {row.get('id', '')})[/dim]" ) @@ -206,17 +198,16 @@ def _print_policy(formatter: Any, row: dict[str, Any]) -> None: rules = row.get("rules") or [] formatter.console.print(f" Rules ({len(rules)}):") for rule in rules: - principal = rule.get("principal") or ", ".join(rule.get("principals") or []) - formatter.console.print(f" - {principal}: {rule.get('condition')}") + formatter.console.print(f" - {_principal_text(rule)}: {group.rule_text(rule)}") -def _print_preview(formatter: Any, result: dict[str, Any]) -> None: +def _print_preview(formatter: Any, group: PolicyGroup, result: dict[str, Any]) -> None: formatter.console.print( f"\n[bold]Preview[/bold] -- {result.get('table', '')} " f"[dim]({result.get('dialect', '')}, scope={result.get('scope', '')})[/dim]" ) for entry in result.get("preview", []): - formatter.console.print(f" {entry.get('principal')}: WHERE {entry.get('condition')}") + formatter.console.print(f" {_principal_text(entry)}: {group.preview_text(entry)}") def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: @@ -226,9 +217,9 @@ def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: def _parse_rules_arg( - formatter: Any, raw: str, shape: str = "{principal|principals, condition}" + formatter: Any, raw: str, shape: str = RLS.rules_shape ) -> list[dict[str, Any]]: - """Parse ``--rules``; ``shape`` is only the hint in the error (``cls`` passes its own).""" + """Parse ``--rules``; ``shape`` is only the hint in the error.""" try: parsed = parse_json_arg(raw, label="--rules") except ValueError as exc: @@ -243,53 +234,39 @@ def _parse_rules_arg( return parsed -# --------------------------------------------------------------------------- -# rls list / detail / schema -# --------------------------------------------------------------------------- - - -@rls_app.command("list") -def rls_list( - ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Project alias"), -) -> None: - """List row-level security policies visible to a project.""" +def list_policies(ctx: typer.Context, group: PolicyGroup, project: str) -> None: formatter = get_formatter(ctx) - result = _call(formatter, get_service(ctx, "rls_service").list_policies, alias=project) + service = get_service(ctx, group.service) + result = _handle_service_call(ctx, service.list_policies, alias=project) if formatter.json_mode: formatter.output(result) elif not result.get("policies"): - formatter.console.print("[dim]No RLS policies found.[/dim]") + formatter.console.print(f"[dim]No {group.label} policies found.[/dim]") else: _format_policy_table(formatter, result["policies"]) -@rls_app.command("detail") -def rls_detail( - ctx: typer.Context, - project: str = typer.Option(..., "--project", help="Project alias"), - policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), -) -> None: - """Show one RLS policy's full rule set.""" +def show_policy(ctx: typer.Context, group: PolicyGroup, project: str, policy_id: str) -> None: formatter = get_formatter(ctx) - service = get_service(ctx, "rls_service") - result = _call(formatter, service.get_policy, alias=project, policy_id=policy_id) + service = get_service(ctx, group.service) + result = _handle_service_call(ctx, service.get_policy, alias=project, policy_id=policy_id) if formatter.json_mode: formatter.output(result) else: - _print_policy(formatter, result) + _print_policy(formatter, group, result) -def print_schema(ctx: typer.Context, service_name: str, item_type: str, project: str) -> None: - """``rls schema`` / ``cls schema``: the live schema, no offline bundled snapshot.""" +def print_schema(ctx: typer.Context, group: PolicyGroup, project: str) -> None: + """The live schema, no offline bundled snapshot.""" formatter = get_formatter(ctx) - # An auth/permission failure is raised (not "schema unavailable") and reported by `_call`. - fetch = _call(formatter, get_service(ctx, service_name).fetch_schema, alias=project) + # An auth/permission failure is raised (not "schema unavailable") and reported by the handler. + service = get_service(ctx, group.service) + fetch = _handle_service_call(ctx, service.fetch_schema, alias=project) if fetch.schema is None: formatter.error( - message=f"Could not fetch the {item_type} schema: {fetch.reason}", + message=f"Could not fetch the {group.name}-policy schema: {fetch.reason}", error_code=ErrorCode.NOT_FOUND, ) raise typer.Exit(code=4) @@ -302,6 +279,136 @@ def print_schema(ctx: typer.Context, service_name: str, item_type: str, project: ) +def create_policy( + ctx: typer.Context, + group: PolicyGroup, + *, + rules: str, + scope: PolicyScope, + target_project: list[str] | None, + dry_run: bool, + yes: bool, + **kwargs: Any, +) -> None: + """``rls create`` / ``cls create``; ``kwargs`` are ``alias``/``table``/``dialect``.""" + gate_scope(ctx, group, scope, target_project) + formatter = get_formatter(ctx) + service = get_service(ctx, group.service) + kwargs |= { + "rules": _parse_rules_arg(formatter, rules, group.rules_shape), + "scope": scope.value, + "target_projects": target_project, + } + + if not dry_run and not yes and not formatter.json_mode: + # A validation/network error here is reported properly by the real call below. + with contextlib.suppress(ConfigError, KeboolaApiError): + _print_preview(formatter, group, service.create_policy(**kwargs, dry_run=True)) + _confirm_or_exit(formatter, f"Create {group.label} policy on {kwargs['table']}?") + + result = _handle_service_call(ctx, service.create_policy, **kwargs, dry_run=dry_run) + + if formatter.json_mode: + formatter.output(result) + return + _print_warnings(formatter, result) + if not dry_run: + formatter.success( + f"Created {group.label} policy {result.get('id', '')} on {kwargs['table']}" + ) + _print_preview(formatter, group, result) + + +def update_policy( + ctx: typer.Context, + group: PolicyGroup, + *, + rules: str | None, + target_project: list[str] | None, + clear_target_projects: bool, + dry_run: bool, + yes: bool, + **kwargs: Any, +) -> None: + """``rls update`` / ``cls update``; ``kwargs`` are ``alias``/``policy_id``/``table``/``dialect``.""" + formatter = get_formatter(ctx) + parsed_rules = ( + _parse_rules_arg(formatter, rules, group.rules_shape) if rules is not None else None + ) + if clear_target_projects and target_project: + formatter.error( + message="--clear-target-projects cannot be combined with --target-project", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + service = get_service(ctx, group.service) + policy_id = kwargs["policy_id"] + + if not dry_run and not yes and not formatter.json_mode: + _confirm_or_exit(formatter, f"Update {group.label} policy {policy_id}?") + + result = _handle_service_call( + ctx, + service.update_policy, + **kwargs, + rules=parsed_rules, + target_projects=[] if clear_target_projects else target_project, + dry_run=dry_run, + ) + + if formatter.json_mode: + formatter.output(result) + return + _print_warnings(formatter, result) + if not dry_run: + formatter.success(f"Updated {group.label} policy {policy_id}") + _print_preview(formatter, group, result) + + +def delete_policy(ctx: typer.Context, group: PolicyGroup, effect: str, **kwargs: Any) -> None: + """``rls delete`` / ``cls delete``; ``--dry-run`` shows the policy that would be deleted.""" + formatter = get_formatter(ctx) + service = get_service(ctx, group.service) + policy_id, dry_run, yes = kwargs["policy_id"], kwargs["dry_run"], kwargs.pop("yes") + + if not dry_run and not yes and not formatter.json_mode: + _confirm_or_exit(formatter, f"Delete {group.label} policy {policy_id}? {effect}") + + result = _handle_service_call(ctx, service.delete_policy, **kwargs) + + if formatter.json_mode: + formatter.output(result) + elif dry_run: + formatter.console.print(f"[bold]Would delete[/bold] {group.label} policy {policy_id}:") + formatter.console.print_json(data=result["policy"]) + else: + formatter.success(f"Deleted {group.label} policy {policy_id}") + + +# --------------------------------------------------------------------------- +# rls commands +# --------------------------------------------------------------------------- + + +@rls_app.command("list") +def rls_list( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), +) -> None: + """List row-level security policies visible to a project.""" + list_policies(ctx, RLS, project) + + +@rls_app.command("detail") +def rls_detail( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + policy_id: str = typer.Option(..., "--policy-id", help="RLS policy ID"), +) -> None: + """Show one RLS policy's full rule set.""" + show_policy(ctx, RLS, project, policy_id) + + @rls_app.command("schema") def rls_schema( ctx: typer.Context, @@ -315,12 +422,7 @@ def rls_schema( schema is authoritative only from the live metastore, and on a stack whose metastore predates it, fetching fails with a clean, classified error. """ - print_schema(ctx, "rls_service", "rls-policy", project) - - -# --------------------------------------------------------------------------- -# rls create / update / delete -# --------------------------------------------------------------------------- + print_schema(ctx, RLS, project) @rls_app.command("create") @@ -347,33 +449,18 @@ def rls_create( yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), ) -> None: """Create one RLS policy for one table (``targeted`` scope unless ``--scope organization``).""" - gate_scope(ctx, "rls", "create", scope, target_project) - formatter = get_formatter(ctx) - service = get_service(ctx, "rls_service") - kwargs = { - "alias": project, - "table": table_id, - "dialect": dialect, - "rules": _parse_rules_arg(formatter, rules), - "scope": scope.value, - "target_projects": target_project, - } - - if not dry_run and not yes and not formatter.json_mode: - # A validation/network error here is reported properly by the real call below. - with contextlib.suppress(ConfigError, KeboolaApiError): - _print_preview(formatter, service.create_policy(**kwargs, dry_run=True)) - _confirm_or_exit(formatter, f"Create RLS policy on {table_id}?") - - result = _call(formatter, service.create_policy, **kwargs, dry_run=dry_run) - - if formatter.json_mode: - formatter.output(result) - return - _print_warnings(formatter, result) - if not dry_run: - formatter.success(f"Created RLS policy {result.get('id', '')} on {table_id}") - _print_preview(formatter, result) + create_policy( + ctx, + RLS, + alias=project, + table=table_id, + dialect=dialect, + rules=rules, + scope=scope, + target_project=target_project, + dry_run=dry_run, + yes=yes, + ) @rls_app.command("update") @@ -406,56 +493,20 @@ def rls_update( Only the flags you pass change -- an omitted flag keeps the policy's current value. The write is a partial update (PATCH) of just those keys. """ - formatter = get_formatter(ctx) - parsed_rules = _parse_rules_arg(formatter, rules) if rules is not None else None - reject_target_conflict(formatter, target_project, clear_target_projects) - service = get_service(ctx, "rls_service") - - if not dry_run and not yes and not formatter.json_mode: - _confirm_or_exit(formatter, f"Update RLS policy {policy_id}?") - - result = _call( - formatter, - service.update_policy, + update_policy( + ctx, + RLS, alias=project, policy_id=policy_id, table=table_id, dialect=dialect, - rules=parsed_rules, - target_projects=[] if clear_target_projects else target_project, + rules=rules, + target_project=target_project, + clear_target_projects=clear_target_projects, dry_run=dry_run, + yes=yes, ) - if formatter.json_mode: - formatter.output(result) - return - _print_warnings(formatter, result) - if not dry_run: - formatter.success(f"Updated RLS policy {policy_id}") - _print_preview(formatter, result) - - -def delete_policy( - ctx: typer.Context, service_name: str, label: str, effect: str, **kwargs: Any -) -> None: - """``rls delete`` / ``cls delete``; ``--dry-run`` shows the policy that would be deleted.""" - formatter = get_formatter(ctx) - service = get_service(ctx, service_name) - policy_id, dry_run, yes = kwargs["policy_id"], kwargs["dry_run"], kwargs.pop("yes") - - if not dry_run and not yes and not formatter.json_mode: - _confirm_or_exit(formatter, f"Delete {label} policy {policy_id}? {effect}") - - result = _call(formatter, service.delete_policy, **kwargs) - - if formatter.json_mode: - formatter.output(result) - elif dry_run: - formatter.console.print(f"[bold]Would delete[/bold] {label} policy {policy_id}:") - formatter.console.print_json(data=result["policy"]) - else: - formatter.success(f"Deleted {label} policy {policy_id}") - @rls_app.command("delete") def rls_delete( @@ -468,8 +519,7 @@ def rls_delete( """Delete an RLS policy -- its table's rows are no longer filtered.""" delete_policy( ctx, - "rls_service", - "RLS", + RLS, "This un-protects its table.", alias=project, policy_id=policy_id, @@ -496,7 +546,7 @@ def rls_delete( " 4) always true (no filtering for this principal)" ) -_COMPARISON_OPS = ("eq", "ne", "gt", "gte", "lt", "lte") +_COMPARISON_OPS = tuple(COMPARISON_SQL) # eq, ne, gt, gte, lt, lte _VALUE_HINT = 'parsed as JSON when it is one (42, 4.5, true); quote it ("42") to force a string' @@ -581,15 +631,16 @@ def rls_setup( """ # `setup` performs `create`'s writes, so an exact `rls.create` denial must cover it too. check_cli_operation(ctx, "rls.create") - gate_scope(ctx, "rls", "create", scope, target_project) + gate_scope(ctx, RLS, scope, target_project) formatter = get_formatter(ctx) - if formatter.json_mode or not _is_interactive(): + if formatter.json_mode or not _stdio_is_tty(): formatter.error(message=_SETUP_HINT, error_code=ErrorCode.INVALID_ARGUMENT) raise typer.Exit(code=2) - tables = _call( - formatter, get_service(ctx, "storage_service").list_tables, aliases=[project] - ).get("tables", []) + storage_service = get_service(ctx, "storage_service") + tables = _handle_service_call(ctx, storage_service.list_tables, aliases=[project]).get( + "tables", [] + ) if not tables: formatter.console.print(f"[dim]No tables found in project '{project}'.[/dim]") raise typer.Exit(code=0) @@ -604,8 +655,7 @@ def rls_setup( formatter.error(message=_SETUP_HINT, error_code=ErrorCode.INVALID_ARGUMENT) raise typer.Exit(code=2) from None - selected = indices or [] - if not selected: + if not indices: formatter.console.print("No tables selected.") raise typer.Exit(code=0) @@ -613,7 +663,8 @@ def rls_setup( _parse_rules_arg(formatter, rules) if rules is not None else _build_rules_interactively() ) service = get_service(ctx, "rls_service") - selected_tables = [str(tables[i]["id"]) for i in selected] + selected_tables = [str(tables[i]["id"]) for i in indices] + policies = f"{len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}" kwargs = { "alias": project, "dialect": dialect, # None = the project backend, resolved by the service @@ -628,13 +679,10 @@ def rls_setup( except (ConfigError, KeboolaApiError) as exc: formatter.console.print(f"[yellow]{table_id}: preview failed ({exc}).[/yellow]") continue - _print_preview(formatter, preview) + _print_preview(formatter, RLS, preview) if not yes: - _confirm_or_exit( - formatter, - f"Create {len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}?", - ) + _confirm_or_exit(formatter, f"Create {policies}?") failed: list[str] = [] exit_codes: set[int] = set() @@ -655,8 +703,7 @@ def rls_setup( if failed: # Automation must not read a partly (or wholly) failed setup as success. formatter.error( - message=f"{len(failed)} of {len(selected_tables)} polic{'y' if len(selected_tables) == 1 else 'ies'}" - f" could not be created: {', '.join(failed)}", + message=f"{len(failed)} of {policies} could not be created: {', '.join(failed)}", error_code=error_codes.pop() if len(error_codes) == 1 else ErrorCode.API_ERROR, ) # Same exit-code contract as `rls create`: when every failure maps to one code (auth -> 3, diff --git a/src/keboola_agent_cli/json_utils.py b/src/keboola_agent_cli/json_utils.py index 50dd0d89..4bb07c5d 100644 --- a/src/keboola_agent_cli/json_utils.py +++ b/src/keboola_agent_cli/json_utils.py @@ -1,4 +1,4 @@ -"""JSON deep-merge and nested-path utilities. +"""JSON deep-merge, nested-path and schema-validation utilities. Used by ``config update`` to patch configuration content without losing sibling keys -- the exact problem that MCP server's @@ -11,6 +11,8 @@ from dataclasses import dataclass from typing import Any +import jsonschema + # Sentinel marking "the key does not exist on this side" in a DiffEntry -- # distinct from an explicit ``None`` value, which is a legal JSON value. _ABSENT: Any = object() @@ -219,3 +221,12 @@ def find_matches_in_json( if obj is not None and match_fn(str(obj)): paths.append(path) return paths + + +def draft7_errors(document: Any, schema: dict[str, Any]) -> list[str]: + """Every Draft7 violation of ``document`` against ``schema``, as "Schema error at PATH: MSG".""" + validator = jsonschema.Draft7Validator(schema) + return [ + f"Schema error at {'/'.join(str(p) for p in err.path) or '(root)'}: {err.message}" + for err in sorted(validator.iter_errors(document), key=lambda e: list(e.path)) + ] diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index b79eba03..5eb0f9eb 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -49,7 +49,6 @@ import logging import re -from collections.abc import Sequence from dataclasses import dataclass from typing import Any, Literal, Self @@ -159,24 +158,25 @@ def _exact_int(value: Any) -> int: return int(value) -def project_ids_as_ints(project_ids: Sequence[Any]) -> list[int]: - """Target project ids as the metastore wants them: positive integers (``targetProjectIds`` is ``[]int``). +_PICK_ANOTHER_NAME = "Pick another name." - Callers may hand over strings (CLI) or ints (REST); a non-numeric or non-positive id is rejected here, - before any request, instead of reaching the backend as a 400 (or, in the grants call, being ignored). + +def _already_exists( + exc: KeboolaApiError, item_type: str, name: str | None, hint: str | None +) -> KeboolaApiError: + """A duplicate-name conflict as a clean ``ALREADY_EXISTS``; ``hint`` is the caller's remedy. + + Without a hint (the semantic-layer callers) the message explains the metastore's uniqueness + rule instead (:func:`_name_conflict`). """ - try: - ids = [_exact_int(project_id) for project_id in project_ids] - except (TypeError, ValueError): - ids = [0] - if any(project_id <= 0 for project_id in ids): - raise KeboolaApiError( - message=f"Target project IDs must be positive integers, got {project_ids!r}", - status_code=400, - error_code=ErrorCode.INVALID_ARGUMENT, - retryable=False, - ) - return ids + if hint is None: + return _name_conflict(item_type, name or "", exc) + return KeboolaApiError( + message=f"{item_type} with name {name!r} already exists. {hint}", + status_code=exc.status_code, + error_code=ErrorCode.ALREADY_EXISTS, + retryable=False, + ) class MetastoreClient(BaseHttpClient): @@ -334,6 +334,7 @@ def post_item( *, scope: ObjectScope = _DEFAULT_SCOPE, target_project_ids: list[int] | None = None, + conflict_hint: str | None = None, ) -> dict[str, Any]: """Create an item. Returns the server's stored representation. @@ -395,7 +396,7 @@ def post_item( exc.status_code == 500 and "Failed to create meta object" in exc.message ) if is_duplicate: - raise _name_conflict(item_type, name, exc) from exc + raise _already_exists(exc, item_type, name, conflict_hint) from exc raise body = response.json() return body.get("data", body) if isinstance(body, dict) else body @@ -406,6 +407,7 @@ def put_item( item_id: str, name: str, data: dict[str, Any], + conflict_hint: str | None = None, ) -> dict[str, Any]: """Replace an item in place via ``PUT`` (revisioned update). @@ -433,7 +435,7 @@ def put_item( ) except KeboolaApiError as exc: if exc.status_code == 409: - raise _name_conflict(item_type, name, exc) from exc + raise _already_exists(exc, item_type, name, conflict_hint) from exc raise body = response.json() return body.get("data", body) if isinstance(body, dict) else body @@ -445,6 +447,7 @@ def patch_item( *, name: str | None = None, data: dict[str, Any] | None = None, + conflict_hint: str = _PICK_ANOTHER_NAME, ) -> dict[str, Any]: """Partially update an item via ``PATCH`` -- only the given fields change. @@ -472,12 +475,7 @@ def patch_item( ) except KeboolaApiError as exc: if exc.status_code == 409: - raise KeboolaApiError( - message=f"{item_type} with name {name!r} already exists. Pick another name.", - status_code=409, - error_code=ErrorCode.ALREADY_EXISTS, - retryable=False, - ) from exc + raise _already_exists(exc, item_type, name, conflict_hint) from exc raise body = response.json() return body.get("data", body) if isinstance(body, dict) else body diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 40815fe2..809080ef 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -359,17 +359,10 @@ "semantic-layer.reference-data.get": "read", "semantic-layer.reference-data.set": "write", "semantic-layer.reference-data.delete": "destructive", - # Row-level security (metastore) — CLI-17. Classified `admin` (not - # merely `write`): RLS policy authorship is an org-level concern (see - # keboola-mcp-server's feature_spec/rls_query_tool/RFC.md) -- `admin` - # is the class this repo's own taxonomy reserves for org-level - # operations (see `project.add` etc. above). `setup` performs the exact - # same write as `create`, just via an interactive picker, so it must - # carry the same class or a permission policy that allows `setup` but - # not `create` (or vice versa) would leave a hole. `delete` is - # `destructive`, like every other remote delete: removing a policy removes - # the row filter (or column allowlist) it enforced. `create --scope - # organization` is escalated to `destructive` too (FLAG_ESCALATIONS). + # Row-level security (metastore) -- CLI-17. Writes are `admin` (policy authorship is an org-level + # concern); `setup` performs `create`'s write, so it carries the same class. `delete` is + # `destructive` like every remote delete; `create --scope organization` is escalated to + # `destructive` too (FLAG_ESCALATIONS). "rls.list": "read", "rls.detail": "read", "rls.schema": "read", diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 73b12caf..56935a03 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -24,12 +24,13 @@ import re from typing import Any -import jsonschema +from ..json_utils import draft7_errors # The RFC's `condition` schema supports these operators; kept as a frozenset # so the CLI/service layer can validate an op before it ever reaches the # structural (schema) check, giving a clearer error for the common typo case. -RLS_COMPARISON_OPS = frozenset({"eq", "ne", "gt", "gte", "lt", "lte"}) +COMPARISON_SQL = {"eq": "=", "ne": "!=", "gt": ">", "gte": ">=", "lt": "<", "lte": "<="} +RLS_COMPARISON_OPS = frozenset(COMPARISON_SQL) RLS_MEMBERSHIP_OPS = frozenset({"in", "not_in"}) RLS_NULLNESS_OPS = frozenset({"is_null", "is_not_null"}) RLS_CONDITION_OPS = RLS_COMPARISON_OPS | RLS_MEMBERSHIP_OPS | RLS_NULLNESS_OPS @@ -47,31 +48,13 @@ # preview must too -- an unquoted preview would hide a case mismatch the real filter does not forgive. _QUOTE = {"snowflake": '"', "bigquery": "`"} -_COMPARISON_SQL = { - "eq": "=", - "ne": "!=", - "gt": ">", - "gte": ">=", - "lt": "<", - "lte": "<=", -} - def validate_policy_structural(policy: dict[str, Any], schema: dict[str, Any]) -> list[str]: - """Draft7-validate a candidate ``rls-policy`` body against a live-fetched schema. + """Draft7-validate a candidate policy body (``{"table", "dialect", "rules"}``) against a live schema. - ``policy`` is the full candidate body (``{"table", "dialect", "rules"}``). - Defense in depth -- the metastore backend validates on write too, per the - RFC. Returns human-readable error strings (empty = valid); never raises - and never reaches the network -- ``schema`` must already be fetched by - the caller (see ``RlsService.fetch_schema``). + Defense in depth -- the metastore validates on write too, but its PATCH only the patched keys. """ - validator = jsonschema.Draft7Validator(schema) - errors: list[str] = [] - for err in sorted(validator.iter_errors(policy), key=lambda e: list(e.path)): - path = "/".join(str(p) for p in err.path) or "(root)" - errors.append(f"Schema error at {path}: {err.message}") - return errors + return draft7_errors(policy, schema) def _is_true_sentinel(condition: dict[Any, Any]) -> bool: @@ -279,8 +262,8 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: quote = _QUOTE.get(dialect, '"') column = f"{quote}{column}{quote}" - if op in _COMPARISON_SQL: - return f"{column} {_COMPARISON_SQL[op]} {_preview_literal(condition.get('value'))}" + if op in COMPARISON_SQL: + return f"{column} {COMPARISON_SQL[op]} {_preview_literal(condition.get('value'))}" if op in RLS_MEMBERSHIP_OPS: values = condition.get("values") or [] rendered = ", ".join(_preview_literal(v) for v in values) diff --git a/src/keboola_agent_cli/services/flow_validation.py b/src/keboola_agent_cli/services/flow_validation.py index 81eda845..ddd387c4 100644 --- a/src/keboola_agent_cli/services/flow_validation.py +++ b/src/keboola_agent_cli/services/flow_validation.py @@ -15,7 +15,7 @@ from collections import deque from typing import Any -import jsonschema +from ..json_utils import draft7_errors # Operand arity per operator (semantic; the schema cannot express these counts). _BINARY_OPERATORS = frozenset( @@ -30,13 +30,7 @@ def _structural_errors( phases: list[dict[str, Any]], tasks: list[dict[str, Any]], schema: dict[str, Any] ) -> list[str]: """Run Draft7 validation against the supplied schema, collecting ALL errors.""" - document = {"phases": phases, "tasks": tasks} - validator = jsonschema.Draft7Validator(schema) - errors: list[str] = [] - for err in sorted(validator.iter_errors(document), key=lambda e: list(e.path)): - path = "/".join(str(p) for p in err.path) or "(root)" - errors.append(f"Schema error at {path}: {err.message}") - return errors + return draft7_errors({"phases": phases, "tasks": tasks}, schema) def _condition_arity_errors(condition: Any) -> list[str]: diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 86e25274..03a76f2e 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -26,8 +26,7 @@ On a stack whose metastore predates the ``rls-policy`` schema, a real project answers with a schema-fetch/list/get/post failure (see ``fetch_schema`` and the -``gotchas.md`` entry). Every method here is unit-tested against a mocked -:class:`MetastoreClient`. +``gotchas.md`` entry). """ from __future__ import annotations @@ -35,7 +34,7 @@ import logging from collections.abc import Callable, Sequence from dataclasses import dataclass -from typing import Any +from typing import Any, NoReturn, cast import jsonschema @@ -86,15 +85,7 @@ class PolicyValidation: @dataclass(frozen=True) class RlsSchemaFetch: - """Result of fetching the live ``rls-policy`` JSON Schema. - - ``schema`` is ``None`` on any failure (network error, ``KeboolaApiError``, - malformed/empty schema -- most likely: the object type isn't registered on - that stack's metastore). A ``None`` schema must NOT block a write; - callers degrade to the schema-independent checks in ``_rls_condition`` - and surface ``reason`` as a warning -- mirrors - ``FlowService._fetch_flow_schema`` / ``FlowSchemaFetch`` exactly. - """ + """The live policy schema, or ``schema=None`` + ``reason`` when it is unavailable (never blocks).""" schema: dict[str, Any] | None reason: str | None @@ -103,17 +94,8 @@ class RlsSchemaFetch: class RlsService(BaseService): """Business logic for the ``rls`` command group. - Inherits multi-project resolution (``resolve_projects``) from - :class:`BaseService`. Adds a dedicated :class:`MetastoreClient` factory, - same pattern as :class:`SemanticLayerService`, so command-layer - operations can target the metastore without polluting the Storage API - client. - - ``item_type`` / ``label`` / ``invalid_error_code`` and the two rule hooks - (:meth:`_local_rule_errors`, :meth:`_preview_rules`) are all that differ - for the column-level sibling, :class:`ClsService`, which subclasses this: - the scope rule, fetch-then-merge update and schema degradation are the - same for both policy types. + :class:`ClsService` overrides only ``item_type`` / ``label`` / ``invalid_error_code`` and the two + rule hooks (:meth:`_local_rule_errors`, :meth:`_preview_rules`). """ item_type: SemanticType = RLS_ITEM_TYPE @@ -131,6 +113,8 @@ def __init__( metastore_client_factory or make_session_aware_client_factory(config_store, MetastoreClient) ) + # Project backend per (stack, token): one `verify_token` per project, not per write. + self._backends: dict[tuple[str, str], str] = {} # ------------------------------------------------------------------ # Helpers @@ -144,11 +128,12 @@ def _new_metastore_client(self, project: ProjectConfig) -> MetastoreClient: return self._metastore_factory(project.stack_url, project.token) def fetch_schema(self, alias: str) -> RlsSchemaFetch: - """Public schema fetch (used by ``rls schema`` and every write path).""" + """The live schema (``rls schema``); writes reuse their open client instead.""" project = self._resolve_one_project(alias) - return self._fetch_schema_for_project(project) + with self._new_metastore_client(project) as client: + return self._fetch_schema(client) - def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: + def _fetch_schema(self, client: MetastoreClient) -> RlsSchemaFetch: """Fetch the live policy schema, degrading when it is merely *unavailable*. A network error, an empty/malformed schema or a non-auth ``KeboolaApiError`` (most likely: the @@ -157,16 +142,15 @@ def _fetch_schema_for_project(self, project: ProjectConfig) -> RlsSchemaFetch: ``MISSING_MASTER_TOKEN``, access denied) is a real problem the user must see, so it is re-raised instead of being reported as a missing schema. """ - with self._new_metastore_client(project) as client: - try: - resolved = fetch_resolved_schema(client, self.item_type) - schema = resolved.schema - except KeboolaApiError as exc: - if exc.error_code in _AUTH_ERROR_CODES or exc.status_code in (401, 403): - raise - return RlsSchemaFetch(schema=None, reason=exc.message) - except Exception as exc: # any fetch failure must degrade, never block a write - return RlsSchemaFetch(schema=None, reason=str(exc)) + try: + resolved = fetch_resolved_schema(client, self.item_type) + schema = resolved.schema + except KeboolaApiError as exc: + if exc.error_code in _AUTH_ERROR_CODES or exc.status_code in (401, 403): + raise + return RlsSchemaFetch(schema=None, reason=exc.message) + except Exception as exc: # any fetch failure must degrade, never block a write + return RlsSchemaFetch(schema=None, reason=str(exc)) if not schema: return RlsSchemaFetch( schema=None, reason=f"metastore returned no schema for {self.item_type}" @@ -208,8 +192,11 @@ def _row_from_item(item: dict[str, Any]) -> dict[str, Any]: def _project_dialect(self, project: ProjectConfig) -> str: """The project's backend -- the only dialect the enforcement accepts for its policies.""" - with self._client_factory(project.stack_url, project.token) as client: - backend = (client.verify_token().default_backend or "").lower() + key = (project.stack_url, project.token) + if key not in self._backends: + with self._client_factory(project.stack_url, project.token) as client: + self._backends[key] = (client.verify_token().default_backend or "").lower() + backend = self._backends[key] if backend not in RLS_DIALECTS: self._raise_invalid( [ @@ -249,7 +236,6 @@ def _taken_principals( def _validate_policy( self, client: MetastoreClient, - project: ProjectConfig, *, policy: dict[str, Any], backend: str, @@ -282,7 +268,7 @@ def _validate_policy( if errors: return PolicyValidation(errors, []) - fetch = self._fetch_schema_for_project(project) + fetch = self._fetch_schema(client) if not fetch.schema: return PolicyValidation([], [f"Live schema validation was skipped: {fetch.reason}"]) return PolicyValidation(_rls_condition.validate_policy_structural(policy, fetch.schema), []) @@ -308,7 +294,7 @@ def _with_warnings(result: dict[str, Any], warnings: list[str]) -> dict[str, Any result["warnings"] = warnings return result - def _raise_invalid(self, errors: list[str]) -> None: + def _raise_invalid(self, errors: list[str]) -> NoReturn: raise KeboolaApiError( message=f"{self.label} policy is invalid: " + "; ".join(errors), status_code=400, @@ -317,22 +303,18 @@ def _raise_invalid(self, errors: list[str]) -> None: ) @staticmethod - def _raise_usage(message: str) -> None: + def _raise_usage(message: str) -> NoReturn: raise KeboolaApiError( message=message, status_code=400, error_code=ErrorCode.INVALID_ARGUMENT, retryable=False ) - def _already_exists(self, table: str, exc: KeboolaApiError) -> KeboolaApiError: - """The client's generic 409 message talks about semantic models; policies are named by table.""" - return KeboolaApiError( - message=( - f"An {self.label} policy for table {table!r} already exists in this project. " - f"Change it with `{self.label.lower()} update --policy-id ...`, " - f"or `{self.label.lower()} delete` it first." - ), - status_code=exc.status_code, - error_code=ErrorCode.ALREADY_EXISTS, - retryable=False, + @property + def _conflict_hint(self) -> str: + """The 409 remedy: policies are named by their table, one per table and project.""" + group = self.label.lower() + return ( + f"This project already has an {self.label} policy for that table: change it with " + f"`{group} update --policy-id ...`, or `{group} delete` it first." ) # ------------------------------------------------------------------ @@ -390,7 +372,7 @@ def create_policy( policy = {"table": table, "dialect": dialect or backend, "rules": rules} with self._new_metastore_client(project) as client: - check = self._validate_policy(client, project, policy=policy, backend=backend) + check = self._validate_policy(client, policy=policy, backend=backend) if check.errors: self._raise_invalid(check.errors) preview = self._preview_rules(rules, policy["dialect"]) @@ -406,18 +388,14 @@ def create_policy( }, check.warnings, ) - try: - created = client.post_item( - self.item_type, - name=table, - data=policy, - scope="organization" if scope == "organization" else "targeted", - target_project_ids=target_ids or None, - ) - except KeboolaApiError as exc: - if exc.error_code == ErrorCode.ALREADY_EXISTS: - raise self._already_exists(table, exc) from exc - raise + created = client.post_item( + self.item_type, + name=table, + data=policy, + scope=cast(ObjectScope, scope), # checked against POLICY_SCOPES above + target_project_ids=target_ids or None, + conflict_hint=self._conflict_hint, + ) row = self._row_from_item(created) row["preview"] = preview return self._with_warnings(row, check.warnings) @@ -477,7 +455,7 @@ def update_policy( **changes, } check = self._validate_policy( - client, project, policy=policy, backend=backend, exclude_id=policy_id + client, policy=policy, backend=backend, exclude_id=policy_id ) if check.errors: self._raise_invalid(check.errors) @@ -502,12 +480,13 @@ def update_policy( if target_ids is not None: client.put_target_projects(self.item_type, policy_id, target_ids) if changes: - try: - client.patch_item(self.item_type, policy_id, name=table, data=changes) - except KeboolaApiError as exc: - if exc.error_code == ErrorCode.ALREADY_EXISTS: - raise self._already_exists(policy["table"], exc) from exc - raise + client.patch_item( + self.item_type, + policy_id, + name=table, + data=changes, + conflict_hint=self._conflict_hint, + ) row = self._detail_row(client.get_item(self.item_type, policy_id)) row["preview"] = preview return self._with_warnings(row, check.warnings) diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index f347af9a..0eb9e2a6 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock import pytest @@ -215,6 +215,7 @@ def test_writes_cls_policy_at_targeted_scope_by_default(self, tmp_path: Path) -> data={"table": "in.c-crm.invoices", "dialect": "snowflake", "rules": _RULES}, scope="targeted", target_project_ids=None, + conflict_hint=ANY, ) assert result["preview"] == [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] @@ -257,7 +258,7 @@ def test_patches_only_the_given_fields(self, tmp_path: Path) -> None: service.update_policy("prod", "p-1", rules=new_rules) mock.patch_item.assert_called_once_with( - "cls-policy", "p-1", name=None, data={"rules": new_rules} + "cls-policy", "p-1", name=None, data={"rules": new_rules}, conflict_hint=ANY ) def test_keeps_existing_targets_and_scope(self, tmp_path: Path) -> None: diff --git a/tests/test_metastore_client.py b/tests/test_metastore_client.py index 1f9fc6cd..abbf0598 100644 --- a/tests/test_metastore_client.py +++ b/tests/test_metastore_client.py @@ -20,7 +20,6 @@ MetastoreClient, ResolvedSchema, fetch_resolved_schema, - project_ids_as_ints, ) STACK_URL_US = "https://connection.keboola.com" @@ -406,22 +405,6 @@ def test_semantic_types_complete(self) -> None: } -class TestProjectIdsAsInts: - """`project_ids_as_ints`: the metastore decodes ``targetProjectIds`` as ``[]int``.""" - - def test_strings_and_ints_become_ints(self) -> None: - assert project_ids_as_ints(["111", 222]) == [111, 222] - assert project_ids_as_ints([2.0]) == [2] - assert project_ids_as_ints([]) == [] - - @pytest.mark.parametrize("bad", [["abc"], ["0"], ["-5"], ["1.5"], [None], [True], [1.5]]) - def test_non_positive_or_non_numeric_ids_are_rejected(self, bad: list) -> None: - with pytest.raises(KeboolaApiError) as excinfo: - project_ids_as_ints(bad) - - assert excinfo.value.error_code == ErrorCode.INVALID_ARGUMENT - - class TestProjectScope401Reclassification: """The metastore's project-admin gate 401 becomes MISSING_MASTER_TOKEN. diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 045fcf76..4efc32e6 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -17,7 +17,7 @@ from typer.testing import CliRunner from keboola_agent_cli.cli import app -from keboola_agent_cli.commands.rls import _is_interactive +from keboola_agent_cli.commands._checkbox_select import _stdio_is_tty from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.errors import ConfigError, ErrorCode, KeboolaApiError from keboola_agent_cli.models import ProjectConfig @@ -502,7 +502,7 @@ def test_setup_is_interactive_only_when_stdin_and_stdout_are_terminals( monkeypatch.setattr("sys.stdin", _Stream(stdin_tty)) monkeypatch.setattr("sys.stdout", _Stream(stdout_tty)) - assert _is_interactive() is expected + assert _stdio_is_tty() is expected class TestRlsReviewFixes: @@ -600,7 +600,7 @@ def _setup_run( args: list[str], store: ConfigStore, service: MagicMock, storage: MagicMock ) -> Any: with ( - patch("keboola_agent_cli.commands.rls._is_interactive", return_value=True), + patch("keboola_agent_cli.commands.rls._stdio_is_tty", return_value=True), patch("keboola_agent_cli.commands.rls.checkbox_select", return_value=[0, 1]), ): return _run(args, store, service, storage_service=storage) @@ -744,7 +744,7 @@ def test_every_table_failing_is_a_non_zero_exit(self, tmp_path: Path) -> None: ) assert result.exit_code == 3 # an auth-class failure keeps the authentication exit code - assert "2 of 2 policies could not be created" in result.output + assert "2 of 2 RLS policies could not be created" in result.output def test_a_partial_failure_is_a_non_zero_exit_and_names_the_failed_table( self, tmp_path: Path @@ -774,7 +774,7 @@ def test_a_partial_failure_is_a_non_zero_exit_and_names_the_failed_table( ) assert result.exit_code == 1 - assert "1 of 2 policies could not be created: in.c-crm.b" in result.output + assert "1 of 2 RLS policies could not be created: in.c-crm.b" in result.output def test_all_tables_created_exits_zero(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index f7e137af..0f035bfd 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -9,7 +9,7 @@ from pathlib import Path from typing import Any, ClassVar -from unittest.mock import MagicMock +from unittest.mock import ANY, MagicMock import pytest @@ -329,23 +329,19 @@ def test_a_backend_without_policy_support_is_refused(self, tmp_path: Path) -> No mock.post_item.assert_not_called() - def test_a_duplicate_name_gets_a_policy_message_not_the_semantic_one( + def test_a_duplicate_name_gets_a_policy_remedy_not_the_semantic_one( self, tmp_path: Path ) -> None: + """Policies are named by table: the 409 remedy names `rls update` / `rls delete`.""" service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.post_item.side_effect = KeboolaApiError( - message="already exists in the target model", - status_code=409, - error_code=ErrorCode.ALREADY_EXISTS, - ) + mock.post_item.return_value = _policy_item() - with pytest.raises(KeboolaApiError) as excinfo: - service.create_policy("prod", table="t.x", rules=_RULES) + service.create_policy("prod", table="t.x", rules=_RULES) - assert excinfo.value.error_code == ErrorCode.ALREADY_EXISTS - assert "rls update --policy-id" in excinfo.value.message - assert "model" not in excinfo.value.message + hint = mock.post_item.call_args.kwargs["conflict_hint"] + assert "rls update --policy-id" in hint + assert "rls delete" in hint @pytest.mark.parametrize( "bad_rules", @@ -447,7 +443,7 @@ def test_patches_only_the_changed_keys_and_validates_the_merged_policy( service.update_policy("prod", "p-1", rules=_RULES) mock.patch_item.assert_called_once_with( - "rls-policy", "p-1", name=None, data={"rules": _RULES} + "rls-policy", "p-1", name=None, data={"rules": _RULES}, conflict_hint=ANY ) mock.put_item.assert_not_called() @@ -459,7 +455,11 @@ def test_a_new_table_renames_the_policy(self, tmp_path: Path) -> None: service.update_policy("prod", "p-1", table="in.c-crm.new") mock.patch_item.assert_called_once_with( - "rls-policy", "p-1", name="in.c-crm.new", data={"table": "in.c-crm.new"} + "rls-policy", + "p-1", + name="in.c-crm.new", + data={"table": "in.c-crm.new"}, + conflict_hint=ANY, ) def test_nothing_to_update_is_a_usage_error(self, tmp_path: Path) -> None: From 6ef1653c0e066ded6749341015c8822dd8ec2c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 7 Oct 2026 19:33:32 +0200 Subject: [PATCH 26/35] refactor(CLI-17): one exit-code mapping, single-source dialect/scope, drop backend memo - exit_code_for() in commands/_helpers.py, used by handle_service_call (moved there; the semantic layer keeps its alias) and by rls setup, so a usage error exits 2 there too, as in rls create - Dialect / PolicyScope enums live in the service layer; the CLI options and REST bodies import them instead of re-typing the values - drop the per-service backend memo: under kbagent serve it grew without bound, keyed by tokens - drop the REST target/scope validator: the service refuses the combination and REST answers 400 - remove dead loggers and the validate_policy_structural wrapper, derive PolicyGroup.label, explicit delete parameters, stale docstrings Co-Authored-By: Claude Opus 5.5 --- src/keboola_agent_cli/commands/_helpers.py | 28 ++++++- .../commands/_semantic_layer_helpers.py | 30 +------- src/keboola_agent_cli/commands/cls.py | 3 +- src/keboola_agent_cli/commands/rls.py | 73 +++++++++---------- src/keboola_agent_cli/server/routers/cls.py | 2 +- src/keboola_agent_cli/server/routers/rls.py | 30 ++++---- .../services/_rls_condition.py | 26 +++---- src/keboola_agent_cli/services/cls_service.py | 2 +- src/keboola_agent_cli/services/rls_service.py | 31 ++++---- tests/test_cls_service.py | 18 +---- tests/test_rls_service.py | 6 +- tests/test_server_rls.py | 11 ++- 12 files changed, 123 insertions(+), 137 deletions(-) diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index db1ed623..4e409728 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -27,7 +27,7 @@ ENV_KBC_TOKEN, EXIT_PERMISSION_DENIED, ) -from ..errors import ErrorCode, KeboolaApiError, PermissionDeniedError, exit_code_for +from ..errors import ConfigError, ErrorCode, KeboolaApiError, PermissionDeniedError, exit_code_for from ..output import OutputFormatter from ..permissions import FLAG_ESCALATIONS from ..update_crossing import print_held_warning, refusal_for @@ -504,6 +504,32 @@ def emit_api_error(formatter: OutputFormatter, exc: KeboolaApiError, *, project: ) +def service_error_exit_code(exc: ConfigError | KeboolaApiError) -> int: + """The CLI exit code for a service error: config 5, a bad option value 2, else by error code.""" + if isinstance(exc, ConfigError): + return 5 + if exc.error_code == ErrorCode.INVALID_ARGUMENT: + return 2 + return map_error_to_exit_code(exc) + + +def handle_service_call[T]( + ctx: typer.Context, func: Callable[..., T], *args: Any, **kwargs: Any +) -> T: + """Run a service call; a ``ConfigError`` / ``KeboolaApiError`` becomes a formatted error + exit.""" + formatter = get_formatter(ctx) + try: + return func(*args, **kwargs) + except ConfigError as exc: + formatter.error( + message=exc.message, error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR) + ) + raise typer.Exit(code=service_error_exit_code(exc)) from None + except KeboolaApiError as exc: + emit_api_error(formatter, exc) + raise typer.Exit(code=service_error_exit_code(exc)) from None + + def item_failure_exit_code(failed: int) -> int | None: """Map the number of failed items of a multi-item operation to a CLI exit code. diff --git a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py index 80f46fb0..7d899893 100644 --- a/src/keboola_agent_cli/commands/_semantic_layer_helpers.py +++ b/src/keboola_agent_cli/commands/_semantic_layer_helpers.py @@ -13,15 +13,9 @@ import typer -from ..errors import ConfigError, ErrorCode, KeboolaApiError +from ..errors import ErrorCode from ._checkbox_select import CheckboxItem, CheckboxUnavailable, checkbox_select -from ._helpers import ( - check_cli_operation, - emit_api_error, - get_formatter, - get_service, - map_error_to_exit_code, -) +from ._helpers import check_cli_operation, get_formatter, get_service, handle_service_call class ScopeChoice(StrEnum): @@ -49,24 +43,8 @@ class ItemType(StrEnum): GLOSSARY = "glossary" -def _handle_service_call(ctx: typer.Context, func, *args, **kwargs): # type: ignore[no-untyped-def] - """Run a service call, mapping ``ConfigError`` / ``KeboolaApiError`` to exit codes. - - Returns the service result on success; on failure, prints the structured - error envelope (JSON mode) or a red error line (human mode) and raises - ``typer.Exit`` with the appropriate code. - """ - formatter = get_formatter(ctx) - try: - return func(*args, **kwargs) - except ConfigError as exc: - formatter.error( - message=exc.message, error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR) - ) - raise typer.Exit(code=5) from None - except KeboolaApiError as exc: - emit_api_error(formatter, exc) - raise typer.Exit(code=map_error_to_exit_code(exc)) from None +# The handler is shared by every metastore-backed command group (semantic-layer, rls, cls). +_handle_service_call = handle_service_call def _is_stdin_tty() -> bool: diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index e3b20371..bcac5a2f 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -8,7 +8,7 @@ - ``cls detail`` -- one policy's full rules. - ``cls schema`` -- the live ``cls-policy`` JSON Schema from the metastore. - ``cls create`` -- author a policy for one table (write). -- ``cls update`` -- fetch-then-merge update of an existing policy (write). +- ``cls update`` -- partial (PATCH) update of an existing policy (write). - ``cls delete`` -- remove a policy (destructive). Each rule is ``{principal|principals, visible_columns: [...]}``: the listed @@ -55,7 +55,6 @@ CLS = PolicyGroup( name="cls", - label="CLS", rules_shape="{principal|principals, visible_columns}", rule_text=lambda rule: ", ".join(rule.get("visible_columns") or []), preview_text=lambda entry: f"SELECT {', '.join(entry.get('visible_columns') or [])}", diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 86475a3a..c5a9cf07 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -6,7 +6,7 @@ - ``rls detail`` -- one policy's full rules. - ``rls schema`` -- the live ``rls-policy`` JSON Schema from the metastore. - ``rls create`` -- author a policy for one table (write). -- ``rls update`` -- fetch-then-merge update of an existing policy (write). +- ``rls update`` -- partial (PATCH) update of an existing policy (write). - ``rls delete`` -- remove a policy (destructive). - ``rls setup`` -- guided, interactive-terminal-only wizard: pick tables via a checkbox picker, build conditions, preview, then create one policy per @@ -33,10 +33,8 @@ import contextlib import json -import logging from collections.abc import Callable from dataclasses import dataclass -from enum import StrEnum from typing import Any import typer @@ -44,19 +42,18 @@ from rich.table import Table from ..errors import ConfigError, ErrorCode, KeboolaApiError -from ..services._rls_condition import COMPARISON_SQL +from ..services._rls_condition import COMPARISON_SQL, Dialect +from ..services.rls_service import PolicyScope from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( check_cli_operation, check_cli_permission, + service_error_exit_code, get_formatter, get_service, - map_error_to_exit_code, + handle_service_call, parse_json_arg, ) -from ._semantic_layer_helpers import _handle_service_call - -logger = logging.getLogger(__name__) rls_app = typer.Typer( help=( @@ -67,20 +64,6 @@ ) -class Dialect(StrEnum): - """``--dialect`` values (the policy schema's enum); omitted = the project backend.""" - - SNOWFLAKE = "snowflake" - BIGQUERY = "bigquery" - - -class PolicyScope(StrEnum): - """``--scope`` values: the schema supports no ``project`` scope for policies.""" - - TARGETED = "targeted" - ORGANIZATION = "organization" - - DIALECT_HELP = "Workspace SQL dialect; must match the project backend (default: the backend)" SCOPE_HELP = ( "targeted (default): the owning project plus --target-project grants. organization: " @@ -99,7 +82,6 @@ class PolicyGroup: """What differs between the ``rls`` and ``cls`` command groups; everything else is shared.""" name: str # "rls" | "cls": the permission prefix, `_service`, `-policy` - label: str # "RLS" | "CLS" rules_shape: str # shown in the --rules parse error rule_text: Callable[[dict[str, Any]], str] # one stored rule's restriction preview_text: Callable[[dict[str, Any]], str] # one --dry-run preview entry's restriction @@ -108,10 +90,13 @@ class PolicyGroup: def service(self) -> str: return f"{self.name}_service" + @property + def label(self) -> str: + return self.name.upper() + RLS = PolicyGroup( name="rls", - label="RLS", rules_shape="{principal|principals, condition}", rule_text=lambda rule: str(rule.get("condition")), preview_text=lambda entry: f"WHERE {entry.get('condition')}", @@ -216,9 +201,7 @@ def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: formatter.warning(warning) -def _parse_rules_arg( - formatter: Any, raw: str, shape: str = RLS.rules_shape -) -> list[dict[str, Any]]: +def _parse_rules_arg(formatter: Any, raw: str, shape: str) -> list[dict[str, Any]]: """Parse ``--rules``; ``shape`` is only the hint in the error.""" try: parsed = parse_json_arg(raw, label="--rules") @@ -237,7 +220,7 @@ def _parse_rules_arg( def list_policies(ctx: typer.Context, group: PolicyGroup, project: str) -> None: formatter = get_formatter(ctx) service = get_service(ctx, group.service) - result = _handle_service_call(ctx, service.list_policies, alias=project) + result = handle_service_call(ctx, service.list_policies, alias=project) if formatter.json_mode: formatter.output(result) @@ -250,7 +233,7 @@ def list_policies(ctx: typer.Context, group: PolicyGroup, project: str) -> None: def show_policy(ctx: typer.Context, group: PolicyGroup, project: str, policy_id: str) -> None: formatter = get_formatter(ctx) service = get_service(ctx, group.service) - result = _handle_service_call(ctx, service.get_policy, alias=project, policy_id=policy_id) + result = handle_service_call(ctx, service.get_policy, alias=project, policy_id=policy_id) if formatter.json_mode: formatter.output(result) @@ -263,7 +246,7 @@ def print_schema(ctx: typer.Context, group: PolicyGroup, project: str) -> None: formatter = get_formatter(ctx) # An auth/permission failure is raised (not "schema unavailable") and reported by the handler. service = get_service(ctx, group.service) - fetch = _handle_service_call(ctx, service.fetch_schema, alias=project) + fetch = handle_service_call(ctx, service.fetch_schema, alias=project) if fetch.schema is None: formatter.error( message=f"Could not fetch the {group.name}-policy schema: {fetch.reason}", @@ -306,7 +289,7 @@ def create_policy( _print_preview(formatter, group, service.create_policy(**kwargs, dry_run=True)) _confirm_or_exit(formatter, f"Create {group.label} policy on {kwargs['table']}?") - result = _handle_service_call(ctx, service.create_policy, **kwargs, dry_run=dry_run) + result = handle_service_call(ctx, service.create_policy, **kwargs, dry_run=dry_run) if formatter.json_mode: formatter.output(result) @@ -347,7 +330,7 @@ def update_policy( if not dry_run and not yes and not formatter.json_mode: _confirm_or_exit(formatter, f"Update {group.label} policy {policy_id}?") - result = _handle_service_call( + result = handle_service_call( ctx, service.update_policy, **kwargs, @@ -365,16 +348,26 @@ def update_policy( _print_preview(formatter, group, result) -def delete_policy(ctx: typer.Context, group: PolicyGroup, effect: str, **kwargs: Any) -> None: +def delete_policy( + ctx: typer.Context, + group: PolicyGroup, + effect: str, + *, + alias: str, + policy_id: str, + dry_run: bool, + yes: bool, +) -> None: """``rls delete`` / ``cls delete``; ``--dry-run`` shows the policy that would be deleted.""" formatter = get_formatter(ctx) service = get_service(ctx, group.service) - policy_id, dry_run, yes = kwargs["policy_id"], kwargs["dry_run"], kwargs.pop("yes") if not dry_run and not yes and not formatter.json_mode: _confirm_or_exit(formatter, f"Delete {group.label} policy {policy_id}? {effect}") - result = _handle_service_call(ctx, service.delete_policy, **kwargs) + result = handle_service_call( + ctx, service.delete_policy, alias=alias, policy_id=policy_id, dry_run=dry_run + ) if formatter.json_mode: formatter.output(result) @@ -638,7 +631,7 @@ def rls_setup( raise typer.Exit(code=2) storage_service = get_service(ctx, "storage_service") - tables = _handle_service_call(ctx, storage_service.list_tables, aliases=[project]).get( + tables = handle_service_call(ctx, storage_service.list_tables, aliases=[project]).get( "tables", [] ) if not tables: @@ -660,9 +653,11 @@ def rls_setup( raise typer.Exit(code=0) parsed_rules = ( - _parse_rules_arg(formatter, rules) if rules is not None else _build_rules_interactively() + _parse_rules_arg(formatter, rules, RLS.rules_shape) + if rules is not None + else _build_rules_interactively() ) - service = get_service(ctx, "rls_service") + service = get_service(ctx, RLS.service) selected_tables = [str(tables[i]["id"]) for i in indices] policies = f"{len(selected_tables)} RLS polic{'y' if len(selected_tables) == 1 else 'ies'}" kwargs = { @@ -693,7 +688,7 @@ def rls_setup( except (ConfigError, KeboolaApiError) as exc: formatter.warning(f"{table_id}: {exc}") failed.append(table_id) - exit_codes.add(5 if isinstance(exc, ConfigError) else map_error_to_exit_code(exc)) + exit_codes.add(service_error_exit_code(exc)) error_codes.add( ErrorCode.CONFIG_ERROR if isinstance(exc, ConfigError) else exc.error_code ) diff --git a/src/keboola_agent_cli/server/routers/cls.py b/src/keboola_agent_cli/server/routers/cls.py index 5fb2d2fa..82b70a8f 100644 --- a/src/keboola_agent_cli/server/routers/cls.py +++ b/src/keboola_agent_cli/server/routers/cls.py @@ -9,4 +9,4 @@ from .rls import build_policy_router -router = build_policy_router("cls", "CLS") +router = build_policy_router("cls") diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py index f2f4914e..455f4f2c 100644 --- a/src/keboola_agent_cli/server/routers/rls.py +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -29,10 +29,12 @@ from typing import Any, Literal from fastapi import APIRouter, Depends -from pydantic import BaseModel, model_validator +from pydantic import BaseModel from ...errors import ErrorCode, KeboolaApiError from ...permissions import PermissionEngine +from ...services._rls_condition import Dialect +from ...services.rls_service import PolicyScope from ..dependencies import ( ServiceRegistry, get_permission_engine, @@ -44,34 +46,32 @@ class PolicyCreate(BaseModel): - """``target_projects`` takes aliases or project IDs, like the semantic-layer bodies.""" + """``target_projects`` takes aliases or project IDs, like the semantic-layer bodies. + + ``target_projects`` with ``scope: organization`` is refused by the service (400). + """ table_id: str rules: list[dict[str, Any]] - dialect: Literal["snowflake", "bigquery"] | None = None # None = the project backend - scope: Literal["targeted", "organization"] = "targeted" + dialect: Dialect | None = None # None = the project backend + scope: PolicyScope = PolicyScope.TARGETED target_projects: list[str | int] | None = None dry_run: bool = False - @model_validator(mode="after") - def _targets_need_targeted_scope(self) -> PolicyCreate: - if self.target_projects and self.scope != "targeted": - raise ValueError("target_projects requires scope='targeted'.") - return self - class PolicyUpdate(BaseModel): """Partial update: an omitted field is unchanged; ``target_projects: []`` revokes every grant.""" table_id: str | None = None - dialect: Literal["snowflake", "bigquery"] | None = None + dialect: Dialect | None = None rules: list[dict[str, Any]] | None = None target_projects: list[str | int] | None = None dry_run: bool = False -def build_policy_router(group: Literal["rls", "cls"], label: str) -> APIRouter: - """The ``/rls`` or ``/cls`` router; ``label`` is "RLS"/"CLS" in summaries.""" +def build_policy_router(group: Literal["rls", "cls"]) -> APIRouter: + """The ``/rls`` or ``/cls`` router.""" + label = group.upper() router = APIRouter(prefix=f"/{group}", tags=[group]) def perm(operation: str) -> Any: @@ -126,7 +126,7 @@ def create_policy( Never `project` scope (the schema does not support it). Mirrors `kbagent create`. """ - if body.scope == "organization": + if body.scope == PolicyScope.ORGANIZATION: engine.check_or_raise(f"{group}.create --scope organization") return service(registry).create_policy( project, @@ -175,4 +175,4 @@ def delete_policy( return router -router = build_policy_router("rls", "RLS") +router = build_policy_router("rls") diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 56935a03..143e11c3 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -22,23 +22,29 @@ from __future__ import annotations import re +from enum import StrEnum from typing import Any -from ..json_utils import draft7_errors - -# The RFC's `condition` schema supports these operators; kept as a frozenset -# so the CLI/service layer can validate an op before it ever reaches the -# structural (schema) check, giving a clearer error for the common typo case. +# The condition operators of the policy schema, checked locally so a typo gets a specific error +# even when the live schema is unavailable. COMPARISON_SQL also renders the preview. COMPARISON_SQL = {"eq": "=", "ne": "!=", "gt": ">", "gte": ">=", "lt": "<", "lte": "<="} RLS_COMPARISON_OPS = frozenset(COMPARISON_SQL) RLS_MEMBERSHIP_OPS = frozenset({"in", "not_in"}) RLS_NULLNESS_OPS = frozenset({"is_null", "is_not_null"}) RLS_CONDITION_OPS = RLS_COMPARISON_OPS | RLS_MEMBERSHIP_OPS | RLS_NULLNESS_OPS + # The metastore's `rls-policy` schema (RFC) restricts `dialect` to these two # -- the same pair `rewrite_query()` on the enforcement side never transpiles # between (predicates are pinned to one workspace dialect per policy). -RLS_DIALECTS: tuple[str, ...] = ("snowflake", "bigquery") +class Dialect(StrEnum): + """The policy schema's ``dialect`` enum (the CLI ``--dialect`` choice and the REST field).""" + + SNOWFLAKE = "snowflake" + BIGQUERY = "bigquery" + + +RLS_DIALECTS: tuple[str, ...] = tuple(Dialect) # The enforcement's principal pattern (mcp-server ``rls.py``): no whitespace or control characters. A # principal that fails it is not ignored there -- it refuses EVERY query in the project. @@ -49,14 +55,6 @@ _QUOTE = {"snowflake": '"', "bigquery": "`"} -def validate_policy_structural(policy: dict[str, Any], schema: dict[str, Any]) -> list[str]: - """Draft7-validate a candidate policy body (``{"table", "dialect", "rules"}``) against a live schema. - - Defense in depth -- the metastore validates on write too, but its PATCH only the patched keys. - """ - return draft7_errors(policy, schema) - - def _is_true_sentinel(condition: dict[Any, Any]) -> bool: """Exactly ``{"true": true}``. ``{"true": false}`` (or ``0``/extra keys) would otherwise read as an always-true policy -- the opposite of what an author who typed ``false`` meant. ``is True`` because diff --git a/src/keboola_agent_cli/services/cls_service.py b/src/keboola_agent_cli/services/cls_service.py index 61ebd9c0..89b713ad 100644 --- a/src/keboola_agent_cli/services/cls_service.py +++ b/src/keboola_agent_cli/services/cls_service.py @@ -6,7 +6,7 @@ (``visible_columns`` -- an allowlist projection; masking is not supported). Everything except the rule shape is identical to :class:`RlsService` (never -``project`` scope, fetch-then-merge update, live-schema validation that +``project`` scope, partial (PATCH) update, live-schema validation that degrades to a warning), so this class only overrides the policy type and the two rule hooks. Enforcement happens in ``keboola-mcp-server``'s ``query_data``, not here -- kbagent only authors policies. diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 03a76f2e..92a63097 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -31,15 +31,16 @@ from __future__ import annotations -import logging from collections.abc import Callable, Sequence from dataclasses import dataclass +from enum import StrEnum from typing import Any, NoReturn, cast import jsonschema from ..config_store import ConfigStore from ..errors import ErrorCode, KeboolaApiError +from ..json_utils import draft7_errors from ..metastore_client import ( MetastoreClient, ObjectScope, @@ -52,11 +53,9 @@ from ._semantic_layer_scope import resolve_target_project_ids from .base import BaseService, ClientFactory, make_session_aware_client_factory -logger = logging.getLogger(__name__) - RLS_ITEM_TYPE: SemanticType = "rls-policy" -__all__ = ["POLICY_SCOPES", "RLS_DIALECTS", "RLS_ITEM_TYPE", "RlsSchemaFetch", "RlsService"] +__all__ = ["RLS_ITEM_TYPE", "PolicyScope", "RlsSchemaFetch", "RlsService"] MetastoreClientFactory = Callable[[str, str], MetastoreClient] @@ -71,8 +70,11 @@ ) -# The scopes a policy may be authored at (the schema's `x-metastore.scope.supported`). -POLICY_SCOPES: tuple[ObjectScope, ...] = ("targeted", "organization") +class PolicyScope(StrEnum): + """The scopes a policy may be authored at (the schema's ``x-metastore.scope.supported``).""" + + TARGETED = "targeted" + ORGANIZATION = "organization" @dataclass(frozen=True) @@ -113,8 +115,6 @@ def __init__( metastore_client_factory or make_session_aware_client_factory(config_store, MetastoreClient) ) - # Project backend per (stack, token): one `verify_token` per project, not per write. - self._backends: dict[tuple[str, str], str] = {} # ------------------------------------------------------------------ # Helpers @@ -192,11 +192,8 @@ def _row_from_item(item: dict[str, Any]) -> dict[str, Any]: def _project_dialect(self, project: ProjectConfig) -> str: """The project's backend -- the only dialect the enforcement accepts for its policies.""" - key = (project.stack_url, project.token) - if key not in self._backends: - with self._client_factory(project.stack_url, project.token) as client: - self._backends[key] = (client.verify_token().default_backend or "").lower() - backend = self._backends[key] + with self._client_factory(project.stack_url, project.token) as client: + backend = (client.verify_token().default_backend or "").lower() if backend not in RLS_DIALECTS: self._raise_invalid( [ @@ -271,7 +268,7 @@ def _validate_policy( fetch = self._fetch_schema(client) if not fetch.schema: return PolicyValidation([], [f"Live schema validation was skipped: {fetch.reason}"]) - return PolicyValidation(_rls_condition.validate_policy_structural(policy, fetch.schema), []) + return PolicyValidation(draft7_errors(policy, fetch.schema), []) def _local_rule_errors(self, rules: Any) -> list[str]: """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" @@ -362,8 +359,8 @@ def create_policy( grants in the same transaction (and needs the organization-admin role). ``dry_run=True`` returns the preview without calling the write endpoint. """ - if scope not in POLICY_SCOPES: - self._raise_usage(f"scope must be one of {POLICY_SCOPES}, got {scope!r}") + if scope not in tuple(PolicyScope): + self._raise_usage(f"scope must be one of {tuple(PolicyScope)}, got {scope!r}") if target_projects and scope != "targeted": self._raise_usage("target projects require scope 'targeted'") target_ids = self._resolve_targets(alias, target_projects) @@ -392,7 +389,7 @@ def create_policy( self.item_type, name=table, data=policy, - scope=cast(ObjectScope, scope), # checked against POLICY_SCOPES above + scope=cast(ObjectScope, str(scope)), # checked against PolicyScope above target_project_ids=target_ids or None, conflict_hint=self._conflict_hint, ) diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index 0eb9e2a6..ec14aa83 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -14,30 +14,18 @@ import pytest -from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.errors import ErrorCode, KeboolaApiError -from keboola_agent_cli.models import ProjectConfig from keboola_agent_cli.services.cls_service import ClsService -from .test_rls_service import storage_client_factory +from .test_rls_service import _make_store, storage_client_factory TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" _RULES = [{"principal": "a@x.com", "visible_columns": ["id", "region"]}] def _make_service(tmp_path: Path) -> tuple[ClsService, MagicMock]: - config_dir = tmp_path / "config" - config_dir.mkdir() - store = ConfigStore(config_dir=config_dir) - store.add_project( - "prod", - ProjectConfig( - stack_url="https://connection.keboola.com", - token=TEST_TOKEN, - project_name="prod", - project_id=5725, - ), - ) + """The RLS fixtures (same store and Storage mock), with a CLS service and no live schema.""" + store = _make_store(tmp_path) mock = MagicMock() mock.__enter__ = MagicMock(return_value=mock) mock.__exit__ = MagicMock(return_value=False) diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 0f035bfd..9da119cc 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -15,11 +15,11 @@ from keboola_agent_cli.config_store import ConfigStore from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.json_utils import draft7_errors from keboola_agent_cli.models import ProjectConfig from keboola_agent_cli.services._rls_condition import ( compile_condition_preview, validate_condition_ops, - validate_policy_structural, validate_rules_local, ) from keboola_agent_cli.services.rls_service import RlsService @@ -781,11 +781,11 @@ def test_valid_policy_passes(self) -> None: }, } policy = {"table": "in.c-x.y", "dialect": "snowflake", "rules": [{"condition": {}}]} - assert validate_policy_structural(policy, schema) == [] + assert draft7_errors(policy, schema) == [] def test_missing_required_field_reported(self) -> None: schema = {"type": "object", "required": ["table", "dialect", "rules"]} - errors = validate_policy_structural({"table": "t"}, schema) + errors = draft7_errors({"table": "t"}, schema) assert errors diff --git a/tests/test_server_rls.py b/tests/test_server_rls.py index 3d0b72e8..0944b863 100644 --- a/tests/test_server_rls.py +++ b/tests/test_server_rls.py @@ -175,8 +175,14 @@ def test_update_is_patch_not_put(tmp_path: Path) -> None: assert res.status_code == 405, res.text -def test_target_projects_with_organization_scope_is_422(tmp_path: Path) -> None: +def test_target_projects_with_organization_scope_is_400(tmp_path: Path) -> None: + """The service refuses the combination (INVALID_ARGUMENT); REST answers it as a 400.""" rls_svc = MagicMock() + rls_svc.create_policy.side_effect = KeboolaApiError( + message="target projects require scope 'targeted'", + status_code=400, + error_code=ErrorCode.INVALID_ARGUMENT, + ) app = _make_app_with_registry(tmp_path, _mock_registry(rls=rls_svc)) body = { "table_id": "t.x", @@ -188,8 +194,7 @@ def test_target_projects_with_organization_scope_is_422(tmp_path: Path) -> None: with TestClient(app) as client: res = client.post(f"/rls/{PROJECT}", headers=AUTH, json=body) - assert res.status_code == 422, res.text - rls_svc.create_policy.assert_not_called() + assert res.status_code == 400, res.text def test_delete_policy_calls_service(tmp_path: Path) -> None: From 96458e0814b63d1af0687a8ec25568d2bbaf8b39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 8 Oct 2026 12:57:42 +0200 Subject: [PATCH 27/35] feat(CLI-17): author rls-policy / cls-policy schema 1.1.0 (groups, OR, default, $identity) Schema 1.1.0 becomes the metastore default (go-monorepo draft PR), so new writes are 1.1.0 without sending a version. - rules select identities by principal, principals or IdP groups - every rule matching one identity applies (RLS conditions OR, CLS columns union), so the duplicate-principal refusal and the cross-policy listing it needed are removed - rls --default (CLI and REST) sets the condition for identities no rule matches; {"false": true} is a new sentinel (no rows) - value {"$identity": "email"} / values {"$identity": "groups"} placeholders are accepted and previewed as / () - a stored policy without dialect (optional in 1.1.0) merges with the project backend on update; a CLS policy refuses a default - docs: CLAUDE.md, AGENT_CONTEXT, commands-reference, gotchas, rls-workflow (new 1.1.0 workflow), keboola-expert Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 31 ++-- plugins/kbagent/agents/keboola-expert.md | 2 +- .../kbagent/references/commands-reference.md | 14 +- .../skills/kbagent/references/gotchas.md | 41 +++-- .../skills/kbagent/references/rls-workflow.md | 67 +++++-- src/keboola_agent_cli/commands/rls.py | 41 ++++- src/keboola_agent_cli/server/routers/rls.py | 4 + .../services/_rls_condition.py | 166 +++++++++--------- src/keboola_agent_cli/services/cls_service.py | 20 ++- src/keboola_agent_cli/services/rls_service.py | 124 ++++++------- tests/test_cls_service.py | 26 ++- tests/test_rls_cli.py | 52 ++++++ tests/test_rls_service.py | 112 +++++++++--- tests/test_server_rls.py | 3 + 14 files changed, 464 insertions(+), 239 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8c6b73a8..0527a481 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1385,8 +1385,8 @@ kbagent semantic-layer reference-data delete --project P --id ID [--yes] kbagent rls list --project P kbagent rls detail --project P --policy-id ID kbagent rls schema --project P -kbagent rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes] -kbagent rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] +kbagent rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--default JSON|@file|-] [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes] +kbagent rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--default JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] kbagent rls delete --project P --policy-id ID [--dry-run] [--yes] kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes] # rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, @@ -1396,13 +1396,17 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # destructive-class (FLAG_ESCALATIONS); there is no project scope. WHO: a project admin may # write `targeted` policies of its own project without grants; grants and organization scope # need the organization-admin role (403 otherwise). Enforced only where the project has the -# `row-level-security` feature. One invalid policy refuses EVERY query of a project in the -# enforcement, so writes refuse a --dialect other than the project backend (it defaults to the -# backend) and a principal with two rules on one table (case-insensitive, also across visible -# policies). --rules is a JSON array of {principal|principals, condition}; condition is a -# declarative primitive tree (column/op/value comparisons, in/not_in, is_null/is_not_null, -# and/or nesting, or {"true": true}) -- never a free-text predicate; a null comparison is -# refused (use is_null). `update` reads the policy, validates the MERGED result, then PATCHes +# `row-level-security` feature. Policy schema 1.1.0 (the metastore default): a rule selects +# identities by principal, principals or IdP `groups`; every rule matching one identity applies +# and the conditions combine with OR; `--default` is the condition for identities no rule +# matches (absent = refused; {"false": true} = no rows; to remove a default, recreate the +# policy). Writes refuse a --dialect other than the project backend (it defaults to the +# backend; the enforcement would refuse every read of that table). --rules is a JSON array of +# {principal|principals|groups, condition}; condition is a declarative primitive tree +# (column/op/value comparisons, in/not_in, is_null/is_not_null, and/or nesting, {"true": true} +# or {"false": true}) -- never a free-text predicate; a null comparison is refused (use +# is_null); `value: {"$identity": "email"}` / `values: {"$identity": "groups"}` are resolved +# per reader by the enforcement. `update` reads the policy, validates the MERGED result, then PATCHes # only the changed keys (grants first); the result is re-read. `delete` is destructive-class. # `--dry-run` previews with columns quoted like the enforcement (keboola-mcp-server # `query_data`), but it is not the enforcement. `rls setup` is interactive-terminal-only @@ -1420,10 +1424,11 @@ kbagent cls create --project P --table-id TABLE_ID --rules JSON|@file|- [--diale kbagent cls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] kbagent cls delete --project P --policy-id ID [--dry-run] [--yes] # cls: column-level security policy authoring (`cls-policy` object type, one object per -# protected table), sibling of `rls`: same scope default, write permissions, dialect and -# duplicate-principal checks, partial update and permission classes. --rules is a JSON array -# of {principal|principals, visible_columns: [col, ...]} objects -- an allowlist projection -# (masking is not supported). `--dry-run` prints each principal's projection. No `setup` +# protected table), sibling of `rls`: same scope default, write permissions, dialect check, +# partial update and permission classes. --rules is a JSON array of +# {principal|principals|groups, visible_columns: [col, ...]} objects -- an allowlist projection +# (masking is not supported); an identity several rules match sees the union of their +# columns; no `--default`. `--dry-run` prints each selector's projection. No `setup` # wizard; the REST routes under /cls/{project} mirror /rls. # New error code: INVALID_CLS_POLICY. Version gate lives in gotchas.md. diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 8b50cb78..c0f6a2e3 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch, or one principal with two rules on a table, is refused: it would block every query of the project). `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch is refused: it would block every read of that table). Schema 1.1.0: rules select by `principal`/`principals`/`groups`, every matching rule applies (RLS conditions OR, CLS columns union), `rls --default '{"false": true}'` gives unmatched readers no rows instead of an error, `{"$identity": "email"|"groups"}` placeholders are resolved per reader. `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index c5d81b02..a3af0dc3 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,24 +469,24 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of per-principal condition primitives. `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend and a principal with two rules on one table (both would make the enforcement refuse every query of the project). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`) plus a condition primitive. Policy schema 1.1.0: every rule matching one identity applies and the conditions combine with OR; an optional policy `default` covers identities no rule matches (absent = refused). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. - `rls list --project P` -- list policies visible to (and applied in) a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, owner_project_id, source_project_id, target_project_ids}, ...]}`. `owner_project_id` is null for an `organization` policy; `target_project_ids` is shown to the owner only. -- `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals, condition}], revision}`. +- `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals|groups, condition}], default?, revision}`. - `rls schema --project P` -- live `rls-policy` JSON Schema fetched from the metastore. Unlike `flow schema`, there is no offline bundled snapshot -- always live-only, `NOT_FOUND` (exit 4) on fetch failure rather than a stale local copy. -- `rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or a `{"true": true}` sentinel) -- never a raw predicate string; a `null` comparison value is refused (use `is_null`). `--dialect` defaults to the project backend and must equal it. `--target-project` (alias or ID, repeatable or comma-separated, `targeted` only) is sent in the create request itself. Validated locally always (rule shape, dialect, duplicate principals across the policies the token can read), plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any write; a policy already on that table is `ALREADY_EXISTS`. `--dry-run` returns the same compiled-condition preview (columns quoted per dialect) a real write would, without writing. -- `rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- reads the policy, validates the MERGED result, then sends a partial `PATCH` of only the changed keys (unknown keys and concurrent changes to other keys survive). `--target-project` replaces the grants of a `targeted` policy, `--clear-target-projects` revokes them; grants change before the rules. The result is re-read after the write. Nothing to change is `INVALID_ARGUMENT` (exit 2). +- `rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--default JSON|@file|-] [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- create one policy for one table. `--rules` is a JSON array of `{principal|principals|groups, condition}` objects; `condition` is a declarative primitive tree (`column`/`op`/`value` comparisons `eq|ne|gt|gte|lt|lte`, `in`/`not_in` with `values`, `is_null`/`is_not_null`, `and`/`or` nesting of 2+ conditions, or the `{"true": true}` / `{"false": true}` sentinels) -- never a raw predicate string; a `null` comparison value is refused (use `is_null`); `value: {"$identity": "email"}` and `values: {"$identity": "groups"}` are resolved per reader by the enforcement. `--default` (a condition, e.g. `{"false": true}` = no rows) applies to identities no rule matches. `--dialect` defaults to the project backend and must equal it. `--target-project` (alias or ID, repeatable or comma-separated, `targeted` only) is sent in the create request itself. Validated locally always (rule and default shape, dialect), plus a live JSON-Schema check when the backend is reachable; failures raise `INVALID_RLS_POLICY` before any write; a policy already on that table is `ALREADY_EXISTS`. `--dry-run` returns the same compiled-condition preview (columns quoted per dialect; `default_preview` for a default) a real write would, without writing. +- `rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--default JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- reads the policy, validates the MERGED result, then sends a partial `PATCH` of only the changed keys (unknown keys and concurrent changes to other keys survive). `--target-project` replaces the grants of a `targeted` policy, `--clear-target-projects` revokes them; grants change before the rules. The result is re-read after the write. A default cannot be removed by an update (recreate the policy). Nothing to change is `INVALID_ARGUMENT` (exit 2). - `rls delete --project P --policy-id ID [--dry-run] [--yes]` -- delete a policy, un-protecting its table. Destructive-class (`--deny-destructive` blocks it). `--dry-run` shows the policy without deleting it. Confirmation prompt unless `--yes`; `--json` skips it too. - `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder (a value is read as JSON when it is one: `42`, `true`) or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table. Under `--json` or without an interactive terminal it exits 2 with an `INVALID_ARGUMENT` error, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. ## Column-Level Security (CLS) -Author `cls-policy` metastore objects -- one object per protected table, each holding per-principal `visible_columns` allowlists (columns not listed are omitted from that principal's result; masking is not supported). Sibling of RLS above: same scope default and options, write permissions, dialect and duplicate-principal checks, partial `update` and `delete --dry-run`; no interactive `setup`. Enforcement (and RLS+CLS composition) happens in `keboola-mcp-server`'s `query_data`. See [rls-workflow.md](rls-workflow.md#column-level-security-cls). +Author `cls-policy` metastore objects -- one object per protected table, each holding per-rule `visible_columns` allowlists (columns not listed are omitted from that identity's result; masking is not supported). Rules select by `principal`, `principals` or `groups`; an identity several rules match sees the union of their columns; there is no `default`. Sibling of RLS above: same scope default and options, write permissions, dialect check, partial `update` and `delete --dry-run`; no interactive `setup`. Enforcement (and RLS+CLS composition) happens in `keboola-mcp-server`'s `query_data`. See [rls-workflow.md](rls-workflow.md#column-level-security-cls). - `cls list --project P` -- list policies visible to a project (same row shape as `rls list`). -- `cls detail --project P --policy-id ID` -- full rule set: `{..., rules: [{principal|principals, visible_columns}], revision}`. +- `cls detail --project P --policy-id ID` -- full rule set: `{..., rules: [{principal|principals|groups, visible_columns}], revision}`. - `cls schema --project P` -- live `cls-policy` JSON Schema from the metastore (`NOT_FOUND`, exit 4, when the stack's metastore does not have the object type). -- `cls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each principal's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. +- `cls create --project P --table-id TABLE_ID --rules JSON|@file|- [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--dry-run] [--yes]` -- `--rules` is a JSON array of `{principal|principals|groups, visible_columns: [col, ...]}` (column names `[A-Za-z0-9_]+`). `--dry-run` prints each selector's allowed projection without writing. Failures raise `INVALID_CLS_POLICY`. - `cls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect ...] [--rules JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- partial update, same as `rls update`. - `cls delete --project P --policy-id ID [--dry-run] [--yes]` -- delete a policy; the table's columns become unrestricted for those principals. Destructive-class. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 72648d63..558a206d 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6772,25 +6772,29 @@ tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full r `--scope organization` need the organization-admin role -- a master token alone gets a 403 (`ACCESS_DENIED`). A project admin reads only the policies its project owns. Policies are enforced only where the consuming project has the `row-level-security` feature. -- **One invalid policy refuses every query of a project.** The enforcement refuses ALL `query_data` - calls of a project when a policy it loads has a `dialect` other than the project backend, or names one - principal (case-insensitive) twice on one table -- in one policy or across policies. The metastore - accepts both, so kbagent refuses them before writing (`INVALID_RLS_POLICY` / `INVALID_CLS_POLICY`). - `--dialect` defaults to the project backend (`verify_token` -> `defaultBackend`). The cross-policy check - only sees policies the token can read. An `organization` policy also reaches projects on the OTHER - backend, where it blocks every query -- prefer `targeted` in a mixed-backend organization. +- **Policy schema 1.1.0 (the metastore default) is a superset of 1.0.0.** A rule selects identities by + `principal`, `principals` or IdP `groups` (exact strings, any listed group matches). Every rule matching + one identity applies: RLS conditions combine with OR, CLS `visible_columns` are united -- so one + identity in several rules (or policies) is no longer an error. RLS `--default` is the condition for + identities no rule matches (absent = refused; `{"false": true}` = no rows); an `update` cannot remove a + default (recreate the policy). `value: {"$identity": "email"}` and `values: {"$identity": "groups"}` are + resolved per reader by the enforcement. `dialect` is optional in the schema; kbagent still sends the + project backend and refuses another one (`INVALID_RLS_POLICY` / `INVALID_CLS_POLICY`), because the + enforcement refuses every read of a table whose policy names another dialect. An `organization` + policy also reaches projects on the OTHER backend -- prefer `targeted` in a mixed-backend organization. - **One `rls-policy` object per protected table**, never one blob per project, named by its table: a second policy on the same table in the same project answers `ALREADY_EXISTS` (update or delete the first). `rls setup` creates one policy per selected table, all sharing the same rules. - **`condition` is a fixed six-shape primitive vocabulary, never a free-text predicate.** Comparison (`eq`/`ne`/`gt`/`gte`/`lt`/`lte`), membership (`in`/`not_in` with `values`), nullness - (`is_null`/`is_not_null`), `and`/`or` (2+ nested conditions each, no `not`), or the `{"true": true}` - sentinel. A `null` comparison value (or `null` in `values`) is refused: the enforcement renders - `col = NULL`, which matches nothing -- use `is_null`. A principal with whitespace or control - characters is refused (the enforcement would reject the whole policy). + (`is_null`/`is_not_null`), `and`/`or` (2+ nested conditions each, no `not`), or the `{"true": true}` / + `{"false": true}` sentinels. A `null` comparison value (or `null` in `values`) is refused: the + enforcement renders `col = NULL`, which matches nothing -- use `is_null`. A principal with whitespace or + control characters, or a group with control characters, is refused. - **The `--dry-run` preview follows the enforcement's rendering but is not the enforcement.** Columns are quoted per dialect (`"col"` / `` `col` ``, so matching is case-exact), booleans render - `TRUE`/`FALSE`. The compiler that decides what a query returns is `keboola-mcp-server`'s + `TRUE`/`FALSE`, `$identity` placeholders `` / `()`, a default as + `default_preview`. The compiler that decides what a query returns is `keboola-mcp-server`'s `rls.py::_compile_primitive` (sqlglot). - **`rls update` sends a partial update (`PATCH`) of only the changed keys.** It still reads the policy first: the metastore's PATCH validates only the keys it receives, so kbagent validates the MERGED @@ -6809,15 +6813,16 @@ tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full r interactive terminal it exits 2 with an `INVALID_ARGUMENT` error (a JSON envelope under `--json`) pointing at `rls create`. A prompted value is read as JSON when it is one (`42`, `true`); quote it to keep a string. Exits non-zero when any selected table's policy could not be created. -- **REST mirror.** `POST /rls/{project}` (`table_id`, `rules`, `dialect?`, `scope`, `target_projects`), - `PATCH /rls/{project}/{policy_id}` (partial; `target_projects: []` revokes), `DELETE ...?dry_run=true`; +- **REST mirror.** `POST /rls/{project}` (`table_id`, `rules`, `default?`, `dialect?`, `scope`, + `target_projects`), `PATCH /rls/{project}/{policy_id}` (partial; `target_projects: []` revokes), `DELETE ...?dry_run=true`; same for `/cls`. Every route is permission-gated; `scope: "organization"` is also checked as `.create --scope organization`. - **`cls` is the column-level sibling of `rls`.** `kbagent cls list|detail|schema|create|update|delete` - author `cls-policy` objects (`{table, dialect, rules: [{principal|principals, visible_columns}]}`) with the - same scope, permissions, checks and partial `update` as `rls`. Differences: `visible_columns` is a - non-empty allowlist of `[A-Za-z0-9_]+` names (masking is not supported), `--dry-run` prints each - principal's projection, validation failures raise `INVALID_CLS_POLICY`, and there is no `cls setup`. + author `cls-policy` objects (`{table, dialect, rules: [{principal|principals|groups, visible_columns}]}`) + with the same scope, permissions, checks and partial `update` as `rls`. Differences: `visible_columns` is + a non-empty allowlist of `[A-Za-z0-9_]+` names (masking is not supported), `--dry-run` prints each + selector's projection, validation failures raise `INVALID_CLS_POLICY`, there is no `default`, and there + is no `cls setup`. - **Reduced validation is reported.** If the live schema could not be fetched, writes still run every local check but add a `warnings` entry; an authentication/permission failure while fetching the schema is raised as that error. diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index 39806991..41dbe2b9 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -35,15 +35,16 @@ deletes the policy objects the enforcement engine reads. the `row-level-security` project feature (it gates CLS too). Without it, queries are unfiltered -- not an error. -**One bad policy blocks every query.** When the enforcement loads an invalid -policy it refuses **every** `query_data` call of the project, not just queries -of that table. Two such cases pass the metastore's own validation, so kbagent -refuses them before any write: -- a `dialect` other than the project backend (`--dialect` defaults to the - backend; a different value is `INVALID_RLS_POLICY`); -- one principal (case-insensitive) with two rules on one table, inside one - policy or across policies. The cross-policy check sees only the policies the - token can read -- for a project admin, those its project owns. +**How rules combine (policy schema 1.1.0, the metastore default).** A rule +selects identities by `principal`, `principals` or IdP `groups`. Every rule +that matches the reader applies: RLS conditions combine with OR, CLS +`visible_columns` are united. An RLS policy may set a `default` condition for +readers no rule matches; without one, their reads are refused. 1.1.0 is a +superset of 1.0.0 -- every older policy keeps its meaning. + +**Dialect.** A policy whose `dialect` differs from the reading workspace makes +the enforcement refuse every read of that table. `--dialect` defaults to the +project backend, and kbagent refuses any other value (`INVALID_RLS_POLICY`). An `organization` policy is loaded by every project in the organization, whatever their backend -- in an organization with both Snowflake and BigQuery @@ -134,7 +135,7 @@ cat > rls_rules.json <<'EOF' EOF # 2. ALWAYS preview first -- --dry-run runs every validation (dialect vs -# backend, rule shape, duplicate principals, live schema check when +# backend, rule and default shape, live schema check when # available) and compiles a condition preview, but writes nothing. kbagent --json rls create \ --project prod \ @@ -144,7 +145,7 @@ kbagent --json rls create \ # -> {"table": "in.c-crm.invoices", "dialect": "snowflake", # "scope": "targeted", "target_project_ids": [], "preview": [ # {"principal": "eu-analyst@example.com", "condition": "\"region\" = 'EU'"}, -# {"principal": ["us-analyst@example.com", "us-lead@example.com"], +# {"principals": ["us-analyst@example.com", "us-lead@example.com"], # "condition": "\"region\" = 'US'"} # ], "dry_run": true} @@ -226,6 +227,39 @@ engine that decides what a query returns lives in `keboola-mcp-server`'s `rls.py` (sqlglot-based). Treat the preview as "does this look like what I meant," not as proof of what will be enforced. +## Workflow 4b -- Groups, a default and identity placeholders (schema 1.1.0) + +One policy for many readers instead of one rule per person. Groups are the IdP +`groups` claim strings exactly as delivered (Entra may deliver object ids); +Keboola never manages membership. + +```bash +cat > orders_rules.json <<'EOF' +[ + {"groups": ["sales-eu"], + "condition": {"column": "region", "op": "in", "values": ["EU"]}}, + {"groups": ["sales-reps"], + "condition": {"column": "owner_email", "op": "eq", "value": {"$identity": "email"}}}, + {"principals": ["auditor@example.com"], + "condition": {"true": true}} +] +EOF +kbagent --json rls create --project prod --table-id in.c-sales.orders \ + --rules @orders_rules.json --default '{"false": true}' --dry-run +``` + +A reader in `sales-eu` and `sales-reps` sees +`"region" IN ('EU') OR "owner_email" = `; the auditor sees every +row; anyone else sees no rows (the `default`), instead of an error. + +- `{"$identity": "email"}` (as `value`) and `{"$identity": "groups"}` (as + `values`) are resolved by the enforcement per reader, as bound literals. +- `{"false": true}` matches no row -- useful as a `default` or a rule. +- A default cannot be removed by `rls update` (the metastore's partial update + cannot delete a key); recreate the policy instead. +- MCP OAuth users have an email but no IdP groups, so `groups` rules only + match readers whose identity carries a groups claim. + ## Workflow 5 -- Share a policy with sibling projects (targeted grants) `--target-project` (on `create`/`setup`/`update`) takes a registered alias or a @@ -297,8 +331,8 @@ kbagent --json rls schema --project prod - **`ACCESS_DENIED` (403)** on `--target-project`, `--clear-target-projects` or `--scope organization` -- the token's user is not an organization admin. - **`INVALID_RLS_POLICY`** -- the policy failed validation: a dialect other than - the project backend, a malformed rule, a null comparison, a principal with two - rules on the table, or the live JSON-Schema check. The message lists every + the project backend, a malformed rule or default, a null comparison, or the + live JSON-Schema check. The message lists every violation; nothing was written. - **`ALREADY_EXISTS`** -- the project already has a policy for this table (policies are named by table). `rls update` or `rls delete` it. @@ -350,6 +384,7 @@ kbagent cls delete --project prod --policy-id --yes ``` - A table can carry both an `rls-policy` and a `cls-policy`; author them separately. -- `INVALID_CLS_POLICY` -- the policy failed validation (exactly one of `principal`/`principals` - per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, the dialect check, duplicate - principals, plus the live JSON Schema). Nothing was written. +- `INVALID_CLS_POLICY` -- the policy failed validation (exactly one of `principal`/`principals`/ + `groups` per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, the dialect check, + no `default`, plus the live JSON Schema). Nothing was written. +- An identity several CLS rules match sees the union of their `visible_columns`. diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index c5a9cf07..d82f2e39 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -75,6 +75,10 @@ "--scope targeted only; granting needs the organization-admin role)" ) TABLE_HELP = "Storage table ID, e.g. in.c-crm.invoices" +DEFAULT_HELP = ( + "JSON|@file|- condition for identities no rule matches, e.g. '{\"false\": true}' (no rows); " + "omitted, their reads are refused" +) @dataclass(frozen=True) @@ -139,6 +143,9 @@ def gate_scope( def _principal_text(item: dict[str, Any]) -> str: + """A rule's (or preview entry's) selector: ``a@x.com``, ``a@x.com, b@x.com`` or ``groups: g1, g2``.""" + if "groups" in item: + return "groups: " + ", ".join(item["groups"]) principal = item.get("principal") or item.get("principals") or [] return principal if isinstance(principal, str) else ", ".join(principal) @@ -184,6 +191,8 @@ def _print_policy(formatter: Any, group: PolicyGroup, row: dict[str, Any]) -> No formatter.console.print(f" Rules ({len(rules)}):") for rule in rules: formatter.console.print(f" - {_principal_text(rule)}: {group.rule_text(rule)}") + if row.get("default") is not None: + formatter.console.print(f" Default: {row['default']}") def _print_preview(formatter: Any, group: PolicyGroup, result: dict[str, Any]) -> None: @@ -193,6 +202,8 @@ def _print_preview(formatter: Any, group: PolicyGroup, result: dict[str, Any]) - ) for entry in result.get("preview", []): formatter.console.print(f" {_principal_text(entry)}: {group.preview_text(entry)}") + if result.get("default_preview") is not None: + formatter.console.print(f" (anyone else): WHERE {result['default_preview']}") def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: @@ -217,6 +228,24 @@ def _parse_rules_arg(formatter: Any, raw: str, shape: str) -> list[dict[str, Any return parsed +def _parse_default_arg(formatter: Any, raw: str | None) -> dict[str, Any] | None: + """Parse ``--default`` (a condition object), or ``None`` when the option was not given.""" + if raw is None: + return None + try: + parsed = parse_json_arg(raw, label="--default") + except ValueError as exc: + formatter.error(message=str(exc), error_code=ErrorCode.INVALID_ARGUMENT) + raise typer.Exit(code=2) from None + if not isinstance(parsed, dict): + formatter.error( + message='--default must be a JSON condition object, e.g. {"false": true}', + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + return parsed + + def list_policies(ctx: typer.Context, group: PolicyGroup, project: str) -> None: formatter = get_formatter(ctx) service = get_service(ctx, group.service) @@ -429,10 +458,11 @@ def rls_create( ..., "--rules", help=( - "JSON|@file|- array of {principal|principals, condition} objects " - "(see `rls schema` / rls-workflow.md for the condition shape)" + "JSON|@file|- array of {principal|principals|groups, condition} objects; rules " + "matching one identity combine with OR (see rls-workflow.md for the condition shape)" ), ), + default: str | None = typer.Option(None, "--default", help=DEFAULT_HELP), dialect: Dialect | None = typer.Option(None, "--dialect", help=DIALECT_HELP), scope: PolicyScope = typer.Option(PolicyScope.TARGETED, "--scope", help=SCOPE_HELP), target_project: list[str] | None = typer.Option(None, "--target-project", help=TARGET_HELP), @@ -447,6 +477,7 @@ def rls_create( RLS, alias=project, table=table_id, + default=_parse_default_arg(get_formatter(ctx), default), dialect=dialect, rules=rules, scope=scope, @@ -468,6 +499,11 @@ def rls_update( rules: str | None = typer.Option( None, "--rules", help="New JSON|@file|- rules array (unset = unchanged)" ), + default: str | None = typer.Option( + None, + "--default", + help="New default condition (unset = unchanged; to remove one, recreate the policy)", + ), target_project: list[str] | None = typer.Option( None, "--target-project", help=f"{TARGET_HELP}; replaces the list (unset = unchanged)" ), @@ -492,6 +528,7 @@ def rls_update( alias=project, policy_id=policy_id, table=table_id, + default=_parse_default_arg(get_formatter(ctx), default), dialect=dialect, rules=rules, target_project=target_project, diff --git a/src/keboola_agent_cli/server/routers/rls.py b/src/keboola_agent_cli/server/routers/rls.py index 455f4f2c..f2ddc717 100644 --- a/src/keboola_agent_cli/server/routers/rls.py +++ b/src/keboola_agent_cli/server/routers/rls.py @@ -54,6 +54,7 @@ class PolicyCreate(BaseModel): table_id: str rules: list[dict[str, Any]] dialect: Dialect | None = None # None = the project backend + default: dict[str, Any] | None = None # rls only: condition for identities no rule matches scope: PolicyScope = PolicyScope.TARGETED target_projects: list[str | int] | None = None dry_run: bool = False @@ -65,6 +66,7 @@ class PolicyUpdate(BaseModel): table_id: str | None = None dialect: Dialect | None = None rules: list[dict[str, Any]] | None = None + default: dict[str, Any] | None = None # rls only; omitted = unchanged target_projects: list[str | int] | None = None dry_run: bool = False @@ -133,6 +135,7 @@ def create_policy( table=body.table_id, rules=body.rules, dialect=body.dialect, + default=body.default, scope=body.scope, target_projects=body.target_projects, dry_run=body.dry_run, @@ -154,6 +157,7 @@ def update_policy( table=body.table_id, dialect=body.dialect, rules=body.rules, + default=body.default, target_projects=body.target_projects, dry_run=body.dry_run, ) diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 143e11c3..805b5b86 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -7,7 +7,11 @@ ``keboola-mcp-server``'s RFC (``feature_spec/rls_query_tool/RFC.md``, "Rule storage" section) -- kbagent never invents its own shape, it authors exactly what the enforcement engine (that repo's ``rls.py::_compile_primitive``, -sqlglot-based) expects to read back from the metastore. +sqlglot-based) expects to read back from the metastore. The checks follow the policy +schema 1.1.0 (go-monorepo ``rls-policy_schema_1.1.0.json``): a rule selects identities by +``principal``, ``principals`` or IdP ``groups``; every rule matching one identity applies (the +conditions combine with OR); ``{"false": true}`` matches no row; a literal may be the +``{"$identity": ...}`` placeholder the engine resolves per reader. ``compile_condition_preview`` below is a **preview** for ``--dry-run`` / confirmation display, not the enforcement engine -- that lives in @@ -15,7 +19,8 @@ enforcement's rendering where an admin could be misled otherwise: columns quoted per dialect (the enforcement matches them case-exactly), booleans as ``TRUE``/``FALSE``. ``null`` comparisons never reach it: the enforcement would -render ``col = NULL`` (matches nothing), so validation refuses them. Adding +render ``col = NULL`` (matches nothing), so validation refuses them. ``$identity`` +placeholders render as ```` / ``()``. Adding sqlglot as a kbagent dependency just for this string is not warranted. """ @@ -46,20 +51,27 @@ class Dialect(StrEnum): RLS_DIALECTS: tuple[str, ...] = tuple(Dialect) -# The enforcement's principal pattern (mcp-server ``rls.py``): no whitespace or control characters. A -# principal that fails it is not ignored there -- it refuses EVERY query in the project. +# The enforcement's principal pattern (mcp-server ``rls.py``): no whitespace or control characters. _PRINCIPAL_RE = re.compile(r"^[^\s\x00-\x1f\x7f]+$") +# IdP group names may contain spaces ("Sales EU"); only control characters are refused. +_GROUP_RE = re.compile(r"^[^\x00-\x1f\x7f]+$") + +# The rule selectors of schema 1.1.0 -- each rule names exactly one. +SELECTORS: tuple[str, ...] = ("principal", "principals", "groups") + +# The `$identity` placeholders: `value` may be the reader's email, `values` the reader's groups. +IDENTITY_EMAIL = {"$identity": "email"} +IDENTITY_GROUPS = {"$identity": "groups"} # How each dialect quotes an identifier. The enforcement compiles columns quoted (case-exact), so the # preview must too -- an unquoted preview would hide a case mismatch the real filter does not forgive. _QUOTE = {"snowflake": '"', "bigquery": "`"} -def _is_true_sentinel(condition: dict[Any, Any]) -> bool: - """Exactly ``{"true": true}``. ``{"true": false}`` (or ``0``/extra keys) would otherwise read as an - always-true policy -- the opposite of what an author who typed ``false`` meant. ``is True`` because - ``1 == True`` in Python.""" - return len(condition) == 1 and condition["true"] is True +def _is_sentinel(condition: dict[Any, Any], key: str) -> bool: + """Exactly ``{"true": true}`` / ``{"false": true}``. ``{"true": false}`` (or ``0``/extra keys) would + otherwise read as the opposite of what the author typed. ``is True`` because ``1 == True``.""" + return len(condition) == 1 and condition[key] is True def _is_scalar(value: Any) -> bool: @@ -78,10 +90,11 @@ def validate_condition_ops(condition: Any) -> list[str]: errors: list[str] = [] if not isinstance(condition, dict): return [f"condition must be an object, got {type(condition).__name__}"] - if "true" in condition: - if not _is_true_sentinel(condition): - errors.append("'true' condition must be exactly {\"true\": true}") - return errors + for sentinel in ("true", "false"): + if sentinel in condition: + if not _is_sentinel(condition, sentinel): + errors.append(f"'{sentinel}' condition must be exactly {{\"{sentinel}\": true}}") + return errors if "and" in condition or "or" in condition: # Exactly one composition key and nothing else: `{"and": [...], "or": [...]}` would otherwise have # one branch silently validated and the other dropped, so the policy would no longer be the tree @@ -116,12 +129,19 @@ def validate_condition_ops(condition: Any) -> list[str]: errors.append( f"condition with op {op!r} cannot compare to null; use op 'is_null' / 'is_not_null'" ) - elif not _is_scalar(condition["value"]): - errors.append(f"condition with op {op!r} needs a string, number or boolean 'value'") + elif not _is_scalar(condition["value"]) and condition["value"] != IDENTITY_EMAIL: + errors.append( + f"condition with op {op!r} needs a string, number or boolean 'value', " + f"or {IDENTITY_EMAIL}" + ) if op in RLS_MEMBERSHIP_OPS: values = condition.get("values") - if not isinstance(values, list) or not values: - errors.append(f"condition with op {op!r} needs a non-empty list 'values'") + if values == IDENTITY_GROUPS: + pass + elif not isinstance(values, list) or not values: + errors.append( + f"condition with op {op!r} needs a non-empty list 'values', or {IDENTITY_GROUPS}" + ) elif any(value is None for value in values): errors.append( f"condition with op {op!r} cannot list null in 'values' (it never matches); use 'is_null'" @@ -139,78 +159,49 @@ def validate_condition_ops(condition: Any) -> list[str]: def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: - """Each rule names exactly one of ``principal``/``principals`` (the schema's ``oneOf``). + """Each rule names exactly one selector: ``principal``, ``principals`` or ``groups``. - Shared by the RLS and CLS (``cls_service``) local checks -- both policy - types use the identical principal shape. + Shared by the RLS and CLS (``cls_service``) local checks -- both policy types use the identical + selector shape. Key PRESENCE decides "exactly one" (a truthiness test would let + ``{"principal": "a", "principals": []}`` through); the value is then checked on its own. """ - # Key PRESENCE decides "exactly one" (a truthiness test would let `{"principal": "a", "principals": []}` - # through as if `principals` were absent); the VALUE is then checked on its own. - has_principal = "principal" in rule - has_principals = "principals" in rule - if has_principal == has_principals: # both or neither - return [f"rules[{index}] must set exactly one of 'principal'/'principals'"] - if has_principal: - principal = rule["principal"] - if not isinstance(principal, str) or not principal: - return [f"rules[{index}].principal must be a non-empty string"] - names = [principal] - else: - names = rule["principals"] - if ( - not isinstance(names, list) - or not names - or not all(isinstance(name, str) and name for name in names) - ): - return [f"rules[{index}].principals must be a non-empty list of non-empty strings"] + present = [key for key in SELECTORS if key in rule] + if len(present) != 1: + return [f"rules[{index}] must set exactly one of {'/'.join(SELECTORS)}"] + key = present[0] + names = [rule[key]] if key == "principal" else rule[key] + if ( + not isinstance(names, list) + or not names + or not all(isinstance(name, str) and name for name in names) + ): + shape = ( + "a non-empty string" if key == "principal" else "a non-empty list of non-empty strings" + ) + return [f"rules[{index}].{key} must be {shape}"] + pattern, problem = ( + (_GROUP_RE, "control characters") + if key == "groups" + else (_PRINCIPAL_RE, "whitespace or control characters") + ) return [ - f"rules[{index}]: principal {name!r} contains whitespace or control characters" + f"rules[{index}]: {key} entry {name!r} contains {problem}" for name in names - if not _PRINCIPAL_RE.fullmatch(name) + if not pattern.fullmatch(name) ] -def rule_principals(rule: dict[str, Any]) -> list[str]: - """The principals one rule names (``principal`` or ``principals``), as written.""" - return [rule["principal"]] if "principal" in rule else list(rule.get("principals") or []) - - -def duplicate_principals( - rules: list[dict[str, Any]], taken: dict[str, str] | None = None -) -> list[str]: - """Principals named twice on one table, case-folded -- the enforcement refuses every query then. - - ``taken`` maps an already-used case-folded principal to where it is used (another policy on the - same table), so the same check covers duplicates inside ``rules`` and across policies. - """ - seen = dict(taken or {}) - errors: list[str] = [] - for index, rule in enumerate(rules): - for name in rule_principals(rule): - folded = name.casefold() - if folded in seen: - errors.append( - f"rules[{index}]: principal {name!r} already has a rule in {seen[folded]}" - ) - else: - seen[folded] = f"rules[{index}]" - return errors - - -def table_key(table: str, dialect: str) -> str: - """How the enforcement compares table keys: case-insensitive on Snowflake, exact on BigQuery.""" - return table.lower() if dialect == "snowflake" else table +def rule_selector(rule: dict[str, Any]) -> dict[str, Any]: + """The rule's selector as written (``{"principal": ...}``, ``{"principals": [...]}`` or ``{"groups": [...]}``).""" + return {key: rule[key] for key in SELECTORS if key in rule} def validate_rules_local(rules: Any) -> list[str]: """Semantic checks on ``rules`` that hold regardless of schema availability. - Runs even when the live schema fetch degraded (see - ``RlsService.fetch_schema``) -- the one check this module can still make - with no network access at all: each rule names exactly one of - ``principal``/``principals`` (the RFC's ``oneOf``, which a Draft7 - validator would otherwise be the only thing checking) and every - ``condition`` uses a known operator (:func:`validate_condition_ops`). + Runs even when the live schema fetch degraded (see ``RlsService._fetch_schema``): each rule names + exactly one selector (:func:`validate_principal_fields`) and a well-formed ``condition`` + (:func:`validate_condition_ops`). Rules naming the same identity are fine -- the engine ORs them. Returns human-readable error strings (empty = valid). """ errors: list[str] = [] @@ -238,10 +229,13 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: validation first; this function stays strict rather than silently rendering something misleading). """ - if "true" in condition: - if not _is_true_sentinel(condition): - raise ValueError(f"'true' condition must be exactly {{\"true\": true}}: {condition!r}") - return "TRUE" + for sentinel in ("true", "false"): + if sentinel in condition: + if not _is_sentinel(condition, sentinel): + raise ValueError( + f"'{sentinel}' condition must be exactly {{\"{sentinel}\": true}}: {condition!r}" + ) + return sentinel.upper() if "and" in condition or "or" in condition: if len(condition) != 1: raise ValueError( @@ -264,7 +258,11 @@ def compile_condition_preview(condition: dict[str, Any], dialect: str) -> str: return f"{column} {COMPARISON_SQL[op]} {_preview_literal(condition.get('value'))}" if op in RLS_MEMBERSHIP_OPS: values = condition.get("values") or [] - rendered = ", ".join(_preview_literal(v) for v in values) + rendered = ( + "" + if values == IDENTITY_GROUPS + else ", ".join(_preview_literal(v) for v in values) + ) keyword = "IN" if op == "in" else "NOT IN" return f"{column} {keyword} ({rendered})" if op == "is_null": @@ -280,10 +278,10 @@ def _preview_literal(value: Any) -> str: on purpose -- this output is never executed, only displayed for human review (see module docstring). """ + if value == IDENTITY_EMAIL: + return "" if isinstance(value, str): return "'" + value.replace("'", "''") + "'" if isinstance(value, bool): # before the generic str(): Python's `True` is SQL's TRUE return "TRUE" if value else "FALSE" - if value is None: - return "NULL" return str(value) diff --git a/src/keboola_agent_cli/services/cls_service.py b/src/keboola_agent_cli/services/cls_service.py index 89b713ad..c62a35f1 100644 --- a/src/keboola_agent_cli/services/cls_service.py +++ b/src/keboola_agent_cli/services/cls_service.py @@ -2,13 +2,15 @@ The ``cls-policy`` metastore object is the column-level sibling of ``rls-policy``: one object per protected table, ``{table, dialect, rules}``, -where each rule names a principal and the columns that principal may see -(``visible_columns`` -- an allowlist projection; masking is not supported). +where each rule selects identities (``principal``, ``principals`` or IdP +``groups``) and the columns they may see (``visible_columns`` -- an allowlist +projection; masking is not supported). Under schema 1.1.0 an identity several +rules match sees the union of their columns; there is no ``default``. Everything except the rule shape is identical to :class:`RlsService` (never ``project`` scope, partial (PATCH) update, live-schema validation that degrades to a warning), so this class only overrides the policy type and the -two rule hooks. Enforcement happens in ``keboola-mcp-server``'s +rule / default hooks. Enforcement happens in ``keboola-mcp-server``'s ``query_data``, not here -- kbagent only authors policies. """ @@ -59,11 +61,13 @@ def _local_rule_errors(self, rules: Any) -> list[str]: return errors def _preview_rules(self, rules: list[dict[str, Any]], dialect: str) -> list[dict[str, Any]]: - """Show each principal's allowed projection (``dialect`` is unused: no SQL is rendered).""" + """Show each selector's allowed projection (``dialect`` is unused: no SQL is rendered).""" return [ - { - "principal": rule.get("principal") or rule.get("principals"), - "visible_columns": rule["visible_columns"], - } + {**_rls_condition.rule_selector(rule), "visible_columns": rule["visible_columns"]} for rule in rules ] + + @staticmethod + def _default_errors(default: Any) -> list[str]: + """A ``cls-policy`` has no ``default``: an identity no rule matches is refused.""" + return [] if default is None else ["'default' is not supported by a CLS policy"] diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 92a63097..90f9d41d 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -19,10 +19,13 @@ WITHOUT target projects; granting target projects and ``organization`` scope need the organization-admin role. -The enforcement (keboola-mcp-server ``query_data``) refuses EVERY query of a -project when one policy it loads is invalid -- a dialect other than the -project backend, or one principal with two rules on one table. Both are -refused here before any write. +Policy schema 1.1.0 (the metastore default): a rule selects identities by +``principal``, ``principals`` or IdP ``groups``; every rule matching one +identity applies and the conditions combine with OR (so one identity in +several rules is fine); an RLS ``default`` condition covers identities no rule +matches (absent = the read is refused). A dialect other than the project +backend makes the enforcement refuse reads of that table, so it is refused +here before any write. On a stack whose metastore predates the ``rls-policy`` schema, a real project answers with a schema-fetch/list/get/post failure (see ``fetch_schema`` and the @@ -208,60 +211,30 @@ def _resolve_targets(self, alias: str, targets: Sequence[int | str] | None) -> l self._config_store, alias, [str(t) for t in targets or []] ) - def _taken_principals( - self, client: MetastoreClient, table: str, dialect: str, exclude_id: str | None - ) -> dict[str, str]: - """Principals other policies on the same table already have a rule for (case-folded). - - Best effort: the metastore lists only the policies this token may read -- for a project admin, - those its project owns -- so a clash with a policy owned elsewhere is not visible here. - """ - key = _rls_condition.table_key(table, dialect) - taken: dict[str, str] = {} - for item in client.list_items(self.item_type): - attrs = item.get("attributes") or {} - if item.get("id") == exclude_id or not isinstance(attrs.get("table"), str): - continue - if _rls_condition.table_key(attrs["table"], dialect) != key: - continue - for rule in attrs.get("rules") or []: - if isinstance(rule, dict): - for name in _rls_condition.rule_principals(rule): - taken.setdefault(str(name).casefold(), f"policy {item.get('id')}") - return taken - def _validate_policy( self, client: MetastoreClient, *, policy: dict[str, Any], backend: str, - exclude_id: str | None = None, ) -> PolicyValidation: """Run every check this module can on the WHOLE policy, local checks first. - Local (schema-independent) checks always run: the dialect matches the project backend, the - rules are well-formed and no principal has two rules on the table (within the policy or across - the other visible policies). Structural (Draft7, against the live schema) checks run when the + Local (schema-independent) checks always run: the dialect matches the project backend, and + the rules (and an RLS ``default``) are well-formed. Structural (Draft7, against the live schema) checks run when the fetch succeeds -- when it doesn't, the caller gets a warning, not a block (see :class:`RlsSchemaFetch`). The whole policy is validated even for a partial update, because the metastore's PATCH validates only the keys it receives. """ - table, dialect, rules = policy["table"], policy["dialect"], policy["rules"] + dialect = policy["dialect"] errors: list[str] = [] if dialect != backend: errors.append( f"dialect {dialect!r} does not match the project backend {backend!r} " - "(the enforcement refuses every query of a project with such a policy)" + "(the enforcement would refuse every read of this table)" ) - errors.extend(self._local_rule_errors(rules)) - if errors: - return PolicyValidation(errors, []) - errors.extend( - _rls_condition.duplicate_principals( - rules, self._taken_principals(client, table, dialect, exclude_id) - ) - ) + errors.extend(self._local_rule_errors(policy["rules"])) + errors.extend(self._default_errors(policy.get("default"))) if errors: return PolicyValidation(errors, []) @@ -274,16 +247,34 @@ def _local_rule_errors(self, rules: Any) -> list[str]: """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" return _rls_condition.validate_rules_local(rules) + @staticmethod + def _default_errors(default: Any) -> list[str]: + """Checks on the policy-level ``default`` condition (hook: :class:`ClsService` has none).""" + if default is None: + return [] + return [f"default.{err}" for err in _rls_condition.validate_condition_ops(default)] + def _preview_rules(self, rules: list[dict[str, Any]], dialect: str) -> list[dict[str, Any]]: """Per-rule display preview shown by ``--dry-run`` (hook for :class:`ClsService`).""" return [ { - "principal": rule.get("principal") or rule.get("principals"), + **_rls_condition.rule_selector(rule), "condition": _rls_condition.compile_condition_preview(rule["condition"], dialect), } for rule in rules ] + def _preview(self, policy: dict[str, Any]) -> dict[str, Any]: + """The ``preview`` (and, for RLS, ``default_preview``) keys of a write/dry-run result.""" + preview: dict[str, Any] = { + "preview": self._preview_rules(policy["rules"], policy["dialect"]) + } + if policy.get("default") is not None: + preview["default_preview"] = _rls_condition.compile_condition_preview( + policy["default"], policy["dialect"] + ) + return preview + @staticmethod def _with_warnings(result: dict[str, Any], warnings: list[str]) -> dict[str, Any]: """Attach validation warnings (e.g. the live schema was unavailable) only when there are any.""" @@ -332,8 +323,11 @@ def get_policy(self, alias: str, policy_id: str) -> dict[str, Any]: return self._detail_row(client.get_item(self.item_type, policy_id)) def _detail_row(self, item: dict[str, Any]) -> dict[str, Any]: + attrs = item.get("attributes") or {} row = self._row_from_item(item) - row["rules"] = (item.get("attributes") or {}).get("rules", []) + row["rules"] = attrs.get("rules", []) + if attrs.get("default") is not None: + row["default"] = attrs["default"] row["revision"] = (item.get("meta") or {}).get("revision") return row @@ -350,11 +344,13 @@ def create_policy( dialect: str | None = None, scope: str = "targeted", target_projects: Sequence[int | str] | None = None, + default: dict[str, Any] | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Create one policy -- at ``targeted`` scope (default) or, explicitly, ``organization``. - ``dialect`` defaults to the project backend and must equal it. ``target_projects`` (aliases or + ``dialect`` defaults to the project backend and must equal it. ``default`` (RLS only) is the + condition for identities no rule matches; omitted, their reads are refused. ``target_projects`` (aliases or project IDs) only with ``targeted``; they go in the create request itself, which stores the grants in the same transaction (and needs the organization-admin role). ``dry_run=True`` returns the preview without calling the write endpoint. @@ -366,13 +362,15 @@ def create_policy( target_ids = self._resolve_targets(alias, target_projects) project = self._resolve_one_project(alias) backend = self._project_dialect(project) - policy = {"table": table, "dialect": dialect or backend, "rules": rules} + policy: dict[str, Any] = {"table": table, "dialect": dialect or backend, "rules": rules} + if default is not None: + policy["default"] = default with self._new_metastore_client(project) as client: check = self._validate_policy(client, policy=policy, backend=backend) if check.errors: self._raise_invalid(check.errors) - preview = self._preview_rules(rules, policy["dialect"]) + preview = self._preview(policy) if dry_run: return self._with_warnings( { @@ -380,7 +378,7 @@ def create_policy( **policy, "scope": scope, "target_project_ids": target_ids, - "preview": preview, + **preview, "dry_run": True, }, check.warnings, @@ -393,9 +391,7 @@ def create_policy( target_project_ids=target_ids or None, conflict_hint=self._conflict_hint, ) - row = self._row_from_item(created) - row["preview"] = preview - return self._with_warnings(row, check.warnings) + return self._with_warnings({**self._detail_row(created), **preview}, check.warnings) def update_policy( self, @@ -406,6 +402,7 @@ def update_policy( dialect: str | None = None, rules: list[dict[str, Any]] | None = None, target_projects: Sequence[int | str] | None = None, + default: dict[str, Any] | None = None, dry_run: bool = False, ) -> dict[str, Any]: """Update one policy: only the given fields change. @@ -416,15 +413,23 @@ def update_policy( ``target_projects``: ``None`` keeps the grants, a list replaces them, ``[]`` revokes them all. Grants change first (an organization-admin-only call): when that is refused, nothing is written. - The result is re-read after the write, so it shows the grants as they now are. + The result is re-read after the write, so it shows the grants as they now are. ``default`` + replaces the RLS default condition; a PATCH cannot remove it (recreate the policy for that). """ changes = { key: value - for key, value in (("table", table), ("dialect", dialect), ("rules", rules)) + for key, value in ( + ("table", table), + ("dialect", dialect), + ("rules", rules), + ("default", default), + ) if value is not None } if not changes and target_projects is None: - self._raise_usage("nothing to update: pass a table, dialect, rules or target projects") + self._raise_usage( + "nothing to update: pass a table, dialect, rules, default or target projects" + ) target_ids = ( self._resolve_targets(alias, target_projects) if target_projects is not None else None ) @@ -445,18 +450,18 @@ def update_policy( ) ] ) + # A 1.1.0 policy may omit `dialect` (the enforcement then uses the workspace backend). policy = { "table": attrs.get("table", ""), - "dialect": attrs.get("dialect", ""), + "dialect": attrs.get("dialect") or backend, "rules": attrs.get("rules", []), + **({"default": attrs["default"]} if attrs.get("default") is not None else {}), **changes, } - check = self._validate_policy( - client, policy=policy, backend=backend, exclude_id=policy_id - ) + check = self._validate_policy(client, policy=policy, backend=backend) if check.errors: self._raise_invalid(check.errors) - preview = self._preview_rules(policy["rules"], policy["dialect"]) + preview = self._preview(policy) if dry_run: return self._with_warnings( { @@ -469,7 +474,7 @@ def update_policy( if target_ids is not None else meta.get("targetProjectIds") or [] ), - "preview": preview, + **preview, "dry_run": True, }, check.warnings, @@ -485,8 +490,7 @@ def update_policy( conflict_hint=self._conflict_hint, ) row = self._detail_row(client.get_item(self.item_type, policy_id)) - row["preview"] = preview - return self._with_warnings(row, check.warnings) + return self._with_warnings({**row, **preview}, check.warnings) def delete_policy(self, alias: str, policy_id: str, *, dry_run: bool = False) -> dict[str, Any]: """Delete one policy by id; ``dry_run`` shows what would be deleted (one read, no write).""" diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index ec14aa83..f9e1596a 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -61,10 +61,10 @@ class TestLocalRuleValidation: (None, "non-empty array"), ([], "non-empty array"), (["x"], "must be an object"), - ([{"visible_columns": ["id"]}], "exactly one of 'principal'/'principals'"), + ([{"visible_columns": ["id"]}], "exactly one of principal/principals/groups"), ( [{"principal": "a@x.com", "principals": ["b@x.com"], "visible_columns": ["id"]}], - "exactly one of 'principal'/'principals'", + "exactly one of principal/principals/groups", ), ([{"principal": "a@x.com"}], "non-empty 'visible_columns'"), ([{"principal": "a@x.com", "visible_columns": []}], "non-empty 'visible_columns'"), @@ -96,9 +96,20 @@ def test_accepts_principals_list(self, tmp_path: Path) -> None: ) assert result["preview"] == [ - {"principal": ["a@x.com", "b@x.com"], "visible_columns": ["id"]} + {"principals": ["a@x.com", "b@x.com"], "visible_columns": ["id"]} ] + def test_accepts_groups_and_rejects_default(self, tmp_path: Path) -> None: + """1.1.0: a groups selector; a cls-policy has no `default`.""" + service, _ = _make_service(tmp_path) + rules = [{"groups": ["sales-eu"], "visible_columns": ["id"]}] + + result = service.create_policy("prod", table="in.c-crm.t", rules=rules, dry_run=True) + + assert result["preview"] == [{"groups": ["sales-eu"], "visible_columns": ["id"]}] + with pytest.raises(KeboolaApiError, match="'default' is not supported"): + service.create_policy("prod", table="in.c-crm.t", rules=rules, default={"false": True}) + def test_a_dialect_other_than_the_backend_is_rejected_before_any_write( self, tmp_path: Path ) -> None: @@ -218,14 +229,15 @@ def test_target_projects_are_granted_in_the_create_request(self, tmp_path: Path) assert mock.post_item.call_args.kwargs["target_project_ids"] == [7, 8] mock.put_target_projects.assert_not_called() - def test_duplicate_principals_are_rejected(self, tmp_path: Path) -> None: + def test_one_identity_in_several_rules_is_accepted(self, tmp_path: Path) -> None: + """1.1.0: an identity several rules match sees the union of their columns.""" service, mock = _make_service(tmp_path) + mock.post_item.return_value = _item() rules = [*_RULES, {"principal": "A@x.com", "visible_columns": ["id"]}] - with pytest.raises(KeboolaApiError, match="already has a rule"): - service.create_policy("prod", table="in.c-crm.t", rules=rules) + service.create_policy("prod", table="in.c-crm.t", rules=rules) - mock.post_item.assert_not_called() + mock.post_item.assert_called_once() def test_dry_run_never_writes(self, tmp_path: Path) -> None: service, mock = _make_service(tmp_path) diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 4efc32e6..5fcf3757 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -192,6 +192,58 @@ def test_fetch_failure_is_exit_4_not_a_crash(self, tmp_path: Path) -> None: # --------------------------------------------------------------------------- +class TestRlsDefaultOption: + """`--default` (schema 1.1.0): a JSON condition passed through; anything else is a usage error.""" + + @pytest.mark.parametrize( + "command", + [["create", "--table-id", "t.x", "--rules", "[]"], ["update", "--policy-id", "p-1"]], + ) + def test_default_is_parsed_and_passed_to_the_service( + self, tmp_path: Path, command: list[str] + ) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + service.create_policy.return_value = {**_policy_row(), "preview": []} + service.update_policy.return_value = {**_policy_row(), "preview": []} + + result = _run( + ["--json", "rls", *command, "--project", "prod", "--default", '{"false": true}'], + store, + service, + ) + + assert result.exit_code == 0, result.output + method = service.create_policy if command[0] == "create" else service.update_policy + assert method.call_args.kwargs["default"] == {"false": True} + + @pytest.mark.parametrize("raw", ["[1]", "not json"]) + def test_a_non_object_default_is_a_usage_error(self, tmp_path: Path, raw: str) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + [ + "--json", + "rls", + "create", + "--project", + "prod", + "--table-id", + "t.x", + "--rules", + "[]", + "--default", + raw, + ], + store, + service, + ) + + assert result.exit_code == 2, result.output + service.create_policy.assert_not_called() + + class TestRlsCreateCli: _RULES_JSON = '[{"principal": "a@x.com", "condition": {"true": true}}]' diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 9da119cc..7efb0e90 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -386,45 +386,100 @@ def test_dry_run_never_calls_post_item(self, tmp_path: Path) -> None: assert result["preview"][0]["condition"] == "\"region\" = 'EU'" -class TestDuplicatePrincipals: - """One principal with two rules on one table makes the enforcement refuse every query.""" +class TestSchema110: + """Policy schema 1.1.0: groups, OR-combined rules, `default`, the false sentinel, `$identity`.""" - def test_within_one_policy_case_insensitively(self, tmp_path: Path) -> None: + def test_one_identity_in_several_rules_is_accepted(self, tmp_path: Path) -> None: + """The 1.1.0 engine ORs the conditions of every rule an identity matches.""" service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.return_value = _policy_item() rules = [ - {"principal": "a@x.com", "condition": {"true": True}}, - {"principals": ["b@x.com", "A@X.com"], "condition": {"true": True}}, + {"principal": "a@x.com", "condition": {"column": "r", "op": "eq", "value": "EU"}}, + {"principals": ["A@X.com"], "condition": {"column": "r", "op": "eq", "value": "US"}}, ] - with pytest.raises(KeboolaApiError, match=r"'A@X.com' already has a rule in rules\[0\]"): - service.create_policy("prod", table="t.x", rules=rules, dry_run=True) + service.create_policy("prod", table="t.x", rules=rules) - mock.post_item.assert_not_called() + mock.post_item.assert_called_once() + mock.list_items.assert_not_called() # no cross-policy principal scan any more + + @pytest.mark.parametrize( + "rule", + [ + {"groups": ["sales-eu", "Sales EU"], "condition": {"true": True}}, + {"groups": ["g"], "condition": {"false": True}}, + { + "groups": ["g"], + "condition": {"column": "o", "op": "eq", "value": {"$identity": "email"}}, + }, + { + "groups": ["g"], + "condition": {"column": "t", "op": "in", "values": {"$identity": "groups"}}, + }, + ], + ) + def test_valid_110_rules(self, rule: dict) -> None: + assert validate_rules_local([rule]) == [] + + @pytest.mark.parametrize( + ("rule", "fragment"), + [ + ({"groups": [], "condition": {"true": True}}, "groups must be a non-empty list"), + ({"groups": ["a\x00"], "condition": {"true": True}}, "control characters"), + ({"principal": "a", "groups": ["g"], "condition": {"true": True}}, "exactly one of"), + ({"groups": ["g"], "condition": {"false": False}}, 'exactly {"false": true}'), + ( + { + "groups": ["g"], + "condition": {"column": "c", "op": "eq", "value": {"$identity": "groups"}}, + }, + "string, number or boolean 'value'", + ), + ( + { + "groups": ["g"], + "condition": {"column": "c", "op": "in", "values": {"$identity": "email"}}, + }, + "non-empty list 'values'", + ), + ], + ) + def test_invalid_110_rules(self, rule: dict, fragment: str) -> None: + errors = validate_rules_local([rule]) + assert errors and fragment in errors[0] + + def test_default_is_validated_sent_and_previewed(self, tmp_path: Path) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) + mock.post_item.return_value = _policy_item() + + result = service.create_policy("prod", table="t.x", rules=_RULES, default={"false": True}) - def test_across_policies_on_the_same_table(self, tmp_path: Path) -> None: + assert mock.post_item.call_args.kwargs["data"]["default"] == {"false": True} + assert result["default_preview"] == "FALSE" + + def test_an_invalid_default_is_refused(self, tmp_path: Path) -> None: service, mock = _make_service(_make_store(tmp_path)) - # Snowflake table keys compare case-insensitively. - mock.list_items.return_value = [_policy_item(item_id="other", table="IN.C-CRM.T")] - with pytest.raises(KeboolaApiError, match="already has a rule in policy other"): - service.create_policy("prod", table="in.c-crm.t", rules=_RULES) + with pytest.raises(KeboolaApiError, match=r"default\.'false' condition"): + service.create_policy("prod", table="t.x", rules=_RULES, default={"false": 1}) mock.post_item.assert_not_called() - def test_policies_on_other_tables_and_the_policy_itself_do_not_count( - self, tmp_path: Path - ) -> None: + def test_update_patches_the_default_and_keeps_a_missing_dialect(self, tmp_path: Path) -> None: + """A 1.1.0 policy may omit `dialect`: the merge falls back to the backend, not to ''.""" service, mock = _make_service(_make_store(tmp_path)) mock.get_schema.side_effect = KeboolaApiError(message="n/a", status_code=404) - mock.list_items.return_value = [ - _policy_item(item_id="other", table="in.c-crm.other"), - _policy_item(item_id="p-1", table="in.c-crm.invoices"), - ] - mock.get_item.return_value = _policy_item(item_id="p-1") + stored = _policy_item() + del stored["attributes"]["dialect"] + mock.get_item.return_value = stored - service.update_policy("prod", "p-1", rules=_RULES) + service.update_policy("prod", "p-1", default={"true": True}) - mock.patch_item.assert_called_once() + mock.patch_item.assert_called_once_with( + "rls-policy", "p-1", name=None, data={"default": {"true": True}}, conflict_hint=ANY + ) # --------------------------------------------------------------------------- @@ -662,6 +717,17 @@ def test_non_true_sentinel_is_never_rendered_as_always_true(self, bad: dict) -> ({"column": "d", "op": "is_not_null"}, "snowflake", '"d" IS NOT NULL'), ({"column": "amount", "op": "gt", "value": 100}, "snowflake", '"amount" > 100'), ({"column": "active", "op": "eq", "value": True}, "snowflake", '"active" = TRUE'), + ({"false": True}, "snowflake", "FALSE"), + ( + {"column": "o", "op": "eq", "value": {"$identity": "email"}}, + "snowflake", + '"o" = ', + ), + ( + {"column": "t", "op": "in", "values": {"$identity": "groups"}}, + "bigquery", + "`t` IN ()", + ), ({"column": "o", "op": "eq", "value": "O'Brien"}, "snowflake", "\"o\" = 'O''Brien'"), ( { diff --git a/tests/test_server_rls.py b/tests/test_server_rls.py index 0944b863..6c697bd0 100644 --- a/tests/test_server_rls.py +++ b/tests/test_server_rls.py @@ -132,6 +132,7 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: "dialect": "snowflake", "rules": [{"principal": "a@x.com", "condition": {"true": True}}], "target_projects": [999, "analytics"], + "default": {"false": True}, "dry_run": False, } @@ -145,6 +146,7 @@ def test_create_policy_passes_kwargs(tmp_path: Path) -> None: assert kwargs["rules"] == body["rules"] assert kwargs["scope"] == "targeted" # the default assert kwargs["target_projects"] == [999, "analytics"] + assert kwargs["default"] == {"false": True} assert kwargs["dry_run"] is False @@ -163,6 +165,7 @@ def test_update_policy_passes_kwargs(tmp_path: Path) -> None: assert args == (PROJECT, POLICY_ID) assert kwargs["table"] == "new.table" assert kwargs["rules"] is None + assert kwargs["default"] is None # omitted = unchanged assert kwargs["target_projects"] is None From 57cdef799cff3aedc79c2993b35138b36db04647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 8 Oct 2026 13:30:33 +0200 Subject: [PATCH 28/35] docs(CLI-17): rls/cls 1.1.0 -- policies combine with AND, engine requirement, vNEXT tags (review) The engine now combines several policies on one table with AND (CLS: intersection), so an added policy can only narrow access, and a reader with no identity is always refused (security review on keboola/mcp-server#709). The docs said rules combine with OR across policies. Also: the CLS section no longer mentions the removed duplicate-principal check, 1.1.0 passages carry (since vNEXT), and the docs say a 1.1.0 policy needs an MCP engine that reads 1.1.0. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 14 ++++---- plugins/kbagent/agents/keboola-expert.md | 2 +- .../kbagent/references/commands-reference.md | 2 +- .../skills/kbagent/references/gotchas.md | 8 +++-- .../skills/kbagent/references/rls-workflow.md | 32 +++++++++++-------- .../services/_rls_condition.py | 4 +-- src/keboola_agent_cli/services/rls_service.py | 9 +++--- 7 files changed, 41 insertions(+), 30 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 0527a481..8921f6b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1397,10 +1397,12 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # write `targeted` policies of its own project without grants; grants and organization scope # need the organization-admin role (403 otherwise). Enforced only where the project has the # `row-level-security` feature. Policy schema 1.1.0 (the metastore default): a rule selects -# identities by principal, principals or IdP `groups`; every rule matching one identity applies -# and the conditions combine with OR; `--default` is the condition for identities no rule -# matches (absent = refused; {"false": true} = no rows; to remove a default, recreate the -# policy). Writes refuse a --dialect other than the project backend (it defaults to the +# identities by principal, principals or IdP `groups`; within a policy every rule matching one +# identity applies and the conditions combine with OR; several policies on one table combine +# with AND (an added policy only narrows); a reader with no identity is always refused; +# `--default` is the condition for an identified reader no rule matches (absent = refused; +# {"false": true} = no rows; to remove a default, recreate the policy). Needs an MCP engine +# that reads 1.1.0 (keboola/mcp-server#709). Writes refuse a --dialect other than the project backend (it defaults to the # backend; the enforcement would refuse every read of that table). --rules is a JSON array of # {principal|principals|groups, condition}; condition is a declarative primitive tree # (column/op/value comparisons, in/not_in, is_null/is_not_null, and/or nesting, {"true": true} @@ -1427,8 +1429,8 @@ kbagent cls delete --project P --policy-id ID [--dry-run] [--yes] # protected table), sibling of `rls`: same scope default, write permissions, dialect check, # partial update and permission classes. --rules is a JSON array of # {principal|principals|groups, visible_columns: [col, ...]} objects -- an allowlist projection -# (masking is not supported); an identity several rules match sees the union of their -# columns; no `--default`. `--dry-run` prints each selector's projection. No `setup` +# (masking is not supported); within a policy an identity several rules match sees the union +# of their columns, several policies intersect; no `--default`. `--dry-run` prints each selector's projection. No `setup` # wizard; the REST routes under /cls/{project} mirror /rls. # New error code: INVALID_CLS_POLICY. Version gate lives in gotchas.md. diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index c0f6a2e3..a457ed4a 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch is refused: it would block every read of that table). Schema 1.1.0: rules select by `principal`/`principals`/`groups`, every matching rule applies (RLS conditions OR, CLS columns union), `rls --default '{"false": true}'` gives unmatched readers no rows instead of an error, `{"$identity": "email"|"groups"}` placeholders are resolved per reader. `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch is refused: it would block every read of that table). Schema 1.1.0: rules select by `principal`/`principals`/`groups`, every matching rule of a policy applies (RLS conditions OR, CLS columns union) while several policies on one table AND (an added policy only narrows; no identity = refused; needs an MCP engine that reads 1.1.0), `rls --default '{"false": true}'` gives unmatched readers no rows instead of an error, `{"$identity": "email"|"groups"}` placeholders are resolved per reader. `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index a3af0dc3..5595b5ee 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,7 +469,7 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`) plus a condition primitive. Policy schema 1.1.0: every rule matching one identity applies and the conditions combine with OR; an optional policy `default` covers identities no rule matches (absent = refused). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`) plus a condition primitive. Policy schema 1.1.0 *(since vNEXT)*: within a policy every rule matching one identity applies and the conditions combine with OR; several policies on one table combine with AND, so an added policy can only narrow access; a reader with no identity is always refused; an optional policy `default` covers an identified reader no rule matches (absent = refused). Needs an MCP engine that reads 1.1.0 (keboola/mcp-server#709). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. - `rls list --project P` -- list policies visible to (and applied in) a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, owner_project_id, source_project_id, target_project_ids}, ...]}`. `owner_project_id` is null for an `organization` policy; `target_project_ids` is shown to the owner only. - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals|groups, condition}], default?, revision}`. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 558a206d..52c48807 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6773,9 +6773,11 @@ tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full r (`ACCESS_DENIED`). A project admin reads only the policies its project owns. Policies are enforced only where the consuming project has the `row-level-security` feature. - **Policy schema 1.1.0 (the metastore default) is a superset of 1.0.0.** A rule selects identities by - `principal`, `principals` or IdP `groups` (exact strings, any listed group matches). Every rule matching - one identity applies: RLS conditions combine with OR, CLS `visible_columns` are united -- so one - identity in several rules (or policies) is no longer an error. RLS `--default` is the condition for + `principal`, `principals` or IdP `groups` (exact strings, any listed group matches). Within a policy, + every rule matching one identity applies: RLS conditions combine with OR, CLS `visible_columns` are + united -- so one identity in several rules is no longer an error. Several policies on one table combine + with AND (CLS: intersection), so an added policy can only narrow access; a reader with no identity is + always refused. Needs a `keboola-mcp-server` whose engine reads schema 1.1.0 (keboola/mcp-server#709): an older engine refuses a policy that uses `groups`, `$identity`, a `default` or `{"false": true}`, and refuses every query of the project when one principal has two rules. RLS `--default` is the condition for identities no rule matches (absent = refused; `{"false": true}` = no rows); an `update` cannot remove a default (recreate the policy). `value: {"$identity": "email"}` and `values: {"$identity": "groups"}` are resolved per reader by the enforcement. `dialect` is optional in the schema; kbagent still sends the diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index 41dbe2b9..b7aebd4d 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -35,12 +35,15 @@ deletes the policy objects the enforcement engine reads. the `row-level-security` project feature (it gates CLS too). Without it, queries are unfiltered -- not an error. -**How rules combine (policy schema 1.1.0, the metastore default).** A rule -selects identities by `principal`, `principals` or IdP `groups`. Every rule -that matches the reader applies: RLS conditions combine with OR, CLS -`visible_columns` are united. An RLS policy may set a `default` condition for -readers no rule matches; without one, their reads are refused. 1.1.0 is a -superset of 1.0.0 -- every older policy keeps its meaning. +**How rules combine (policy schema 1.1.0, the metastore default; since vNEXT).** A +rule selects identities by `principal`, `principals` or IdP `groups`. Within a +policy, every rule that matches the reader applies: RLS conditions combine with +OR, CLS `visible_columns` are united. An RLS policy may set a `default` condition +for an identified reader no rule matches; without one, the policy refuses the +read. Several policies on one table combine with AND (CLS: intersection), so an +added policy can only narrow access, never widen it. A reader with no identity +is always refused. 1.1.0 is a superset of 1.0.0 -- every older policy keeps its +meaning. Needs a `keboola-mcp-server` whose engine reads schema 1.1.0 (keboola/mcp-server#709): an older engine refuses a policy that uses `groups`, `$identity`, a `default` or `{"false": true}`, and refuses every query of the project when one principal has two rules. **Dialect.** A policy whose `dialect` differs from the reading workspace makes the enforcement refuse every read of that table. `--dialect` defaults to the @@ -229,6 +232,8 @@ meant," not as proof of what will be enforced. ## Workflow 4b -- Groups, a default and identity placeholders (schema 1.1.0) +*(since vNEXT)* + One policy for many readers instead of one rule per person. Groups are the IdP `groups` claim strings exactly as delivered (Entra may deliver object ids); Keboola never manages membership. @@ -360,12 +365,12 @@ kbagent --json rls schema --project prod ## Column-Level Security (CLS) -`kbagent cls` authors `cls-policy` objects: one per protected table, each rule naming a -principal and the `visible_columns` that principal may read (an allowlist -- unlisted -columns are omitted from the result; masking is not supported). It mirrors `rls` -(`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same scope rule and -default, the same write permissions, dialect check, duplicate-principal check, -`--dry-run`/`--yes` flags and partial `update`; there is no `cls setup` wizard. +`kbagent cls` authors `cls-policy` objects: one per protected table, each rule a selector +(`principal`, `principals` or IdP `groups`) and the `visible_columns` those readers may see +(an allowlist -- unlisted columns are omitted from the result; masking is not supported). +It mirrors `rls` (`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same +scope rule and default, the same write permissions, dialect check, `--dry-run`/`--yes` +flags and partial `update`; there is no `cls setup` wizard and no `default`. Enforcement and RLS+CLS composition happen in `keboola-mcp-server`'s `query_data`, gated by the same `row-level-security` feature. @@ -387,4 +392,5 @@ kbagent cls delete --project prod --policy-id --yes - `INVALID_CLS_POLICY` -- the policy failed validation (exactly one of `principal`/`principals`/ `groups` per rule, non-empty `visible_columns` of `[A-Za-z0-9_]+` names, the dialect check, no `default`, plus the live JSON Schema). Nothing was written. -- An identity several CLS rules match sees the union of their `visible_columns`. +- Within one CLS policy, an identity several rules match sees the union of their + `visible_columns`; several CLS policies on one table intersect (since vNEXT). diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 805b5b86..6baee172 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -9,8 +9,8 @@ what the enforcement engine (that repo's ``rls.py::_compile_primitive``, sqlglot-based) expects to read back from the metastore. The checks follow the policy schema 1.1.0 (go-monorepo ``rls-policy_schema_1.1.0.json``): a rule selects identities by -``principal``, ``principals`` or IdP ``groups``; every rule matching one identity applies (the -conditions combine with OR); ``{"false": true}`` matches no row; a literal may be the +``principal``, ``principals`` or IdP ``groups``; within a policy every rule matching one identity +applies (the conditions combine with OR; policies on one table combine with AND); ``{"false": true}`` matches no row; a literal may be the ``{"$identity": ...}`` placeholder the engine resolves per reader. ``compile_condition_preview`` below is a **preview** for ``--dry-run`` / diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 90f9d41d..6a8ae033 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -20,10 +20,11 @@ need the organization-admin role. Policy schema 1.1.0 (the metastore default): a rule selects identities by -``principal``, ``principals`` or IdP ``groups``; every rule matching one -identity applies and the conditions combine with OR (so one identity in -several rules is fine); an RLS ``default`` condition covers identities no rule -matches (absent = the read is refused). A dialect other than the project +``principal``, ``principals`` or IdP ``groups``; within a policy every rule +matching one identity applies and the conditions combine with OR (so one +identity in several rules is fine); several policies on one table combine +with AND, so an added policy can only narrow access; an RLS ``default`` +covers an identified reader no rule matches (absent = the read is refused). A dialect other than the project backend makes the enforcement refuse reads of that table, so it is refused here before any write. From fc87b6b5d80defde9313b3b53d91600dc7675686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 13 Oct 2026 07:09:51 +0200 Subject: [PATCH 29/35] fix(CLI-17): fit the SKILL.md description and the agent prompt in their size limits after the rebase Main already leaves the SKILL.md description 1 character under the 1024 limit Claude Desktop enforces, and the expert agent prompt 1055 bytes under its 70000-byte budget, so the rls/cls additions pushed both over once combined with main (1060 characters, 71063 bytes). The description now names the feature as RLS/CLS in the covered topics and drops the separate rls/cls trigger words, with two wording trims that keep their meaning ("large (200 GB) file uploads" is "200 GB file uploads", "setup and logout" is "setup, logout"). The expert agent's rls/cls table row keeps the command shape, the scope rules, the 1.1.0 semantics and the pitfalls and points to rls-workflow.md for the rest (default policy, $identity placeholders, dialect, dry-run, the cls rules example), which already documents them. Co-Authored-By: Claude Sonnet 5.5 --- plugins/kbagent/agents/keboola-expert.md | 2 +- plugins/kbagent/skills/kbagent/SKILL.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index a457ed4a..b1c6fcc6 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (owning project + `--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin only) -- use it only when the user explicitly asks for org-wide; never `project` scope. `--dialect` defaults to the project backend (a mismatch is refused: it would block every read of that table). Schema 1.1.0: rules select by `principal`/`principals`/`groups`, every matching rule of a policy applies (RLS conditions OR, CLS columns union) while several policies on one table AND (an added policy only narrows; no identity = refused; needs an MCP engine that reads 1.1.0), `rls --default '{"false": true}'` gives unmatched readers no rows instead of an error, `{"$identity": "email"|"groups"}` placeholders are resolved per reader. `--rules` is a declarative JSON condition tree, never free-text SQL; use `is_null`, never `value: null`. `--dry-run` previews (follows the enforcement's quoting, but the enforcement is `keboola-mcp-server`'s `query_data`). Grants / org scope need the organization-admin role (403 otherwise); enforced only where the project has the `row-level-security` feature. `cls` takes `--rules '[{"principal":"a@x.com","visible_columns":["id"]}]'` (column allowlist) and has no `setup`. A metastore that predates the `rls-policy`/`cls-policy` schemas answers a classified `NOT_FOUND` schema-fetch error -- not a kbagent bug. Guided flow: `kbagent rls setup --project P` (interactive-terminal-only; exit 2 under `--json`/non-TTY) or read [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P` to inspect the live shape first; `rls delete --dry-run` before a delete | hand-writing a raw predicate string as `condition` (no such field exists in the schema); assuming an `organization` policy only applies to its own project (it applies org-wide); assuming `rls setup` works headless (it doesn't -- use `rls create` there) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin): only on an explicit request; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups`, matching rules OR (CLS: union), several policies AND, no identity = refused; needs an MCP engine that reads 1.1.0. `--rules` is a JSON condition tree, never SQL. Enforced only in projects with the `row-level-security` feature. Details, `rls --default`, `$identity`, `setup`: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` before a write | a raw predicate as `condition`; assuming an `organization` policy stays in its own project; `rls setup` headless (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index cd2d41b6..20b15243 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -5,11 +5,11 @@ description: > Covers: searching configurations, job history, data lineage, dev branches, merge requests, workspace SQL debugging, GitOps sync, bucket sharing/linking, encrypting secrets, - Storage tables/files/snapshots, large (200 GB) file uploads, data apps, + Storage tables/files/snapshots, 200 GB file uploads, data apps, flows/schedules, invitations, feature flags, OTLP data streams, scoped Storage tokens, semantic - layer, row/column-level security, Developer Portal, - first-time setup and logout. + layer, RLS/CLS, Developer Portal, + first-time setup, logout. Triggers: kbagent, Keboola, keboola config, keboola job, keboola lineage, keboola sync, gitops, dev branch, merge request, @@ -18,7 +18,7 @@ description: > feature flag, flow schedule, invite member, SQL transformation edit, sync action, keboola docs, table snapshot, auth, login, sign in, PAYG credits, flow notifications, alert recipients, config trash, - restore config, zero-copy clone, workspace load type, rls, cls, + restore config, zero-copy clone, workspace load type, multipart, import job, job-detail, upload-table no-wait, set up keboola, connect project, logout. --- From 6338cf6653d10f92ab5a31d77bc30a7da37c6260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 13 Oct 2026 08:06:03 +0200 Subject: [PATCH 30/35] feat(CLI-17): accept the "*" wildcard selector in rls/cls policies (schema 1.1.0) A rule may select every identified reader with "*" (principal, principals or groups). The selector must be the only entry of a principals/groups list, and it is refused locally when the live schema is older than 1.1.0 (in a 1.0.0 object "*" is a literal that matches nobody). The rule table shows it as "every identified reader", and the workflow, gotchas and agent docs explain that it opens a table under the deny-all default of a governed workspace. Tests cover the single-entry rule, the schema gate and the rendering for both rls and cls. Co-Authored-By: Claude Sonnet 5.5 --- CLAUDE.md | 2 + docs/error-codes.md | 2 +- plugins/kbagent/agents/keboola-expert.md | 2 +- .../kbagent/references/commands-reference.md | 2 +- .../skills/kbagent/references/gotchas.md | 7 +++ .../skills/kbagent/references/rls-workflow.md | 30 ++++++++++++ src/keboola_agent_cli/commands/rls.py | 7 ++- .../services/_rls_condition.py | 39 +++++++++++++++ src/keboola_agent_cli/services/rls_service.py | 10 +++- tests/test_cls_service.py | 21 +++++++++ tests/test_rls_service.py | 47 +++++++++++++++++++ 11 files changed, 163 insertions(+), 6 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 8921f6b9..f87d0218 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1400,6 +1400,8 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # identities by principal, principals or IdP `groups`; within a policy every rule matching one # identity applies and the conditions combine with OR; several policies on one table combine # with AND (an added policy only narrows); a reader with no identity is always refused; +# the wildcard `"*"` (sole entry of `principal`/`principals`/`groups`) matches every identified reader and is how a table is opened to everyone under default deny; in a governed workspace +# (the platform denies by default, admins never exempt) a table with no rls-policy is refused; # `--default` is the condition for an identified reader no rule matches (absent = refused; # {"false": true} = no rows; to remove a default, recreate the policy). Needs an MCP engine # that reads 1.1.0 (keboola/mcp-server#709). Writes refuse a --dialect other than the project backend (it defaults to the diff --git a/docs/error-codes.md b/docs/error-codes.md index 8a6c05fb..19eb316a 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -233,4 +233,4 @@ A command that fails with an error code exits with the code's `Exit` value below | Code | Exit | Type | Description | |---|---|---|---| | `INVALID_CLS_POLICY` | 1 | `api` | A `cls create`/`update` candidate body (`table`/`dialect`/`rules`/`visible_columns`) failed validation -- either the schema-independent local checks (`principal`/`principals` not exactly one, empty or non-identifier `visible_columns`) or the live JSON-Schema check against the metastore's `cls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | -| `INVALID_RLS_POLICY` | 1 | `api` | An `rls create`/`update` candidate body (`table`/`dialect`/`rules`/`condition`) failed validation -- either the schema-independent local checks (unknown op, `principal`/`principals` not exactly one) or the live JSON-Schema check against the metastore's `rls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | +| `INVALID_RLS_POLICY` | 1 | `api` | An `rls create`/`update` candidate body (`table`/`dialect`/`rules`/`condition`) failed validation -- either the schema-independent local checks (unknown op, `principal`/`principals` not exactly one, a `"*"` wildcard that is not the sole entry of its list, or one used against a metastore whose `rls-policy` schema is older than 1.1.0) or the live JSON-Schema check against the metastore's `rls-policy` schema when reachable. The message lists every violation found; nothing was written. Not retryable | diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index b1c6fcc6..dcdb7283 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin): only on an explicit request; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups`, matching rules OR (CLS: union), several policies AND, no identity = refused; needs an MCP engine that reads 1.1.0. `--rules` is a JSON condition tree, never SQL. Enforced only in projects with the `row-level-security` feature. Details, `rls --default`, `$identity`, `setup`: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` before a write | a raw predicate as `condition`; assuming an `organization` policy stays in its own project; `rls setup` headless (use `rls create`) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin): only on an explicit request; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups` (`"*"` as sole entry = every identified reader), matching rules OR (CLS: union), several policies AND, no identity = refused; needs an MCP engine that reads 1.1.0. `--rules` is a JSON condition tree, never SQL. Enforced only in projects with the `row-level-security` feature. Details, `rls --default`, `$identity`, `setup`: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` before a write | a raw predicate as `condition`; assuming an `organization` policy stays in its own project; `rls setup` headless (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 5595b5ee..88d613d6 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,7 +469,7 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`) plus a condition primitive. Policy schema 1.1.0 *(since vNEXT)*: within a policy every rule matching one identity applies and the conditions combine with OR; several policies on one table combine with AND, so an added policy can only narrow access; a reader with no identity is always refused; an optional policy `default` covers an identified reader no rule matches (absent = refused). Needs an MCP engine that reads 1.1.0 (keboola/mcp-server#709). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`; `"*"` as the sole entry matches every identified reader, *(since vNEXT)*) plus a condition primitive. Policy schema 1.1.0 *(since vNEXT)*: within a policy every rule matching one identity applies and the conditions combine with OR; several policies on one table combine with AND, so an added policy can only narrow access; a reader with no identity is always refused; an optional policy `default` covers an identified reader no rule matches (absent = refused). Needs an MCP engine that reads 1.1.0 (keboola/mcp-server#709). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. - `rls list --project P` -- list policies visible to (and applied in) a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, owner_project_id, source_project_id, target_project_ids}, ...]}`. `owner_project_id` is null for an `organization` policy; `target_project_ids` is shown to the owner only. - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals|groups, condition}], default?, revision}`. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 52c48807..89658024 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6784,6 +6784,13 @@ tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full r project backend and refuses another one (`INVALID_RLS_POLICY` / `INVALID_CLS_POLICY`), because the enforcement refuses every read of a table whose policy names another dialect. An `organization` policy also reaches projects on the OTHER backend -- prefer `targeted` in a mixed-backend organization. +- **Wildcard `"*"` and default deny (since vNEXT).** `principal: "*"`, `principals: ["*"]` or `groups: ["*"]` + (schema 1.1.0) matches every identified reader (never a reader with no identity); it must be the only entry + of its list (`INVALID_RLS_POLICY` / `INVALID_CLS_POLICY` otherwise) and is refused against a metastore + whose schema is older than 1.1.0, where `"*"` is a literal that matches nobody. In a governed workspace + the platform denies by default: a table with no `rls-policy` is refused (a `cls-policy` alone grants no + rows), admins included, so a table is opened to everyone only by an explicit wildcard policy with + `{"true": true}`. Workflow: `rls-workflow.md` > "Default deny and the wildcard". - **One `rls-policy` object per protected table**, never one blob per project, named by its table: a second policy on the same table in the same project answers `ALREADY_EXISTS` (update or delete the first). `rls setup` creates one policy per selected table, all sharing the same rules. diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index b7aebd4d..b5a39580 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -265,6 +265,36 @@ row; anyone else sees no rows (the `default`), instead of an error. - MCP OAuth users have an email but no IdP groups, so `groups` rules only match readers whose identity carries a groups claim. +## Workflow 4c -- Default deny and the wildcard (governed workspaces) + +*(since vNEXT)* + +In a **governed workspace** (the platform creates every workspace governed once +RLS/CLS is switched on for the organization or project) nothing is readable +until a policy says so: a table with **no** `rls-policy` is refused, and so is a +table whose policies match the reader in no rule. Admins and organization admins +are not exempt. An admin who sees no data has not authored the policies yet -- +author them before switching RLS/CLS on. + +To open a table to every identified reader, use the wildcard selector `"*"` +(`principal`, `principals` or `groups`, schema 1.1.0) with an explicit condition: + +```bash +kbagent --json rls create --project prod --table-id in.c-ref.countries \ + --rules '[{"principal": "*", "condition": {"true": true}}]' --dry-run +``` + +- `"*"` must be the **only** entry of a `principals` / `groups` list + (`["*", "a@example.com"]` is refused locally); it never matches a reader with + no identity; it cannot widen a stricter policy on the same table (policies + AND together). +- It needs a metastore whose `rls-policy` schema is 1.1.0 or later: on a 1.0.0 + schema `"*"` is a literal that matches nobody, so kbagent refuses it. +- A debug or "see everything" user is a normal rule (for example + `{"groups": ["debug"], "condition": {"true": true}}`), not a bypass flag. +- A `cls-policy` alone never grants rows: a table needs an `rls-policy` (the + wildcard one if everyone may read all rows) before its columns matter. + ## Workflow 5 -- Share a policy with sibling projects (targeted grants) `--target-project` (on `create`/`setup`/`update`) takes a registered alias or a diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index d82f2e39..6da30a57 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -147,7 +147,8 @@ def _principal_text(item: dict[str, Any]) -> str: if "groups" in item: return "groups: " + ", ".join(item["groups"]) principal = item.get("principal") or item.get("principals") or [] - return principal if isinstance(principal, str) else ", ".join(principal) + text = principal if isinstance(principal, str) else ", ".join(principal) + return f"{text} (every identified reader)" if text == "*" else text def _format_policy_table(formatter: Any, policies: list[dict[str, Any]]) -> None: @@ -623,7 +624,9 @@ def _prompt_condition() -> dict[str, Any]: def _build_rules_interactively() -> list[dict[str, Any]]: rules: list[dict[str, Any]] = [] while True: - principal = typer.prompt("Principal (user email, or comma-separated list for a group)") + principal = typer.prompt( + "Principal (user email, comma-separated list for a group, or * for every identified reader)" + ) principals = [p.strip() for p in principal.split(",") if p.strip()] condition = _prompt_condition() rule: dict[str, Any] = {"condition": condition} diff --git a/src/keboola_agent_cli/services/_rls_condition.py b/src/keboola_agent_cli/services/_rls_condition.py index 6baee172..ef6b9b9c 100644 --- a/src/keboola_agent_cli/services/_rls_condition.py +++ b/src/keboola_agent_cli/services/_rls_condition.py @@ -59,6 +59,12 @@ class Dialect(StrEnum): # The rule selectors of schema 1.1.0 -- each rule names exactly one. SELECTORS: tuple[str, ...] = ("principal", "principals", "groups") +# Schema 1.1.0 wildcard selector: matches every IDENTIFIED reader (never an anonymous one). It must be the +# only entry of a `principals` / `groups` list, and exists only from schema 1.1.0: in a 1.0.0 object it is a +# literal that no email equals, so it matches nobody. +WILDCARD = "*" +WILDCARD_MIN_SCHEMA_VERSION = (1, 1, 0) + # The `$identity` placeholders: `value` may be the reader's email, `values` the reader's groups. IDENTITY_EMAIL = {"$identity": "email"} IDENTITY_GROUPS = {"$identity": "groups"} @@ -179,6 +185,10 @@ def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: "a non-empty string" if key == "principal" else "a non-empty list of non-empty strings" ) return [f"rules[{index}].{key} must be {shape}"] + if key != "principal" and WILDCARD in names and len(names) > 1: + return [ + f"rules[{index}].{key}: {WILDCARD!r} (every identified reader) must be the only entry" + ] pattern, problem = ( (_GROUP_RE, "control characters") if key == "groups" @@ -191,6 +201,35 @@ def validate_principal_fields(rule: dict[Any, Any], index: int) -> list[str]: ] +def rules_use_wildcard(rules: Any) -> bool: + """Whether any rule selects by the ``"*"`` wildcard (``principal``, ``principals`` or ``groups``).""" + if not isinstance(rules, list): + return False + return any( + isinstance(rule, dict) + and any( + (rule[key] == WILDCARD) if key == "principal" else (WILDCARD in (rule[key] or [])) + for key in SELECTORS + if key in rule and isinstance(rule[key], str | list) + ) + for rule in rules + ) + + +def schema_lacks_wildcard(schema: dict[str, Any]) -> str | None: + """The schema's ``version`` when it is older than the one that defines ``"*"``, else ``None``. + + An unreadable or absent ``version`` is not proof of anything, so it returns ``None`` (the caller's + structural validation against the live schema still runs). + """ + version = schema.get("version") + try: + parsed = tuple(int(part) for part in str(version).split(".")[:3]) + except ValueError: + return None + return str(version) if parsed < WILDCARD_MIN_SCHEMA_VERSION else None + + def rule_selector(rule: dict[str, Any]) -> dict[str, Any]: """The rule's selector as written (``{"principal": ...}``, ``{"principals": [...]}`` or ``{"groups": [...]}``).""" return {key: rule[key] for key in SELECTORS if key in rule} diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 6a8ae033..35bd2e2d 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -242,7 +242,15 @@ def _validate_policy( fetch = self._fetch_schema(client) if not fetch.schema: return PolicyValidation([], [f"Live schema validation was skipped: {fetch.reason}"]) - return PolicyValidation(draft7_errors(policy, fetch.schema), []) + errors = draft7_errors(policy, fetch.schema) + if _rls_condition.rules_use_wildcard(policy["rules"]) and ( + old := _rls_condition.schema_lacks_wildcard(fetch.schema) + ): + errors.append( + f"the wildcard {_rls_condition.WILDCARD!r} needs {self.item_type} schema 1.1.0 or later; " + f"this metastore serves {old}, where it is a literal that matches nobody" + ) + return PolicyValidation(errors, []) def _local_rule_errors(self, rules: Any) -> list[str]: """Schema-independent checks on ``rules`` (hook for :class:`ClsService`).""" diff --git a/tests/test_cls_service.py b/tests/test_cls_service.py index f9e1596a..439c0202 100644 --- a/tests/test_cls_service.py +++ b/tests/test_cls_service.py @@ -74,6 +74,11 @@ class TestLocalRuleValidation: "not a valid column name", ), ([{"principal": "a@x.com", "visible_columns": [1]}], "not a valid column name"), + ( + [{"principals": ["*", "a@x.com"], "visible_columns": ["id"]}], + "must be the only entry", + ), + ([{"groups": ["*", "sales"], "visible_columns": ["id"]}], "must be the only entry"), ], ) def test_rejects(self, tmp_path: Path, rules: Any, fragment: str) -> None: @@ -87,6 +92,22 @@ def test_rejects(self, tmp_path: Path, rules: Any, fragment: str) -> None: assert exc.value.message.startswith("CLS policy is invalid") mock.post_item.assert_not_called() + @pytest.mark.parametrize( + "selector", [{"principal": "*"}, {"principals": ["*"]}, {"groups": ["*"]}] + ) + def test_accepts_the_wildcard_selector_alone(self, tmp_path: Path, selector: dict) -> None: + service, _ = _make_service(tmp_path) + + result = service.create_policy( + "prod", + table="in.c-crm.t", + dialect="snowflake", + rules=[{**selector, "visible_columns": ["id"]}], + dry_run=True, + ) + + assert result["preview"] == [{**selector, "visible_columns": ["id"]}] + def test_accepts_principals_list(self, tmp_path: Path) -> None: service, _ = _make_service(tmp_path) rules = [{"principals": ["a@x.com", "b@x.com"], "visible_columns": ["id"]}] diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index 7efb0e90..ec0b1c21 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -1029,6 +1029,12 @@ class TestPrincipalFieldsAreValidatedByValueNotTruthiness: ({"principals": ["a", ""], "condition": _COND}, "principals must be a non-empty list"), ({"principals": ["a", 2], "condition": _COND}, "principals must be a non-empty list"), ({"condition": _COND}, "exactly one of"), + # The wildcard must be the sole entry of its list (schema 1.1.0). + ({"principals": ["*", "a@x.com"], "condition": _COND}, "must be the only entry"), + ({"principals": ["a@x.com", "*"], "condition": _COND}, "must be the only entry"), + ({"principals": ["*", "*"], "condition": _COND}, "must be the only entry"), + ({"groups": ["*", "sales"], "condition": _COND}, "must be the only entry"), + ({"groups": [], "condition": _COND}, "groups must be a non-empty list"), ], ) def test_malformed_principal_fields_are_rejected_locally( @@ -1042,11 +1048,52 @@ def test_malformed_principal_fields_are_rejected_locally( [ {"principal": "a@x.com", "condition": _COND}, {"principals": ["a@x.com", "b@x.com"], "condition": _COND}, + {"principal": "*", "condition": _COND}, + {"principals": ["*"], "condition": _COND}, + {"groups": ["*"], "condition": _COND}, ], ) def test_well_formed_principal_fields_pass(self, rule: dict) -> None: assert validate_rules_local([rule]) == [] + @pytest.mark.parametrize( + ("schema_version", "blocked"), + [("1.0.0", True), ("1.1.0", False), ("1.2.0", False), (None, False), ("v1", False)], + ) + def test_wildcard_is_refused_on_a_schema_older_than_1_1_0( + self, tmp_path: Path, schema_version: str | None, blocked: bool + ) -> None: + """In a 1.0.0 object `*` is a literal no email equals: it would silently match nobody.""" + service, mock = _make_service(_make_store(tmp_path)) + schema: dict[str, Any] = {"type": "object"} + if schema_version: + schema["version"] = schema_version + mock.get_schema.return_value = schema + mock.post_item.return_value = _policy_item() + rules = [{"principal": "*", "condition": self._COND}] + + if blocked: + with pytest.raises(KeboolaApiError) as excinfo: + service.create_policy("prod", table="t.x", rules=rules) + assert "needs rls-policy schema 1.1.0" in excinfo.value.message + mock.post_item.assert_not_called() + else: + service.create_policy("prod", table="t.x", rules=rules) + mock.post_item.assert_called_once() + + def test_a_policy_without_a_wildcard_is_not_blocked_by_an_old_schema( + self, tmp_path: Path + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.get_schema.return_value = {"type": "object", "version": "1.0.0"} + mock.post_item.return_value = _policy_item() + + service.create_policy( + "prod", table="t.x", rules=[{"principal": "a@x.com", "condition": self._COND}] + ) + + mock.post_item.assert_called_once() + def test_a_malformed_principal_never_reaches_a_write_when_the_schema_is_unavailable( self, tmp_path: Path ) -> None: From 22e760bc03d5b5f2b56e22686bfd2c2f575dc4db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Thu, 15 Oct 2026 07:01:33 +0200 Subject: [PATCH 31/35] fix(CLI-17): keep the keboola-expert prompt under its byte budget after the rebase Main's lineage notes grew the agent prompt, so the RLS/CLS tool-selection row pushed it to 70250 bytes against the 70000-byte budget (tests/test_agent_prompt.py). Shorten the row; the dropped detail (--default, $identity, setup) stays in rls-workflow.md. Co-Authored-By: Claude Sonnet 5.5 --- plugins/kbagent/agents/keboola-expert.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index dcdb7283..688e4527 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` governs the table in EVERY project of the org (destructive-class, org-admin): only on an explicit request; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups` (`"*"` as sole entry = every identified reader), matching rules OR (CLS: union), several policies AND, no identity = refused; needs an MCP engine that reads 1.1.0. `--rules` is a JSON condition tree, never SQL. Enforced only in projects with the `row-level-security` feature. Details, `rls --default`, `$identity`, `setup`: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` before a write | a raw predicate as `condition`; assuming an `organization` policy stays in its own project; `rls setup` headless (use `rls create`) | +| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` applies in EVERY org project (org-admin): only if asked; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups` (`"*"` = every identified reader), matching rules OR, several policies AND, no identity = refused; needs an MCP engine reading 1.1.0. Needs the `row-level-security` feature. Details: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` first | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | From 368c2c8558a5f60414650326d020e9cda4dc13c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Mon, 19 Oct 2026 08:18:03 +0200 Subject: [PATCH 32/35] fix(CLI-17): satisfy the CLI interface spec gate and the agent prompt budget Main gained a CLI spec gate and a 70000-byte limit for the expert agent prompt while this branch was open. - rls setup prompts print through the formatter instead of typer.echo - handle_service_call keeps the ConfigError subclass code and reports API errors through emit_api_error - rls setup is recorded as a terminal-only exception for serve routes - docs/cli-manifest.json and the spec baseline are regenerated - the RLS/CLS row of the expert agent prompt is shortened (details stay in rls-workflow.md) and a few redundant phrases are trimmed Co-Authored-By: Claude Sonnet 5.5 --- docs/cli-manifest.json | 187 ++++++++++++ plugins/kbagent/agents/keboola-expert.md | 8 +- src/keboola_agent_cli/commands/_helpers.py | 335 ++------------------- src/keboola_agent_cli/commands/rls.py | 24 +- 4 files changed, 234 insertions(+), 320 deletions(-) diff --git a/docs/cli-manifest.json b/docs/cli-manifest.json index 989e129e..0f9b9a7e 100644 --- a/docs/cli-manifest.json +++ b/docs/cli-manifest.json @@ -419,6 +419,91 @@ {"kind": "option", "name": "full", "opts": ["--full", "-v"], "type": "boolean", "is_flag": true, "default": false, "help": "Show complete notes for each version (default: headlines of the BREAKING and first notes)."} ] }, + { + "path": "cls", + "kind": "group", + "hidden": false, + "help": "Manage column-level security policies (metastore-backed, per-principal column allowlist). 'list'/'detail'/'schema' are read-only; 'create'/'update'/'delete' are gated as admin -- see CONTRIBUTING.md." + }, + { + "path": "cls create", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Create one CLS policy for one table (``targeted`` scope unless ``--scope organization``).", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias -- the project that owns the policy"}, + {"kind": "option", "name": "table_id", "opts": ["--table-id"], "type": "text", "required": true, "default": null, "help": "Storage table ID, e.g. in.c-crm.invoices"}, + {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "required": true, "default": null, "help": "JSON|@file|- array of {principal|principals, visible_columns} objects; visible_columns is the allowlist of columns that principal can read"}, + {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the projection without writing"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, + { + "path": "cls delete", + "kind": "command", + "hidden": false, + "category": "destructive", + "help": "Delete a CLS policy -- its table's columns become unrestricted.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "CLS policy ID"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Show the policy without deleting it"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, + { + "path": "cls detail", + "kind": "command", + "hidden": false, + "category": "read", + "help": "Show one CLS policy's full rule set.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "CLS policy ID"} + ] + }, + { + "path": "cls list", + "kind": "command", + "hidden": false, + "category": "read", + "help": "List column-level security policies visible to a project.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"} + ] + }, + { + "path": "cls schema", + "kind": "command", + "hidden": false, + "category": "read", + "help": "Print the live ``cls-policy`` JSON Schema fetched from the metastore.\n\nLive-only (no bundled snapshot): if the metastore of that stack does not\nhave the ``cls-policy`` object type registered, this fails with a clean,\nclassified ``NOT_FOUND`` error.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias -- fetch the live schema from this stack"} + ] + }, + { + "path": "cls update", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Update an existing CLS policy.\n\nOnly the flags you pass change -- an omitted flag keeps the policy's\ncurrent value. The write is a partial update (PATCH) of just those keys.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "CLS policy ID"}, + {"kind": "option", "name": "table_id", "opts": ["--table-id"], "type": "text", "default": null, "help": "New table (unset = unchanged)"}, + {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "New dialect, must match the project backend (unset = unchanged)"}, + {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "default": null, "help": "New JSON|@file|- rules array (unset = unchanged)"}, + {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role); replaces the list (unset = unchanged)"}, + {"kind": "option", "name": "clear_target_projects", "opts": ["--clear-target-projects"], "type": "boolean", "is_flag": true, "default": false, "help": "Revoke every project the policy is shared with (cannot be combined with --target-project)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the projection without writing"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, { "path": "component", "kind": "group", @@ -2453,6 +2538,108 @@ "help": "Start interactive REPL mode for kbagent.", "params": [] }, + { + "path": "rls", + "kind": "group", + "hidden": false, + "help": "Manage row-level security policies (metastore-backed). 'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' are gated as admin -- see CONTRIBUTING.md." + }, + { + "path": "rls create", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Create one RLS policy for one table (``targeted`` scope unless ``--scope organization``).", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias -- the project that owns the policy"}, + {"kind": "option", "name": "table_id", "opts": ["--table-id"], "type": "text", "required": true, "default": null, "help": "Storage table ID, e.g. in.c-crm.invoices"}, + {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "required": true, "default": null, "help": "JSON|@file|- array of {principal|principals|groups, condition} objects; rules matching one identity combine with OR (see rls-workflow.md for the condition shape)"}, + {"kind": "option", "name": "default", "opts": ["--default"], "type": "text", "default": null, "help": "JSON|@file|- condition for identities no rule matches, e.g. '{\"false\": true}' (no rows); omitted, their reads are refused"}, + {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the compiled condition without writing"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, + { + "path": "rls delete", + "kind": "command", + "hidden": false, + "category": "destructive", + "help": "Delete an RLS policy -- its table's rows are no longer filtered.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "RLS policy ID"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Show the policy without deleting it"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, + { + "path": "rls detail", + "kind": "command", + "hidden": false, + "category": "read", + "help": "Show one RLS policy's full rule set.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "RLS policy ID"} + ] + }, + { + "path": "rls list", + "kind": "command", + "hidden": false, + "category": "read", + "help": "List row-level security policies visible to a project.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"} + ] + }, + { + "path": "rls schema", + "kind": "command", + "hidden": false, + "category": "read", + "help": "Print the live ``rls-policy`` JSON Schema fetched from the metastore.\n\nUnlike ``flow schema``, there is no offline bundled snapshot -- the\nschema is authoritative only from the live metastore, and on a stack whose\nmetastore predates it, fetching fails with a clean, classified error.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias -- fetch the live schema from this stack"} + ] + }, + { + "path": "rls setup", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Guided RLS setup: pick tables, build a condition, preview, then write.\n\nInteractive-terminal-only (with ``--json`` or without a terminal it exits 2\nwith a usage error -- use ``rls create`` there). Creates one RLS policy per\nselected table, all sharing the same rules, via the exact same write path\n``rls create`` uses.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias whose tables you're protecting"}, + {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, + {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "default": null, "help": "JSON|@file|- rules array -- skips the interactive condition builder"}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip the final confirmation"} + ] + }, + { + "path": "rls update", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Update an existing RLS policy.\n\nOnly the flags you pass change -- an omitted flag keeps the policy's\ncurrent value. The write is a partial update (PATCH) of just those keys.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "policy_id", "opts": ["--policy-id"], "type": "text", "required": true, "default": null, "help": "RLS policy ID"}, + {"kind": "option", "name": "table_id", "opts": ["--table-id"], "type": "text", "default": null, "help": "New table (unset = unchanged)"}, + {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "New dialect, must match the project backend (unset = unchanged)"}, + {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "default": null, "help": "New JSON|@file|- rules array (unset = unchanged)"}, + {"kind": "option", "name": "default", "opts": ["--default"], "type": "text", "default": null, "help": "New default condition (unset = unchanged; to remove one, recreate the policy)"}, + {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role); replaces the list (unset = unchanged)"}, + {"kind": "option", "name": "clear_target_projects", "opts": ["--clear-target-projects"], "type": "boolean", "is_flag": true, "default": false, "help": "Revoke every project the policy is shared with (cannot be combined with --target-project)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the compiled condition without writing"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, { "path": "schedule", "kind": "group", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 688e4527..b56ad20c 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -161,10 +161,10 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Call the running `kbagent serve` from a scheduled-agent subprocess | `kbagent http get/post/patch/delete ` -- uses the `KBAGENT_SERVE_URL` + `KBAGENT_SERVE_TOKEN` the scheduler injects; `http get /openapi.json` discovers endpoints | forking `kbagent ` (also fine -- `KBAGENT_CONFIG_DIR` is propagated) | `curl $KBAGENT_SERVE_URL/...` by hand (loses the auth header, error mapping and JSON formatting `kbagent http` adds) | | Launch the web UI for an end-user | `kbagent serve --ui [--port PORT]` -- one FastAPI process mounts the bundled React SPA and sets an HttpOnly `kbagent_session` cookie, so SSE works with no token in the URL or JS heap | the legacy Vite + Node BFF dev flow (`web/README.md`) | inventing a `--token-in-url` flag; running uvicorn directly against the dist (the cookie bootstrap only fires from `serve --ui`) | | Schedule / manage Agent Tasks | `kbagent agent ` -- CRUD, `run [--stream]`, history, `test`/`cron-preview`/`prompt-improve`. Local-only; cron firing needs `kbagent serve`. See [agent-tasks-cli-workflow](../skills/kbagent/references/agent-tasks-cli-workflow.md) | `kbagent http /agents...` inside scheduled subprocesses | hand-editing `agents.json` | -| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List); a stale `MISSING_MASTER_TOKEN` on a plain read means an outdated kbagent still carrying the old reclassification, not a real gate | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | -| ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (the metastore has no soft-delete and no version history -- the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a token whose Storage Admin role is `admin` on the project (a master token qualifies, but so does any project-admin user token as of PSGO-282) -- a non-admin token still gets `MISSING_MASTER_TOKEN`/403 on writes, just not on reads. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: it carries the per-verb recipes, the rename cascade and the promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | -| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project (a merge), `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` (preview with `--dry-run`) -- **irreversible, no downgrade endpoint, destructive-class**. `add ` without `--scope` inherits its model's scope. `scope get --type T --context-id ID` shows the current state first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Author or change a row-level (RLS) or column-level (CLS) security policy | `kbagent {rls,cls} create/update/delete --project P --table-id T ...` -- `targeted` scope by default (`--target-project` grants); `--scope organization` applies in EVERY org project (org-admin): only if asked; never `project` scope. Schema 1.1.0: rules select by `principal`/`principals`/`groups` (`"*"` = every identified reader), matching rules OR, several policies AND, no identity = refused; needs an MCP engine reading 1.1.0. Needs the `row-level-security` feature. Details: [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` / `cls schema --project P`; `--dry-run` first | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | +| Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List) | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | +| ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (no soft-delete or history: the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a project-admin token (master or admin-role user token, PSGO-282); a non-admin token gets `MISSING_MASTER_TOKEN`/403 on writes. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: per-verb recipes, rename cascade, promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | +| Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project, `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` -- **irreversible, destructive-class**. `scope get --type T --context-id ID` first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | +| Row/column-level (RLS/CLS) security policy | `kbagent {rls,cls} create\|update\|delete --project P --table-id T` (`--dry-run` first; never `--scope project`, `--scope organization` only if asked); rules pick by `principal`/`principals`/`groups`, `"*"` = every identified reader; needs `row-level-security`. [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index 4e409728..fce85df2 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -12,13 +12,11 @@ import os import secrets import sys -from collections.abc import Callable, Iterable, Sized -from dataclasses import dataclass +from collections.abc import Callable from pathlib import Path from typing import Any import typer -import yaml from .. import telemetry from ..config_store import ConfigStore @@ -29,7 +27,6 @@ ) from ..errors import ConfigError, ErrorCode, KeboolaApiError, PermissionDeniedError, exit_code_for from ..output import OutputFormatter -from ..permissions import FLAG_ESCALATIONS from ..update_crossing import print_held_warning, refusal_for @@ -81,107 +78,34 @@ def resolve_manage_token(*, allow_env: bool = False) -> str: raise typer.Exit(code=2) -def parse_json_arg(raw: str, *, label: str, allow_yaml: bool = False) -> Any: +def parse_json_arg(raw: str, *, label: str) -> Any: """Parse a ``JSON|@file|-`` argument: inline JSON, ``@path``, or ``-`` for stdin. - The house input contract for structured flags (spec rule I1): ``config - update --configuration``, ``transformation edit --op``, ``merge-request - resolve --resolved``, ``encrypt values --input``, ``http post --body`` and - the others. ``allow_yaml`` parses the text as YAML, a superset of JSON - (``flow new/update/validate --file``). + The house input contract for structured flags (``config update + --configuration``, ``transformation edit --op``, ``merge-request resolve + --resolved``). Hoisted from ``transformation.py``'s private copy so the + merge-request group did not add another; ``config.py`` still carries an + older dict-only variant (``_parse_json_input``) -- folding it in is a + follow-up, not this PR. Raises: ValueError: On a missing/unreadable file or malformed JSON -- the message names ``label`` (the flag) so the caller can print it at exit 2 as-is. """ - loads: Callable[[str], Any] = yaml.safe_load if allow_yaml else json.loads try: if raw == "-": - return loads(sys.stdin.read()) + return json.loads(sys.stdin.read()) if raw.startswith("@"): file_path = Path(raw[1:]) if not file_path.is_file(): raise ValueError(f"{label}: file not found: {file_path}") - return loads(file_path.read_text(encoding="utf-8")) - return loads(raw) - except (json.JSONDecodeError, yaml.YAMLError) as exc: - raise ValueError(f"{label}: invalid {'YAML' if allow_yaml else 'JSON'}: {exc}") from exc - except (OSError, UnicodeDecodeError) as exc: - # a directory, a permission problem, a binary file -- a usage error, not a crash - raise ValueError(f"{label}: cannot read input: {exc}") from exc - - -def split_multi(values: Iterable[str] | None) -> list[str]: - """Flatten a repeatable option that also takes a comma list (spec rule A4). - - ``--x a --x b,c`` gives ``["a", "b", "c"]``. Empty parts are dropped, and a - value given twice is kept once, in first-seen order. Use it only for IDs - and fixed values: a free-form value (JSON, ``KEY=VALUE``, SQL, text) can - contain a comma, so those options are repeat-only and stay unsplit. - """ - parts = (part.strip() for value in values or () for part in value.split(",")) - return list(dict.fromkeys(part for part in parts if part)) - - -def split_int_multi(values: Iterable[str] | None, *, label: str) -> list[int]: - """``split_multi`` for integer IDs; a part that is not an integer raises ``ValueError``.""" - parts = split_multi(values) - bad = [part for part in parts if not (part.isascii() and part.isdigit())] - if bad: - raise ValueError(f"{label} takes integer IDs, got: {', '.join(bad)}") - return [int(part) for part in parts] - - -def read_text_arg(formatter: OutputFormatter, path: Path, *, label: str) -> str: - """Read a UTF-8 file option; an unreadable file is ``INVALID_ARGUMENT`` naming ``label``.""" - try: - return path.read_text(encoding="utf-8") - except (OSError, UnicodeDecodeError) as exc: - formatter.error( - message=f"{label}: cannot read file {path}: {exc}", - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) from None - - -def _is_file(path: str) -> bool: - """``Path.is_file`` that answers False where it would raise (a name too long).""" - try: - return Path(path).is_file() - except (OSError, ValueError): - return False - - -class RefuseAtFile: - """Option callback for free text that refuses ``@`` (spec rule I3). - - Without it, ``--text @notes.md`` stores the literal text ``@notes.md``. - The refusal is ``INVALID_ARGUMENT`` (exit 2) with a hint that names the - file option of the command, also for the case where the text itself starts - with ``@``. ``@`` followed by anything that is not a file (a relative path - is relative to the current directory) stays text. Use it as - ``typer.Option(..., callback=RefuseAtFile("--file").callback)``; - ``scripts/check_cli_spec.py`` finds the free-text options without it. - """ - - def __init__(self, file_option: str) -> None: - self.file_option = file_option - - def callback( - self, ctx: typer.Context, param: typer.CallbackParam, value: str | None - ) -> str | None: - if value is not None and value.startswith("@") and _is_file(value[1:]): - get_formatter(ctx).error( - message=( - f"{param.opts[0]} got {value!r}, the name of an existing file (relative " - f"to the current directory). Use {self.file_option} {value[1:]} to read " - f"the file. To store the text {value!r} itself, write it to a file and " - f"pass that file with {self.file_option}." - ), - error_code=ErrorCode.INVALID_ARGUMENT, - ) - raise typer.Exit(code=2) - return value + return json.loads(file_path.read_text(encoding="utf-8")) + return json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{label}: invalid JSON: {exc}") from exc + except OSError as exc: + # a directory, a permission problem -- a usage error, not a crash + raise ValueError(f"{label}: cannot read file: {exc}") from exc def read_password_stdin(label: str = "Password: ") -> str: @@ -522,7 +446,8 @@ def handle_service_call[T]( return func(*args, **kwargs) except ConfigError as exc: formatter.error( - message=exc.message, error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR) + message=exc.message, + error_code=getattr(exc, "error_code", ErrorCode.CONFIG_ERROR), ) raise typer.Exit(code=service_error_exit_code(exc)) from None except KeboolaApiError as exc: @@ -530,6 +455,18 @@ def handle_service_call[T]( raise typer.Exit(code=service_error_exit_code(exc)) from None +def emit_project_warnings(formatter: OutputFormatter, result: dict) -> None: + """Emit warnings from multi-project operation results. + + Iterates the 'errors' list in the result dict (if present) and prints + each entry as a warning via the formatter. + """ + for err in result.get("errors", []): + alias = err.get("project_alias", "unknown") + message = err.get("message", "Unknown error") + formatter.warning(f"Project '{alias}': {message}") + + def item_failure_exit_code(failed: int) -> int | None: """Map the number of failed items of a multi-item operation to a CLI exit code. @@ -552,68 +489,6 @@ def item_failure_exit_code(failed: int) -> int | None: return 1 -def finish_items( - formatter: OutputFormatter, - result: Any, - *, - failed: int, - total: int, - noun: str, - summary: str, - render: Callable[[], object] | None = None, -) -> None: - """End a command that processes several items: output, one headline, exit code. - - The one way to end such a command (HO2, E4): ``--json`` gets the envelope - with ``exit_code``; human mode gets ``render()`` (the per-item lines) and - then one headline, ``Success: `` or ``Failed: N of M .`` - The command exits 1 when an item failed. ``noun`` is plural ("tables"). - - One wording for every summary (HO2): `` (s) ``, - with details after a comma and a closing period, for example - ``2 table(s) deleted.`` or ``3 config(s) pushed: 1 created, 2 updated, 0 deleted.`` - A dry run says ``would be <...>``. - The fan-out reads end through ``_all_projects.finish_fanout``, which - counts failed projects (E4: ``PAYG_NOT_AVAILABLE`` is not a failure). - """ - code = item_failure_exit_code(failed) - if formatter.json_mode: - formatter.output(result, exit_code=code or 0) - else: - if render is not None: - render() - formatter.item_headline(failed=failed, total=total, noun=noun, summary=summary) - if code: - raise typer.Exit(code=code) - - -def finish_bulk( - formatter: OutputFormatter, - result: dict[str, Any], - done: Sized, - noun: str, - done_as: str, - render: Callable[[], object], -) -> None: - """:func:`finish_items` for a bulk command whose result lists its failures in ``failed``. - - ``done`` holds the items that succeeded; ``noun`` is singular ("table"), - ``done_as`` what happened to them ("deleted", "would be deleted"), so the - headline reads ``Success: 2 table(s) deleted.`` or - ``Failed: 1 of 3 tables. 2 table(s) deleted.`` - """ - failed = len(result["failed"]) - finish_items( - formatter, - result, - failed=failed, - total=len(done) + failed, - noun=f"{noun}s", - summary=f"{len(done)} {noun}(s) {done_as}.", - render=render, - ) - - def _is_help_request(ctx: typer.Context) -> bool: """Check if the current invocation is a --help request. @@ -687,13 +562,7 @@ def check_cli_operation(ctx: typer.Context, operation: str) -> None: higher risk class than its command (see `permissions.FLAG_ESCALATIONS`). The callback has already cleared the bare command by the time a command body runs, so this is an additional gate, not a replacement. - - A command calls this exactly when the escalating flag (or an inherited - organization scope) applies, so it also raises the command category that - the ``TARGET_FROM_ACTIVE_CHOICE`` warning reads (spec T12). """ - if FLAG_ESCALATIONS.get(operation) == "destructive": - get_formatter(ctx).command_category = "destructive" engine = ctx.obj.get("permission_engine") if engine is None or not engine.active: return @@ -761,150 +630,6 @@ def validate_branch_requires_project( raise typer.Exit(code=2) from None -def is_interactive_terminal() -> bool: - """True when a person can see a prompt on stderr and answer it on stdin.""" - return bool( - hasattr(sys.stdin, "isatty") - and sys.stdin.isatty() - and hasattr(sys.stderr, "isatty") - and sys.stderr.isatty() - ) - - -CONFIRMATION_HINT = ( - "Run the command again with --yes to confirm. " - "AI agents: ask the user before you retry with --yes." -) - - -def _confirm_or_abort(formatter: OutputFormatter, yes: bool, question: str) -> None: - """The shared safety prompt (spec S4). - - ``--yes`` and ``--json`` skip it: both mean consent. On a terminal the - question goes to stderr, and a declined prompt prints ``Aborted.`` and - exits 0 (E6). Without a terminal nobody can answer, and an open stdin - pipe must not answer for the user (D6): the command refuses with - ``CONFIRMATION_REQUIRED`` and exit 2, and the message names ``--yes``. - """ - if yes or formatter.json_mode: - return - if not is_interactive_terminal(): - formatter.error( - message=f"{ErrorCode.CONFIRMATION_REQUIRED.value}: {question} " - f"There is no terminal to answer it. {CONFIRMATION_HINT}", - error_code=ErrorCode.CONFIRMATION_REQUIRED, - ) - raise typer.Exit(code=2) - if not typer.confirm(question, err=True): - formatter.console.print("Aborted.") - raise typer.Exit(code=0) - - -# The ``code`` of a ``warnings[]`` entry for a deprecated form (spec C3, O3). -DEPRECATED_CODE = "DEPRECATED" - - -@dataclass(frozen=True) -class Deprecation: - """One deprecated command-line form (spec C3). - - ``replacement`` is the form to use instead, or None when the old form - has no effect and the caller only drops it. ``remove_after`` is the ISO - date after which a minor release may remove the old form. It is None - while that date is unknown: the window of a form that 1.0.0 hides - starts at the migration time M, which exists only once v1.0.0 is - published. - """ - - message: str - hint: str - replacement: str | None - remove_after: str | None = None - - -# C3: every deprecated form, keyed by its command path and option. A removed -# form stays here, so that its error can name the replacement. -DEPRECATIONS: dict[str, Deprecation] = { - "stream delete --force": Deprecation( - message="stream delete --force is deprecated. Use --yes to skip the confirmation.", - hint="--yes", - replacement="--yes", - ), - "semantic-layer import --yes": Deprecation( - message="semantic-layer import --yes is deprecated and has no effect. Omit it.", - hint="omit --yes (skip on conflict is the default)", - replacement=None, - ), - # V9: `kbagent serve` partial updates move from PUT to PATCH. A REST form - # is keyed "METHOD /path"; its warning goes into the Kbagent-Warnings header. - "PUT /semantic-layer/items/{kind}/{name}": Deprecation( - message="PUT /semantic-layer/items/{kind}/{name} is deprecated. Use PATCH.", - hint="PATCH /semantic-layer/items/{kind}/{name}", - replacement="PATCH /semantic-layer/items/{kind}/{name}", - ), - "PUT /merge-requests/{project}/{merge_request_id}": Deprecation( - message="PUT /merge-requests/{project}/{merge_request_id} is deprecated. Use PATCH.", - hint="PATCH /merge-requests/{project}/{merge_request_id}", - replacement="PATCH /merge-requests/{project}/{merge_request_id}", - ), - "PUT /data-apps/{project}/{app_id}/secrets": Deprecation( - message="PUT /data-apps/{project}/{app_id}/secrets is deprecated. Use PATCH.", - hint="PATCH /data-apps/{project}/{app_id}/secrets", - replacement="PATCH /data-apps/{project}/{app_id}/secrets", - ), - # Only the merge is deprecated: PUT with `replace: true` stays the full replace. - "PUT /configs/{project}/{component_id}/{config_id}/variables": Deprecation( - message="PUT .../variables without replace: true merges the variables, which is " - "deprecated. Use PATCH to merge, or PUT with replace: true to replace all variables. " - "After the window, a PUT without replace: true is refused, never run as a replace.", - hint="PATCH /configs/{project}/{component_id}/{config_id}/variables", - replacement="PATCH /configs/{project}/{component_id}/{config_id}/variables", - ), -} - - -def deprecation_entry(form: str) -> tuple[str, dict[str, Any]]: - """The message and the extra fields of the ``DEPRECATED`` warning of ``form``.""" - entry = DEPRECATIONS[form] - when = ( - f"It can be removed after {entry.remove_after}." - if entry.remove_after - else "It keeps working until at least 28 days after the 1.0 migration date." - ) - extra = { - "hint": entry.hint, - "replacement": entry.replacement, - "remove_after": entry.remove_after, - } - return f"{entry.message} {when}", extra - - -def warn_deprecated(formatter: OutputFormatter, form: str) -> None: - """Add the ``DEPRECATED`` warning of a ``DEPRECATIONS`` entry (one stderr line in human mode).""" - message, extra = deprecation_entry(form) - formatter.warning(message, code=DEPRECATED_CODE, **extra) - - -def deprecated_option(form: str, *param_decls: str) -> Any: - """A hidden flag for a deprecated form in ``DEPRECATIONS`` (spec C3). - - The vendored click has no option-level ``deprecated``. The flag keeps - working: the command reads its value (for an alias, ``yes = yes or - force``), and the callback adds the ``DEPRECATED`` warning when the - caller gives the flag. - """ - if form not in DEPRECATIONS: - raise KeyError(f"{form!r} has no DEPRECATIONS entry") - - def _warn(ctx: typer.Context, value: bool) -> bool: - formatter = (ctx.obj or {}).get("formatter") - if value and formatter is not None and not ctx.resilient_parsing: - warn_deprecated(formatter, form) - return value - - return typer.Option(False, *param_decls, hidden=True, callback=_warn) - - _CONFIRM_CODE_LENGTH = 4 diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 6da30a57..866b86c3 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -581,12 +581,14 @@ def rls_delete( _VALUE_HINT = 'parsed as JSON when it is one (42, 4.5, true); quote it ("42") to force a string' -def _prompt_choice(text: str, choices: tuple[str, ...], default: str) -> str: +def _prompt_choice(formatter: Any, text: str, choices: tuple[str, ...], default: str) -> str: while True: value = typer.prompt(text, default=default) if value in choices: return value - typer.echo(f"Enter one of: {', '.join(choices)}") + formatter.err_console.print( + f"Enter one of: {', '.join(choices)}", markup=False, highlight=False + ) def _typed_value(raw: str) -> Any: @@ -602,33 +604,33 @@ def _typed_value(raw: str) -> Any: return value if isinstance(value, str | int | float | bool) else raw -def _prompt_condition() -> dict[str, Any]: - typer.echo(_CONDITION_MENU) - choice = _prompt_choice("Choice", ("1", "2", "3", "4"), default="1") +def _prompt_condition(formatter: Any) -> dict[str, Any]: + formatter.err_console.print(_CONDITION_MENU, markup=False, highlight=False) + choice = _prompt_choice(formatter, "Choice", ("1", "2", "3", "4"), default="1") if choice == "4": return {"true": True} column = typer.prompt("Column name") if choice == "1": - op = _prompt_choice("Operator", _COMPARISON_OPS, default="eq") + op = _prompt_choice(formatter, "Operator", _COMPARISON_OPS, default="eq") value = _typed_value(typer.prompt(f"Value ({_VALUE_HINT})")) return {"column": column, "op": op, "value": value} if choice == "2": - op = _prompt_choice("Operator", ("in", "not_in"), default="in") + op = _prompt_choice(formatter, "Operator", ("in", "not_in"), default="in") raw_values = typer.prompt(f"Values (comma-separated; each {_VALUE_HINT})") values = [_typed_value(v.strip()) for v in raw_values.split(",") if v.strip()] return {"column": column, "op": op, "values": values} - op = _prompt_choice("Operator", ("is_null", "is_not_null"), default="is_null") + op = _prompt_choice(formatter, "Operator", ("is_null", "is_not_null"), default="is_null") return {"column": column, "op": op} -def _build_rules_interactively() -> list[dict[str, Any]]: +def _build_rules_interactively(formatter: Any) -> list[dict[str, Any]]: rules: list[dict[str, Any]] = [] while True: principal = typer.prompt( "Principal (user email, comma-separated list for a group, or * for every identified reader)" ) principals = [p.strip() for p in principal.split(",") if p.strip()] - condition = _prompt_condition() + condition = _prompt_condition(formatter) rule: dict[str, Any] = {"condition": condition} if len(principals) == 1: rule["principal"] = principals[0] @@ -695,7 +697,7 @@ def rls_setup( parsed_rules = ( _parse_rules_arg(formatter, rules, RLS.rules_shape) if rules is not None - else _build_rules_interactively() + else _build_rules_interactively(formatter) ) service = get_service(ctx, RLS.service) selected_tables = [str(tables[i]["id"]) for i in indices] From 158c0fbd83fefb52cc7fd9657530bfa7dcc56120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Tue, 20 Oct 2026 08:43:28 +0200 Subject: [PATCH 33/35] CLI-17: sort the imports of rls.py after renaming the service error exit code helper The helper was renamed to service_error_exit_code so that it does not clash with errors.exit_code_for, the one exit-code table of CLI-47. Co-Authored-By: Claude Sonnet 5.5 --- src/keboola_agent_cli/commands/rls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 866b86c3..61510189 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -48,11 +48,11 @@ from ._helpers import ( check_cli_operation, check_cli_permission, - service_error_exit_code, get_formatter, get_service, handle_service_call, parse_json_arg, + service_error_exit_code, ) rls_app = typer.Typer( From 1b110ce78bb86acdcb4055fd479453dbb2392768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 21 Oct 2026 13:50:09 +0200 Subject: [PATCH 34/35] feat(CLI-17): governed workspace end-to-end for rls/cls; adapt to main after rebase Rebased onto the `workspace create --governed` branch (AJDA-3460), which carries the latest main. Integration: - rls-workflow.md: an end-to-end flow -- `workspace create --governed`, `rls create` / `cls create`, `workspace load`, `workspace query` through Query Service -- with the expected filtered result and the gotchas (no password or key, `--ui` exit 2, non-Snowflake exit 5, Query Service decides enforcement per project, the `row-level-security` feature gates only the MCP `query_data` path). Statements that named `query_data` as the only enforcement are qualified in the workflow, gotchas.md, commands-reference.md, CLAUDE.md, keboola-expert.md and the --scope help. Adapted to main: - `kbagent context`: the rls/cls notes move into GROUP_NOTES (the command lists are generated from the manifest now). - rls/cls confirmations use `_confirm_or_abort`: without a terminal they refuse with CONFIRMATION_REQUIRED (exit 2) instead of reading stdin; the setup wizard's "add another rule" question uses the existing choice prompt. Tests use the `interactive_terminal` fixture, and the contract test lists the new confirming commands. - `handle_service_call` keeps main's ConfigError error-code passthrough. - Regenerated docs/cli-manifest.json, _context_commands.py, docs/web-server-endpoints.md and the CLI spec baseline (rls/cls list without --limit, `rls setup` terminal-only exception). Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 5 +- docs/cli-manifest.json | 6 +- docs/web-server-endpoints.md | 2 +- plugins/kbagent/agents/keboola-expert.md | 2 +- .../kbagent/references/commands-reference.md | 2 +- .../skills/kbagent/references/gotchas.md | 15 +- .../skills/kbagent/references/rls-workflow.md | 75 +++- scripts/cli_spec_baseline.json | 3 + .../commands/_context_commands.py | 68 ++++ src/keboola_agent_cli/commands/_helpers.py | 332 ++++++++++++++++-- src/keboola_agent_cli/commands/cls.py | 4 +- src/keboola_agent_cli/commands/context.py | 83 +++++ src/keboola_agent_cli/commands/rls.py | 23 +- tests/test_cli_contract.py | 13 + tests/test_cls_cli.py | 5 + tests/test_rls_cli.py | 17 + 16 files changed, 585 insertions(+), 70 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f87d0218..c11c9994 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1395,8 +1395,9 @@ kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file # table in EVERY project of the org (the enforcement applies every listed policy) and is # destructive-class (FLAG_ESCALATIONS); there is no project scope. WHO: a project admin may # write `targeted` policies of its own project without grants; grants and organization scope -# need the organization-admin role (403 otherwise). Enforced only where the project has the -# `row-level-security` feature. Policy schema 1.1.0 (the metastore default): a rule selects +# need the organization-admin role (403 otherwise). Enforced by Query Service on a governed +# workspace (`workspace create --governed`; Query Service decides per project), or by +# keboola-mcp-server `query_data` where the project has the `row-level-security` feature. Policy schema 1.1.0 (the metastore default): a rule selects # identities by principal, principals or IdP `groups`; within a policy every rule matching one # identity applies and the conditions combine with OR; several policies on one table combine # with AND (an added policy only narrows); a reader with no identity is always refused; diff --git a/docs/cli-manifest.json b/docs/cli-manifest.json index 0f9b9a7e..7632b5bb 100644 --- a/docs/cli-manifest.json +++ b/docs/cli-manifest.json @@ -436,7 +436,7 @@ {"kind": "option", "name": "table_id", "opts": ["--table-id"], "type": "text", "required": true, "default": null, "help": "Storage table ID, e.g. in.c-crm.invoices"}, {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "required": true, "default": null, "help": "JSON|@file|- array of {principal|principals, visible_columns} objects; visible_columns is the allowlist of columns that principal can read"}, {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, - {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization -- an explicit, organization-admin-only choice, gated as destructive."}, {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the projection without writing"}, {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} @@ -2556,7 +2556,7 @@ {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "required": true, "default": null, "help": "JSON|@file|- array of {principal|principals|groups, condition} objects; rules matching one identity combine with OR (see rls-workflow.md for the condition shape)"}, {"kind": "option", "name": "default", "opts": ["--default"], "type": "text", "default": null, "help": "JSON|@file|- condition for identities no rule matches, e.g. '{\"false\": true}' (no rows); omitted, their reads are refused"}, {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, - {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization -- an explicit, organization-admin-only choice, gated as destructive."}, {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Preview the compiled condition without writing"}, {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} @@ -2616,7 +2616,7 @@ {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias whose tables you're protecting"}, {"kind": "option", "name": "dialect", "opts": ["--dialect"], "type": "choice", "choices": ["snowflake", "bigquery"], "default": null, "help": "Workspace SQL dialect; must match the project backend (default: the backend)"}, {"kind": "option", "name": "rules", "opts": ["--rules"], "type": "text", "default": null, "help": "JSON|@file|- rules array -- skips the interactive condition builder"}, - {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization that has the row-level-security feature -- an explicit, organization-admin-only choice, gated as destructive."}, + {"kind": "option", "name": "scope", "opts": ["--scope"], "type": "choice", "choices": ["targeted", "organization"], "default": "targeted", "help": "targeted (default): the owning project plus --target-project grants. organization: EVERY project of the organization -- an explicit, organization-admin-only choice, gated as destructive."}, {"kind": "option", "name": "target_project", "opts": ["--target-project"], "type": "text", "multiple": true, "default": null, "help": "Project alias or ID the policy also applies to (repeatable or comma-separated; --scope targeted only; granting needs the organization-admin role)"}, {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip the final confirmation"} ] diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 26ebbc30..2af707bd 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -9,7 +9,7 @@ auth, and the concepts behind these routes live in [`web-server.md`](web-server.md); a running server serves the same spec interactively at `/docs` (Swagger) and `/openapi.json`. -**261 operations** across **222 paths** and **32 routers**. +**273 operations** across **228 paths** and **34 routers**. Paths are shown as the server registers them. Reaching them through the Node BFF (or single-process `--ui` mode) prefixes every path with `/api`. diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index b56ad20c..39a9e7a8 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List) | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (no soft-delete or history: the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a project-admin token (master or admin-role user token, PSGO-282); a non-admin token gets `MISSING_MASTER_TOKEN`/403 on writes. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: per-verb recipes, rename cascade, promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project, `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` -- **irreversible, destructive-class**. `scope get --type T --context-id ID` first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Row/column-level (RLS/CLS) security policy | `kbagent {rls,cls} create\|update\|delete --project P --table-id T` (`--dry-run` first; never `--scope project`, `--scope organization` only if asked); rules pick by `principal`/`principals`/`groups`, `"*"` = every identified reader; needs `row-level-security`. [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | +| Row/column-level (RLS/CLS) security policy | `kbagent {rls,cls} create\|update\|delete --project P --table-id T` (`--dry-run` first; never `--scope project`, `--scope organization` only if asked); rules pick by `principal`/`principals`/`groups`, `"*"` = every identified reader; enforced by QS on a `workspace create --governed` workspace or by MCP (`row-level-security`). [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 88d613d6..cd68bb7b 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -469,7 +469,7 @@ There is no bulk-elevate endpoint: promoting every semantic-layer object in an e ## Row-Level Security (RLS) -Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`; `"*"` as the sole entry matches every identified reader, *(since vNEXT)*) plus a condition primitive. Policy schema 1.1.0 *(since vNEXT)*: within a policy every rule matching one identity applies and the conditions combine with OR; several policies on one table combine with AND, so an added policy can only narrow access; a reader with no identity is always refused; an optional policy `default` covers an identified reader no rule matches (absent = refused). Needs an MCP engine that reads 1.1.0 (keboola/mcp-server#709). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. +Author `rls-policy` metastore objects -- one object per protected table, each holding a list of rules, each a selector (`principal`, `principals` or IdP `groups`; `"*"` as the sole entry matches every identified reader, *(since vNEXT)*) plus a condition primitive. Policy schema 1.1.0 *(since vNEXT)*: within a policy every rule matching one identity applies and the conditions combine with OR; several policies on one table combine with AND, so an added policy can only narrow access; a reader with no identity is always refused; an optional policy `default` covers an identified reader no rule matches (absent = refused). Needs an MCP engine that reads 1.1.0 (keboola/mcp-server#709). `kbagent` only **authors** these; the actual SQL rewrite/enforcement happens in Query Service on a governed workspace (`workspace create --governed`, Query Service decides per project; *(since vNEXT)*) or in `keboola-mcp-server`'s `query_data` tool (a different repo), only in projects with the `row-level-security` feature. **Scope is `targeted` unless `--scope organization`; never `project`.** `targeted` governs the owning project plus its `--target-project` grants; `organization` governs the table in every project of the organization (destructive-class flag). A project admin (master token) may write `targeted` policies of its own project without grants; grants and organization scope need the organization-admin role. Every write refuses a `dialect` other than the project backend (the enforcement would refuse every read of that table). A stack whose metastore predates the `rls-policy`/`cls-policy` schema migrations answers `rls schema`/`cls schema` with a classified `NOT_FOUND` -- see [rls-workflow.md](rls-workflow.md) Workflow 7. - `rls list --project P` -- list policies visible to (and applied in) a project. Output: `{project, policies: [{id, table, dialect, rule_count, scope, owner_project_id, source_project_id, target_project_ids}, ...]}`. `owner_project_id` is null for an `organization` policy; `target_project_ids` is shown to the owner only. - `rls detail --project P --policy-id ID` -- full rule set for one policy: `{..., rules: [{principal|principals|groups, condition}], default?, revision}`. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 89658024..8c9fcd66 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6753,8 +6753,10 @@ The package was named `keboola-agent-cli` up to 0.62.0 and is named `keboola-cli *(since vNEXT)* `kbagent rls` authors metastore `rls-policy` objects -- one per protected table. It never -enforces anything itself: the actual SQL rewrite happens in `keboola-mcp-server`'s `query_data` -tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full recipes. +enforces anything itself: the actual SQL rewrite happens in Query Service on a governed workspace +(`workspace create --governed`) or in `keboola-mcp-server`'s `query_data` tool, a separate +repo/runtime. See [rls-workflow.md](rls-workflow.md) for full recipes, including the end-to-end +governed-workspace flow. - **Availability depends on the stack's metastore.** The `rls-policy` and `cls-policy` object types are registered by the metastore's `2026-09-29` schema migrations. A stack whose metastore predates them @@ -6762,16 +6764,17 @@ tool, a separate repo/runtime. See [rls-workflow.md](rls-workflow.md) for full r there, not a kbagent bug. Check with `kbagent --json rls schema --project P` / `cls schema`. - **Scope is `targeted` unless you pass `--scope organization`; there is no `project` scope.** `targeted` (the metastore schema's own default) governs the owning project plus the `--target-project` grants. - `organization` governs the table in EVERY project of the organization with the `row-level-security` - feature: the enforcement applies every policy the metastore lists for a project and does not re-check + `organization` governs the table in EVERY project of the organization where it is enforced: the enforcement applies every policy the metastore lists for a project and does not re-check the owner, so an organization policy is never "only where it matches". Every listed policy applies to the listing project. `--scope organization` is destructive-class (`rls.create --scope organization` / `cls.create --scope organization` in `FLAG_ESCALATIONS`). - **Who may write.** A project admin (master token, admin role) may create/update/delete `targeted` policies of its own project without grants. Grants (`--target-project`, `--clear-target-projects`) and `--scope organization` need the organization-admin role -- a master token alone gets a 403 - (`ACCESS_DENIED`). A project admin reads only the policies its project owns. Policies are enforced only - where the consuming project has the `row-level-security` feature. + (`ACCESS_DENIED`). A project admin reads only the policies its project owns. On the MCP `query_data` path + policies are enforced only where the consuming project has the `row-level-security` feature; on a + governed workspace Query Service decides per project, and the identity running `workspace query` needs + a matching rule (default deny). - **Policy schema 1.1.0 (the metastore default) is a superset of 1.0.0.** A rule selects identities by `principal`, `principals` or IdP `groups` (exact strings, any listed group matches). Within a policy, every rule matching one identity applies: RLS conditions combine with OR, CLS `visible_columns` are diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index b5a39580..a8c77307 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -4,10 +4,11 @@ Row-level security in Keboola is authored as metastore `rls-policy` objects -- one object per protected table, holding a list of per-principal condition primitives (never a hand-written SQL predicate). `kbagent` only **authors** these objects; enforcement (the actual SQL rewrite on a query) happens -inside `keboola-mcp-server`'s `query_data` tool, a different repo and a -different runtime. `kbagent rls` never executes a query and never decides -whether a real query gets filtered -- it only creates, reads, updates and -deletes the policy objects the enforcement engine reads. +outside kbagent, in one of two readers: Query Service on a governed workspace +(see [End-to-end](#end-to-end----governed-workspace-policies-query-service); +since vNEXT) or `keboola-mcp-server`'s `query_data` tool. `kbagent rls` never +decides whether a real query gets filtered -- it only creates, reads, updates +and deletes the policy objects the enforcement reads. **Scope: `targeted` by default, `organization` only when asked for, never `project`.** @@ -15,7 +16,7 @@ deletes the policy objects the enforcement engine reads. table in the owning project (`--project`) plus every project granted with `--target-project` (alias or project ID, repeatable or comma-separated). - `--scope organization` governs the table in **every project of the - organization** that has the `row-level-security` project feature. The + organization** where it is enforced (see the prerequisite below). The enforcement applies every policy the metastore lists for a project -- it does not re-check the owning project -- so an organization policy is never "only for my project". It is gated as `destructive` (`rls.create --scope @@ -31,9 +32,11 @@ deletes the policy objects the enforcement engine reads. | `--scope organization` | no (403) | yes | | Read (`list` / `detail`) | only the policies its project owns | everything visible to the project | -**Enforcement prerequisite.** Policies do nothing until the consuming project has -the `row-level-security` project feature (it gates CLS too). Without it, queries -are unfiltered -- not an error. +**Enforcement prerequisite.** On the MCP path (`query_data`), policies do nothing +until the consuming project has the `row-level-security` project feature (it +gates CLS too); without it, queries are unfiltered -- not an error. On a governed +workspace, Query Service decides per project whether it enforces; that feature +flag is not what switches it (since vNEXT). **How rules combine (policy schema 1.1.0, the metastore default; since vNEXT).** A rule selects identities by `principal`, `principals` or IdP `groups`. Within a @@ -81,6 +84,50 @@ Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). | Govern the table in every project of the organization (organization admin) | `rls create ... --scope organization` | | Preview the compiled condition before writing anything | any write command + `--dry-run` | | Remove a policy (un-protect a table) | `rls delete --project P --policy-id ID` (`--dry-run` shows it first) | +| Query a table with the policies applied, no MCP server | governed workspace -- see [End-to-end](#end-to-end----governed-workspace-policies-query-service) | + +## End-to-end -- governed workspace, policies, Query Service + +*(since vNEXT)* + +Everything below is kbagent commands: no MCP server, no warehouse credentials. +Workspace details and gotchas: [workspace-workflow.md](workspace-workflow.md#governed-workspace). + +```bash +# 1. A governed workspace (Snowflake, headless). The result has no password and +# no private_key: there is nothing to save. Note workspace_id. +kbagent --json workspace create --project prod --name "governed-ws" --governed + +# 2. Policies on the Storage table (preview with --dry-run first, see Workflow 2). +# Under default deny a table with no rls-policy is refused, so give every +# reader a rule -- including the identity you will query with in step 4. +kbagent --json rls create --project prod --table-id in.c-crm.invoices \ + --rules '[{"principal": "eu-analyst@example.com", + "condition": {"column": "region", "op": "eq", "value": "EU"}}]' +kbagent --json cls create --project prod --table-id in.c-crm.invoices \ + --rules '[{"principal": "eu-analyst@example.com", + "visible_columns": ["id", "region", "amount"]}]' + +# 3. Load the table into the workspace (or let Storage put it there). +kbagent --json workspace load --project prod --workspace-id WS_ID \ + --tables in.c-crm.invoices + +# 4. Read it through Query Service -- the only way into a governed workspace. +kbagent --json workspace query --project prod --workspace-id WS_ID \ + --sql 'SELECT * FROM "invoices"' +``` + +Expected (illustrative): for `eu-analyst@example.com` only the `region = 'EU'` +rows, and only the columns `id`, `region`, `amount`. An identity with no +matching rule is refused, not handed an empty result; a reader with no identity +is always refused. + +- `workspace password` is refused on a governed workspace, and so is any direct + warehouse login: Query Service is the only reader. +- `--governed` with `--ui` is `INVALID_ARGUMENT` (exit 2); on a non-Snowflake + backend it is `CONFIG_ERROR` (exit 5). Nothing is created in either case. +- Query Service decides per project whether it enforces the policies. The + `row-level-security` project feature gates only the MCP `query_data` path. --- @@ -161,7 +208,7 @@ A caller with no rule on a governed table is refused, not silently returned empty -- fail-closed is the enforcement engine's whole design point (see the RFC in `keboola-mcp-server`). Before protecting a table, make sure every principal who legitimately needs access has a matching rule, including -yourself if you plan to verify the result via `query_data`. A principal is the +yourself if you plan to verify the result (`query_data` or a governed workspace). A principal is the user's email, matched case-insensitively. ## Workflow 3 -- Inspect existing policies before making a change @@ -269,8 +316,9 @@ row; anyone else sees no rows (the `default`), instead of an error. *(since vNEXT)* -In a **governed workspace** (the platform creates every workspace governed once -RLS/CLS is switched on for the organization or project) nothing is readable +In a **governed workspace** (`workspace create --governed`, or a workspace the +platform creates governed once RLS/CLS is switched on for the organization or +project) nothing is readable until a policy says so: a table with **no** `rls-policy` is refused, and so is a table whose policies match the reader in no rule. Admins and organization admins are not exempt. An admin who sees no data has not authored the policies yet -- @@ -401,8 +449,9 @@ kbagent --json rls schema --project prod It mirrors `rls` (`list`, `detail`, `schema`, `create`, `update`, `delete`) with the same scope rule and default, the same write permissions, dialect check, `--dry-run`/`--yes` flags and partial `update`; there is no `cls setup` wizard and no `default`. -Enforcement and RLS+CLS composition happen in `keboola-mcp-server`'s `query_data`, -gated by the same `row-level-security` feature. +Enforcement and RLS+CLS composition happen in the same two readers as RLS: Query +Service on a governed workspace, or `keboola-mcp-server`'s `query_data` (gated by +the `row-level-security` feature). ```bash kbagent --json cls schema --project prod # live shape diff --git a/scripts/cli_spec_baseline.json b/scripts/cli_spec_baseline.json index 07ff575d..1d83c983 100644 --- a/scripts/cli_spec_baseline.json +++ b/scripts/cli_spec_baseline.json @@ -503,6 +503,7 @@ "agent list", "branch list", "branch metadata-list", + "cls list", "component list", "config list", "config metadata-list", @@ -519,6 +520,7 @@ "project invitation-list", "project list", "project member-list", + "rls list", "schedule list", "semantic-layer model list", "semantic-layer reference-data list", @@ -661,6 +663,7 @@ "context": "terminal-only: prints the agent instructions of this kbagent install; no Keboola object", "init": "terminal-only: manages kbagent itself, not Keboola", "repl": "terminal-only: manages kbagent itself, not Keboola", + "rls setup": "terminal-only: an interactive wizard that needs a real terminal (it exits 2 with --json); the same policies are written over REST by POST/PATCH /rls/{project}", "semantic-layer add constraint": "served by /semantic-layer/items/{kind}, which one telemetry label cannot name", "semantic-layer add dataset": "served by /semantic-layer/items/{kind}, which one telemetry label cannot name", "semantic-layer add glossary": "served by /semantic-layer/items/{kind}, which one telemetry label cannot name", diff --git a/src/keboola_agent_cli/commands/_context_commands.py b/src/keboola_agent_cli/commands/_context_commands.py index 2200a3d6..d8a17f41 100644 --- a/src/keboola_agent_cli/commands/_context_commands.py +++ b/src/keboola_agent_cli/commands/_context_commands.py @@ -145,6 +145,37 @@ kbagent changelog [--limit N] [--full] Show recent changelog (what changed in each version). +""", + "cls": """\ +### cls + +Manage column-level security policies (metastore-backed, per-principal column +allowlist). 'list'/'detail'/'schema' are read-only; 'create'/'update'/'delete' +are gated as admin -- see CONTRIBUTING.md. + + kbagent cls create --project PROJECT --table-id ID --rules RULES + [--dialect snowflake|bigquery] [--scope targeted|organization] + [--target-project PROJECT ...] [--dry-run] [--yes] + Create one CLS policy for one table (``targeted`` scope unless ``--scope + organization``). + + kbagent cls delete --project PROJECT --policy-id ID [--dry-run] [--yes] + Delete a CLS policy -- its table's columns become unrestricted. + + kbagent cls detail --project PROJECT --policy-id ID + Show one CLS policy's full rule set. + + kbagent cls list --project PROJECT + List column-level security policies visible to a project. + + kbagent cls schema --project PROJECT + Print the live ``cls-policy`` JSON Schema fetched from the metastore. + + kbagent cls update --project PROJECT --policy-id ID [--table-id ID] + [--dialect snowflake|bigquery] [--rules RULES] + [--target-project PROJECT ...] [--clear-target-projects] [--dry-run] + [--yes] + Update an existing CLS policy. """, "component": """\ ### component @@ -825,6 +856,43 @@ kbagent repl Start interactive REPL mode for kbagent. +""", + "rls": """\ +### rls + +Manage row-level security policies (metastore-backed). +'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' are +gated as admin -- see CONTRIBUTING.md. + + kbagent rls create --project PROJECT --table-id ID --rules RULES + [--default DEFAULT] [--dialect snowflake|bigquery] + [--scope targeted|organization] [--target-project PROJECT ...] + [--dry-run] [--yes] + Create one RLS policy for one table (``targeted`` scope unless ``--scope + organization``). + + kbagent rls delete --project PROJECT --policy-id ID [--dry-run] [--yes] + Delete an RLS policy -- its table's rows are no longer filtered. + + kbagent rls detail --project PROJECT --policy-id ID + Show one RLS policy's full rule set. + + kbagent rls list --project PROJECT + List row-level security policies visible to a project. + + kbagent rls schema --project PROJECT + Print the live ``rls-policy`` JSON Schema fetched from the metastore. + + kbagent rls setup --project PROJECT [--dialect snowflake|bigquery] + [--rules RULES] [--scope targeted|organization] + [--target-project PROJECT ...] [--yes] + Guided RLS setup: pick tables, build a condition, preview, then write. + + kbagent rls update --project PROJECT --policy-id ID [--table-id ID] + [--dialect snowflake|bigquery] [--rules RULES] [--default DEFAULT] + [--target-project PROJECT ...] [--clear-target-projects] [--dry-run] + [--yes] + Update an existing RLS policy. """, "schedule": """\ ### schedule diff --git a/src/keboola_agent_cli/commands/_helpers.py b/src/keboola_agent_cli/commands/_helpers.py index fce85df2..8dad47d2 100644 --- a/src/keboola_agent_cli/commands/_helpers.py +++ b/src/keboola_agent_cli/commands/_helpers.py @@ -12,11 +12,13 @@ import os import secrets import sys -from collections.abc import Callable +from collections.abc import Callable, Iterable, Sized +from dataclasses import dataclass from pathlib import Path from typing import Any import typer +import yaml from .. import telemetry from ..config_store import ConfigStore @@ -27,6 +29,7 @@ ) from ..errors import ConfigError, ErrorCode, KeboolaApiError, PermissionDeniedError, exit_code_for from ..output import OutputFormatter +from ..permissions import FLAG_ESCALATIONS from ..update_crossing import print_held_warning, refusal_for @@ -78,34 +81,107 @@ def resolve_manage_token(*, allow_env: bool = False) -> str: raise typer.Exit(code=2) -def parse_json_arg(raw: str, *, label: str) -> Any: +def parse_json_arg(raw: str, *, label: str, allow_yaml: bool = False) -> Any: """Parse a ``JSON|@file|-`` argument: inline JSON, ``@path``, or ``-`` for stdin. - The house input contract for structured flags (``config update - --configuration``, ``transformation edit --op``, ``merge-request resolve - --resolved``). Hoisted from ``transformation.py``'s private copy so the - merge-request group did not add another; ``config.py`` still carries an - older dict-only variant (``_parse_json_input``) -- folding it in is a - follow-up, not this PR. + The house input contract for structured flags (spec rule I1): ``config + update --configuration``, ``transformation edit --op``, ``merge-request + resolve --resolved``, ``encrypt values --input``, ``http post --body`` and + the others. ``allow_yaml`` parses the text as YAML, a superset of JSON + (``flow new/update/validate --file``). Raises: ValueError: On a missing/unreadable file or malformed JSON -- the message names ``label`` (the flag) so the caller can print it at exit 2 as-is. """ + loads: Callable[[str], Any] = yaml.safe_load if allow_yaml else json.loads try: if raw == "-": - return json.loads(sys.stdin.read()) + return loads(sys.stdin.read()) if raw.startswith("@"): file_path = Path(raw[1:]) if not file_path.is_file(): raise ValueError(f"{label}: file not found: {file_path}") - return json.loads(file_path.read_text(encoding="utf-8")) - return json.loads(raw) - except json.JSONDecodeError as exc: - raise ValueError(f"{label}: invalid JSON: {exc}") from exc - except OSError as exc: - # a directory, a permission problem -- a usage error, not a crash - raise ValueError(f"{label}: cannot read file: {exc}") from exc + return loads(file_path.read_text(encoding="utf-8")) + return loads(raw) + except (json.JSONDecodeError, yaml.YAMLError) as exc: + raise ValueError(f"{label}: invalid {'YAML' if allow_yaml else 'JSON'}: {exc}") from exc + except (OSError, UnicodeDecodeError) as exc: + # a directory, a permission problem, a binary file -- a usage error, not a crash + raise ValueError(f"{label}: cannot read input: {exc}") from exc + + +def split_multi(values: Iterable[str] | None) -> list[str]: + """Flatten a repeatable option that also takes a comma list (spec rule A4). + + ``--x a --x b,c`` gives ``["a", "b", "c"]``. Empty parts are dropped, and a + value given twice is kept once, in first-seen order. Use it only for IDs + and fixed values: a free-form value (JSON, ``KEY=VALUE``, SQL, text) can + contain a comma, so those options are repeat-only and stay unsplit. + """ + parts = (part.strip() for value in values or () for part in value.split(",")) + return list(dict.fromkeys(part for part in parts if part)) + + +def split_int_multi(values: Iterable[str] | None, *, label: str) -> list[int]: + """``split_multi`` for integer IDs; a part that is not an integer raises ``ValueError``.""" + parts = split_multi(values) + bad = [part for part in parts if not (part.isascii() and part.isdigit())] + if bad: + raise ValueError(f"{label} takes integer IDs, got: {', '.join(bad)}") + return [int(part) for part in parts] + + +def read_text_arg(formatter: OutputFormatter, path: Path, *, label: str) -> str: + """Read a UTF-8 file option; an unreadable file is ``INVALID_ARGUMENT`` naming ``label``.""" + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeDecodeError) as exc: + formatter.error( + message=f"{label}: cannot read file {path}: {exc}", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) from None + + +def _is_file(path: str) -> bool: + """``Path.is_file`` that answers False where it would raise (a name too long).""" + try: + return Path(path).is_file() + except (OSError, ValueError): + return False + + +class RefuseAtFile: + """Option callback for free text that refuses ``@`` (spec rule I3). + + Without it, ``--text @notes.md`` stores the literal text ``@notes.md``. + The refusal is ``INVALID_ARGUMENT`` (exit 2) with a hint that names the + file option of the command, also for the case where the text itself starts + with ``@``. ``@`` followed by anything that is not a file (a relative path + is relative to the current directory) stays text. Use it as + ``typer.Option(..., callback=RefuseAtFile("--file").callback)``; + ``scripts/check_cli_spec.py`` finds the free-text options without it. + """ + + def __init__(self, file_option: str) -> None: + self.file_option = file_option + + def callback( + self, ctx: typer.Context, param: typer.CallbackParam, value: str | None + ) -> str | None: + if value is not None and value.startswith("@") and _is_file(value[1:]): + get_formatter(ctx).error( + message=( + f"{param.opts[0]} got {value!r}, the name of an existing file (relative " + f"to the current directory). Use {self.file_option} {value[1:]} to read " + f"the file. To store the text {value!r} itself, write it to a file and " + f"pass that file with {self.file_option}." + ), + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + return value def read_password_stdin(label: str = "Password: ") -> str: @@ -455,18 +531,6 @@ def handle_service_call[T]( raise typer.Exit(code=service_error_exit_code(exc)) from None -def emit_project_warnings(formatter: OutputFormatter, result: dict) -> None: - """Emit warnings from multi-project operation results. - - Iterates the 'errors' list in the result dict (if present) and prints - each entry as a warning via the formatter. - """ - for err in result.get("errors", []): - alias = err.get("project_alias", "unknown") - message = err.get("message", "Unknown error") - formatter.warning(f"Project '{alias}': {message}") - - def item_failure_exit_code(failed: int) -> int | None: """Map the number of failed items of a multi-item operation to a CLI exit code. @@ -489,6 +553,68 @@ def item_failure_exit_code(failed: int) -> int | None: return 1 +def finish_items( + formatter: OutputFormatter, + result: Any, + *, + failed: int, + total: int, + noun: str, + summary: str, + render: Callable[[], object] | None = None, +) -> None: + """End a command that processes several items: output, one headline, exit code. + + The one way to end such a command (HO2, E4): ``--json`` gets the envelope + with ``exit_code``; human mode gets ``render()`` (the per-item lines) and + then one headline, ``Success: `` or ``Failed: N of M .`` + The command exits 1 when an item failed. ``noun`` is plural ("tables"). + + One wording for every summary (HO2): `` (s) ``, + with details after a comma and a closing period, for example + ``2 table(s) deleted.`` or ``3 config(s) pushed: 1 created, 2 updated, 0 deleted.`` + A dry run says ``would be <...>``. + The fan-out reads end through ``_all_projects.finish_fanout``, which + counts failed projects (E4: ``PAYG_NOT_AVAILABLE`` is not a failure). + """ + code = item_failure_exit_code(failed) + if formatter.json_mode: + formatter.output(result, exit_code=code or 0) + else: + if render is not None: + render() + formatter.item_headline(failed=failed, total=total, noun=noun, summary=summary) + if code: + raise typer.Exit(code=code) + + +def finish_bulk( + formatter: OutputFormatter, + result: dict[str, Any], + done: Sized, + noun: str, + done_as: str, + render: Callable[[], object], +) -> None: + """:func:`finish_items` for a bulk command whose result lists its failures in ``failed``. + + ``done`` holds the items that succeeded; ``noun`` is singular ("table"), + ``done_as`` what happened to them ("deleted", "would be deleted"), so the + headline reads ``Success: 2 table(s) deleted.`` or + ``Failed: 1 of 3 tables. 2 table(s) deleted.`` + """ + failed = len(result["failed"]) + finish_items( + formatter, + result, + failed=failed, + total=len(done) + failed, + noun=f"{noun}s", + summary=f"{len(done)} {noun}(s) {done_as}.", + render=render, + ) + + def _is_help_request(ctx: typer.Context) -> bool: """Check if the current invocation is a --help request. @@ -562,7 +688,13 @@ def check_cli_operation(ctx: typer.Context, operation: str) -> None: higher risk class than its command (see `permissions.FLAG_ESCALATIONS`). The callback has already cleared the bare command by the time a command body runs, so this is an additional gate, not a replacement. + + A command calls this exactly when the escalating flag (or an inherited + organization scope) applies, so it also raises the command category that + the ``TARGET_FROM_ACTIVE_CHOICE`` warning reads (spec T12). """ + if FLAG_ESCALATIONS.get(operation) == "destructive": + get_formatter(ctx).command_category = "destructive" engine = ctx.obj.get("permission_engine") if engine is None or not engine.active: return @@ -630,6 +762,150 @@ def validate_branch_requires_project( raise typer.Exit(code=2) from None +def is_interactive_terminal() -> bool: + """True when a person can see a prompt on stderr and answer it on stdin.""" + return bool( + hasattr(sys.stdin, "isatty") + and sys.stdin.isatty() + and hasattr(sys.stderr, "isatty") + and sys.stderr.isatty() + ) + + +CONFIRMATION_HINT = ( + "Run the command again with --yes to confirm. " + "AI agents: ask the user before you retry with --yes." +) + + +def _confirm_or_abort(formatter: OutputFormatter, yes: bool, question: str) -> None: + """The shared safety prompt (spec S4). + + ``--yes`` and ``--json`` skip it: both mean consent. On a terminal the + question goes to stderr, and a declined prompt prints ``Aborted.`` and + exits 0 (E6). Without a terminal nobody can answer, and an open stdin + pipe must not answer for the user (D6): the command refuses with + ``CONFIRMATION_REQUIRED`` and exit 2, and the message names ``--yes``. + """ + if yes or formatter.json_mode: + return + if not is_interactive_terminal(): + formatter.error( + message=f"{ErrorCode.CONFIRMATION_REQUIRED.value}: {question} " + f"There is no terminal to answer it. {CONFIRMATION_HINT}", + error_code=ErrorCode.CONFIRMATION_REQUIRED, + ) + raise typer.Exit(code=2) + if not typer.confirm(question, err=True): + formatter.console.print("Aborted.") + raise typer.Exit(code=0) + + +# The ``code`` of a ``warnings[]`` entry for a deprecated form (spec C3, O3). +DEPRECATED_CODE = "DEPRECATED" + + +@dataclass(frozen=True) +class Deprecation: + """One deprecated command-line form (spec C3). + + ``replacement`` is the form to use instead, or None when the old form + has no effect and the caller only drops it. ``remove_after`` is the ISO + date after which a minor release may remove the old form. It is None + while that date is unknown: the window of a form that 1.0.0 hides + starts at the migration time M, which exists only once v1.0.0 is + published. + """ + + message: str + hint: str + replacement: str | None + remove_after: str | None = None + + +# C3: every deprecated form, keyed by its command path and option. A removed +# form stays here, so that its error can name the replacement. +DEPRECATIONS: dict[str, Deprecation] = { + "stream delete --force": Deprecation( + message="stream delete --force is deprecated. Use --yes to skip the confirmation.", + hint="--yes", + replacement="--yes", + ), + "semantic-layer import --yes": Deprecation( + message="semantic-layer import --yes is deprecated and has no effect. Omit it.", + hint="omit --yes (skip on conflict is the default)", + replacement=None, + ), + # V9: `kbagent serve` partial updates move from PUT to PATCH. A REST form + # is keyed "METHOD /path"; its warning goes into the Kbagent-Warnings header. + "PUT /semantic-layer/items/{kind}/{name}": Deprecation( + message="PUT /semantic-layer/items/{kind}/{name} is deprecated. Use PATCH.", + hint="PATCH /semantic-layer/items/{kind}/{name}", + replacement="PATCH /semantic-layer/items/{kind}/{name}", + ), + "PUT /merge-requests/{project}/{merge_request_id}": Deprecation( + message="PUT /merge-requests/{project}/{merge_request_id} is deprecated. Use PATCH.", + hint="PATCH /merge-requests/{project}/{merge_request_id}", + replacement="PATCH /merge-requests/{project}/{merge_request_id}", + ), + "PUT /data-apps/{project}/{app_id}/secrets": Deprecation( + message="PUT /data-apps/{project}/{app_id}/secrets is deprecated. Use PATCH.", + hint="PATCH /data-apps/{project}/{app_id}/secrets", + replacement="PATCH /data-apps/{project}/{app_id}/secrets", + ), + # Only the merge is deprecated: PUT with `replace: true` stays the full replace. + "PUT /configs/{project}/{component_id}/{config_id}/variables": Deprecation( + message="PUT .../variables without replace: true merges the variables, which is " + "deprecated. Use PATCH to merge, or PUT with replace: true to replace all variables. " + "After the window, a PUT without replace: true is refused, never run as a replace.", + hint="PATCH /configs/{project}/{component_id}/{config_id}/variables", + replacement="PATCH /configs/{project}/{component_id}/{config_id}/variables", + ), +} + + +def deprecation_entry(form: str) -> tuple[str, dict[str, Any]]: + """The message and the extra fields of the ``DEPRECATED`` warning of ``form``.""" + entry = DEPRECATIONS[form] + when = ( + f"It can be removed after {entry.remove_after}." + if entry.remove_after + else "It keeps working until at least 28 days after the 1.0 migration date." + ) + extra = { + "hint": entry.hint, + "replacement": entry.replacement, + "remove_after": entry.remove_after, + } + return f"{entry.message} {when}", extra + + +def warn_deprecated(formatter: OutputFormatter, form: str) -> None: + """Add the ``DEPRECATED`` warning of a ``DEPRECATIONS`` entry (one stderr line in human mode).""" + message, extra = deprecation_entry(form) + formatter.warning(message, code=DEPRECATED_CODE, **extra) + + +def deprecated_option(form: str, *param_decls: str) -> Any: + """A hidden flag for a deprecated form in ``DEPRECATIONS`` (spec C3). + + The vendored click has no option-level ``deprecated``. The flag keeps + working: the command reads its value (for an alias, ``yes = yes or + force``), and the callback adds the ``DEPRECATED`` warning when the + caller gives the flag. + """ + if form not in DEPRECATIONS: + raise KeyError(f"{form!r} has no DEPRECATIONS entry") + + def _warn(ctx: typer.Context, value: bool) -> bool: + formatter = (ctx.obj or {}).get("formatter") + if value and formatter is not None and not ctx.resilient_parsing: + warn_deprecated(formatter, form) + return value + + return typer.Option(False, *param_decls, hidden=True, callback=_warn) + + _CONFIRM_CODE_LENGTH = 4 diff --git a/src/keboola_agent_cli/commands/cls.py b/src/keboola_agent_cli/commands/cls.py index bcac5a2f..3b1ed376 100644 --- a/src/keboola_agent_cli/commands/cls.py +++ b/src/keboola_agent_cli/commands/cls.py @@ -15,8 +15,8 @@ columns are the only ones that principal can read (allowlist projection; masking is not supported). As with ``rls``, every write is authored at ``targeted`` scope unless ``--scope organization`` is passed -- never -``project``. Enforcement happens in ``keboola-mcp-server``'s -``query_data``; kbagent only authors policies. +``project``. Enforcement happens in Query Service on a governed workspace +or in ``keboola-mcp-server``'s ``query_data``; kbagent only authors policies. ``list``/``detail``/``schema`` are read-only and safe under ``--deny-writes``; ``create``/``update`` are gated as ``admin``, ``delete`` and ``--scope diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index 22f1f1e4..f9302747 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -423,6 +423,26 @@ Show recent changelog (what changed in each version). Default: last 5 versions, the first sentence of every BREAKING note, plus of the first other notes until at least two show; --full (-v) expands every note. +""", + "cls": """\ +#### Column-Level Security (CLS) + +Author `cls-policy` metastore objects: one object per protected table, a +`visible_columns` allowlist per rule (unlisted columns are omitted from +that identity's result; masking is not supported). Rules select by +principal, principals or `groups`; within a policy an identity several rules +match sees the union of their columns, several policies intersect; there is +no `default`. Sibling of RLS -- same scope default, permissions, checks, +partial update and enforcement paths (Query Service on a governed workspace, +or `keboola-mcp-server`'s `query_data`, which composes RLS and CLS); no +`setup` wizard. A principal with no rule for a governed table is refused. A +`cls-policy` alone never grants rows: in a governed workspace the table also +needs an `rls-policy`. + + cls create + --rules is a JSON array of {principal|principals|groups, + visible_columns: [col, ...]} objects. --dry-run prints each selector's + allowed projection without writing. """, "component": """\ #### Component Discovery @@ -1543,6 +1563,69 @@ project member-set-role Change an existing member's role via PATCH /manage/projects/{id}/users/{userId}. +""", + "rls": """\ +#### Row-Level Security (RLS) + +Author `rls-policy` metastore objects: one object per protected table, a +declarative condition primitive per rule (never free-text SQL). Policy schema +1.1.0: a rule selects identities by principal, principals or IdP `groups`; +within a policy every rule matching one identity applies and the conditions +combine with OR; several policies on one table combine with AND (an added +policy only narrows); a reader with no identity is always refused; the +wildcard `"*"` (sole entry of `principal`/`principals`/`groups`) matches every +identified reader and is how a table is opened to everyone under default deny. +In a governed workspace (the platform denies by default, admins never exempt) +a table with no rls-policy is refused. An optional policy `default` covers an +identified reader no rule matches (absent = refused). Same metastore/master-token +requirements as Semantic Layer. +**Scope**: `targeted` by default (the owning project plus --target-project +grants); `--scope organization` governs the table in EVERY project of the +organization (destructive-class, organization-admin only) -- use it only +when the user asks for org-wide. There is NO project scope. A project admin +may write `targeted` policies of its own project without grants; grants and +organization scope need the organization-admin role (403 otherwise). +Writes refuse a dialect other than the project backend (the enforcement +would refuse every read of that table). A stack whose metastore predates the +`rls-policy`/`cls-policy` schemas answers `rls schema` with a classified +NOT_FOUND schema-fetch error -- expected there, not a kbagent bug. +**Enforcement** (the actual SQL rewrite) is never done by kbagent, which only +authors policies. Two readers apply them: Query Service on a governed +workspace (`workspace create --governed`, then `workspace load` and +`workspace query`; Query Service decides per project whether to enforce), and +`keboola-mcp-server`'s `query_data`, which enforces only where the project has +the `row-level-security` feature and needs an engine that reads schema 1.1.0 +(since vNEXT). The identity that queries needs a matching rule. + + rls list + Policies visible to -- and applied in -- a project (id, table, dialect, + rule_count, scope, owner_project_id, target_project_ids). + rls create + --rules is a JSON array of {principal|principals|groups, condition} + objects -- condition is a primitive tree (column/op/value, in/not_in, + is_null/is_not_null, and/or nesting, `{"true": true}` or + `{"false": true}`), never a raw predicate string; use is_null, never + value null; `value: {"$identity": "email"}` / `values: {"$identity": + "groups"}` are resolved per reader. --default (e.g. `{"false": true}` = + no rows) covers identities no rule matches. --dialect defaults to the + project backend. --target-project (alias or ID, repeatable or + comma-separated) grants other projects. --dry-run previews the compiled + condition (columns quoted like the enforcement) without writing. + rls update + Partial update (PATCH of the changed keys only); the merged policy is + validated first and the result is re-read after the write. A default + cannot be removed by an update -- recreate the policy. + rls delete + Un-protects its table (destructive-class). + rls schema + Live-only: no offline bundled snapshot (unlike `flow schema`). + rls setup + INTERACTIVE-TERMINAL-ONLY wizard: checkbox table picker (reuses + `storage tables`), then the interactive column/op/value condition + builder or the same --rules escape hatch `create` takes, then a preview + + confirm, then one `rls create` call per selected table. Under --json or + without an interactive terminal it exits 2 (INVALID_ARGUMENT) pointing at + `rls create`. """, "schedule": """\ #### Schedule Discovery & Audit (Fleet-Wide) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index 61510189..e2cff418 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -46,6 +46,7 @@ from ..services.rls_service import PolicyScope from ._checkbox_select import CheckboxItem, CheckboxUnavailable, _stdio_is_tty, checkbox_select from ._helpers import ( + _confirm_or_abort, check_cli_operation, check_cli_permission, get_formatter, @@ -67,7 +68,7 @@ DIALECT_HELP = "Workspace SQL dialect; must match the project backend (default: the backend)" SCOPE_HELP = ( "targeted (default): the owning project plus --target-project grants. organization: " - "EVERY project of the organization that has the row-level-security feature -- an explicit, " + "EVERY project of the organization -- an explicit, " "organization-admin-only choice, gated as destructive." ) TARGET_HELP = ( @@ -117,12 +118,6 @@ def _rls_permission_check(ctx: typer.Context) -> None: # --------------------------------------------------------------------------- -def _confirm_or_exit(formatter: Any, question: str) -> None: - if not typer.confirm(question): - formatter.console.print("[yellow]Aborted.[/yellow]") - raise typer.Exit(code=0) - - def gate_scope( ctx: typer.Context, group: PolicyGroup, scope: PolicyScope, target_project: list[str] | None ) -> None: @@ -317,7 +312,7 @@ def create_policy( # A validation/network error here is reported properly by the real call below. with contextlib.suppress(ConfigError, KeboolaApiError): _print_preview(formatter, group, service.create_policy(**kwargs, dry_run=True)) - _confirm_or_exit(formatter, f"Create {group.label} policy on {kwargs['table']}?") + _confirm_or_abort(formatter, yes, f"Create {group.label} policy on {kwargs['table']}?") result = handle_service_call(ctx, service.create_policy, **kwargs, dry_run=dry_run) @@ -358,7 +353,7 @@ def update_policy( policy_id = kwargs["policy_id"] if not dry_run and not yes and not formatter.json_mode: - _confirm_or_exit(formatter, f"Update {group.label} policy {policy_id}?") + _confirm_or_abort(formatter, yes, f"Update {group.label} policy {policy_id}?") result = handle_service_call( ctx, @@ -393,7 +388,7 @@ def delete_policy( service = get_service(ctx, group.service) if not dry_run and not yes and not formatter.json_mode: - _confirm_or_exit(formatter, f"Delete {group.label} policy {policy_id}? {effect}") + _confirm_or_abort(formatter, yes, f"Delete {group.label} policy {policy_id}? {effect}") result = handle_service_call( ctx, service.delete_policy, alias=alias, policy_id=policy_id, dry_run=dry_run @@ -637,7 +632,10 @@ def _build_rules_interactively(formatter: Any) -> list[dict[str, Any]]: else: rule["principals"] = principals rules.append(rule) - if not typer.confirm("Add another rule to this policy?", default=False): + if ( + _prompt_choice(formatter, "Add another rule to this policy? (y/n)", ("y", "n"), "n") + == "n" + ): return rules @@ -718,8 +716,7 @@ def rls_setup( continue _print_preview(formatter, RLS, preview) - if not yes: - _confirm_or_exit(formatter, f"Create {policies}?") + _confirm_or_abort(formatter, yes, f"Create {policies}?") failed: list[str] = [] exit_codes: set[int] = set() diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py index 95856fbc..d7d6a788 100644 --- a/tests/test_cli_contract.py +++ b/tests/test_cli_contract.py @@ -630,6 +630,12 @@ def test_a_stream_that_fails_stays_ndjson( "auth logout": "auth logout", "branch delete": "branch delete --project p --branch 1", "branch metadata-delete": "branch metadata-delete --project p --metadata-id 1", + "cls create": ( + 'cls create --project p --table-id in.c-a.b --rules [{"principal":"a@example.com",' + '"visible_columns":["id"]}]' + ), + "cls delete": "cls delete --project p --policy-id x", + "cls update": "cls update --project p --policy-id x", "config delete-metadata": ( "config delete-metadata --project p --component-id c --config-id 1 --metadata-id 1" ), @@ -654,6 +660,12 @@ def test_a_stream_that_fails_stays_ndjson( ), "project invitation-cancel": "project invitation-cancel --project p --email a@example.com", "project member-remove": "project member-remove --project p --email a@example.com", + "rls create": ( + 'rls create --project p --table-id in.c-a.b --rules [{"principal":"a@example.com",' + '"condition":{"true":true}}]' + ), + "rls delete": "rls delete --project p --policy-id x", + "rls update": "rls update --project p --policy-id x", "semantic-layer edit glossary": "semantic-layer edit glossary --project p --term a --new-term b", "semantic-layer model delete": "semantic-layer model delete --project p --model m", "semantic-layer promote": "semantic-layer promote --from-project a --to-project b", @@ -687,6 +699,7 @@ def test_a_stream_that_fails_stays_ndjson( "merge-request merge": "resolves the merge request first, then asks", "org setup": "needs the manage token first, then previews and asks", "project refresh": "needs the manage token first, then previews and asks", + "rls setup": "terminal-only wizard: without a terminal it exits 2 before any question", "semantic-layer add metric": "dataset guard in the service; refuses without a terminal", "semantic-layer edit metric": "rename-cascade guard in the service; refuses without a terminal", "semantic-layer import": "hidden deprecated --yes with no effect (C3)", diff --git a/tests/test_cls_cli.py b/tests/test_cls_cli.py index 1aaf272c..cf83788a 100644 --- a/tests/test_cls_cli.py +++ b/tests/test_cls_cli.py @@ -205,6 +205,7 @@ def test_dry_run_prints_projection_without_prompt(self, tmp_path: Path) -> None: assert "a@x.com: SELECT id, region" in result.output assert service.create_policy.call_args.kwargs["dry_run"] is True + @pytest.mark.usefixtures("interactive_terminal") def test_confirmed_prompt_previews_then_writes(self, tmp_path: Path) -> None: service = MagicMock() service.create_policy.return_value = {**_row(), "preview": _preview()["preview"]} @@ -215,6 +216,7 @@ def test_confirmed_prompt_previews_then_writes(self, tmp_path: Path) -> None: assert "Created CLS policy p-1" in result.output assert [c.kwargs["dry_run"] for c in service.create_policy.call_args_list] == [True, False] + @pytest.mark.usefixtures("interactive_terminal") def test_declined_prompt_aborts_without_writing(self, tmp_path: Path) -> None: service = MagicMock() service.create_policy.return_value = _preview(dry_run=True) @@ -301,6 +303,7 @@ def test_passes_only_given_flags(self, tmp_path: Path) -> None: assert kwargs["dialect"] is None assert kwargs["rules"][0]["principal"] == "a@x.com" + @pytest.mark.usefixtures("interactive_terminal") def test_declined_prompt_aborts(self, tmp_path: Path) -> None: service = MagicMock() @@ -314,6 +317,7 @@ def test_declined_prompt_aborts(self, tmp_path: Path) -> None: assert "Aborted" in result.output service.update_policy.assert_not_called() + @pytest.mark.usefixtures("interactive_terminal") def test_confirmed_human_output(self, tmp_path: Path) -> None: service = MagicMock() service.update_policy.return_value = {**_row(), "preview": _preview()["preview"]} @@ -391,6 +395,7 @@ def test_json_skips_prompt(self, tmp_path: Path) -> None: assert result.exit_code == 0, result.output assert json.loads(result.output)["data"]["deleted"] is True + @pytest.mark.usefixtures("interactive_terminal") def test_declined_prompt_does_not_delete(self, tmp_path: Path) -> None: service = MagicMock() diff --git a/tests/test_rls_cli.py b/tests/test_rls_cli.py index 5fcf3757..1723e3c6 100644 --- a/tests/test_rls_cli.py +++ b/tests/test_rls_cli.py @@ -350,6 +350,7 @@ def test_dry_run_uses_dry_run_flag_and_needs_no_confirm(self, tmp_path: Path) -> _, kwargs = service.create_policy.call_args assert kwargs["dry_run"] is True + @pytest.mark.usefixtures("interactive_terminal") def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) service = MagicMock() @@ -461,6 +462,7 @@ def test_yes_flag_skips_confirm(self, tmp_path: Path) -> None: assert result.exit_code == 0, result.output service.delete_policy.assert_called_once_with(alias="prod", policy_id="p-1", dry_run=False) + @pytest.mark.usefixtures("interactive_terminal") def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) service = MagicMock() @@ -476,6 +478,21 @@ def test_human_mode_without_yes_aborts_on_no(self, tmp_path: Path) -> None: assert "Aborted" in result.output service.delete_policy.assert_not_called() + def test_without_terminal_refuses_instead_of_reading_stdin(self, tmp_path: Path) -> None: + store = _setup_config(tmp_path / "cfg", {"prod": {}}) + service = MagicMock() + + result = _run( + ["rls", "delete", "--project", "prod", "--policy-id", "p-1"], + store, + service, + input="y\n", + ) + + assert result.exit_code == 2 + assert "CONFIRMATION_REQUIRED" in result.output + service.delete_policy.assert_not_called() + def test_json_mode_needs_no_confirm(self, tmp_path: Path) -> None: store = _setup_config(tmp_path / "cfg", {"prod": {}}) service = MagicMock() From 61b6afa56454bc68d04d75b594ea5dfd16e865b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Va=C5=A1ko?= Date: Wed, 21 Oct 2026 15:55:53 +0200 Subject: [PATCH 35/35] feat(CLI-17): rls bind/unbind/bindings/bindings-sweep for workspace policy bindings Query Service applies the policy-binding of a governed workspace: a metastore `policy-binding` object named by the workspace KID (KID::project-:workspace-) that lists the rls/cls policies the workspace gets. No binding = every policy of the project; a binding only narrows and applies to the next query. - `rls bind`: idempotent upsert (PUT replaces both lists). Validates at least one policy, every UUID readable as an rls/cls policy of the project, and the workspace in production or a dev branch; a non-governed login type is a warning. Prints the KID. - `rls unbind`: destructive-class, looks the binding up, then confirms through _confirm_or_abort. - `rls bindings`: kid, workspace id, policy ids, last update and whether the workspace still exists. - `rls bindings-sweep`: reports orphaned bindings (workspace 404 in production and every dev branch, older than --older-than days, a workspace KID of this project); --yes deletes them (destructive via FLAG_ESCALATIONS). Any other check error is a failed item, never a delete; app KIDs are never swept. The metastore exposes no reverse-reference lookup, so a referenced binding is only detected by the metastore refusing the delete (409). - The stack segment is exposed by no Storage/Manage/Connection API, so --stack-id (env KBAGENT_STACK_ID) is required; added to the env var registry with added_in vNEXT (the release PR resolves it). - `rls delete` / `cls delete` of a bound policy (409, onDelete restrict) now names `rls bindings` / `rls bind` / `rls unbind`. - REST: GET /rls/{project}/bindings, PUT|DELETE /rls/{project}/bindings/{workspace_id}, POST /rls/{project}/bindings/sweep, mounted before the rls router so /bindings is not read as a policy id. - Docs: context notes, CLAUDE.md, commands-reference.md, gotchas.md, rls-workflow.md (end-to-end section), keboola-expert.md; regenerated manifest, context commands, SKILL.md table and endpoint reference. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 23 + docs/cli-manifest.json | 53 ++ docs/web-server-endpoints.md | 10 +- plugins/kbagent/agents/keboola-expert.md | 2 +- plugins/kbagent/skills/kbagent/SKILL.md | 4 + .../kbagent/references/commands-reference.md | 4 + .../skills/kbagent/references/gotchas.md | 22 + .../skills/kbagent/references/rls-workflow.md | 35 ++ src/keboola_agent_cli/cli.py | 3 + .../commands/_context_commands.py | 19 + .../commands/_rls_bindings.py | 245 +++++++++ src/keboola_agent_cli/commands/context.py | 17 + src/keboola_agent_cli/commands/rls.py | 7 + src/keboola_agent_cli/constants.py | 8 + src/keboola_agent_cli/env_vars.py | 8 + src/keboola_agent_cli/metastore_client.py | 2 + src/keboola_agent_cli/permissions.py | 8 + .../server/_serve_command_map.py | 4 + src/keboola_agent_cli/server/app.py | 6 +- src/keboola_agent_cli/server/dependencies.py | 3 + .../server/routers/rls_bindings.py | 113 ++++ src/keboola_agent_cli/services/cls_service.py | 4 +- .../services/policy_binding_service.py | 467 +++++++++++++++++ src/keboola_agent_cli/services/rls_service.py | 18 +- tests/test_cli_contract.py | 2 + tests/test_e2e.py | 28 + tests/test_env_vars.py | 3 +- tests/test_policy_binding_service.py | 481 ++++++++++++++++++ tests/test_project_ref.py | 1 + tests/test_rls_bindings_cli.py | 315 ++++++++++++ tests/test_rls_service.py | 14 + tests/test_server_rls_bindings.py | 150 ++++++ 32 files changed, 2070 insertions(+), 9 deletions(-) create mode 100644 src/keboola_agent_cli/commands/_rls_bindings.py create mode 100644 src/keboola_agent_cli/server/routers/rls_bindings.py create mode 100644 src/keboola_agent_cli/services/policy_binding_service.py create mode 100644 tests/test_policy_binding_service.py create mode 100644 tests/test_rls_bindings_cli.py create mode 100644 tests/test_server_rls_bindings.py diff --git a/CLAUDE.md b/CLAUDE.md index c11c9994..1f0a096b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1389,6 +1389,29 @@ kbagent rls create --project P --table-id TABLE_ID --rules JSON|@file|- [--defau kbagent rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--default JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes] kbagent rls delete --project P --policy-id ID [--dry-run] [--yes] kbagent rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes] +kbagent rls bind --project P --workspace-id ID [--rls-policy UUID ...] [--cls-policy UUID ...] [--stack-id STACK] [--dry-run] +kbagent rls unbind --project P --workspace-id ID [--stack-id STACK] [--dry-run] [--yes] +kbagent rls bindings --project P [--workspace-id ID] +kbagent rls bindings-sweep --project P [--older-than DAYS] [--dry-run | --yes] +# rls bind/unbind/bindings/bindings-sweep: a `policy-binding` metastore object restricts ONE governed +# workspace to some of the project's rls/cls policies; named by the workspace KID +# `KID::project-:workspace-`. No binding = the workspace gets EVERY policy of the +# project; a binding only narrows, never grants; Query Service re-reads it on every governed query, +# so a change applies to the next query. `bind` is an idempotent upsert (PUT replaces both lists), +# validates first (>=1 policy, every UUID readable as an rls/cls policy of the project, the workspace +# exists in production or a dev branch; a non-governed login type is only a warning) and prints the +# KID. --stack-id (env KBAGENT_STACK_ID) is REQUIRED: no API exposes the stack segment, it must equal +# Query Service's QUERY_RLS_STACK_ID (AWS dev: dev-keboola-aws-eu-west-1); a wrong one writes a +# binding nothing reads, so the workspace silently keeps every policy. `unbind` is destructive-class +# (looks the binding up, then confirms). `bindings-sweep` only reports unless --yes (destructive via +# FLAG_ESCALATIONS): orphaned = a workspace KID of this project whose workspace 404s in production +# and every dev branch AND last updated more than --older-than days (default 30) ago; any other +# check error is a failed item (exit 1), never a delete; app KIDs are never swept. References to a +# binding cannot be checked first (no reverse-reference API); the metastore refuses (409) a delete +# a reference blocks. A bound policy cannot be deleted (onDelete restrict): `rls delete`/`cls delete` +# then names `rls bindings`. REST: GET /rls/{project}/bindings, PUT|DELETE +# /rls/{project}/bindings/{workspace_id} (stack_id required), POST /rls/{project}/bindings/sweep. +# Version gate lives in gotchas.md. # rls: row-level security policy authoring, metastore-backed (`rls-policy` object type, # one object per protected table, named by it). SCOPE: `targeted` by default (the schema's own # default: the owning project plus --target-project grants); `--scope organization` governs the diff --git a/docs/cli-manifest.json b/docs/cli-manifest.json index 7632b5bb..83f0ee15 100644 --- a/docs/cli-manifest.json +++ b/docs/cli-manifest.json @@ -2544,6 +2544,45 @@ "hidden": false, "help": "Manage row-level security policies (metastore-backed). 'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' are gated as admin -- see CONTRIBUTING.md." }, + { + "path": "rls bind", + "kind": "command", + "hidden": false, + "category": "admin", + "help": "Bind policies to a governed workspace: it then gets only these policies.\n\nCreates the binding, or replaces both of its lists when the workspace is already bound.\nWithout a binding the workspace gets every policy of the project.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "workspace_id", "opts": ["--workspace-id"], "type": "integer", "required": true, "default": null, "help": "Governed workspace ID"}, + {"kind": "option", "name": "rls_policy", "opts": ["--rls-policy"], "type": "text", "multiple": true, "default": null, "help": "RLS policy ID to bind (repeatable or comma-separated)"}, + {"kind": "option", "name": "cls_policy", "opts": ["--cls-policy"], "type": "text", "multiple": true, "default": null, "help": "CLS policy ID to bind (repeatable or comma-separated)"}, + {"kind": "option", "name": "stack_id", "opts": ["--stack-id"], "type": "text", "envvar": "KBAGENT_STACK_ID", "default": null, "help": "Stack segment of the workspace KID -- the value Query Service is configured with, for example dev-keboola-aws-eu-west-1 (no API exposes it)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Validate and show the binding without writing"} + ] + }, + { + "path": "rls bindings", + "kind": "command", + "hidden": false, + "category": "read", + "help": "List the policy bindings of a project and whether each workspace still exists.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "workspace_id", "opts": ["--workspace-id"], "type": "integer", "default": null, "help": "Only the binding of this workspace"} + ] + }, + { + "path": "rls bindings-sweep", + "kind": "command", + "hidden": false, + "category": "read", + "help": "Find the bindings whose workspace no longer exists; with --yes, delete them.\n\nA binding is orphaned when its workspace is gone (404 in production and every dev branch),\nit was last updated more than --older-than days ago, and the KID is a workspace KID of this\nproject. App KIDs are never swept. Without --yes nothing is deleted.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "older_than", "opts": ["--older-than"], "type": "integer", "default": 30, "help": "Only bindings last updated more than this many days ago"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Only report the orphaned bindings (the default)"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Delete the orphaned bindings"} + ] + }, { "path": "rls create", "kind": "command", @@ -2621,6 +2660,20 @@ {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip the final confirmation"} ] }, + { + "path": "rls unbind", + "kind": "command", + "hidden": false, + "category": "destructive", + "help": "Delete the policy binding of a workspace: it then gets every policy of the project.", + "params": [ + {"kind": "option", "name": "project", "opts": ["--project"], "type": "text", "required": true, "default": null, "help": "Project alias"}, + {"kind": "option", "name": "workspace_id", "opts": ["--workspace-id"], "type": "integer", "required": true, "default": null, "help": "Workspace ID"}, + {"kind": "option", "name": "stack_id", "opts": ["--stack-id"], "type": "text", "envvar": "KBAGENT_STACK_ID", "default": null, "help": "Stack segment of the workspace KID -- the value Query Service is configured with, for example dev-keboola-aws-eu-west-1 (no API exposes it)"}, + {"kind": "option", "name": "dry_run", "opts": ["--dry-run"], "type": "boolean", "is_flag": true, "default": false, "help": "Show the binding without deleting it"}, + {"kind": "option", "name": "yes", "opts": ["--yes", "-y"], "type": "boolean", "is_flag": true, "default": false, "help": "Skip confirmation prompt"} + ] + }, { "path": "rls update", "kind": "command", diff --git a/docs/web-server-endpoints.md b/docs/web-server-endpoints.md index 2af707bd..e7753677 100644 --- a/docs/web-server-endpoints.md +++ b/docs/web-server-endpoints.md @@ -9,7 +9,7 @@ auth, and the concepts behind these routes live in [`web-server.md`](web-server.md); a running server serves the same spec interactively at `/docs` (Swagger) and `/openapi.json`. -**273 operations** across **228 paths** and **34 routers**. +**277 operations** across **231 paths** and **34 routers**. Paths are shown as the server registers them. Reaching them through the Node BFF (or single-process `--ui` mode) prefixes every path with `/api`. @@ -428,12 +428,16 @@ Model, validate, import/export, diff, promote, and build semantic layer artifact | `PUT` | `/semantic-layer/scope/{context_id}/elevation-request` | Request scope elevation | | `DELETE` | `/semantic-layer/scope/{context_id}/elevation-request` | Withdraw scope elevation request | -### `rls` (6 operations) +### `rls` (10 operations) -Row-level security policies (metastore-backed `rls-policy` objects), authored at `targeted` scope unless `organization` is asked for -- never `project` scope. Mirrors `kbagent rls list|detail|schema|create|update|delete` (`rls setup`'s interactive picker has no endpoint of its own, same carve-out as `auth register-projects`). +Row-level security policies (metastore-backed `rls-policy` objects), authored at `targeted` scope unless `organization` is asked for -- never `project` scope. Mirrors `kbagent rls list|detail|schema|create|update|delete|bind|unbind|bindings|bindings-sweep` (`rls setup`'s interactive picker has no endpoint of its own, same carve-out as `auth register-projects`). | Method | Path | Summary | |---|---|---| +| `GET` | `/rls/{project}/bindings` | List the policy bindings of a project | +| `POST` | `/rls/{project}/bindings/sweep` | Find (and delete) orphaned policy bindings | +| `PUT` | `/rls/{project}/bindings/{workspace_id}` | Bind policies to a governed workspace | +| `DELETE` | `/rls/{project}/bindings/{workspace_id}` | Delete the policy binding of a workspace | | `GET` | `/rls/{project}` | List RLS policies | | `POST` | `/rls/{project}` | Create a RLS policy | | `GET` | `/rls/{project}/schema` | Fetch the live rls-policy JSON Schema | diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 39a9e7a8..267a69ee 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -164,7 +164,7 @@ been retired, so its absence is NOT a promise (see §1 Rule 6). | Read a semantic-layer model (models, metrics, datasets, constraints) | `kbagent --json semantic-layer show --project P [--model M] [--type metric\|dataset\|relationship\|constraint\|glossary]`; `model list`; `search-context` / `get-context` for glob/id lookup; `validate [--deep]` before trusting one. Any valid, non-disabled, non-expired Storage token works for reads (0.97.0, PSGO-282 -- the metastore no longer requires a master token for GET/List) | -- | hand-rolled `httpx` loops against `metastore.*.keboola.com` (bypasses retry/backoff and the kbagent error envelope) | | ANY semantic-layer write (add / edit / remove / import / promote / build / scope add|remove|set|request-create) | `kbagent semantic-layer export` FIRST (no soft-delete or history: the snapshot is the only restore path), then the write, `--dry-run` where offered. Needs a project-admin token (master or admin-role user token, PSGO-282); a non-admin token gets `MISSING_MASTER_TOKEN`/403 on writes. `Read` [semantic-layer-workflow.md](../skills/kbagent/references/semantic-layer-workflow.md) before starting: per-verb recipes, rename cascade, promote classification | `semantic-layer diff` (`--project-a/-b` or `--file-a/-b`) to confirm what a write would change | raw metastore REST (no rollback, no orphan scan, no modelUUID rewrite); a write with no export taken | | Share a semantic-layer object across projects, or make it org-wide | **ASK THE USER which project(s) before EVER passing `--scope organization\|targeted` (on `model create`/`add `) or running `scope set --scope organization`/`scope request-create` -- never guess or default to widening visibility.** Once the user names the target(s): `--scope targeted --target-project ALIAS\|ID` grants specific projects; `scope add\|remove --project P --type T --context-id ID --target-project ALIAS\|ID` adjusts an existing item's grants from the owning project, `scope set --target-project ...` / `--clear` replaces the list; org-wide needs `scope request-create` then an org-admin's `scope set --scope organization --yes` -- **irreversible, destructive-class**. `scope get --type T --context-id ID` first. See [metastore-scope-workflow.md](../skills/kbagent/references/metastore-scope-workflow.md) | `scope request-list --type T` to find items an org-admin should act on | passing `--scope organization\|targeted` or running `scope set --scope organization` without the user having named the target project(s) first; assuming `--scope organization` succeeds on a non-admin token (it 403s -- creating directly at organization scope needs the SAME organization-admin role as elevating); forgetting that `import`/`promote`/`build --model` into a shared model also share every new item (they take the model's scope unless `import`/`promote` get `--scope`); trying to elevate an item created before 0.97.0 (schema version 1.0.0: recreate it via export, delete, create, import) | -| Row/column-level (RLS/CLS) security policy | `kbagent {rls,cls} create\|update\|delete --project P --table-id T` (`--dry-run` first; never `--scope project`, `--scope organization` only if asked); rules pick by `principal`/`principals`/`groups`, `"*"` = every identified reader; enforced by QS on a `workspace create --governed` workspace or by MCP (`row-level-security`). [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | +| Row/column-level (RLS/CLS) security policy | `kbagent {rls,cls} create\|update\|delete --project P --table-id T` (`--dry-run` first; never `--scope project`, `--scope organization` only if asked); rules pick by `principal`/`principals`/`groups`, `"*"` = every identified reader; enforced by QS on a `workspace create --governed` workspace (`rls bind` narrows one; none = all policies) or MCP (`row-level-security`). [rls-workflow.md](../skills/kbagent/references/rls-workflow.md) | `rls schema` | a raw predicate as `condition`; headless `rls setup` (use `rls create`) | | User asks to "log in" / authenticate via browser / register a session's projects | ATTENDED session + a background shell: run `kbagent auth login --device-code --stack URL --register-projects` in a **BACKGROUND** shell (capture stdout+stderr, human mode -- `--json` puts the panel on stderr), relay the verification URL + user code to the user, then poll `kbagent --json auth status` (exit 0 = signed in, exit 3 = not yet). The human's part is approving in the browser, not typing the command. No background shell -> hand the plain `auth login --stack URL --register-projects` to the user's terminal. To register projects from an EXISTING session, `kbagent auth register-projects --all` or `--project-id ID` is non-interactive and agent-safe | -- | running `auth login` in a FOREGROUND tool shell (~120 s timeout kills it mid-flight); running it unattended (nobody can approve); re-running it blind without checking `auth status` first (orphans a session); the flagless `register-projects` picker unattended; reading the token out of `auth.json`; using a numeric project id as an alias | | User has NO Keboola account / project at all and asks to get started | `kbagent project create --url URL [--project ALIAS] [--name NAME] [--backend snowflake\|bigquery]` (0.95.0+) -- provisions a real project, stores its session and registers the alias in one call, agent-runnable. **Then relay the result's `confirm_url` to the human verbatim and say the project is owned by nobody until they open it**; after they confirm, the agent session is revoked by design -> `kbagent auth login --stack URL` | the user creating the project in the Keboola UI, then `auth login` / `project add --token-stdin` | calling it on a stack where a session already exists (exit 5 -- one session per stack); reporting success without the confirm link (an unclaimed project is a billable orphan); retrying it after a 5xx/429/503 without being asked (each success creates a new organization + project + credit grant) | | CI task has account credentials | `kbagent auth login-password --email E (--password-stdin \| --password P) [--totp-secret SEED]` (0.84.0+), agent-runnable | a static Storage token | `auth login` unattended | diff --git a/plugins/kbagent/skills/kbagent/SKILL.md b/plugins/kbagent/skills/kbagent/SKILL.md index 20b15243..a87ee3d0 100644 --- a/plugins/kbagent/skills/kbagent/SKILL.md +++ b/plugins/kbagent/skills/kbagent/SKILL.md @@ -378,6 +378,10 @@ When working inside a git repository or project directory, run `kbagent init` (o | Update an existing RLS policy | `kbagent rls update --project PROJECT --policy-id POLICY-ID` | | Delete an RLS policy -- its table's rows are no longer filtered | `kbagent rls delete --project PROJECT --policy-id POLICY-ID` | | Guided RLS setup: pick tables, build a condition, preview, then write | `kbagent rls setup --project PROJECT` | +| Bind policies to a governed workspace: it then gets only these policies | `kbagent rls bind --project PROJECT --workspace-id WORKSPACE-ID` | +| Delete the policy binding of a workspace: it then gets every policy of the project | `kbagent rls unbind --project PROJECT --workspace-id WORKSPACE-ID` | +| List the policy bindings of a project and whether each workspace still exists | `kbagent rls bindings --project PROJECT` | +| Find the bindings whose workspace no longer exists; with --yes, delete them | `kbagent rls bindings-sweep --project PROJECT` | | List column-level security policies visible to a project | `kbagent cls list --project PROJECT` | | Show one CLS policy's full rule set | `kbagent cls detail --project PROJECT --policy-id POLICY-ID` | | Print the live ``cls-policy`` JSON Schema fetched from the metastore | `kbagent cls schema --project PROJECT` | diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index cd68bb7b..9b5ea2c1 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -478,6 +478,10 @@ Author `rls-policy` metastore objects -- one object per protected table, each ho - `rls update --project P --policy-id ID [--table-id TABLE_ID] [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--default JSON|@file|-] [--target-project ALIAS|ID ... | --clear-target-projects] [--dry-run] [--yes]` -- reads the policy, validates the MERGED result, then sends a partial `PATCH` of only the changed keys (unknown keys and concurrent changes to other keys survive). `--target-project` replaces the grants of a `targeted` policy, `--clear-target-projects` revokes them; grants change before the rules. The result is re-read after the write. A default cannot be removed by an update (recreate the policy). Nothing to change is `INVALID_ARGUMENT` (exit 2). - `rls delete --project P --policy-id ID [--dry-run] [--yes]` -- delete a policy, un-protecting its table. Destructive-class (`--deny-destructive` blocks it). `--dry-run` shows the policy without deleting it. Confirmation prompt unless `--yes`; `--json` skips it too. - `rls setup --project P [--dialect snowflake|bigquery] [--rules JSON|@file|-] [--scope targeted|organization] [--target-project ALIAS|ID ...] [--yes]` -- guided, **interactive-terminal-only** wizard: checkbox table picker (reuses `storage tables`), then either an interactive column/op/value condition builder (a value is read as JSON when it is one: `42`, `true`) or the `--rules` escape hatch, a compiled-condition preview per table, a single confirm, then one `rls create` call per selected table. Under `--json` or without an interactive terminal it exits 2 with an `INVALID_ARGUMENT` error, and it has **no REST route** (`kbagent serve`), same carve-out as `auth register-projects`'s picker. +- `rls bind --project P --workspace-id ID [--rls-policy UUID ...] [--cls-policy UUID ...] [--stack-id STACK] [--dry-run]` *(since vNEXT)* -- restrict one governed workspace to the listed policies (a `policy-binding` object named by the workspace KID `KID::project-:workspace-`, which the output prints). Idempotent upsert: creates the binding, or replaces both lists. Validated first: at least one policy, every UUID readable as an rls/cls policy of the project (`NOT_FOUND` otherwise; a project admin reads only policies its project owns), the workspace exists in production or a dev branch (`NOT_FOUND`); a login type other than `governed` is a warning. `--stack-id` (env `KBAGENT_STACK_ID`) is required -- no API exposes it; it must equal Query Service's `QUERY_RLS_STACK_ID` (AWS dev: `dev-keboola-aws-eu-west-1`), else nothing reads the binding. `--rls-policy`/`--cls-policy` are repeatable or comma-separated. Admin-class like `create`. +- `rls unbind --project P --workspace-id ID [--stack-id STACK] [--dry-run] [--yes]` *(since vNEXT)* -- delete the binding: the workspace gets every policy of the project again. Destructive-class; looks the binding up, then confirms (refuses without a terminal unless `--yes`/`--json`). No binding: `bound: false`, exit 0. +- `rls bindings --project P [--workspace-id ID]` *(since vNEXT)* -- `{project, bindings: [{id, kid, kind, workspace_id, rls_policy_ids, cls_policy_ids, updated, workspace_exists}]}`; `workspace_exists` is null for an app KID, a KID of another project, or when the check failed (`workspace_check_error` says which). +- `rls bindings-sweep --project P [--older-than DAYS] [--dry-run | --yes]` *(since vNEXT)* -- report (default) or, with `--yes`, delete the orphaned bindings: a workspace KID of this project, last updated more than `--older-than` days ago (default 30), whose workspace 404s in production and every dev branch. Any other check error is a failed item (exit 1), never a delete; app KIDs are listed, never swept. `references_checked: false`: the metastore has no reverse-reference lookup and refuses (409) a delete a reference blocks (reported as kept). `--yes` is destructive-class (`--deny-destructive` blocks it; the report does not); `--dry-run` with `--yes` is exit 2. ## Column-Level Security (CLS) diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 8c9fcd66..37f9dfa7 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -6839,3 +6839,25 @@ governed-workspace flow. local check but add a `warnings` entry; an authentication/permission failure while fetching the schema is raised as that error. +## Policy bindings (`rls bind`): no binding means every policy; a wrong stack id fails silently + +*(since vNEXT)* + +- A `policy-binding` restricts one governed workspace to some of the project's rls/cls policies. It is + named by the workspace KID `KID::project-:workspace-` (no branch segment). **No binding = + the workspace gets every policy of the project**; a binding only narrows, it never grants (a governed + workspace is deny-by-default anyway). Query Service re-reads it on every governed query, so `bind` / + `unbind` apply to the next query. +- **`--stack-id` (env `KBAGENT_STACK_ID`) is required and kbagent cannot check it.** No Storage, Manage or + Connection API exposes the stack segment; it must equal Query Service's `QUERY_RLS_STACK_ID` (AWS dev: + `dev-keboola-aws-eu-west-1`). A different value writes a binding nothing reads: the workspace silently + keeps every policy. Verify a binding with a `workspace query` by the bound identity. +- `bind` replaces both lists (a rebind with only `--rls-policy` drops the old CLS list). A policy listed in + a binding cannot be deleted (`onDelete: restrict`, HTTP 409); `rls delete` / `cls delete` then name + `rls bindings` -- rebind without it or `rls unbind` first. +- Deleting a workspace does not delete its binding: unbind first, or run `rls bindings-sweep` (report; + `--yes` deletes). The sweep deletes only a workspace KID of this project whose workspace 404s in + production and every dev branch and that was last updated more than `--older-than` days ago (default + 30); any other error is a failed item, never a delete, and app KIDs are never swept. The metastore has no + reverse-reference lookup, so references to a binding are not checked first; it refuses (409) a delete + that one blocks. diff --git a/plugins/kbagent/skills/kbagent/references/rls-workflow.md b/plugins/kbagent/skills/kbagent/references/rls-workflow.md index a8c77307..dee14b7a 100644 --- a/plugins/kbagent/skills/kbagent/references/rls-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/rls-workflow.md @@ -85,6 +85,7 @@ Non-obvious behaviors and version gates: [gotchas.md](gotchas.md). | Preview the compiled condition before writing anything | any write command + `--dry-run` | | Remove a policy (un-protect a table) | `rls delete --project P --policy-id ID` (`--dry-run` shows it first) | | Query a table with the policies applied, no MCP server | governed workspace -- see [End-to-end](#end-to-end----governed-workspace-policies-query-service) | +| Give one workspace only some of the policies | `rls bind --project P --workspace-id WS --rls-policy ID --stack-id S` (`rls unbind`, `rls bindings`, `rls bindings-sweep`) | ## End-to-end -- governed workspace, policies, Query Service @@ -129,6 +130,40 @@ is always refused. - Query Service decides per project whether it enforces the policies. The `row-level-security` project feature gates only the MCP `query_data` path. +### Restrict one workspace to some policies (bindings) + +Without a binding a workspace gets **every** policy of the project. A binding +names the policies one workspace gets; it only narrows, never grants, and +Query Service re-reads it on every query, so a change applies to the next query. + +```bash +# Policy ids come from `rls list` / `cls list`. --stack-id (or KBAGENT_STACK_ID) +# must equal Query Service's QUERY_RLS_STACK_ID; no API exposes it. +kbagent --json rls bind --project prod --workspace-id WS_ID \ + --rls-policy RLS_POLICY_ID --cls-policy CLS_POLICY_ID \ + --stack-id dev-keboola-aws-eu-west-1 --dry-run # validate, then rerun without --dry-run +# -> {"kid": "KID:dev-keboola-aws-eu-west-1:project-1234:workspace-WS_ID", "action": "create", ...} + +kbagent --json rls bindings --project prod # kid, policy ids, updated, workspace_exists +kbagent rls unbind --project prod --workspace-id WS_ID --stack-id dev-keboola-aws-eu-west-1 +# -> the workspace gets every policy of the project again + +# Deleting a workspace leaves its binding behind: unbind first, or sweep the leftovers. +kbagent --json rls bindings-sweep --project prod # report only +kbagent --json rls bindings-sweep --project prod --older-than 30 --yes # delete the orphans +``` + +- `bind` validates first (at least one policy, every id is a policy of the + project, the workspace exists) and replaces both lists when the workspace is + already bound. A non-governed workspace is only a warning. +- A wrong `--stack-id` writes a binding nothing reads: the workspace silently + keeps every policy. Check with a `workspace query` as the bound identity. +- A policy listed in a binding cannot be deleted until it is unbound. +- The sweep deletes only a binding whose workspace is gone (404 in production + and every dev branch) and that is older than `--older-than` days; app KIDs are + never swept. References to a binding cannot be checked first; the metastore + refuses a delete that one blocks. + --- ## Workflow 1 -- Guided setup (the default path for a human admin) diff --git a/src/keboola_agent_cli/cli.py b/src/keboola_agent_cli/cli.py index 479af683..68ea81fe 100644 --- a/src/keboola_agent_cli/cli.py +++ b/src/keboola_agent_cli/cli.py @@ -82,6 +82,7 @@ from .services.merge_request_service import MergeRequestService from .services.notification_service import NotificationService from .services.org_service import OrgService +from .services.policy_binding_service import PolicyBindingService from .services.project_service import ProjectService from .services.repo_validate_service import RepoValidateService from .services.rls_service import RlsService @@ -370,6 +371,7 @@ def main( semantic_layer_service = SemanticLayerService(config_store=config_store) rls_service = RlsService(config_store=config_store) cls_service = ClsService(config_store=config_store) + policy_binding_service = PolicyBindingService(config_store=config_store) repo_validate_service = RepoValidateService(config_store=config_store) kai_service = KaiService(config_store=config_store) docs_service = DocsService(config_store=config_store) @@ -433,6 +435,7 @@ def main( ctx.obj["semantic_layer_service"] = semantic_layer_service ctx.obj["rls_service"] = rls_service ctx.obj["cls_service"] = cls_service + ctx.obj["policy_binding_service"] = policy_binding_service ctx.obj["repo_validate_service"] = repo_validate_service ctx.obj["kai_service"] = kai_service ctx.obj["docs_service"] = docs_service diff --git a/src/keboola_agent_cli/commands/_context_commands.py b/src/keboola_agent_cli/commands/_context_commands.py index d8a17f41..e84dec32 100644 --- a/src/keboola_agent_cli/commands/_context_commands.py +++ b/src/keboola_agent_cli/commands/_context_commands.py @@ -864,6 +864,20 @@ 'list'/'detail'/'schema' are read-only; 'create'/'update'/'setup'/'delete' are gated as admin -- see CONTRIBUTING.md. + kbagent rls bind --project PROJECT --workspace-id N + [--rls-policy POLICY ...] [--cls-policy POLICY ...] [--stack-id ID] + [--dry-run] + Bind policies to a governed workspace: it then gets only these policies. + + kbagent rls bindings --project PROJECT [--workspace-id N] + List the policy bindings of a project and whether each workspace still + exists. + + kbagent rls bindings-sweep --project PROJECT [--older-than N] [--dry-run] + [--yes] + Find the bindings whose workspace no longer exists; with --yes, delete + them. + kbagent rls create --project PROJECT --table-id ID --rules RULES [--default DEFAULT] [--dialect snowflake|bigquery] [--scope targeted|organization] [--target-project PROJECT ...] @@ -888,6 +902,11 @@ [--target-project PROJECT ...] [--yes] Guided RLS setup: pick tables, build a condition, preview, then write. + kbagent rls unbind --project PROJECT --workspace-id N [--stack-id ID] + [--dry-run] [--yes] + Delete the policy binding of a workspace: it then gets every policy of the + project. + kbagent rls update --project PROJECT --policy-id ID [--table-id ID] [--dialect snowflake|bigquery] [--rules RULES] [--default DEFAULT] [--target-project PROJECT ...] [--clear-target-projects] [--dry-run] diff --git a/src/keboola_agent_cli/commands/_rls_bindings.py b/src/keboola_agent_cli/commands/_rls_bindings.py new file mode 100644 index 00000000..6c315365 --- /dev/null +++ b/src/keboola_agent_cli/commands/_rls_bindings.py @@ -0,0 +1,245 @@ +"""Policy-binding commands of the ``rls`` group: ``bind``, ``unbind``, ``bindings``, ``bindings-sweep``. + +A binding restricts one governed workspace to some of the project's rls/cls policies; a workspace +without one gets every policy of the project. Mounted onto ``rls_app`` by ``rls.py`` (that module +is near the commands-file size ceiling), so the commands share the ``rls.*`` permission namespace. +Business logic: :class:`~keboola_agent_cli.services.policy_binding_service.PolicyBindingService`. +""" + +from __future__ import annotations + +from typing import Any + +import typer +from rich.table import Table + +from ..constants import ENV_STACK_ID, STACK_ID_EXAMPLE +from ..errors import ErrorCode +from ._helpers import ( + _confirm_or_abort, + check_cli_operation, + finish_items, + get_formatter, + get_service, + handle_service_call, + split_multi, +) + +SERVICE = "policy_binding_service" +STACK_ID_HELP = ( + "Stack segment of the workspace KID -- the value Query Service is configured with, " + f"for example {STACK_ID_EXAMPLE} (no API exposes it)" +) +_ALL_POLICIES = "The workspace now gets every policy of the project again." + + +def _print_warnings(formatter: Any, result: dict[str, Any]) -> None: + for warning in result.get("warnings", []): + formatter.warning(warning) + + +def _ids(values: list[str]) -> str: + return ", ".join(values) or "-" + + +def _bindings_table(formatter: Any, rows: list[dict[str, Any]], *, sweep: bool) -> None: + table = Table(title="Policy bindings" if not sweep else "Policy bindings sweep") + for column in ("KID", "RLS policies", "CLS policies", "Updated"): + table.add_column(column) + table.add_column("Status" if sweep else "Workspace exists") + if sweep: + table.add_column("Reason") + for row in rows: + cells = [row["kid"], _ids(row["rls_policy_ids"]), _ids(row["cls_policy_ids"])] + cells.append(str(row.get("updated") or "-")) + if sweep: + cells += [row["status"], row.get("reason", "")] + else: + exists = row.get("workspace_exists") + cells.append("-" if exists is None else ("yes" if exists else "NO")) + table.add_row(*cells) + formatter.console.print(table) + + +def register(app: typer.Typer) -> None: + """Mount the binding commands onto ``app`` (the ``rls`` Typer group).""" + + @app.command("bind") + def rls_bind( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + workspace_id: int = typer.Option(..., "--workspace-id", help="Governed workspace ID"), + rls_policy: list[str] | None = typer.Option( + None, + "--rls-policy", + help="RLS policy ID to bind (repeatable or comma-separated)", + ), + cls_policy: list[str] | None = typer.Option( + None, + "--cls-policy", + help="CLS policy ID to bind (repeatable or comma-separated)", + ), + stack_id: str | None = typer.Option( + None, "--stack-id", envvar=ENV_STACK_ID, help=STACK_ID_HELP + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Validate and show the binding without writing" + ), + ) -> None: + """Bind policies to a governed workspace: it then gets only these policies. + + Creates the binding, or replaces both of its lists when the workspace is already bound. + Without a binding the workspace gets every policy of the project. + """ + formatter = get_formatter(ctx) + result = handle_service_call( + ctx, + get_service(ctx, SERVICE).bind, + project, + workspace_id, + stack_id=stack_id, + rls_policies=split_multi(rls_policy), + cls_policies=split_multi(cls_policy), + dry_run=dry_run, + ) + if formatter.json_mode: + formatter.output(result) + return + _print_warnings(formatter, result) + lists = f"RLS: {_ids(result['rls_policy_ids'])}; CLS: {_ids(result['cls_policy_ids'])}" + if dry_run: + formatter.console.print( + f"Would {result['action']} the binding {result['kid']} ({lists})." + ) + else: + formatter.success(f"Binding {result['action']}: {result['kid']} ({lists})") + + @app.command("unbind") + def rls_unbind( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + workspace_id: int = typer.Option(..., "--workspace-id", help="Workspace ID"), + stack_id: str | None = typer.Option( + None, "--stack-id", envvar=ENV_STACK_ID, help=STACK_ID_HELP + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Show the binding without deleting it" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation prompt"), + ) -> None: + """Delete the policy binding of a workspace: it then gets every policy of the project.""" + formatter = get_formatter(ctx) + service = get_service(ctx, SERVICE) + found = handle_service_call( + ctx, service.unbind, project, workspace_id, stack_id=stack_id, dry_run=True + ) + if not found["bound"]: + result = {**found, "dry_run": dry_run} + if formatter.json_mode: + formatter.output(result) + else: + formatter.console.print( + f"No binding for {found['kid']}: the workspace already gets every policy." + ) + return + if dry_run: + if formatter.json_mode: + formatter.output(found) + else: + formatter.console.print(f"Would delete the binding {found['kid']}. {_ALL_POLICIES}") + return + _confirm_or_abort( + formatter, yes, f"Delete the policy binding {found['kid']}? {_ALL_POLICIES}" + ) + result = handle_service_call(ctx, service.unbind, project, workspace_id, stack_id=stack_id) + if formatter.json_mode: + formatter.output(result) + else: + formatter.success(f"Deleted the binding {result['kid']}. {_ALL_POLICIES}") + + @app.command("bindings") + def rls_bindings( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + workspace_id: int | None = typer.Option( + None, "--workspace-id", help="Only the binding of this workspace" + ), + ) -> None: + """List the policy bindings of a project and whether each workspace still exists.""" + formatter = get_formatter(ctx) + result = handle_service_call( + ctx, get_service(ctx, SERVICE).list_bindings, project, workspace_id + ) + if formatter.json_mode: + formatter.output(result) + elif not result["bindings"]: + formatter.console.print( + "[dim]No policy bindings: every workspace gets every policy of the project.[/dim]" + ) + else: + _bindings_table(formatter, result["bindings"], sweep=False) + + @app.command("bindings-sweep") + def rls_bindings_sweep( + ctx: typer.Context, + project: str = typer.Option(..., "--project", help="Project alias"), + older_than: int = typer.Option( + 30, + "--older-than", + min=0, + help="Only bindings last updated more than this many days ago", + ), + dry_run: bool = typer.Option( + False, "--dry-run", help="Only report the orphaned bindings (the default)" + ), + yes: bool = typer.Option(False, "--yes", "-y", help="Delete the orphaned bindings"), + ) -> None: + """Find the bindings whose workspace no longer exists; with --yes, delete them. + + A binding is orphaned when its workspace is gone (404 in production and every dev branch), + it was last updated more than --older-than days ago, and the KID is a workspace KID of this + project. App KIDs are never swept. Without --yes nothing is deleted. + """ + formatter = get_formatter(ctx) + if dry_run and yes: + formatter.error( + message="--dry-run and --yes cannot be combined: --yes deletes, --dry-run reports.", + error_code=ErrorCode.INVALID_ARGUMENT, + ) + raise typer.Exit(code=2) + if yes: + check_cli_operation(ctx, "rls.bindings-sweep --yes") + result = handle_service_call( + ctx, + get_service(ctx, SERVICE).sweep, + project, + older_than_days=older_than, + delete=yes, + ) + counts = result["summary"] + rows = result["bindings"] + if yes: + summary = ( + f"{len(rows)} binding(s) checked: {counts['deleted']} deleted, " + f"{counts['kept']} kept." + ) + else: + summary = ( + f"{len(rows)} binding(s) checked: {counts['orphaned']} would be deleted, " + f"{counts['kept']} kept." + ) + + def render() -> None: + if rows: + _bindings_table(formatter, rows, sweep=True) + formatter.console.print(f"[dim]{result['references_note']}[/dim]") + + finish_items( + formatter, + result, + failed=counts["failed"], + total=len(rows), + noun="bindings", + summary=summary, + render=render, + ) diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index f9302747..887139cd 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -1626,6 +1626,23 @@ + confirm, then one `rls create` call per selected table. Under --json or without an interactive terminal it exits 2 (INVALID_ARGUMENT) pointing at `rls create`. + rls bind + Restricts ONE governed workspace to the listed policies (a `policy-binding` + named by the KID `KID::project-:workspace-`, printed). No + binding = the workspace gets every policy of the project; a binding only + narrows; Query Service re-reads it on every query (since vNEXT). Upsert: + replaces both lists. --stack-id (env KBAGENT_STACK_ID) is required -- no + API exposes it; it must equal Query Service's QUERY_RLS_STACK_ID (AWS dev: + dev-keboola-aws-eu-west-1), or nothing reads the binding and the workspace + silently keeps every policy. A bound policy cannot be deleted. + rls unbind + Destructive-class; the workspace gets every policy again. + rls bindings-sweep + Reports orphaned bindings (workspace gone in production and every dev + branch, older than --older-than days); --yes deletes them (destructive). + Never deletes on any other error; app KIDs are never swept; references to + a binding cannot be checked first (the metastore refuses such a delete). + Delete a workspace's binding with it: `rls unbind` before `workspace delete`. """, "schedule": """\ #### Schedule Discovery & Audit (Fleet-Wide) diff --git a/src/keboola_agent_cli/commands/rls.py b/src/keboola_agent_cli/commands/rls.py index e2cff418..3e3b8b48 100644 --- a/src/keboola_agent_cli/commands/rls.py +++ b/src/keboola_agent_cli/commands/rls.py @@ -743,3 +743,10 @@ def rls_setup( # Same exit-code contract as `rls create`: when every failure maps to one code (auth -> 3, # config -> 5, ...) use it; a mix is a general failure. raise typer.Exit(code=exit_codes.pop() if len(exit_codes) == 1 else 1) + + +# The policy-binding commands live in a private module (rls.py is near the commands-file size +# ceiling). Mounted here so they share the `rls.*` permission namespace and `kbagent rls --help`. +from ._rls_bindings import register as _register_binding_commands # noqa: E402 + +_register_binding_commands(rls_app) diff --git a/src/keboola_agent_cli/constants.py b/src/keboola_agent_cli/constants.py index df0c55ad..a41ecd8b 100644 --- a/src/keboola_agent_cli/constants.py +++ b/src/keboola_agent_cli/constants.py @@ -740,6 +740,14 @@ def _resolve_app_name() -> str: # Deliberately NOT in QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES until a query against one has been # verified on a stack (that list is "confirmed", not "expected"). GOVERNED_WORKSPACE_LOGIN_TYPE: str = "governed" + +# --- Policy bindings (`rls bind`) --- +# The stack segment of a workspace KID, `KID::project-:workspace-`. Query Service takes +# it from its own configuration (QUERY_RLS_STACK_ID); no Storage, Manage or Connection API exposes it, +# so kbagent needs it from --stack-id or this variable. A value that differs from the Query Service +# one writes a binding nothing reads: the workspace silently keeps every policy of the project. +ENV_STACK_ID: str = "KBAGENT_STACK_ID" +STACK_ID_EXAMPLE: str = "dev-keboola-aws-eu-west-1" QUERY_SERVICE_COMPATIBLE_LOGIN_TYPES: frozenset[str] = frozenset( { "snowflake-service-keypair", diff --git a/src/keboola_agent_cli/env_vars.py b/src/keboola_agent_cli/env_vars.py index 7f380738..56145ae1 100644 --- a/src/keboola_agent_cli/env_vars.py +++ b/src/keboola_agent_cli/env_vars.py @@ -62,6 +62,14 @@ class EnvVar: secret=False, added_in="0.4.0", ), + EnvVar( + "KBAGENT_STACK_ID", + "Stack segment of a workspace KID for 'rls bind' and 'rls unbind', when --stack-id " + "is not given.", + secret=False, + added_in="vNEXT", + option="rls bind --stack-id, rls unbind --stack-id", + ), EnvVar( "KBC_TOKEN", "Storage API token for 'project add' and for KBAGENT_PROJECT_FROM_ENV.", diff --git a/src/keboola_agent_cli/metastore_client.py b/src/keboola_agent_cli/metastore_client.py index 5eb0f9eb..e319442d 100644 --- a/src/keboola_agent_cli/metastore_client.py +++ b/src/keboola_agent_cli/metastore_client.py @@ -95,6 +95,8 @@ "semantic-reference-data", "rls-policy", "cls-policy", + # Not in SEMANTIC_TYPES: a binding is not a semantic-layer object (`rls bind`). + "policy-binding", ] diff --git a/src/keboola_agent_cli/permissions.py b/src/keboola_agent_cli/permissions.py index 809080ef..bc01345b 100644 --- a/src/keboola_agent_cli/permissions.py +++ b/src/keboola_agent_cli/permissions.py @@ -370,6 +370,12 @@ "rls.update": "admin", "rls.delete": "destructive", "rls.setup": "admin", + # Policy bindings (`rls bind` & co.): `bind` writes like `create`; `unbind` deletes; + # `bindings-sweep` only reports unless --yes (FLAG_ESCALATIONS), which deletes. + "rls.bind": "admin", + "rls.unbind": "destructive", + "rls.bindings": "read", + "rls.bindings-sweep": "read", # Column-level security: same classes as rls.* for the same reason. "cls.list": "read", "cls.detail": "read", @@ -502,6 +508,8 @@ # organization (one invalid policy refuses all of their queries). "rls.create --scope organization": "destructive", "cls.create --scope organization": "destructive", + # Deletes every orphaned policy binding it finds. + "rls.bindings-sweep --yes": "destructive", } # Operations that exist ONLY on the `kbagent serve` REST surface. They are real diff --git a/src/keboola_agent_cli/server/_serve_command_map.py b/src/keboola_agent_cli/server/_serve_command_map.py index 4941e8b6..06ae1773 100644 --- a/src/keboola_agent_cli/server/_serve_command_map.py +++ b/src/keboola_agent_cli/server/_serve_command_map.py @@ -29,6 +29,7 @@ ("DELETE", "/flows/{project}/{config_id}/schedule"): "flow schedule-remove", ("DELETE", "/notifications/{project}/{subscription_id}"): "notification delete", ("DELETE", "/projects/{alias}"): "project remove", + ("DELETE", "/rls/{project}/bindings/{workspace_id}"): "rls unbind", ("DELETE", "/rls/{project}/{policy_id}"): "rls delete", ("DELETE", "/semantic-layer/items/{kind}/{name}"): "", ( @@ -132,6 +133,7 @@ ("GET", "/projects/{alias}/description"): "project description-get", ("GET", "/projects/{alias}/info"): "project info", ("GET", "/rls/{project}"): "rls list", + ("GET", "/rls/{project}/bindings"): "rls bindings", ("GET", "/rls/{project}/schema"): "rls schema", ("GET", "/rls/{project}/{policy_id}"): "rls detail", ("GET", "/schedules"): "schedule list", @@ -264,6 +266,7 @@ ("POST", "/projects/bulk-delete"): "project remove", ("POST", "/projects/use/{alias}"): "project use", ("POST", "/rls/{project}"): "rls create", + ("POST", "/rls/{project}/bindings/sweep"): "rls bindings-sweep", ("POST", "/semantic-layer/build"): "semantic-layer build", ("POST", "/semantic-layer/diff"): "semantic-layer diff", ("POST", "/semantic-layer/import"): "semantic-layer import", @@ -313,6 +316,7 @@ ("PUT", "/configs/{project}/{component_id}/{config_id}/variables"): "config variables-set", ("PUT", "/data-apps/{project}/{app_id}/secrets"): "data-app secrets-set", ("PUT", "/projects/{alias}/description"): "project description-set", + ("PUT", "/rls/{project}/bindings/{workspace_id}"): "rls bind", ("PUT", "/semantic-layer/items/{kind}/{name}"): "", ("PUT", "/semantic-layer/reference-data"): "semantic-layer reference-data set", ("PUT", "/semantic-layer/scope/{context_id}"): "semantic-layer scope set", diff --git a/src/keboola_agent_cli/server/app.py b/src/keboola_agent_cli/server/app.py index fd63f0da..5b2a0502 100644 --- a/src/keboola_agent_cli/server/app.py +++ b/src/keboola_agent_cli/server/app.py @@ -79,6 +79,7 @@ org, projects, rls, + rls_bindings, schedules, search, semantic_layer, @@ -363,7 +364,8 @@ "Row-level security policies (metastore-backed `rls-policy` " "objects), authored at `targeted` scope unless `organization` is " "asked for -- never `project` scope. Mirrors `kbagent rls list|detail|schema|" - "create|update|delete` (`rls setup`'s interactive picker has no " + "create|update|delete|bind|unbind|bindings|bindings-sweep` (`rls setup`'s " + "interactive picker has no " "endpoint of its own, same carve-out as " "`auth register-projects`)." ), @@ -1041,6 +1043,8 @@ async def _generic_handler(request: Request, exc: Exception): app.include_router(encrypt.router) app.include_router(search.router) app.include_router(semantic_layer.router) + # Before rls.router: GET /rls/{project}/{policy_id} would capture /rls/{project}/bindings. + app.include_router(rls_bindings.router) app.include_router(rls.router) app.include_router(cls.router) app.include_router(transformation.router) diff --git a/src/keboola_agent_cli/server/dependencies.py b/src/keboola_agent_cli/server/dependencies.py index 8218683f..ae592c14 100644 --- a/src/keboola_agent_cli/server/dependencies.py +++ b/src/keboola_agent_cli/server/dependencies.py @@ -47,6 +47,7 @@ from ..services.merge_request_service import MergeRequestService from ..services.notification_service import NotificationService from ..services.org_service import OrgService +from ..services.policy_binding_service import PolicyBindingService from ..services.project_service import ProjectService from ..services.repo_validate_service import RepoValidateService from ..services.rls_service import RlsService @@ -129,6 +130,7 @@ class ServiceRegistry: semantic_layer: SemanticLayerService = field(init=False) rls: RlsService = field(init=False) cls: ClsService = field(init=False) + policy_binding: PolicyBindingService = field(init=False) repo_validate: RepoValidateService = field(init=False) kai: KaiService = field(init=False) encrypt: EncryptService = field(init=False) @@ -177,6 +179,7 @@ def __post_init__(self) -> None: self.semantic_layer = SemanticLayerService(config_store=cs) self.rls = RlsService(config_store=cs) self.cls = ClsService(config_store=cs) + self.policy_binding = PolicyBindingService(config_store=cs) self.repo_validate = RepoValidateService(config_store=cs) self.kai = KaiService(config_store=cs) self.encrypt = EncryptService(config_store=cs) diff --git a/src/keboola_agent_cli/server/routers/rls_bindings.py b/src/keboola_agent_cli/server/routers/rls_bindings.py new file mode 100644 index 00000000..076eaf3f --- /dev/null +++ b/src/keboola_agent_cli/server/routers/rls_bindings.py @@ -0,0 +1,113 @@ +"""Policy-binding endpoints -- the REST mirror of ``kbagent rls bind|unbind|bindings|bindings-sweep``. + +Included in the app BEFORE the ``rls`` router: ``GET /rls/{project}/{policy_id}`` would otherwise +capture ``GET /rls/{project}/bindings`` (Starlette matches routes in registration order). + +``stack_id`` is a required field here: the service never reads the environment (the CLI's +``KBAGENT_STACK_ID`` fallback is a CLI option default). A sweep with ``dry_run: false`` deletes, so +it is also checked as ``rls.bindings-sweep --yes`` (destructive), like the CLI flag. +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, Depends +from pydantic import BaseModel, Field + +from ...permissions import PermissionEngine +from ..dependencies import ( + ServiceRegistry, + get_permission_engine, + get_registry, + require_permission, +) + +router = APIRouter(prefix="/rls", tags=["rls"]) + + +class BindingPut(BaseModel): + """The policies bound to the workspace; both lists are replaced, at least one is non-empty.""" + + stack_id: str + rls_policies: list[str] = Field(default_factory=list) + cls_policies: list[str] = Field(default_factory=list) + dry_run: bool = False + + +class BindingSweep(BaseModel): + """``dry_run: false`` deletes the orphaned bindings (``rls bindings-sweep --yes``).""" + + older_than_days: int = Field(default=30, ge=0) + dry_run: bool = True + + +@router.get( + "/{project}/bindings", + summary="List the policy bindings of a project", + dependencies=[Depends(require_permission("rls.bindings"))], +) +def list_bindings( + project: str, + workspace_id: int | None = None, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Bindings with whether each workspace still exists. Mirrors `kbagent rls bindings`.""" + return registry.policy_binding.list_bindings(project, workspace_id) + + +@router.post( + "/{project}/bindings/sweep", + summary="Find (and delete) orphaned policy bindings", + dependencies=[Depends(require_permission("rls.bindings-sweep"))], +) +def sweep_bindings( + project: str, + body: BindingSweep, + registry: ServiceRegistry = Depends(get_registry), + engine: PermissionEngine = Depends(get_permission_engine), +) -> dict[str, Any]: + """Reports by default; ``dry_run: false`` deletes. Mirrors `kbagent rls bindings-sweep`.""" + if not body.dry_run: + engine.check_or_raise("rls.bindings-sweep --yes") + return registry.policy_binding.sweep( + project, older_than_days=body.older_than_days, delete=not body.dry_run + ) + + +@router.put( + "/{project}/bindings/{workspace_id}", + summary="Bind policies to a governed workspace", + dependencies=[Depends(require_permission("rls.bind"))], +) +def bind_workspace( + project: str, + workspace_id: int, + body: BindingPut, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """Create or replace the binding of one workspace. Mirrors `kbagent rls bind`.""" + return registry.policy_binding.bind( + project, + workspace_id, + stack_id=body.stack_id, + rls_policies=body.rls_policies, + cls_policies=body.cls_policies, + dry_run=body.dry_run, + ) + + +@router.delete( + "/{project}/bindings/{workspace_id}", + summary="Delete the policy binding of a workspace", + dependencies=[Depends(require_permission("rls.unbind"))], +) +def unbind_workspace( + project: str, + workspace_id: int, + stack_id: str, + dry_run: bool = False, + registry: ServiceRegistry = Depends(get_registry), +) -> dict[str, Any]: + """The workspace then gets every policy of the project. Mirrors `kbagent rls unbind`.""" + return registry.policy_binding.unbind(project, workspace_id, stack_id=stack_id, dry_run=dry_run) diff --git a/src/keboola_agent_cli/services/cls_service.py b/src/keboola_agent_cli/services/cls_service.py index c62a35f1..d3a327e9 100644 --- a/src/keboola_agent_cli/services/cls_service.py +++ b/src/keboola_agent_cli/services/cls_service.py @@ -10,8 +10,8 @@ Everything except the rule shape is identical to :class:`RlsService` (never ``project`` scope, partial (PATCH) update, live-schema validation that degrades to a warning), so this class only overrides the policy type and the -rule / default hooks. Enforcement happens in ``keboola-mcp-server``'s -``query_data``, not here -- kbagent only authors policies. +rule / default hooks. Enforcement happens in Query Service on a governed workspace or in +``keboola-mcp-server``'s ``query_data``, not here -- kbagent only authors policies. """ from __future__ import annotations diff --git a/src/keboola_agent_cli/services/policy_binding_service.py b/src/keboola_agent_cli/services/policy_binding_service.py new file mode 100644 index 00000000..cf2d4006 --- /dev/null +++ b/src/keboola_agent_cli/services/policy_binding_service.py @@ -0,0 +1,467 @@ +"""Policy bindings -- business logic for ``kbagent rls bind|unbind|bindings|bindings-sweep``. + +A ``policy-binding`` metastore object restricts one governed workspace (or data app) to a subset of +the project's ``rls-policy`` / ``cls-policy`` objects. It is named by the canonical KID of the +workspace, ``KID::project-:workspace-``, and holds ``kid`` plus ``rlsPolicyUUIDs`` +and/or ``clsPolicyUUIDs``. Query Service reads it on every governed query: + +- a KID with no binding gets EVERY policy of the project; +- a KID with a binding gets only the listed policies -- a binding narrows, it never grants. + +The stack segment is not exposed by any API kbagent can call (Query Service takes it from its own +configuration), so the caller passes it. The metastore exposes no reverse-reference lookup: whether +something still references a binding is known only when a delete is refused (HTTP 409). +""" + +from __future__ import annotations + +import re +import uuid +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from enum import StrEnum +from typing import Any, NoReturn + +from ..config_store import ConfigStore +from ..constants import ENV_STACK_ID, GOVERNED_WORKSPACE_LOGIN_TYPE, STACK_ID_EXAMPLE +from ..errors import ErrorCode, KeboolaApiError +from ..metastore_client import MetastoreClient, SemanticType +from ..models import ProjectConfig +from .base import BaseService, ClientFactory, make_session_aware_client_factory + +BINDING_TYPE: SemanticType = "policy-binding" +POLICY_TYPES: dict[str, SemanticType] = {"rls": "rls-policy", "cls": "cls-policy"} +MAX_POLICIES_PER_LIST = 100 # the schema's maxItems +_SWEEP_STATUSES = ("orphaned", "deleted", "kept", "failed") + +STACK_ID_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +WORKSPACE_KID_RE = re.compile( + r"^KID:(?P[a-z0-9]+(?:-[a-z0-9]+)*):project-(?P0|[1-9][0-9]*)" + r":workspace-(?P0|[1-9][0-9]*)$" +) +_APP_SEGMENT_RE = re.compile(r":app-[A-Za-z0-9._%-]+") + +REFERENCES_NOTE = ( + "The metastore exposes no reverse-reference lookup, so references to a binding are not " + "checked before a delete; the metastore refuses (HTTP 409) a delete that a reference blocks." +) + +__all__ = [ + "BINDING_TYPE", + "REFERENCES_NOTE", + "KidKind", + "PolicyBindingService", + "parse_kid", + "workspace_kid", +] + +MetastoreClientFactory = Callable[[str, str], MetastoreClient] + + +class KidKind(StrEnum): + WORKSPACE = "workspace" + APP = "app" + OTHER = "other" + + +class WorkspaceState(StrEnum): + EXISTS = "exists" + GONE = "gone" + UNKNOWN = "unknown" + + +@dataclass(frozen=True) +class ParsedKid: + kind: KidKind + project_id: int | None = None + workspace_id: int | None = None + + +@dataclass(frozen=True) +class WorkspaceCheck: + """Whether a workspace exists; ``error`` is set when the check itself failed (``UNKNOWN``).""" + + state: WorkspaceState + workspace: dict[str, Any] | None = None + error: KeboolaApiError | None = None + + +def _usage(message: str) -> NoReturn: + raise KeboolaApiError( + message=message, status_code=400, error_code=ErrorCode.INVALID_ARGUMENT, retryable=False + ) + + +def workspace_kid(stack_id: str | None, project_id: int, workspace_id: int) -> str: + """The canonical KID of a workspace; INVALID_ARGUMENT when the stack id is missing or malformed.""" + if not stack_id: + _usage( + f"--stack-id (or {ENV_STACK_ID}) is required: the stack segment of the workspace KID, " + f"the same value Query Service is configured with (QUERY_RLS_STACK_ID), for example " + f"{STACK_ID_EXAMPLE!r} on the AWS dev stack. No API exposes it, so kbagent cannot " + "derive it." + ) + if not STACK_ID_RE.match(stack_id): + _usage( + f"--stack-id {stack_id!r} is not a stack id: lowercase letters and digits in " + f"dash-separated words, for example {STACK_ID_EXAMPLE!r}." + ) + if workspace_id < 0: + _usage(f"--workspace-id must not be negative, got {workspace_id}.") + kid = f"KID:{stack_id}:project-{project_id}:workspace-{workspace_id}" + if not WORKSPACE_KID_RE.match(kid): + _usage(f"{kid!r} is not a canonical workspace KID.") + return kid + + +def parse_kid(kid: str) -> ParsedKid: + """Classify a stored KID: a canonical workspace KID, an app KID, or anything else.""" + match = WORKSPACE_KID_RE.match(kid) + if match: + return ParsedKid( + KidKind.WORKSPACE, int(match.group("project")), int(match.group("workspace")) + ) + if _APP_SEGMENT_RE.search(kid): + return ParsedKid(KidKind.APP) + return ParsedKid(KidKind.OTHER) + + +def _policy_ids(raw: Sequence[str] | None, flag: str) -> list[str]: + """Deduplicated, lower-cased UUIDs in input order; INVALID_ARGUMENT on a malformed one.""" + seen: list[str] = [] + for value in raw or []: + try: + normalized = str(uuid.UUID(value)) + except ValueError: + _usage(f"{flag} {value!r} is not a UUID (a policy id from `rls list` / `cls list`).") + if normalized not in seen: + seen.append(normalized) + if len(seen) > MAX_POLICIES_PER_LIST: + _usage(f"{flag}: at most {MAX_POLICIES_PER_LIST} policies per binding, got {len(seen)}.") + return seen + + +def _binding_row(item: dict[str, Any]) -> dict[str, Any]: + attrs = item.get("attributes") or {} + meta = item.get("meta") or {} + kid = str(attrs.get("kid") or meta.get("name") or "") + parsed = parse_kid(kid) + return { + "id": item.get("id", ""), + "kid": kid, + "kind": parsed.kind.value, + "workspace_id": parsed.workspace_id, + "kid_project_id": parsed.project_id, + "rls_policy_ids": list(attrs.get("rlsPolicyUUIDs") or []), + "cls_policy_ids": list(attrs.get("clsPolicyUUIDs") or []), + "updated": meta.get("lastUpdated"), + } + + +def _parse_time(value: Any) -> datetime | None: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC) + + +class PolicyBindingService(BaseService): + """Business logic for the policy-binding commands of the ``rls`` group.""" + + def __init__( + self, + config_store: ConfigStore, + client_factory: ClientFactory | None = None, + metastore_client_factory: MetastoreClientFactory | None = None, + now: Callable[[], datetime] | None = None, + ) -> None: + super().__init__(config_store=config_store, client_factory=client_factory) + self._metastore_factory: MetastoreClientFactory = ( + metastore_client_factory + or make_session_aware_client_factory(config_store, MetastoreClient) + ) + self._now = now or (lambda: datetime.now(UTC)) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _project(self, alias: str) -> ProjectConfig: + return self.resolve_projects([alias])[alias] + + def _project_id(self, project: ProjectConfig, storage: Any) -> int: + if project.project_id is not None: + return project.project_id + project_id = storage.verify_token().project_id + if project_id is None: + raise KeboolaApiError( + message="The token does not name its project, so the workspace KID cannot be built.", + status_code=400, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) + return int(project_id) + + @staticmethod + def _find_binding(client: MetastoreClient, kid: str) -> dict[str, Any] | None: + for item in client.list_items(BINDING_TYPE): + if _binding_row(item)["kid"] == kid: + return item + return None + + @staticmethod + def _check_workspace(storage: Any, workspace_id: int) -> WorkspaceCheck: + """Production first; on a 404 every dev branch too. A failed check is ``UNKNOWN``, never gone.""" + try: + return WorkspaceCheck(WorkspaceState.EXISTS, storage.get_workspace(workspace_id)) + except KeboolaApiError as exc: + if exc.status_code != 404: + return WorkspaceCheck(WorkspaceState.UNKNOWN, error=exc) + try: + branches = [b for b in storage.list_dev_branches() if not b.get("isDefault")] + for branch in branches: + try: + found = storage.get_workspace(workspace_id, branch_id=int(branch["id"])) + except KeboolaApiError as exc: + if exc.status_code == 404: + continue + raise + return WorkspaceCheck(WorkspaceState.EXISTS, found) + except KeboolaApiError as exc: + return WorkspaceCheck(WorkspaceState.UNKNOWN, error=exc) + return WorkspaceCheck(WorkspaceState.GONE) + + @staticmethod + def _missing_policies(client: MetastoreClient, kind: str, ids: list[str]) -> list[str]: + missing: list[str] = [] + for policy_id in ids: + try: + client.get_item(POLICY_TYPES[kind], policy_id) + except KeboolaApiError as exc: + if exc.status_code != 404: + raise + missing.append(f"{kind}-policy {policy_id}") + return missing + + # ------------------------------------------------------------------ + # Commands + # ------------------------------------------------------------------ + + def bind( + self, + alias: str, + workspace_id: int, + *, + stack_id: str | None, + rls_policies: Sequence[str] | None = None, + cls_policies: Sequence[str] | None = None, + dry_run: bool = False, + ) -> dict[str, Any]: + """Create or replace the binding of one workspace (idempotent upsert).""" + rls_ids = _policy_ids(rls_policies, "--rls-policy") + cls_ids = _policy_ids(cls_policies, "--cls-policy") + if not rls_ids and not cls_ids: + _usage("Pass at least one --rls-policy or --cls-policy: a binding lists policies.") + project = self._project(alias) + warnings: list[str] = [] + with self._client_factory(project.stack_url, project.token) as storage: + kid = workspace_kid(stack_id, self._project_id(project, storage), workspace_id) + check = self._check_workspace(storage, workspace_id) + if check.state is WorkspaceState.UNKNOWN and check.error is not None: + raise check.error + if check.state is WorkspaceState.GONE: + raise KeboolaApiError( + message=f"Workspace {workspace_id} does not exist in project '{alias}' " + "(production or any dev branch).", + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + login_type = ((check.workspace or {}).get("connection") or {}).get("loginType", "") + if login_type != GOVERNED_WORKSPACE_LOGIN_TYPE: + warnings.append( + f"Workspace {workspace_id} has login type {login_type or 'unknown'!r}, not " + f"{GOVERNED_WORKSPACE_LOGIN_TYPE!r}: Query Service applies a binding only to a " + "governed workspace." + ) + + data: dict[str, Any] = {"kid": kid} + if rls_ids: + data["rlsPolicyUUIDs"] = rls_ids + if cls_ids: + data["clsPolicyUUIDs"] = cls_ids + with self._metastore_factory(project.stack_url, project.token) as client: + missing = self._missing_policies(client, "rls", rls_ids) + missing += self._missing_policies(client, "cls", cls_ids) + if missing: + raise KeboolaApiError( + message=( + f"Not found in project '{alias}': {', '.join(missing)}. A binding lists " + "existing policies only (`rls list` / `cls list`); a project admin reads " + "only the policies its own project owns." + ), + status_code=404, + error_code=ErrorCode.NOT_FOUND, + retryable=False, + ) + existing = self._find_binding(client, kid) + result: dict[str, Any] = { + "project": alias, + "workspace_id": workspace_id, + "kid": kid, + "rls_policy_ids": rls_ids, + "cls_policy_ids": cls_ids, + "login_type": login_type, + } + if existing is not None: + previous = _binding_row(existing) + result["previous"] = { + "rls_policy_ids": previous["rls_policy_ids"], + "cls_policy_ids": previous["cls_policy_ids"], + } + if dry_run: + result |= { + "action": "replace" if existing else "create", + "binding_id": existing.get("id") if existing else None, + "dry_run": True, + } + return self._with_warnings(result, warnings) + stored, action = self._upsert(client, kid, data, existing) + result |= {"action": action, "binding_id": stored.get("id", "")} + return self._with_warnings(result, warnings) + + def _upsert( + self, + client: MetastoreClient, + kid: str, + data: dict[str, Any], + existing: dict[str, Any] | None, + ) -> tuple[dict[str, Any], str]: + """PUT (replaces both lists) when bound, else POST; a POST that lost a race is PUT once.""" + if existing is None: + try: + return client.post_item(BINDING_TYPE, name=kid, data=data), "created" + except KeboolaApiError as exc: + if exc.error_code != ErrorCode.ALREADY_EXISTS: + raise + existing = self._find_binding(client, kid) + if existing is None: + raise + return client.put_item(BINDING_TYPE, existing["id"], name=kid, data=data), "replaced" + + def unbind( + self, alias: str, workspace_id: int, *, stack_id: str | None, dry_run: bool = False + ) -> dict[str, Any]: + """Delete the binding of one workspace; ``dry_run`` only looks it up.""" + project = self._project(alias) + with self._client_factory(project.stack_url, project.token) as storage: + kid = workspace_kid(stack_id, self._project_id(project, storage), workspace_id) + result: dict[str, Any] = {"project": alias, "workspace_id": workspace_id, "kid": kid} + with self._metastore_factory(project.stack_url, project.token) as client: + existing = self._find_binding(client, kid) + if existing is None: + return result | {"bound": False, "deleted": False} + row = _binding_row(existing) + result |= {"bound": True, "binding_id": row["id"], "binding": row} + if dry_run: + return result | {"dry_run": True} + client.delete_item(BINDING_TYPE, row["id"]) + return result | {"deleted": True} + + def list_bindings(self, alias: str, workspace_id: int | None = None) -> dict[str, Any]: + """Every binding of the project, with whether the workspace of its KID still exists.""" + project = self._project(alias) + with self._metastore_factory(project.stack_url, project.token) as client: + rows = [_binding_row(item) for item in client.list_items(BINDING_TYPE)] + if workspace_id is not None: + rows = [r for r in rows if r["workspace_id"] == workspace_id] + with self._client_factory(project.stack_url, project.token) as storage: + project_id = self._project_id(project, storage) + for row in rows: + row["workspace_exists"] = None + if row["kind"] != KidKind.WORKSPACE or row["workspace_id"] is None: + continue + if row["kid_project_id"] != project_id: + row["workspace_check_error"] = ( + f"the KID names project {row['kid_project_id']}, not this one" + ) + continue + check = self._check_workspace(storage, row["workspace_id"]) + if check.state is WorkspaceState.UNKNOWN: + row["workspace_check_error"] = check.error.message if check.error else "" + else: + row["workspace_exists"] = check.state is WorkspaceState.EXISTS + return {"project": alias, "bindings": rows} + + def sweep(self, alias: str, *, older_than_days: int, delete: bool) -> dict[str, Any]: + """Find (and with ``delete``, remove) the bindings whose workspace is gone. + + A binding is orphaned when its KID is a canonical workspace KID of this project, it was last + updated more than ``older_than_days`` ago, and the workspace returns 404 in production and + in every dev branch. Any other outcome of that check is a failed item, never a delete. App + KIDs are listed and never swept. + """ + if older_than_days < 0: + _usage(f"--older-than must not be negative, got {older_than_days}.") + project = self._project(alias) + cutoff = self._now() - timedelta(days=older_than_days) + with ( + self._metastore_factory(project.stack_url, project.token) as client, + self._client_factory(project.stack_url, project.token) as storage, + ): + project_id = self._project_id(project, storage) + rows = [_binding_row(item) for item in client.list_items(BINDING_TYPE)] + for row in rows: + row |= self._sweep_decision(storage, row, project_id, cutoff) + if row["status"] != "orphaned" or not delete: + continue + try: + client.delete_item(BINDING_TYPE, row["id"]) + row["status"] = "deleted" + except KeboolaApiError as exc: + if exc.status_code == 409: + row |= {"status": "kept", "reason": f"the metastore refused: {exc.message}"} + else: + row |= {"status": "failed", "reason": exc.message} + counts = {s: sum(1 for r in rows if r["status"] == s) for s in _SWEEP_STATUSES} + return { + "project": alias, + "older_than_days": older_than_days, + "dry_run": not delete, + "references_checked": False, + "references_note": REFERENCES_NOTE, + "bindings": rows, + "summary": counts, + } + + def _sweep_decision( + self, storage: Any, row: dict[str, Any], project_id: int, cutoff: datetime + ) -> dict[str, str]: + if row["kind"] == KidKind.APP: + return {"status": "kept", "reason": "app KID: never swept"} + if row["kind"] != KidKind.WORKSPACE: + return {"status": "kept", "reason": "not a canonical workspace KID"} + if row["kid_project_id"] != project_id: + return {"status": "kept", "reason": f"the KID names project {row['kid_project_id']}"} + updated = _parse_time(row["updated"]) + if updated is None: + return {"status": "kept", "reason": "no last-update timestamp"} + if updated > cutoff: + return {"status": "kept", "reason": "updated within --older-than"} + check = self._check_workspace(storage, row["workspace_id"]) + if check.state is WorkspaceState.EXISTS: + return {"status": "kept", "reason": "the workspace exists"} + if check.state is WorkspaceState.UNKNOWN: + message = check.error.message if check.error else "unknown error" + return {"status": "failed", "reason": f"workspace check failed: {message}"} + return {"status": "orphaned", "reason": "the workspace no longer exists"} + + @staticmethod + def _with_warnings(result: dict[str, Any], warnings: list[str]) -> dict[str, Any]: + if warnings: + result["warnings"] = warnings + return result diff --git a/src/keboola_agent_cli/services/rls_service.py b/src/keboola_agent_cli/services/rls_service.py index 35bd2e2d..e71408a7 100644 --- a/src/keboola_agent_cli/services/rls_service.py +++ b/src/keboola_agent_cli/services/rls_service.py @@ -511,5 +511,21 @@ def delete_policy(self, alias: str, policy_id: str, *, dry_run: bool = False) -> "policy": self._detail_row(client.get_item(self.item_type, policy_id)), "dry_run": True, } - client.delete_item(self.item_type, policy_id) + try: + client.delete_item(self.item_type, policy_id) + except KeboolaApiError as exc: + if exc.status_code != 409: + raise + # A policy-binding lists it (references are onDelete: restrict). + raise KeboolaApiError( + message=( + f"The metastore refused to delete {self.label} policy {policy_id}: a " + "policy-binding still lists it. Find it with `rls bindings`, then rebind " + "the workspace without it (`rls bind`) or remove the binding " + f"(`rls unbind`). {exc.message}" + ), + status_code=409, + error_code=ErrorCode.API_ERROR, + retryable=False, + ) from exc return {"project": alias, "policy_id": policy_id, "deleted": True} diff --git a/tests/test_cli_contract.py b/tests/test_cli_contract.py index d7d6a788..6ec69b78 100644 --- a/tests/test_cli_contract.py +++ b/tests/test_cli_contract.py @@ -699,7 +699,9 @@ def test_a_stream_that_fails_stays_ndjson( "merge-request merge": "resolves the merge request first, then asks", "org setup": "needs the manage token first, then previews and asks", "project refresh": "needs the manage token first, then previews and asks", + "rls bindings-sweep": "--yes deletes the orphaned bindings; it answers no question", "rls setup": "terminal-only wizard: without a terminal it exits 2 before any question", + "rls unbind": "looks the binding up first, then asks", "semantic-layer add metric": "dataset guard in the service; refuses without a terminal", "semantic-layer edit metric": "rename-cascade guard in the service; refuses without a terminal", "semantic-layer import": "hidden deprecated --yes with no effect (C3)", diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 104782df..cd659974 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -689,6 +689,7 @@ def test_full_cli_e2e(self) -> None: ) self._test_rls_commands() self._test_rls_cls_crud() + self._test_rls_bindings() # ============================================================== # PHASE 15: Cleanup @@ -3819,6 +3820,33 @@ def _test_rls_cls_crud(self) -> None: assert refused.exit_code == 2, refused.output assert "INVALID_ARGUMENT" in refused.output + def _test_rls_bindings(self) -> None: + """`rls bindings` / `bindings-sweep` (report only) / `bind --dry-run` / `unbind --dry-run`. + + Nothing is written: the sweep runs without --yes, `bind` targets a workspace that does not + exist (NOT_FOUND before any write), and `unbind --dry-run` only looks the binding up. The + stack id is a syntactically valid placeholder, so the KID matches no real binding. A stack + whose metastore lacks the `policy-binding` type answers the reads with NOT_FOUND (a skip). + """ + listed = self._run("rls", "bindings", "--project", self.alias) + if listed.exit_code != 0: + assert "NOT_FOUND" in listed.output, listed.output + print(f" {_YELLOW}SKIP: rls bindings (policy-binding not registered here){_RESET}") + return + assert isinstance(_json_ok(listed)["data"]["bindings"], list) + swept = self._run_ok("rls", "bindings-sweep", "--project", self.alias) + assert swept["data"]["dry_run"] is True + stack = ("--stack-id", "kbagent-e2e-placeholder") + bound = self._run( + "rls", "bind", "--project", self.alias, "--workspace-id", "1", *stack, + "--rls-policy", "00000000-0000-4000-8000-000000000000", "--dry-run", + ) # fmt: skip + assert bound.exit_code != 0 and "NOT_FOUND" in bound.output, bound.output + unbound = self._run_ok( + "rls", "unbind", "--project", self.alias, "--workspace-id", "1", *stack, "--dry-run" + ) + assert unbound["data"]["bound"] is False + def _test_job_commands(self) -> None: """Verify job listing structure and detail (if jobs exist).""" # job list diff --git a/tests/test_env_vars.py b/tests/test_env_vars.py index d09cd27a..4fe9b83b 100644 --- a/tests/test_env_vars.py +++ b/tests/test_env_vars.py @@ -35,7 +35,8 @@ def test_registry_entries_are_complete_and_unique() -> None: assert len(names) == len(set(names)) for var in ENV_VARS: assert var.purpose.endswith("."), var.name - assert re.fullmatch(r"\d+\.\d+\.\d+", var.added_in), var.name + # `vNEXT` until the release PR: `make vnext-resolve` rewrites it in src/ like any gate. + assert re.fullmatch(r"\d+\.\d+\.\d+|vNEXT", var.added_in), var.name def _envvar_options() -> list[tuple[list[str], str]]: diff --git a/tests/test_policy_binding_service.py b/tests/test_policy_binding_service.py new file mode 100644 index 00000000..75375706 --- /dev/null +++ b/tests/test_policy_binding_service.py @@ -0,0 +1,481 @@ +"""Service-layer tests for ``PolicyBindingService`` (``rls bind|unbind|bindings|bindings-sweep``). + +The metastore and Storage clients are ``MagicMock``s: the tests check the KID, the validation +order, the upsert (POST vs PUT), and every sweep condition, without HTTP. +""" + +from __future__ import annotations + +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig +from keboola_agent_cli.services.policy_binding_service import ( + BINDING_TYPE, + KidKind, + PolicyBindingService, + parse_kid, + workspace_kid, +) + +TEST_TOKEN = "901-55555-fakeTestTokenDoNotUseXXXXXXXX" +STACK = "dev-keboola-aws-eu-west-1" +PROJECT_ID = 5725 +KID = f"KID:{STACK}:project-{PROJECT_ID}:workspace-42" +RLS_ID = "11111111-1111-4111-8111-111111111111" +RLS_ID_2 = "22222222-2222-4222-8222-222222222222" +CLS_ID = "33333333-3333-4333-8333-333333333333" +NOW = datetime(2026, 10, 21, tzinfo=UTC) +OLD = "2026-08-01T00:00:00Z" +RECENT = "2026-10-20T00:00:00Z" + + +def _api_error(status: int, code: ErrorCode = ErrorCode.API_ERROR) -> KeboolaApiError: + return KeboolaApiError(message=f"HTTP {status}", status_code=status, error_code=code) + + +def _ctx(mock: MagicMock) -> MagicMock: + mock.__enter__ = MagicMock(return_value=mock) + mock.__exit__ = MagicMock(return_value=False) + return mock + + +def _binding( + kid: str = KID, + item_id: str = "b-1", + rls: list[str] | None = None, + cls: list[str] | None = None, + updated: str | None = OLD, +) -> dict[str, Any]: + attrs: dict[str, Any] = {"kid": kid} + if rls: + attrs["rlsPolicyUUIDs"] = rls + if cls: + attrs["clsPolicyUUIDs"] = cls + return { + "type": BINDING_TYPE, + "id": item_id, + "attributes": attrs, + "meta": {"name": kid, "lastUpdated": updated}, + } + + +def _make( + tmp_path: Path, + *, + bindings: list[dict[str, Any]] | None = None, + workspace: dict[str, Any] | Exception | None = None, + project_id: int | None = PROJECT_ID, +) -> tuple[PolicyBindingService, MagicMock, MagicMock]: + config_dir = tmp_path / "config" + config_dir.mkdir() + store = ConfigStore(config_dir=config_dir) + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=project_id, + ), + ) + metastore = _ctx(MagicMock()) + metastore.list_items.return_value = bindings or [] + metastore.post_item.return_value = {"id": "b-new"} + metastore.put_item.side_effect = lambda _t, item_id, **_kw: {"id": item_id} + storage = _ctx(MagicMock()) + if isinstance(workspace, Exception): + storage.get_workspace.side_effect = workspace + else: + storage.get_workspace.return_value = workspace or { + "id": 42, + "connection": {"loginType": "governed"}, + } + storage.list_dev_branches.return_value = [{"id": 1, "isDefault": True}] + storage.verify_token.return_value.project_id = PROJECT_ID + service = PolicyBindingService( + config_store=store, + client_factory=lambda url, token: storage, + metastore_client_factory=lambda url, token: metastore, + now=lambda: NOW, + ) + return service, metastore, storage + + +# --------------------------------------------------------------------------- +# KID +# --------------------------------------------------------------------------- + + +class TestKid: + def test_workspace_kid_is_canonical(self) -> None: + assert workspace_kid(STACK, PROJECT_ID, 42) == KID + + @pytest.mark.parametrize("stack", [None, "", "Dev_Stack", "-dev", "dev--stack", "dev stack"]) + def test_missing_or_malformed_stack_id_is_a_usage_error(self, stack: str | None) -> None: + with pytest.raises(KeboolaApiError) as exc_info: + workspace_kid(stack, PROJECT_ID, 42) + assert exc_info.value.error_code == ErrorCode.INVALID_ARGUMENT + assert "dev-keboola-aws-eu-west-1" in exc_info.value.message + + @pytest.mark.parametrize( + ("kid", "kind", "project", "workspace"), + [ + (KID, KidKind.WORKSPACE, PROJECT_ID, 42), + (f"KID:{STACK}:project-1:app-abc", KidKind.APP, None, None), + (f"KID:{STACK}:project-1:workspace-01", KidKind.OTHER, None, None), + (f"KID:{STACK}:project-1:branch-2:workspace-3", KidKind.OTHER, None, None), + ("not-a-kid", KidKind.OTHER, None, None), + ], + ) + def test_parse_kid( + self, kid: str, kind: KidKind, project: int | None, workspace: int | None + ) -> None: + parsed = parse_kid(kid) + assert (parsed.kind, parsed.project_id, parsed.workspace_id) == (kind, project, workspace) + + +# --------------------------------------------------------------------------- +# bind +# --------------------------------------------------------------------------- + + +class TestBind: + def test_creates_when_unbound(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path) + + result = service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID]) + + metastore.post_item.assert_called_once_with( + BINDING_TYPE, name=KID, data={"kid": KID, "rlsPolicyUUIDs": [RLS_ID]} + ) + metastore.put_item.assert_not_called() + assert result["action"] == "created" + assert result["kid"] == KID + assert result["binding_id"] == "b-new" + assert "warnings" not in result + + def test_replaces_both_lists_when_bound(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding(rls=[RLS_ID], cls=[CLS_ID])]) + + result = service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID_2]) + + # PUT, not PATCH: the old clsPolicyUUIDs must not survive a rebind without --cls-policy. + metastore.put_item.assert_called_once_with( + BINDING_TYPE, "b-1", name=KID, data={"kid": KID, "rlsPolicyUUIDs": [RLS_ID_2]} + ) + metastore.post_item.assert_not_called() + assert result["action"] == "replaced" + assert result["previous"] == {"rls_policy_ids": [RLS_ID], "cls_policy_ids": [CLS_ID]} + + def test_post_that_lost_a_race_is_put_once(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path) + metastore.post_item.side_effect = _api_error(409, ErrorCode.ALREADY_EXISTS) + metastore.list_items.side_effect = [[], [_binding()]] + + result = service.bind("prod", 42, stack_id=STACK, cls_policies=[CLS_ID]) + + assert result["action"] == "replaced" + metastore.put_item.assert_called_once() + + def test_dry_run_validates_and_writes_nothing(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding(rls=[RLS_ID])]) + + result = service.bind("prod", 42, stack_id=STACK, cls_policies=[CLS_ID], dry_run=True) + + assert result["dry_run"] is True + assert result["action"] == "replace" + assert result["binding_id"] == "b-1" + metastore.get_item.assert_called_once_with("cls-policy", CLS_ID) + metastore.post_item.assert_not_called() + metastore.put_item.assert_not_called() + + def test_ids_are_deduplicated_and_normalized(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path) + + service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID, RLS_ID.upper(), RLS_ID]) + + assert metastore.post_item.call_args.kwargs["data"]["rlsPolicyUUIDs"] == [RLS_ID] + + @pytest.mark.parametrize( + ("rls", "cls", "needle"), + [ + ([], [], "at least one"), + (["not-a-uuid"], [], "not a UUID"), + ([f"{i:08d}-0000-4000-8000-000000000000" for i in range(101)], [], "at most 100"), + ], + ) + def test_usage_errors_before_any_call( + self, tmp_path: Path, rls: list[str], cls: list[str], needle: str + ) -> None: + service, metastore, storage = _make(tmp_path) + + with pytest.raises(KeboolaApiError) as exc_info: + service.bind("prod", 42, stack_id=STACK, rls_policies=rls, cls_policies=cls) + + assert exc_info.value.error_code == ErrorCode.INVALID_ARGUMENT + assert needle in exc_info.value.message + storage.get_workspace.assert_not_called() + metastore.list_items.assert_not_called() + + def test_missing_stack_id_is_a_usage_error(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path) + + with pytest.raises(KeboolaApiError) as exc_info: + service.bind("prod", 42, stack_id=None, rls_policies=[RLS_ID]) + + assert exc_info.value.error_code == ErrorCode.INVALID_ARGUMENT + metastore.post_item.assert_not_called() + + def test_unknown_policy_fails_without_writing(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path) + metastore.get_item.side_effect = [{"id": RLS_ID}, _api_error(404, ErrorCode.NOT_FOUND)] + + with pytest.raises(KeboolaApiError) as exc_info: + service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID], cls_policies=[CLS_ID]) + + assert exc_info.value.error_code == ErrorCode.NOT_FOUND + assert f"cls-policy {CLS_ID}" in exc_info.value.message + metastore.post_item.assert_not_called() + + def test_missing_workspace_fails(self, tmp_path: Path) -> None: + service, metastore, storage = _make( + tmp_path, workspace=_api_error(404, ErrorCode.NOT_FOUND) + ) + storage.list_dev_branches.return_value = [{"id": 1, "isDefault": True}, {"id": 7}] + + with pytest.raises(KeboolaApiError) as exc_info: + service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID]) + + assert exc_info.value.error_code == ErrorCode.NOT_FOUND + storage.get_workspace.assert_any_call(42, branch_id=7) + metastore.post_item.assert_not_called() + + def test_workspace_found_in_a_dev_branch(self, tmp_path: Path) -> None: + service, _, storage = _make(tmp_path) + storage.get_workspace.side_effect = [ + _api_error(404, ErrorCode.NOT_FOUND), + {"id": 42, "connection": {"loginType": "governed"}}, + ] + storage.list_dev_branches.return_value = [{"id": 7}] + + result = service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID]) + + assert result["action"] == "created" + + def test_non_governed_workspace_warns_but_binds(self, tmp_path: Path) -> None: + service, _, _ = _make( + tmp_path, workspace={"id": 42, "connection": {"loginType": "snowflake-person-keypair"}} + ) + + result = service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID]) + + assert result["action"] == "created" + assert "not 'governed'" in result["warnings"][0] + + def test_project_id_comes_from_the_token_when_not_stored(self, tmp_path: Path) -> None: + service, _, storage = _make(tmp_path, project_id=None) + + result = service.bind("prod", 42, stack_id=STACK, rls_policies=[RLS_ID]) + + storage.verify_token.assert_called_once() + assert result["kid"] == KID + + +# --------------------------------------------------------------------------- +# unbind +# --------------------------------------------------------------------------- + + +class TestUnbind: + def test_deletes_the_binding(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding(rls=[RLS_ID])]) + + result = service.unbind("prod", 42, stack_id=STACK) + + metastore.delete_item.assert_called_once_with(BINDING_TYPE, "b-1") + assert result["deleted"] is True + assert result["kid"] == KID + + def test_dry_run_only_looks_it_up(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding(rls=[RLS_ID])]) + + result = service.unbind("prod", 42, stack_id=STACK, dry_run=True) + + assert result["bound"] is True + assert result["dry_run"] is True + metastore.delete_item.assert_not_called() + + def test_no_binding_is_not_an_error(self, tmp_path: Path) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding(kid=KID + "1")]) + + result = service.unbind("prod", 42, stack_id=STACK) + + assert result == { + "project": "prod", + "workspace_id": 42, + "kid": KID, + "bound": False, + "deleted": False, + } + metastore.delete_item.assert_not_called() + + +# --------------------------------------------------------------------------- +# bindings +# --------------------------------------------------------------------------- + + +class TestListBindings: + def test_rows_with_workspace_existence(self, tmp_path: Path) -> None: + gone = _binding(kid=f"KID:{STACK}:project-{PROJECT_ID}:workspace-9", item_id="b-2") + app = _binding(kid=f"KID:{STACK}:project-{PROJECT_ID}:app-x", item_id="b-3") + service, _, storage = _make(tmp_path, bindings=[_binding(rls=[RLS_ID]), gone, app]) + storage.get_workspace.side_effect = lambda ws, branch_id=None: ( + {"id": ws} if ws == 42 else (_ for _ in ()).throw(_api_error(404)) + ) + + rows = service.list_bindings("prod")["bindings"] + + assert [(r["id"], r["workspace_exists"]) for r in rows] == [ + ("b-1", True), + ("b-2", False), + ("b-3", None), + ] + assert rows[0]["rls_policy_ids"] == [RLS_ID] + assert rows[0]["updated"] == OLD + + def test_kid_of_another_project_is_not_checked(self, tmp_path: Path) -> None: + other = _binding(kid=f"KID:{STACK}:project-999:workspace-42") + service, _, storage = _make(tmp_path, bindings=[other]) + + row = service.list_bindings("prod")["bindings"][0] + + assert row["workspace_exists"] is None + assert "999" in row["workspace_check_error"] + storage.get_workspace.assert_not_called() + + def test_filter_and_failed_check(self, tmp_path: Path) -> None: + other = _binding(kid=f"KID:{STACK}:project-{PROJECT_ID}:workspace-9", item_id="b-2") + service, _, _ = _make(tmp_path, bindings=[_binding(), other], workspace=_api_error(500)) + + rows = service.list_bindings("prod", workspace_id=42)["bindings"] + + assert [r["id"] for r in rows] == ["b-1"] + assert rows[0]["workspace_exists"] is None + assert rows[0]["workspace_check_error"] == "HTTP 500" + + +# --------------------------------------------------------------------------- +# bindings-sweep +# --------------------------------------------------------------------------- + + +def _ws_kid(workspace_id: int, project_id: int = PROJECT_ID) -> str: + return f"KID:{STACK}:project-{project_id}:workspace-{workspace_id}" + + +class TestSweep: + @pytest.mark.parametrize( + ("binding", "workspace", "status", "reason"), + [ + (_binding(updated=OLD), _api_error(404), "orphaned", "no longer exists"), + (_binding(updated=RECENT), _api_error(404), "kept", "within --older-than"), + (_binding(updated=OLD), {"id": 42}, "kept", "workspace exists"), + (_binding(updated=OLD), _api_error(500), "failed", "workspace check failed"), + (_binding(updated=OLD), _api_error(403), "failed", "workspace check failed"), + (_binding(updated=None), _api_error(404), "kept", "no last-update"), + (_binding(updated="yesterday"), _api_error(404), "kept", "no last-update"), + ( + _binding(kid=f"KID:{STACK}:project-{PROJECT_ID}:app-7", updated=OLD), + _api_error(404), + "kept", + "app KID", + ), + (_binding(kid=_ws_kid(42, 999), updated=OLD), _api_error(404), "kept", "999"), + (_binding(kid="KID:x:workspace-01", updated=OLD), _api_error(404), "kept", "canonical"), + ], + ) + def test_conditions( + self, + tmp_path: Path, + binding: dict[str, Any], + workspace: Any, + status: str, + reason: str, + ) -> None: + service, metastore, _ = _make(tmp_path, bindings=[binding], workspace=workspace) + + result = service.sweep("prod", older_than_days=30, delete=False) + + row = result["bindings"][0] + assert row["status"] == status + assert reason in row["reason"] + assert result["dry_run"] is True + assert result["references_checked"] is False + metastore.delete_item.assert_not_called() + + def test_gone_only_when_every_dev_branch_also_404s(self, tmp_path: Path) -> None: + service, _, storage = _make(tmp_path, bindings=[_binding()]) + storage.get_workspace.side_effect = [_api_error(404), {"id": 42}] + storage.list_dev_branches.return_value = [{"id": 1, "isDefault": True}, {"id": 7}] + + row = service.sweep("prod", older_than_days=30, delete=False)["bindings"][0] + + assert row["status"] == "kept" + + def test_failed_branch_listing_is_never_a_delete(self, tmp_path: Path) -> None: + service, metastore, storage = _make( + tmp_path, bindings=[_binding()], workspace=_api_error(404) + ) + storage.list_dev_branches.side_effect = _api_error(500) + + result = service.sweep("prod", older_than_days=30, delete=True) + + assert result["bindings"][0]["status"] == "failed" + metastore.delete_item.assert_not_called() + + def test_delete_removes_only_orphans(self, tmp_path: Path) -> None: + bindings = [ + _binding(kid=_ws_kid(1), item_id="gone"), + _binding(kid=_ws_kid(2), item_id="young", updated=RECENT), + _binding(kid=f"KID:{STACK}:project-{PROJECT_ID}:app-3", item_id="app"), + ] + service, metastore, _ = _make(tmp_path, bindings=bindings, workspace=_api_error(404)) + + result = service.sweep("prod", older_than_days=30, delete=True) + + metastore.delete_item.assert_called_once_with(BINDING_TYPE, "gone") + assert result["summary"] == {"orphaned": 0, "deleted": 1, "kept": 2, "failed": 0} + + @pytest.mark.parametrize(("status", "outcome"), [(409, "kept"), (500, "failed")]) + def test_refused_delete(self, tmp_path: Path, status: int, outcome: str) -> None: + service, metastore, _ = _make(tmp_path, bindings=[_binding()], workspace=_api_error(404)) + metastore.delete_item.side_effect = _api_error(status) + + row = service.sweep("prod", older_than_days=30, delete=True)["bindings"][0] + + assert row["status"] == outcome + + def test_older_than_zero_takes_every_age(self, tmp_path: Path) -> None: + service, _, _ = _make( + tmp_path, bindings=[_binding(updated=RECENT)], workspace=_api_error(404) + ) + + row = service.sweep("prod", older_than_days=0, delete=False)["bindings"][0] + + assert row["status"] == "orphaned" + + def test_negative_age_is_a_usage_error(self, tmp_path: Path) -> None: + service, _, _ = _make(tmp_path) + + with pytest.raises(KeboolaApiError) as exc_info: + service.sweep("prod", older_than_days=-1, delete=False) + + assert exc_info.value.error_code == ErrorCode.INVALID_ARGUMENT diff --git a/tests/test_project_ref.py b/tests/test_project_ref.py index 2cd64527..4d3c1f0f 100644 --- a/tests/test_project_ref.py +++ b/tests/test_project_ref.py @@ -157,6 +157,7 @@ def test_no_shadow_notice_for_aliases_of_the_same_project(self) -> None: "--clear-target-projects": "a boolean flag (rls/cls update): revoke every grant, takes no project", "--register-projects": "a boolean flag", "--remove-projects": "a boolean flag", + "--stack-id": "rls bind/unbind: the stack segment of a workspace KID, not a registered alias", } diff --git a/tests/test_rls_bindings_cli.py b/tests/test_rls_bindings_cli.py new file mode 100644 index 00000000..256a50fc --- /dev/null +++ b/tests/test_rls_bindings_cli.py @@ -0,0 +1,315 @@ +"""CLI tests for `kbagent rls bind|unbind|bindings|bindings-sweep` (CliRunner, mocked service).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from keboola_agent_cli.cli import app +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.constants import ENV_STACK_ID +from keboola_agent_cli.errors import ErrorCode, KeboolaApiError +from keboola_agent_cli.models import ProjectConfig + +runner = CliRunner() +TEST_TOKEN = "999-token-abc" +STACK = "dev-keboola-aws-eu-west-1" +KID = f"KID:{STACK}:project-1234:workspace-42" +RLS_ID = "11111111-1111-4111-8111-111111111111" +CLS_ID = "33333333-3333-4333-8333-333333333333" + + +@pytest.fixture(autouse=True) +def _no_stack_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(ENV_STACK_ID, raising=False) + + +def _run(tmp_path: Path, args: list[str], service: MagicMock, input: str | None = None) -> Any: + store = ConfigStore(config_dir=tmp_path / "cfg") + store.add_project( + "prod", + ProjectConfig( + stack_url="https://connection.keboola.com", + token=TEST_TOKEN, + project_name="prod", + project_id=1234, + ), + ) + with ( + patch("keboola_agent_cli.cli.ConfigStore") as mock_store, + patch("keboola_agent_cli.cli.PolicyBindingService") as mock_service, + ): + mock_store.return_value = store + mock_service.return_value = service + return runner.invoke(app, args, input=input) + + +def _bound() -> dict[str, Any]: + return { + "project": "prod", + "workspace_id": 42, + "kid": KID, + "bound": True, + "binding_id": "b-1", + "binding": {"id": "b-1", "kid": KID}, + "dry_run": True, + } + + +class TestBind: + def test_json_passes_the_split_lists(self, tmp_path: Path) -> None: + service = MagicMock() + service.bind.return_value = {"kid": KID, "action": "created", "binding_id": "b-1"} + + result = _run( + tmp_path, + [ + "--json", "rls", "bind", "--project", "prod", "--workspace-id", "42", + "--rls-policy", f"{RLS_ID},{RLS_ID}", "--cls-policy", CLS_ID, + "--stack-id", STACK, + ], + service, + ) # fmt: skip + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["data"]["kid"] == KID + service.bind.assert_called_once_with( + "prod", + 42, + stack_id=STACK, + rls_policies=[RLS_ID], + cls_policies=[CLS_ID], + dry_run=False, + ) + + def test_stack_id_from_the_environment( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv(ENV_STACK_ID, STACK) + service = MagicMock() + service.bind.return_value = { + "kid": KID, + "action": "replace", + "rls_policy_ids": [RLS_ID], + "cls_policy_ids": [], + "warnings": ["not governed"], + } + + result = _run( + tmp_path, + [ + "rls", "bind", "--project", "prod", "--workspace-id", "42", + "--rls-policy", RLS_ID, "--dry-run", + ], + service, + ) # fmt: skip + + assert result.exit_code == 0, result.output + assert service.bind.call_args.kwargs["stack_id"] == STACK + assert service.bind.call_args.kwargs["dry_run"] is True + assert "Would replace the binding" in result.output + assert "not governed" in result.output + + def test_usage_error_from_the_service_exits_2(self, tmp_path: Path) -> None: + service = MagicMock() + service.bind.side_effect = KeboolaApiError( + message="--stack-id is required", status_code=400, error_code=ErrorCode.INVALID_ARGUMENT + ) + + result = _run( + tmp_path, + ["--json", "rls", "bind", "--project", "prod", "--workspace-id", "42"], + service, + ) + + assert result.exit_code == 2 + assert json.loads(result.stdout)["error"]["code"] == "INVALID_ARGUMENT" + + +class TestUnbind: + ARGS: tuple[str, ...] = ( + "rls", "unbind", "--project", "prod", "--workspace-id", "42", "--stack-id", STACK, + ) # fmt: skip + + def test_not_bound_needs_no_confirmation(self, tmp_path: Path) -> None: + service = MagicMock() + service.unbind.return_value = {"kid": KID, "bound": False, "deleted": False} + + result = _run(tmp_path, list(self.ARGS), service) + + assert result.exit_code == 0, result.output + assert "already gets every policy" in result.output + service.unbind.assert_called_once() + + def test_dry_run_deletes_nothing(self, tmp_path: Path) -> None: + service = MagicMock() + service.unbind.return_value = _bound() + + result = _run(tmp_path, ["--json", *self.ARGS, "--dry-run"], service) + + assert result.exit_code == 0, result.output + service.unbind.assert_called_once_with("prod", 42, stack_id=STACK, dry_run=True) + + def test_without_terminal_refuses(self, tmp_path: Path) -> None: + service = MagicMock() + service.unbind.return_value = _bound() + + result = _run(tmp_path, list(self.ARGS), service, input="y\n") + + assert result.exit_code == 2 + assert "CONFIRMATION_REQUIRED" in result.output + assert service.unbind.call_count == 1 # the lookup only + + @pytest.mark.usefixtures("interactive_terminal") + @pytest.mark.parametrize(("answer", "calls"), [("n\n", 1), ("y\n", 2)]) + def test_prompt(self, tmp_path: Path, answer: str, calls: int) -> None: + service = MagicMock() + service.unbind.side_effect = [_bound(), {"kid": KID, "deleted": True}] + + result = _run(tmp_path, list(self.ARGS), service, input=answer) + + assert result.exit_code == 0, result.output + assert service.unbind.call_count == calls + if calls == 2: + assert "every policy of the project again" in result.output + + def test_yes_deletes(self, tmp_path: Path) -> None: + service = MagicMock() + service.unbind.side_effect = [_bound(), {"kid": KID, "deleted": True}] + + result = _run(tmp_path, ["--json", *self.ARGS, "--yes"], service) + + assert result.exit_code == 0, result.output + assert json.loads(result.stdout)["data"]["deleted"] is True + + def test_deny_destructive_blocks_it(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run(tmp_path, ["--json", "--deny-destructive", *self.ARGS, "--yes"], service) + + assert result.exit_code == 6 + service.unbind.assert_not_called() + + +class TestBindings: + def test_empty_human(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_bindings.return_value = {"project": "prod", "bindings": []} + + result = _run(tmp_path, ["rls", "bindings", "--project", "prod"], service) + + assert result.exit_code == 0, result.output + assert "every workspace gets every policy" in result.output + + def test_table_and_filter(self, tmp_path: Path) -> None: + service = MagicMock() + service.list_bindings.return_value = { + "project": "prod", + "bindings": [ + { + "kid": KID, + "rls_policy_ids": [RLS_ID], + "cls_policy_ids": [], + "updated": "2026-10-01T00:00:00Z", + "workspace_exists": False, + } + ], + } + + result = _run( + tmp_path, + ["rls", "bindings", "--project", "prod", "--workspace-id", "42"], + service, + ) + + assert result.exit_code == 0, result.output + assert "NO" in result.output + service.list_bindings.assert_called_once_with("prod", 42) + + +def _sweep_result(**counts: int) -> dict[str, Any]: + summary = {"orphaned": 0, "deleted": 0, "kept": 0, "failed": 0} | counts + return { + "project": "prod", + "dry_run": True, + "references_checked": False, + "references_note": "no reverse-reference lookup", + "bindings": [ + { + "kid": KID, + "rls_policy_ids": [], + "cls_policy_ids": [CLS_ID], + "updated": None, + "status": "orphaned", + "reason": "gone", + } + ], + "summary": summary, + } + + +class TestSweep: + def test_reports_by_default(self, tmp_path: Path) -> None: + service = MagicMock() + service.sweep.return_value = _sweep_result(orphaned=1) + + result = _run(tmp_path, ["rls", "bindings-sweep", "--project", "prod"], service) + + assert result.exit_code == 0, result.output + service.sweep.assert_called_once_with("prod", older_than_days=30, delete=False) + assert "would be deleted" in result.output + assert "no reverse-reference lookup" in result.output + + def test_yes_deletes(self, tmp_path: Path) -> None: + service = MagicMock() + service.sweep.return_value = _sweep_result(deleted=1) + + result = _run( + tmp_path, + ["--json", "rls", "bindings-sweep", "--project", "prod", "--older-than", "7", "-y"], + service, + ) + + assert result.exit_code == 0, result.output + service.sweep.assert_called_once_with("prod", older_than_days=7, delete=True) + + def test_a_failed_item_exits_1(self, tmp_path: Path) -> None: + service = MagicMock() + service.sweep.return_value = _sweep_result(failed=1) + + result = _run(tmp_path, ["--json", "rls", "bindings-sweep", "--project", "prod"], service) + + assert result.exit_code == 1 + + def test_dry_run_and_yes_conflict(self, tmp_path: Path) -> None: + service = MagicMock() + + result = _run( + tmp_path, + ["--json", "rls", "bindings-sweep", "--project", "prod", "--dry-run", "--yes"], + service, + ) + + assert result.exit_code == 2 + service.sweep.assert_not_called() + + @pytest.mark.parametrize(("flag", "exit_code"), [([], 0), (["--yes"], 6)]) + def test_deny_destructive_blocks_only_the_delete( + self, tmp_path: Path, flag: list[str], exit_code: int + ) -> None: + service = MagicMock() + service.sweep.return_value = _sweep_result() + + result = _run( + tmp_path, + ["--json", "--deny-destructive", "rls", "bindings-sweep", "--project", "prod", *flag], + service, + ) + + assert result.exit_code == exit_code, result.output + assert service.sweep.called is (exit_code == 0) diff --git a/tests/test_rls_service.py b/tests/test_rls_service.py index ec0b1c21..f6408e62 100644 --- a/tests/test_rls_service.py +++ b/tests/test_rls_service.py @@ -668,6 +668,20 @@ def test_dry_run_shows_the_policy_and_deletes_nothing(self, tmp_path: Path) -> N assert result["dry_run"] is True assert result["policy"]["id"] == "p-1" + @pytest.mark.parametrize(("status", "hint"), [(409, True), (500, False)]) + def test_a_bound_policy_names_the_binding_commands( + self, tmp_path: Path, status: int, hint: bool + ) -> None: + service, mock = _make_service(_make_store(tmp_path)) + mock.delete_item.side_effect = KeboolaApiError( + message="refused", status_code=status, error_code=ErrorCode.API_ERROR + ) + + with pytest.raises(KeboolaApiError) as exc_info: + service.delete_policy("prod", "p-1") + + assert ("rls bindings" in exc_info.value.message) is hint + # --------------------------------------------------------------------------- # Pure condition helpers diff --git a/tests/test_server_rls_bindings.py b/tests/test_server_rls_bindings.py new file mode 100644 index 00000000..68c5c5e8 --- /dev/null +++ b/tests/test_server_rls_bindings.py @@ -0,0 +1,150 @@ +"""Tests for `server/routers/rls_bindings.py`: router -> service parity, route order, permissions.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +if importlib.util.find_spec("fastapi") is None: # pragma: no cover + pytest.skip( + "FastAPI not installed; run `uv pip install -e '.[server]'`", allow_module_level=True + ) + +from fastapi.testclient import TestClient + +from keboola_agent_cli.config_store import ConfigStore +from keboola_agent_cli.models import PermissionPolicy +from keboola_agent_cli.server import create_app +from keboola_agent_cli.server.dependencies import ServiceRegistry, get_registry + +AUTH = {"Authorization": "Bearer test-token"} +STACK = "dev-keboola-aws-eu-west-1" +RLS_ID = "11111111-1111-4111-8111-111111111111" + + +def _client(tmp_path: Path, service: MagicMock, rls: MagicMock | None = None) -> TestClient: + registry = ServiceRegistry.__new__(ServiceRegistry) + registry.policy_binding = service + registry.rls = rls or MagicMock() + app = create_app(config_dir=str(tmp_path), auth_token="test-token") + app.dependency_overrides[get_registry] = lambda: registry + return TestClient(app) + + +def _deny(tmp_path: Path, deny: str) -> None: + store = ConfigStore(config_dir=tmp_path) + config = store.load() + config.permissions = PermissionPolicy(mode="allow", deny=[deny]) + store.save(config) + + +def test_list_is_not_captured_by_the_policy_detail_route(tmp_path: Path) -> None: + service, rls = MagicMock(), MagicMock() + service.list_bindings.return_value = {"project": "prod", "bindings": []} + + with _client(tmp_path, service, rls) as client: + res = client.get("/rls/prod/bindings?workspace_id=42", headers=AUTH) + + assert res.status_code == 200, res.text + service.list_bindings.assert_called_once_with("prod", 42) + rls.get_policy.assert_not_called() + + +def test_bind_calls_service(tmp_path: Path) -> None: + service = MagicMock() + service.bind.return_value = {"kid": "k", "action": "created"} + + with _client(tmp_path, service) as client: + res = client.put( + "/rls/prod/bindings/42", + headers=AUTH, + json={"stack_id": STACK, "rls_policies": [RLS_ID], "dry_run": True}, + ) + + assert res.status_code == 200, res.text + service.bind.assert_called_once_with( + "prod", 42, stack_id=STACK, rls_policies=[RLS_ID], cls_policies=[], dry_run=True + ) + + +def test_bind_requires_stack_id(tmp_path: Path) -> None: + service = MagicMock() + + with _client(tmp_path, service) as client: + res = client.put("/rls/prod/bindings/42", headers=AUTH, json={"rls_policies": [RLS_ID]}) + + assert res.status_code in (400, 422), res.text + service.bind.assert_not_called() + + +def test_unbind_calls_service(tmp_path: Path) -> None: + service = MagicMock() + service.unbind.return_value = {"deleted": True} + + with _client(tmp_path, service) as client: + res = client.delete(f"/rls/prod/bindings/42?stack_id={STACK}", headers=AUTH) + + assert res.status_code == 200, res.text + service.unbind.assert_called_once_with("prod", 42, stack_id=STACK, dry_run=False) + + +@pytest.mark.parametrize( + ("body", "delete"), [({}, False), ({"dry_run": False, "older_than_days": 7}, True)] +) +def test_sweep_calls_service(tmp_path: Path, body: dict[str, Any], delete: bool) -> None: + service = MagicMock() + service.sweep.return_value = {"bindings": []} + + with _client(tmp_path, service) as client: + res = client.post("/rls/prod/bindings/sweep", headers=AUTH, json=body) + + assert res.status_code == 200, res.text + service.sweep.assert_called_once_with( + "prod", older_than_days=body.get("older_than_days", 30), delete=delete + ) + + +@pytest.mark.parametrize(("dry_run", "status"), [(True, 200), (False, 403)]) +def test_denying_destructive_blocks_only_a_deleting_sweep( + tmp_path: Path, dry_run: bool, status: int +) -> None: + _deny(tmp_path, "cli:destructive") + service = MagicMock() + service.sweep.return_value = {"bindings": []} + + with _client(tmp_path, service) as client: + res = client.post("/rls/prod/bindings/sweep", headers=AUTH, json={"dry_run": dry_run}) + + assert res.status_code == status, res.text + assert service.sweep.called is (status == 200) + + +@pytest.mark.parametrize(("deny", "status"), [("cli:destructive", 403), ("cli:admin", 200)]) +def test_unbind_is_destructive(tmp_path: Path, deny: str, status: int) -> None: + _deny(tmp_path, deny) + service = MagicMock() + service.unbind.return_value = {"deleted": True} + + with _client(tmp_path, service) as client: + res = client.delete(f"/rls/prod/bindings/42?stack_id={STACK}", headers=AUTH) + + assert res.status_code == status, res.text + + +def test_bind_is_admin(tmp_path: Path) -> None: + _deny(tmp_path, "cli:admin") + service = MagicMock() + + with _client(tmp_path, service) as client: + res = client.put( + "/rls/prod/bindings/42", + headers=AUTH, + json={"stack_id": STACK, "rls_policies": [RLS_ID]}, + ) + + assert res.status_code == 403, res.text + service.bind.assert_not_called()