From 0eb4c04f5b4c1b4b3e606f48b89be770da26cd6b Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 9 Oct 2026 21:38:10 +0200 Subject: [PATCH 1/3] chore(release): 0.98.0 --- .claude-plugin/marketplace.json | 2 +- plugins/kbagent/.claude-plugin/plugin.json | 2 +- plugins/kbagent/agents/keboola-expert.md | 4 +- .../kbagent/references/commands-reference.md | 48 +++++++++---------- .../skills/kbagent/references/gotchas.md | 22 ++++----- .../references/large-upload-workflow.md | 4 +- .../kbagent/references/member-workflow.md | 2 +- .../references/metastore-scope-workflow.md | 2 +- .../references/semantic-layer-workflow.md | 2 +- .../references/storage-files-workflow.md | 2 +- .../kbagent/references/sync-workflow.md | 2 +- .../kbagent/references/workspace-workflow.md | 2 +- pyproject.toml | 2 +- src/keboola_agent_cli/changelog.py | 38 +++++++++++++++ src/keboola_agent_cli/commands/context.py | 12 ++--- uv.lock | 2 +- 16 files changed, 93 insertions(+), 55 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index efc92183..a1e99447 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -10,7 +10,7 @@ "plugins": [ { "name": "kbagent", - "version": "0.97.0", + "version": "0.98.0", "source": "./plugins/kbagent", "description": "DEPRECATED — install from keboola/ai-kit: /plugin marketplace add keboola/ai-kit && /plugin install kbagent@keboola-claude-kit — AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, sync configs as files, manage dev branches, and debug SQL in workspaces", "category": "development" diff --git a/plugins/kbagent/.claude-plugin/plugin.json b/plugins/kbagent/.claude-plugin/plugin.json index 46f82bf3..2310dc35 100644 --- a/plugins/kbagent/.claude-plugin/plugin.json +++ b/plugins/kbagent/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "kbagent", - "version": "0.97.0", + "version": "0.98.0", "description": "AI-friendly interface to Keboola Connection projects — explore configs, jobs, lineage, sync configs as files, manage dev branches, and debug SQL in workspaces", "author": { "name": "Keboola", diff --git a/plugins/kbagent/agents/keboola-expert.md b/plugins/kbagent/agents/keboola-expert.md index 6c73c427..58984aea 100644 --- a/plugins/kbagent/agents/keboola-expert.md +++ b/plugins/kbagent/agents/keboola-expert.md @@ -474,7 +474,7 @@ its absence is NOT a promise the entry is version-independent (see §1 Rule 6). projects succeed. Branch on the code, never on the message. The fan-out readers work on a session now, so a per-project `AUTH_NOT_SUPPORTED_ON_STACK` comes only from one of the three static-only features. -- **Exit 1 with a full `--json` payload = a partial failure (vNEXT+, #745)**: +- **Exit 1 with a full `--json` payload = a partial failure (0.98.0+, #745)**: a write that keeps going after one item fails (`sync push`, `sync clone`, `sync push/pull/diff --all-projects`, `org setup`, `project refresh`, `project invite --from-csv`, `workspace gc`, `semantic-layer @@ -516,7 +516,7 @@ its absence is NOT a promise the entry is version-independent (see §1 Rule 6). are `cli:*` categories, exact operation names, or globs matching >=1 operation (check with `kbagent permissions list`). -**Big `storage upload-table` (since vNEXT)** +**Big `storage upload-table` (since 0.98.0)** - Use `--no-wait`, keep `job_id` + `file_id`, poll `storage job-detail --wait`. `STORAGE_JOB_TIMEOUT` here = import still running: NEVER re-run the upload (duplicate rows); recover with `load-file --file-id`. diff --git a/plugins/kbagent/skills/kbagent/references/commands-reference.md b/plugins/kbagent/skills/kbagent/references/commands-reference.md index 848205f0..88075317 100644 --- a/plugins/kbagent/skills/kbagent/references/commands-reference.md +++ b/plugins/kbagent/skills/kbagent/references/commands-reference.md @@ -80,7 +80,7 @@ end-to-end walkthrough and troubleshooting. - `project remove --project NAME` -- disconnect a project - `project edit --project NAME [--url URL] [--token TOKEN] [--new-alias NEW] [--dry-run]` -- update connection details and/or rename the alias. `--new-alias` cascades through config.json (`projects` key + `default_project` if matched) and the nested sync directory `//` when present (-2 collision suffix, git-mv with shutil fallback). Lineage cache rebuild is manual (see gotchas, since v0.31.0). Combined with `--url` / `--token` in one call, those mutations target the new alias post-rename. `--dry-run` previews everything (collision check, planned disk-rename method, lineage-cache warning) without mutating state -- same exit codes as live for validation errors. `--token` on a browser-login (session) project is allowed and converts it to a static-token project, but warns that `auth logout --remove-projects` will no longer clean that alias up (`project remove` does); `--json` carries the text in an additive top-level `warnings` array, and `--dry-run` previews the identical warning (since v0.80.0) - `project status [--project NAME]` -- test connectivity and response time. Same guaranteed `auth_mode` field as `project list`, with the `Auth` column right after `Status` and present on ERROR rows too (since v0.80.0) -- `project refresh --project ALIAS | --all [--dry-run] [--force] [--yes] [--token-description DESC] [--token-expires-in N]` -- mint replacement Storage tokens via the Manage API for projects whose token is expired or invalid. `--force` also replaces still-valid non-expiring tokens. Browser-login (session) projects land in `skipped` with the reason that there is no static token to replace -- their credential lives in `auth.json` and rotates on its own -- and `--force` does not convert them either; use `project edit --token` for a deliberate single-project conversion (since v0.80.0). **Since vNEXT (#745)** a non-empty `projects_failed` exits **1** (was 0); `--dry-run` exits **1** on a failure too +- `project refresh --project ALIAS | --all [--dry-run] [--force] [--yes] [--token-description DESC] [--token-expires-in N]` -- mint replacement Storage tokens via the Manage API for projects whose token is expired or invalid. `--force` also replaces still-valid non-expiring tokens. Browser-login (session) projects land in `skipped` with the reason that there is no static token to replace -- their credential lives in `auth.json` and rotates on its own -- and `--force` does not convert them either; use `project edit --token` for a deliberate single-project conversion (since v0.80.0). **Since 0.98.0 (#745)** a non-empty `projects_failed` exits **1** (was 0); `--dry-run` exits **1** on a failure too - `project description-get --project NAME` -- read the dashboard project description (KBC.projectDescription on the default branch). Returns `{"description": ""}` if not set, not an error - `project description-set --project NAME [--text STR | --file PATH | --stdin]` -- set the dashboard project description (markdown). Pass exactly one of `--text`, `--file`, or `--stdin`. Writes to `KBC.projectDescription` on the default branch -- always the main branch, regardless of any active dev branch - `project use ALIAS` -- pin `ALIAS` as the persistent default project. Stored as `default_project` in config.json. A registered project ID pins that project's alias *(since 0.96.1)*. Overridden at runtime by `KBAGENT_PROJECT=ALIAS` (env, beats pin) and by `--project ALIAS` (CLI flag, beats both) @@ -92,7 +92,7 @@ end-to-end walkthrough and troubleshooting. All seven commands authenticate via `KBC_MANAGE_API_TOKEN` (Manage API), not the project's Storage token. Allowed roles are exactly `admin`, `guest`, `readOnly`, `share` -- the API self-reports this list in its 400 validation error and `constants.PROJECT_ROLES` mirrors it. - `project invite --project ALIAS --email EMAIL --role admin|guest|readOnly|share [--reason TEXT] [--dry-run]` -- single-shot invitation. Returns `{"status": "ok", "invitation_id": ..., ...}`. Re-inviting an already-invited or already-member email returns `{"status": "noop", "note": "already_invited" | "already_member"}` (HTTP 400 from the Manage API, normalised to a no-op). -- `project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run]` -- bulk invitation. CSV header required; columns: `email`, `project` (alias, or a project ID -- an alias wins *(since 0.96.1)*, and a row fails when that alias hides another project's ID) or `project_id` (numeric, ID only), `role` (optional with `--default-role`), `reason` (optional). Parallelised via `ThreadPoolExecutor` (default 8 workers). Single-stack-URL invariant per file: rows referencing different stacks raise `ConfigError` upfront. Result is `{"total","succeeded","noop","failed","rows":[...]}`; `rows[]` order is *not deterministic*. **Since vNEXT (#745)** a run with `failed > 0` exits **1**; `--dry-run` exits **1** on a failure too. The full JSON payload is still emitted first, so parse it and then check the exit code. On 0.97.0 and older the exit was 0 regardless -- inspect `failed` there. +- `project invite --from-csv FILE [--default-role ROLE] [--workers N] [--dry-run]` -- bulk invitation. CSV header required; columns: `email`, `project` (alias, or a project ID -- an alias wins *(since 0.96.1)*, and a row fails when that alias hides another project's ID) or `project_id` (numeric, ID only), `role` (optional with `--default-role`), `reason` (optional). Parallelised via `ThreadPoolExecutor` (default 8 workers). Single-stack-URL invariant per file: rows referencing different stacks raise `ConfigError` upfront. Result is `{"total","succeeded","noop","failed","rows":[...]}`; `rows[]` order is *not deterministic*. **Since 0.98.0 (#745)** a run with `failed > 0` exits **1**; `--dry-run` exits **1** on a failure too. The full JSON payload is still emitted first, so parse it and then check the exit code. On 0.97.0 and older the exit was 0 regardless -- inspect `failed` there. - `project member-list --project ALIAS [--include-pending]` -- list active members. Each member dict carries `id`, `email`, `name`, `role`, `status`, `mfa_enabled`. With `--include-pending`, the response also includes `pending_invitations: [...]`. - `project invitation-list --project ALIAS` -- list pending (unaccepted) invitations only. - `project invitation-cancel --project ALIAS --email EMAIL [--invitation-id ID] [--yes]` -- cancel a pending invitation. Without `--invitation-id`, the service resolves it by listing pending invitations and matching `--email` (case-insensitive). 204 No Content on success; `KeboolaApiError(NOT_FOUND)` if the email has no pending invitation. @@ -114,7 +114,7 @@ The `permissions` subcommands persist a write/destructive policy to config.json - `permissions check OPERATION` -- check if a specific operation is allowed (e.g. `permissions check storage.delete-table`) ## Organization -- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org. **Since vNEXT (#745)** a non-empty `projects_failed` exits **1** (was 0); with `--refresh`, a failed token refresh of an already-registered project is listed under `projects_refresh_failed` (keyed by `alias`) and also exits **1**; `--dry-run` exits **1** on a failure too. (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) +- `org setup --org-id ID --url URL [--dry-run] [--yes]` -- bulk-onboard all projects from an org. **Since 0.98.0 (#745)** a non-empty `projects_failed` exits **1** (was 0); with `--refresh`, a failed token refresh of an already-registered project is listed under `projects_refresh_failed` (keyed by `alias`) and also exits **1**; `--dry-run` exits **1** on a failure too. (org admin; manage token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) - `org setup --project-ids 1,2,3 --url URL [--dry-run] [--yes]` -- onboard specific projects by ID (any project member; manage token / Personal Access Token via interactive prompt by default, or `--allow-env-manage-token` + `KBC_MANAGE_API_TOKEN` for CI on 0.29.0+) ## Billing (PAYG Credits) (since v0.84.2) @@ -179,8 +179,8 @@ Requires a **super-admin** Manage API token (same kind as `org setup`). Same def - `storage table-detail --project NAME --table-id ID [--branch ID]` -- table detail with columns, types, primary key, row count (branch-aware). Since 0.88.0 (#621) also returns the raw Storage API `definition`: on BigQuery that carries `timePartitioning` / `rangePartitioning` / `clustering` / `requirePartitionFilter` / `partitions[]`, and it is the only way to verify a repartition landed. Human mode prints the layout and a partition COUNT; `--json` passes `definition` through verbatim. Present on every response (untyped tables too), so `null` means the stack omitted the key, not "untyped". Since 0.88.0 (#624) it also RESOLVES column descriptions written by anyone -- the UI, a component, or kbagent -- into `column_details[].description`, with precedence native definition -> `columnMetadata` `KBC.description` -> legacy flat `KBC.column.*` (an alias table falls back to the source table's `columnMetadata`, matching the MCP server). The response always carries `legacy_column_descriptions`, naming the columns still backed by the pre-0.88.0 convention -- human mode warns and points at `storage describe-migrate`. *(since v0.89.0)* Human mode also renders a `Description` column in the Columns table (only when at least one column has one; long text wraps rather than truncating); on 0.88.0 descriptions were visible in `--json` only. Reading never writes, so it is safe under a read-only token or `--deny-writes`. *(since 0.95.0, #761)* The response also carries `backend_path` (the owning bucket's Storage `backendPath`, verbatim: Snowflake `[database, schema]`, BigQuery `[dataset]`) and `sql_path` (the quoted, directly queryable table path, same convention as `storage bucket-detail`'s per-table `sql_path`; `null` when Storage reports no location or the backend is not Snowflake/BigQuery) - `storage create-bucket --project NAME --stage STAGE --name NAME [--description D] [--backend B] [--branch ID]` -- create bucket (branch-aware). With `--branch ID` on a project lacking the `storage-branches` feature (legacy fake-branch), response carries `legacy_branch_storage: true` and human mode prints a warning -- the runner will create a parallel `out.c--*` bucket at job time. See `storage-types-workflow.md` - `storage create-table --project NAME --bucket-id ID --name NAME [--column col:TYPE[(length)] ...] [--primary-key COL] [--not-null COL ...] [--default NAME=VALUE ...] [--source-table-id ID] [--source-branch-id N] [--time-partitioning-type DAY|HOUR|MONTH|YEAR] [--time-partitioning-field COL] [--time-partitioning-expiration-ms MS] [--range-partitioning-field COL --range-partitioning-start S --range-partitioning-end E --range-partitioning-interval I] [--clustering-field COL ...] [--branch ID] [--if-not-exists]` -- create typed table. Base types `STRING/INTEGER/NUMERIC/FLOAT/BOOLEAN/DATE/TIMESTAMP` plus native backend types with length (`VARCHAR(40)`, `NUMBER(18,2)`, `TIMESTAMP_TZ`, `VARIANT`, etc.) -- type/length validation delegated to the Storage API. `--not-null` marks a column `nullable=false`; `--default NAME=VALUE` sets a DEFAULT expression (booleans must be lowercase `true`/`false`). In a dev branch, the target bucket is auto-materialized if it has not yet been written to there -- response surfaces this via `auto_created_bucket: bool`. On legacy fake-branch projects (no `storage-branches` feature), `legacy_branch_storage: true` flags that the runner will use a separate `out.c--*` bucket at job time. `--if-not-exists` (0.47.0+) turns a duplicate-display-name failure into `action: skipped` when the table really exists at the expected id (safe for parallel workers). Since 0.47.1 the skipped envelope reports the EXISTING table's actual `columns`/`primary_key`/`name`, mirrors the request under `requested_columns`/`requested_primary_key`, and sets `schema_drift: true` when they diverge. **`--source-table-id` (0.66.0+, BigQuery only)** copies an existing table's data into the requested partition/clustering layout instead of building from `--column` (schema derived from source -> `--column`/`--not-null`/`--default` forbidden; the two are mutually exclusive). This is the supported way to repartition a populated BigQuery table -- then promote it with `storage swap-tables`. Partition/clustering flags (`--time-partitioning-*`, `--range-partitioning-*`, `--clustering-field`) also work on a plain `--column` create (BigQuery only); time vs range partitioning are mutually exclusive and range bounds are strings. When any source/partition/clustering flag is used, a one-call backend pre-flight rejects non-BigQuery projects (exit 2) before the create. See `storage-types-workflow.md` -- `storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--delimiter D] [--enclosure E] [--no-auto-create] [--wait/--no-wait] [--timeout SECONDS] [--branch ID] [--progress]` -- upload a CSV (or a `.csv.gz`, uploaded as-is) into a table; auto-creates the bucket/table from the header unless `--no-auto-create` (branch-aware). Two phases: cloud upload, then an async Storage import job. *(since vNEXT, #834)* On AWS stacks a file above 64 MiB is sent as an S3 **multipart** upload (64 MiB parts, auto-scaled to fit 10,000 parts, 4 in parallel, per-part retry; peak memory ~4 x part size) -- the old 5 GiB single-PUT ceiling and whole-file-in-RAM read are gone; Azure/GCP unchanged. Human mode shows an upload progress bar on stderr. *(since vNEXT)* `--wait` (default) waits `--timeout SECONDS` (default 600) for the import; `--no-wait` still waits for the upload, then returns `file_id`, `job_id`, `job_status` (`imported_rows` null while pending) -- follow it with `storage job-detail --job-id ID --wait`. A wait timeout is `STORAGE_JOB_TIMEOUT` (exit 4, `retryable: false`): the import keeps running server-side, so never re-run the upload -- re-import a failed one from the uploaded file with `storage load-file --file-id`. Big files: see `large-upload-workflow.md`. *(since vNEXT)* `--progress` always reports progress on stderr -- also with `--json` and without a terminal (a bar on a terminal, otherwise one line every 10 s plus a final line: percent, transferred/total, speed over a trailing 30 s window, elapsed, ETA); stdout is untouched, so `--json ... --progress 2>progress.log` keeps a machine-readable result and a readable log -- `storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--where-column COL --where-value VAL ... [--where-operator eq|neq]] [--changed-since WHEN] [--changed-until WHEN] [--branch ID] [--progress]` -- export table to CSV (branch-aware). `--where-column` + `--where-value` (repeatable, OR within the set) + `--where-operator eq|neq` filter rows server-side; `--changed-since`/`--changed-until` (unix ts or strtotime like `-2 days`) filter by import time -- the credential-only, no-workspace way to pull a filtered/incremental slice (0.62.0+). *(since vNEXT)* `--progress` as on `upload-table`; counts network (compressed) bytes and starts the clock when the download starts (the export job wait is excluded); a sliced export whose manifest has no sizes shows bytes + speed without percent/ETA +- `storage upload-table --project NAME --table-id ID --file PATH [--incremental] [--delimiter D] [--enclosure E] [--no-auto-create] [--wait/--no-wait] [--timeout SECONDS] [--branch ID] [--progress]` -- upload a CSV (or a `.csv.gz`, uploaded as-is) into a table; auto-creates the bucket/table from the header unless `--no-auto-create` (branch-aware). Two phases: cloud upload, then an async Storage import job. *(since 0.98.0, #834)* On AWS stacks a file above 64 MiB is sent as an S3 **multipart** upload (64 MiB parts, auto-scaled to fit 10,000 parts, 4 in parallel, per-part retry; peak memory ~4 x part size) -- the old 5 GiB single-PUT ceiling and whole-file-in-RAM read are gone; Azure/GCP unchanged. Human mode shows an upload progress bar on stderr. *(since 0.98.0)* `--wait` (default) waits `--timeout SECONDS` (default 600) for the import; `--no-wait` still waits for the upload, then returns `file_id`, `job_id`, `job_status` (`imported_rows` null while pending) -- follow it with `storage job-detail --job-id ID --wait`. A wait timeout is `STORAGE_JOB_TIMEOUT` (exit 4, `retryable: false`): the import keeps running server-side, so never re-run the upload -- re-import a failed one from the uploaded file with `storage load-file --file-id`. Big files: see `large-upload-workflow.md`. *(since 0.98.0)* `--progress` always reports progress on stderr -- also with `--json` and without a terminal (a bar on a terminal, otherwise one line every 10 s plus a final line: percent, transferred/total, speed over a trailing 30 s window, elapsed, ETA); stdout is untouched, so `--json ... --progress 2>progress.log` keeps a machine-readable result and a readable log +- `storage download-table --project NAME --table-id ID [--output FILE] [--columns COL ...] [--limit N] [--where-column COL --where-value VAL ... [--where-operator eq|neq]] [--changed-since WHEN] [--changed-until WHEN] [--branch ID] [--progress]` -- export table to CSV (branch-aware). `--where-column` + `--where-value` (repeatable, OR within the set) + `--where-operator eq|neq` filter rows server-side; `--changed-since`/`--changed-until` (unix ts or strtotime like `-2 days`) filter by import time -- the credential-only, no-workspace way to pull a filtered/incremental slice (0.62.0+). *(since 0.98.0)* `--progress` as on `upload-table`; counts network (compressed) bytes and starts the clock when the download starts (the export job wait is excluded); a sliced export whose manifest has no sizes shows bytes + speed without percent/ETA - `storage add-column --project NAME --table-id ID --column COL:TYPE[(length)] [--not-null] [--default VALUE] [--branch ID]` -- add a single column to an existing table (0.62.0+). Same `name:TYPE(length)` grammar as `create-table --column`; a bare `name` adds an untyped STRING column. Synchronous endpoint (no job to wait on). `--not-null` needs an empty table or a `--default`. Mirror of `delete-column` - `storage delete-table --project NAME --table-id ID [--table-id ...] [--force] [--dry-run] [--yes] [--branch ID]` -- delete tables, --force cascade-deletes aliased tables (branch-aware) - `storage truncate-table --project NAME --table-id ID [--table-id ...] [--dry-run] [--yes] [--branch ID]` -- delete all rows while preserving table schema, primary key, descriptions, sharing edges, and downstream dependents. Batch via repeated `--table-id`. Endpoint is uniformly async-via-job on every branch (returns a queued `tableRowsDelete` job; client polls via `_wait_for_storage_job` before returning). Idempotent (truncating an empty table is a no-op). Use when re-seeding a table without losing the schema contract @@ -196,19 +196,19 @@ Requires a **super-admin** Manage API token (same kind as `org setup`). Same def - `storage describe-bucket --project NAME --bucket-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a bucket description (stored as `KBC.description` in bucket metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage bucket-detail` - `storage describe-table --project NAME --table-id ID [--text STR | --file PATH | --stdin] [--branch ID]` -- set a table description (stored as `KBC.description` in table metadata, upsert). Provide exactly one of `--text`, `--file`, `--stdin`. Read back via `storage table-detail` - `storage describe-column --project NAME --table-id ID --column NAME=DESCRIPTION [--column ...] [--branch ID]` -- set one or more column descriptions. *(since v0.88.0)* Writes through the native `PUT /v2/storage/branch/{branch}/tables/{id}/definition` endpoint (the one the web UI uses; async `tableDefinitionUpdate` storage job) with `isDescriptionSystemManaged: false`, so the next component run's Output Mapping cannot overwrite the text. The backend mirrors the value into `columnMetadata` `KBC.description`, so the Keboola UI, the MCP server (`get_tables`) and the Snowflake `COMMENT` / BigQuery column description all see it. Unknown column names are rejected BEFORE any write (behavior change -- the pre-0.88.0 flat-metadata write accepted typos silently). Legacy flat `KBC.column.{name}.description` entries on the same table are migrated in the same write and then deleted. Read back in `storage table-detail` under `column_details[].description` -- `storage describe-batch --project NAME --from-file PATH [--branch ID]` -- apply bucket/table/column descriptions from a YAML file (top-level `buckets`, `tables`, `columns` sections, all optional). Column items go through the same native write (and same fail-fast + auto-migration) as `describe-column`. Partial-failure tolerant for API errors: per-item errors are collected and reported, the batch does not abort. Non-zero exit only when at least one item failed. **Since vNEXT (#745)** this also holds under `--json` (on 0.97.0 and older `--json` exited 0 even with errors), and the human headline reads `Failed:` instead of a green `Batch complete:`. *(since v0.89.0)* A malformed file (a section that is not a mapping of ID to description, a non-mapping `columns` entry, a `null` description, a document that is not a mapping, two keys colliding after `str()` coercion) is rejected whole before the first write -- `INVALID_ARGUMENT`, exit 2, message naming the offending key and its actual type. On 0.88.0 and earlier the same input raised an `AttributeError` traceback partway through the batch, leaving it half-applied -- `storage describe-migrate --project ALIAS [--table-id ID ...] [--bucket-id ID] [--prune-orphans] [--dry-run] [--yes] [--branch ID]` *(since v0.88.0)* -- bulk-convert legacy pre-0.88.0 flat `KBC.column.*.description` metadata to the native definition endpoint. Scope is explicit `--table-id` (repeatable), a single `--bucket-id`, or every table in the project; the two scope flags are mutually exclusive (exit 2). Scans first and prints the summary, then asks for confirmation -- `--dry-run` reports without writing, `--yes` skips the prompt. A column whose currently visible description already differs is skipped as `conflict` (the newer value wins); an entry for a column that no longer exists is skipped as `orphan` unless `--prune-orphans` deletes it. Migrated flat entries are deleted after a successful write, so a later `describe-column` clearing the text cannot be resurrected by the read fallback. Per-table failures are accumulated into `errors[]` and never abort the run; a non-empty `errors[]` exits 1, and **since vNEXT (#745)** the human headline then reads `Failed: Migrated N table(s) of M scanned, K failed` instead of a green `Migrated:`. Permission class `write` +- `storage describe-batch --project NAME --from-file PATH [--branch ID]` -- apply bucket/table/column descriptions from a YAML file (top-level `buckets`, `tables`, `columns` sections, all optional). Column items go through the same native write (and same fail-fast + auto-migration) as `describe-column`. Partial-failure tolerant for API errors: per-item errors are collected and reported, the batch does not abort. Non-zero exit only when at least one item failed. **Since 0.98.0 (#745)** this also holds under `--json` (on 0.97.0 and older `--json` exited 0 even with errors), and the human headline reads `Failed:` instead of a green `Batch complete:`. *(since v0.89.0)* A malformed file (a section that is not a mapping of ID to description, a non-mapping `columns` entry, a `null` description, a document that is not a mapping, two keys colliding after `str()` coercion) is rejected whole before the first write -- `INVALID_ARGUMENT`, exit 2, message naming the offending key and its actual type. On 0.88.0 and earlier the same input raised an `AttributeError` traceback partway through the batch, leaving it half-applied +- `storage describe-migrate --project ALIAS [--table-id ID ...] [--bucket-id ID] [--prune-orphans] [--dry-run] [--yes] [--branch ID]` *(since v0.88.0)* -- bulk-convert legacy pre-0.88.0 flat `KBC.column.*.description` metadata to the native definition endpoint. Scope is explicit `--table-id` (repeatable), a single `--bucket-id`, or every table in the project; the two scope flags are mutually exclusive (exit 2). Scans first and prints the summary, then asks for confirmation -- `--dry-run` reports without writing, `--yes` skips the prompt. A column whose currently visible description already differs is skipped as `conflict` (the newer value wins); an entry for a column that no longer exists is skipped as `orphan` unless `--prune-orphans` deletes it. Migrated flat entries are deleted after a successful write, so a later `describe-column` clearing the text cannot be resurrected by the read fallback. Per-table failures are accumulated into `errors[]` and never abort the run; a non-empty `errors[]` exits 1, and **since 0.98.0 (#745)** the human headline then reads `Failed: Migrated N table(s) of M scanned, K failed` instead of a green `Migrated:`. Permission class `write` ## Storage Files - `storage files --project NAME [--tag TAG ...] [--limit N] [--offset N] [--query Q] [--branch ID]` -- list Storage Files, optionally filtered by tag/query - `storage file-detail --project NAME --file-id ID` -- file metadata (size, tags, sliced, provider) -- `storage file-upload --project NAME --file PATH [--name NAME] [--tag TAG ...] [--permanent] [--branch ID] [--progress]` -- upload a file to Storage. *(since vNEXT, #834)* Uses the same S3 multipart upload as `upload-table` on AWS stacks (above 64 MiB; no 5 GiB ceiling). `--progress` as on `upload-table` *(since vNEXT)* -- `storage file-download --project NAME [--file-id ID | --tag TAG ...] [--output FILE|DIR] [--progress]` -- download a Storage File. Auto-detects sliced `.parquet` files and writes per-slice into a directory (never concatenates -- parquet slices have their own footers). `--progress` as on `download-table` *(since vNEXT)* +- `storage file-upload --project NAME --file PATH [--name NAME] [--tag TAG ...] [--permanent] [--branch ID] [--progress]` -- upload a file to Storage. *(since 0.98.0, #834)* Uses the same S3 multipart upload as `upload-table` on AWS stacks (above 64 MiB; no 5 GiB ceiling). `--progress` as on `upload-table` *(since 0.98.0)* +- `storage file-download --project NAME [--file-id ID | --tag TAG ...] [--output FILE|DIR] [--progress]` -- download a Storage File. Auto-detects sliced `.parquet` files and writes per-slice into a directory (never concatenates -- parquet slices have their own footers). `--progress` as on `download-table` *(since 0.98.0)* - `storage file-tag --project NAME --file-id ID [--add TAG ...] [--remove TAG ...]` -- add/remove tags on a file - `storage file-delete --project NAME --file-id ID [--file-id ...] [--dry-run] [--yes]` -- delete Storage Files -- `storage load-file --project NAME --file-id ID --table-id ID [--incremental] [--delimiter D] [--enclosure E] [--wait/--no-wait] [--timeout SECONDS] [--branch ID]` -- import a Storage File into a table (CSV / `.csv.gz`). The recovery path after a failed `upload-table` import: the file is already uploaded, so re-import it instead of uploading again. *(since vNEXT, #834)* `--wait/--no-wait` + `--timeout SECONDS` (default 600) behave as on `upload-table` (`--no-wait` returns `job_id`; a wait timeout is `STORAGE_JOB_TIMEOUT`, exit 4, the import keeps running) -- `storage job-detail --project NAME --job-id ID [--wait] [--timeout SECONDS]` *(since vNEXT, #834)* -- show one Storage API job: `status`, `operation_name`, `table_id`, `file_id`, created/start/end times, `imported_rows`, `warnings`, `results`, `error`. `--wait` polls until the job finishes (within `--timeout`; `--timeout` without `--wait` is rejected, exit 2 `INVALID_ARGUMENT` -- serve answers 422). Exit 0 for `success` / `waiting` / `processing`; exit 1 `STORAGE_JOB_FAILED` when the job ended in `error` (details still returned); exit 4 on a `--wait` timeout. Job ids are project-scoped -- no `--branch`, the active branch does not matter. Permission class `read`. Serve: `GET /storage/jobs/{project}/{job_id}?wait=&timeout=` -- `storage unload-table --project NAME --table-id ID [--columns COL ...] [--limit N] [--tag TAG ...] [--download] [--output FILE|DIR] [--file-type csv|parquet] [--branch ID] [--progress]` -- export a table to a Storage File. `--file-type parquet` produces sliced Parquet; `--download` saves each slice as its own file under `./{project}/{table_id}.parquet/` (default) together with `_manifest.json`. `--progress` reports the `--download` transfer as on `download-table` *(since vNEXT)* +- `storage load-file --project NAME --file-id ID --table-id ID [--incremental] [--delimiter D] [--enclosure E] [--wait/--no-wait] [--timeout SECONDS] [--branch ID]` -- import a Storage File into a table (CSV / `.csv.gz`). The recovery path after a failed `upload-table` import: the file is already uploaded, so re-import it instead of uploading again. *(since 0.98.0, #834)* `--wait/--no-wait` + `--timeout SECONDS` (default 600) behave as on `upload-table` (`--no-wait` returns `job_id`; a wait timeout is `STORAGE_JOB_TIMEOUT`, exit 4, the import keeps running) +- `storage job-detail --project NAME --job-id ID [--wait] [--timeout SECONDS]` *(since 0.98.0, #834)* -- show one Storage API job: `status`, `operation_name`, `table_id`, `file_id`, created/start/end times, `imported_rows`, `warnings`, `results`, `error`. `--wait` polls until the job finishes (within `--timeout`; `--timeout` without `--wait` is rejected, exit 2 `INVALID_ARGUMENT` -- serve answers 422). Exit 0 for `success` / `waiting` / `processing`; exit 1 `STORAGE_JOB_FAILED` when the job ended in `error` (details still returned); exit 4 on a `--wait` timeout. Job ids are project-scoped -- no `--branch`, the active branch does not matter. Permission class `read`. Serve: `GET /storage/jobs/{project}/{job_id}?wait=&timeout=` +- `storage unload-table --project NAME --table-id ID [--columns COL ...] [--limit N] [--tag TAG ...] [--download] [--output FILE|DIR] [--file-type csv|parquet] [--branch ID] [--progress]` -- export a table to a Storage File. `--file-type parquet` produces sliced Parquet; `--download` saves each slice as its own file under `./{project}/{table_id}.parquet/` (default) together with `_manifest.json`. `--progress` reports the `--download` transfer as on `download-table` *(since 0.98.0)* ## Data Streams (OTLP) Uses the per-project Storage token (no manage token). Control plane = `stream.` (derived from `connection.`). The OTLP ingest endpoint (`stream-in./otlp///`) is returned in `source.otlp.url` with the secret in the path -- **masked by default, `--reveal` to print it**. `create-source --type otlp` auto-provisions the logs/metrics/traces sinks (bucket `in.c-otlp-`) so data lands; `--no-sinks` opts out. See `stream-workflow.md`. @@ -280,7 +280,7 @@ Non-SOX Branches 2.0: merge a dev branch into production with review. Alias `mr` - `workspace password --project ALIAS --workspace-id ID` -- reset and return new password - `workspace load --project ALIAS --workspace-id ID --tables TABLE_ID [...] [--preserve] [--load-type clone|copy|view] [--force] [--timeout SECONDS]` -- load storage tables. **Since 0.91.0 (#687)**: default is per-table auto-decide -- zero-copy `clone` when eligible (same backend, full load, no external-schema bucket), else plain `copy`; `--load-type` forces one type for every table and fails loudly (never silently degrades) if a table is ineligible for it. A `copy` over 1 GiB needs interactive confirmation or `--force` in non-interactive mode. `--timeout` defaults to 300s (other storage-job commands keep 60s); on timeout the job keeps running server-side and the error now exits 4, not 1 - `workspace query --project ALIAS --workspace-id ID --sql "..." [--file F] [--transactional] [--full] [--limit N]` -- run SQL via Query Service. **Fast inline results since v0.59.0**: default reads the result set inline via `GET /api/v1/queries/{job}/{stmt}/results` (JSON `columns`+`rows`, no CSV-file materialization), capped at `--limit` rows (default 500) and marked `truncated` when there are more; pass `--full` for the complete CSV export (slower, uncapped). Each statement still carries `csv_data` (synthesized from the inline rows) so older parsers keep working. **Backend-agnostic since v0.58.0**: runs against both Snowflake and BigQuery workspaces. Mind the dialect: Snowflake quotes identifiers with `"..."`, BigQuery with backticks `` `...` `` -- `workspace gc [--project NAME ...] [--dry-run] [--yes]` -- garbage-collect orphaned workspaces (and any lingering `keboola.sandboxes` configs). `--dry-run` previews without deleting; `--project` repeatable, omit to GC across all connected projects. **Since vNEXT (#745)** a non-empty `errors[]` (a failed delete, or a project that could not be listed) exits **1**; `--dry-run` exits **1** on a failure too +- `workspace gc [--project NAME ...] [--dry-run] [--yes]` -- garbage-collect orphaned workspaces (and any lingering `keboola.sandboxes` configs). `--dry-run` previews without deleting; `--project` repeatable, omit to GC across all connected projects. **Since 0.98.0 (#745)** a non-empty `errors[]` (a failed delete, or a project that could not be listed) exits **1**; `--dry-run` exits **1** on a failure too - `workspace from-transformation --project ALIAS --component-id ID --config-id ID [--row-id ID]` -- workspace from existing transform ## Data Apps (Python/JS by default; Streamlit via --type) @@ -336,7 +336,7 @@ Requires the project to be added with its **master ('owner') Storage API token** - `flow update --project NAME --flow-id ID [--name N] [--description D] [--file @path.yaml|-|JSON] [--branch ID]` -- update name, description, or phases/tasks; `--file` is a full-replace of phases+tasks; merge-aware validation against the live CF schema (same graceful semantic-only degradation on fetch failure); requires at least one of --name/--description/--file - `flow delete --project NAME --flow-id ID [--branch ID] [--yes]` -- delete a flow config (confirmation guard) - `flow schedule --project NAME --flow-id ID --cron "0 6 * * *" [--timezone TZ] [--disabled] [--branch ID]` -- attach a cron schedule (stored as keboola.scheduler config, target.componentId=keboola.flow); replaces any existing schedule. Since v0.66.1 the config is also activated on the Scheduler Service so the trigger fires; an activation failure keeps the config written and surfaces a warning (`activated: false`, exit 0) -- `flow schedule-remove --project NAME --flow-id ID [--branch ID] [--yes]` -- remove all cron schedules attached to a flow (since v0.66.1 also deregisters them from the Scheduler Service); idempotent. When no schedule could be deleted it fails with `SCHEDULE_DELETE_FAILED`. **Since vNEXT (#745)** a schedule whose delete failed while others were deleted is listed in a new `errors[]` (`{schedule_id, error}`), the human headline reads `Failed: Removed N schedule(s) ..., M failed`, and the exit is **1**; before, that failure was dropped and the command printed `Success:` and exited 0 +- `flow schedule-remove --project NAME --flow-id ID [--branch ID] [--yes]` -- remove all cron schedules attached to a flow (since v0.66.1 also deregisters them from the Scheduler Service); idempotent. When no schedule could be deleted it fails with `SCHEDULE_DELETE_FAILED`. **Since 0.98.0 (#745)** a schedule whose delete failed while others were deleted is listed in a new `errors[]` (`{schedule_id, error}`), the human headline reads `Failed: Removed N schedule(s) ..., M failed`, and the exit is **1**; before, that failure was dropped and the command printed `Success:` and exited 0 - `flow triggers --project NAME --flow-id ID [--branch ID]` -- **the honest answer to "what starts this flow automatically"** (since 0.92.0, #714). Returns `cron_schedules` (the `keboola.scheduler` configs `schedule list` already saw) **and** `table_triggers` (`GET /v2/storage/triggers` -- a separate Storage resource, NOT a component config, which is why no kbagent command saw them before). Cross-project triggers (a trigger-queue app config in ANOTHER project) are **not** covered: the result carries `cross_project_triggers_checked: false`, deliberately not an empty list -- an empty list reads as "checked, found none", the exact false negative this command exists to prevent. **Check that flag before concluding a flow has no trigger.** Table triggers are **production-only** (the Storage route is declared `isAvailableInBranch: false`), so `--branch` narrows the cron half only and `table_triggers_branch_scoped` is always `false`. A `last_run` of `null` means the trigger exists but has never fired -- not that it is disabled ## Schedule Discovery & Audit (Fleet-Wide) @@ -362,10 +362,10 @@ Requires the project to be added with its **master ('owner') Storage API token** ## Sync (GitOps) - `sync init --project ALIAS [--directory DIR] [--git-branching] [--adopt-existing] [--with-workspaces]` -- initialize sync working directory; `--adopt-existing` adopts a `.keboola/manifest.json` already written by the kbc Go CLI without overwriting (idempotent; validates `project_id` against the alias token). `--with-workspaces` *(since 0.96.1)* sets `syncWorkspaces` in the manifest, so pull/diff/push/clone also sync shared SQL workspaces (`keboola.sandboxes` without `parameters.id`, `runtime.shared: true`), config only; a `push --force` delete removes the workspace's SQL editor sessions first (a plain push holds it back under `skipped_deletions`). With `--adopt-existing` it turns the key on in an existing manifest. See `sync-workflow.md` > "Shared SQL workspaces". -- `sync pull --project ALIAS [--all-projects] [--force] [--theirs] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID]` -- download configs to local files. **Auto-inits:** if the target directory has no `.keboola/manifest.json`, pull runs `init` first, so a separate `sync init` is not needed for a first checkout of a project. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient. `--force` is conflict-aware (since 0.53.0): a locally-modified config whose remote is unchanged is **preserved** (pending delta stays pushable, never silently re-stamped); a true merge conflict (local AND remote both changed since last pull) **aborts** the pull (exit 1, `SYNC_CONFLICT`; `--json` lists `details.conflicts`); local-untouched + remote-changed takes remote. `--theirs` (since v0.72.0) is the supported "discard local, take production" reconcile path: overwrites locally-modified configs/rows, restores deleted/missing files, resolves conflicts by taking remote (no abort, no manifest surgery). Since v0.72.0 plain pull also re-materializes a tracked config whose local dir was deleted (manifest<->disk invariant), so delete-dir-then-pull refetches. Config-level `isDisabled` is kept through pull and push (since v0.72.0) as sparse `is_disabled: true` in `_config.yml` -- absent key = enabled. `--branch` (0.47.0+) per-invocation dev-branch override, beats every other branch source. Ignored components (since 0.91.0): `keboola.sandboxes` + `keboola.mcp-server-tool` are always excluded (`keboola.sandboxes` shared SQL workspaces are synced when the manifest sets `syncWorkspaces`, *since 0.96.1*), unioned with the manifest's `ignoredComponents` list; a component newly ignored has its manifest entry dropped and local directory removed, reported with pull action `"ignored"` (distinct from `"removed"` = genuinely deleted on remote). Config-folder round-trip *(since 0.94.0)*: pull captures each config's UI folder (`KBC.configuration.folderName`, from the branch-only `search/component-configurations` endpoint) into the manifest, and reports `folder_lookup_failed` when that lookup fails, keeping the previously captured folder. **Since vNEXT (#745)** `--all-projects` exits **1** when `summary.failed > 0` (a project whose pull raised); `--dry-run` exits **1** on a failure too. -- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings]` -- push local changes (auto-encrypts secrets, fails if encryption fails). Workspace delete *(since 0.96.1)*: in a `syncWorkspaces` tree, `push --force` deleting a shared SQL workspace also deletes its SQL editor sessions (every user's, push branch) and their backend workspaces, which `config restore` does not bring back, so check `push --dry-run --force` (`warnings[]` `workspace_sessions`) first; `--force` is destructive-class (`sync.push --force`, blocked by `--deny-destructive`). Fresh-CREATE writeback updates placeholder manifest entries in place (since 0.47.0) and propagates any `KBC.configuration.*` metadata via `set_config_metadata`. Fresh-CREATE variable binding (since 0.47.2): when a `keboola.variables` config + its values row are created alongside a transformation in the same push, the transformation's `variables_id` / `variables_values_id` placeholders are rebound to the assigned ULIDs and the row's `values` are hoisted even without a `_keboola` block, so `job run` succeeds with no post-push `config variables-set` step (unresolvable/ambiguous links add a `variable_link` entry in `errors[]`, never a broken link). Since 0.96.1 push also remaps shared-code links, flow and orchestrator task `configId`s / `configRowIds` and schedule targets to the configs created in the same push. A link it cannot set is an `errors[]` entry with `change_type` `shared_code_link`, `flow_task_link` or `schedule_target_link` (error code `LINK_UNRESOLVED` for a row id without a new row, `API_ERROR` for a failed PUT). After a failed PUT the local files already hold the new ids and the manifest hash stays stale, so the next `sync push` sends them; this also applies to `variable_link`. The result carries `link_remaps` (`flow_tasks`, `orchestrator_tasks`, `schedule_targets`, `shared_code`, `config_row_ids`); `flow_task_remaps` counts flow tasks only. Never-fetched guard (since v0.72.0): a manifest entry with an empty `pull_hash` and no local files (pre-0.72 name-collision phantom) is **never** planned as a remote DELETE -- diff/push exclude it and report it under `never_fetched` with a warning (run `sync pull` to materialize); local deletion of a properly-pulled config deletes on `push --force`; since 0.96.1 (#792) a plain push deletes nothing and lists the deletion under `skipped_deletions` (also in `--dry-run`), and a config or row deleted on the remote since the last pull is `remote_deleted`, which push never re-creates. Adopted-by-id writeback (since v0.72.0): pushing an untracked file whose `_keboola.config_id` resolves on the branch also writes the manifest entry, so follow-up diffs are stable. `--branch` (0.47.0+) per-invocation override; when no `/` subtree exists on disk (since 0.47.2) the local default tree (`main/`) is promoted to the target branch (API writes still target the branch id); `--no-name-drift-warnings` (0.47.0+) drops the cosmetic warnings array. Branch-scoped since v0.89.0 (issue #649): push consumes the diff's changeset, so configs tracked on another branch's tree are never planned as creates -- they ride along on the result envelope under `orphaned` instead (see `sync diff`). **Since 0.91.0 (#686)** the manifest baseline `pull_config_hash` is stamped from the API response (or a read-back), not from the files on disk, so a pushed multi-statement SQL transformation -- or anything disabled in the UI whose local YAML lacks `is_disabled` -- no longer shows permanent phantom `REMOTE MODIFIED` drift; if the config cannot be read back after the write the baseline is left UNTOUCHED and a `warnings[]` entry says to run `sync pull` (never a disk-derived fallback). One legacy change is refused per-change with `SYNC_LEGACY_BOUNDARY`: a tree pulled before statement-boundary markers existed whose only difference from the remote is the lost boundaries (pushing it would collapse separate SQL statements into one) -- run `sync pull` for that project first. Ignored components (since 0.91.0) are filtered out on both sides of the diff push builds on, so a stale local directory for an ignored component (e.g. `keboola.mcp-server-tool`) is never classified as `DELETED` and can never be pushed as a remote deletion. **Since vNEXT (#745)** a push with a non-empty `errors[]` exits **1** and the human headline reads `Failed: Pushed: N created, ..., M failed` instead of a green `Success:`. `--all-projects` exits 1 when `summary.failed > 0`, and `summary.failed` now also counts a project whose push returned a non-empty `errors[]` (before, it counted as `success` and its line read `OK`; now the line is marked `x` and states the failed count). A `--dry-run` exits 1 on a failure too. `--json` output is still emitted before the non-zero exit. -- `sync clone --source DIR --target ALIAS --target-dir DIR [--bucket-map FILE] [--variable-values FILE] [--instance-rename FILE] [--no-create-buckets] [--dry-run] [--branch ID]` -- clone a reference synced project into a **fresh** target project and parameterize it. Copies the reference tree at `--source` into `--target-dir`, applies declarative overrides from JSON/YAML files (`--bucket-map` `{old_bucket_id: new_bucket_id}` rewrites storage input/output table refs; `--variable-values` `{var_name: value}` overrides `keboola.variables` rows; `--instance-rename` `{old_path_prefix: new_path_prefix}` renames config dirs + manifest paths), re-points the manifest at the target project, and pushes. Because the reference's config ids do not exist in the fresh target, every config is CREATEd fresh and **keboola.flow task `configId`s + transformation variable links are remapped reference->ULID** by push Phase C/D (the push result carries `flow_task_remaps`). Since 0.96.1 push also remaps shared-code links (`shared_code_id`, `shared_code_row_ids`, `{{}}` script placeholders), legacy `keboola.orchestrator` task `configId`s, task `configRowIds` and a schedule's `target.configurationId`; the clone result carries `link_remaps` per kind (see `sync push`). **`warnings[]` (since 0.96.1)**: the clone result (also `--dry-run`; human mode prints them) lists `missing_task_target` (a flow or orchestrator task runs a config that is not in the tree), `encrypted_values_copied` (the `KBC::` paths the target cannot decrypt, as `_config.yml` paths: `secret_keys` get a plaintext + `sync push`, `unencryptable_keys` need `kbagent encrypt values`, `oauth_keys` a new authorization), `data_app_not_deployed` (run `data-app deploy`) and `schedule_not_active` (clone never activates a schedule; `flow schedule` does, and the hint is left out when several schedules run one flow), plus the push warnings. Only the run that creates the configs reports them: a re-run returns `warnings: []`, so keep them from the first run. **Idempotent**: re-running with an existing `--target-dir` skips copy/overrides and just pushes, reporting `no_changes` / `created: 0`. Fails fast (`CONFIG_ERROR`) if the target already contains the reference's configs -- clone requires a fresh/empty target. `SyncService.clone_project(...)` returns a typed `CloneResult` for in-process SDK callers. Override files must be flat `{id: scalar}` mappings *(since v0.89.0)* -- a nested mapping, list, or null value is rejected with `CONFIG_ERROR` (exit 5) naming the key and its actual type. `--branch` is optional on a fresh clone *(since v0.93.1)*. It defaults to the target's production branch, resolved from the API the same way `sync init` does. Pass `--branch ` only to target a dev branch. The config folder (`KBC.configuration.folderName`) is recreated in the target *(since 0.94.0)* -- clone re-points its production configs onto the branch push resolves, so the create-path writeback carries the folder for a plain, `--branch`, and git-branching production clone. **Data-app runtime type (since 0.94.0)**: a `keboola.data-apps` config's type (`python-js` / `streamlit`) lives only on the Data Science `/apps` record, so `sync pull` records it in `_keboola.data_app_type` and clone sends it through `create_app`. A config with no recorded type is created as `python-js`, the default, with a `data_app_type_default` warning *(since 0.96.1)*. Re-pull a tree pulled by an older version before cloning a Streamlit app, or it is created as `python-js`. **Storage buckets (since 0.96.1)**: clone copies configs, not storage, so a cloned config's input/output mappings point at buckets a fresh target lacks. Clone reads the `storage/buckets.json` pull export and creates the missing buckets in the target **by default** (`--no-create-buckets` skips it) -- idempotent (an existing bucket is skipped, a per-bucket API failure is collected in `bucket_errors`), and the created id is `--bucket-map`-remapped so it matches the rewritten config refs. Each bucket is created on the backend the export recorded. A linked (shared) bucket is linked to the same source as in the reference, under the same id, and listed in `linked_buckets` with its source (an empty bucket in its place would stay empty). The source project's sharing settings decide whether the target may link it; a refused link lands in `bucket_errors`. A tree pulled by an older version does not record which buckets are linked, so clone creates no bucket from it and records one `bucket_errors` entry -- re-pull the reference, or pass `--no-create-buckets`. Only the buckets are created, never their tables or data (the export has no table data) -- populate tables by bucket sharing / `storage upload-table`, or by running the flows. **Since vNEXT (#745)** a clone with a non-empty `errors[]` or `bucket_errors[]` exits **1** and the human headline reads `Failed: ...` with the failed count -- a clone where every config failed used to print `Success ... 0 created` and exit 0. A re-run (`no_changes`) with a `bucket_errors[]` entry exits 1 too. -- `sync diff --project ALIAS [--all-projects] [--branch ID]` -- 3-way diff (local vs base vs remote), detects conflicts. `--branch` (0.47.0+) per-invocation dev-branch override. Branch-scoped since v0.89.0 (issue #649): the local side is read from exactly ONE tree (the target branch's subtree, or `main/` when the target has none). Manifest entries belonging to another branch's tree -- what `sync pull --branch ` leaves behind when it re-targets the manifest -- are excluded from the changeset and reported under `orphaned` (`summary.orphaned` + details with `component_id`, `config_id`, `path`, `branch_id`, `branch_path`, `exists_on_target`, `reason`, `hint`); human mode previews the first 10. An orphaned FILE whose `_keboola.config_id` still resolves on the target is adopted (diffed as `unchanged`/`modified`), never re-created; same-tree id claims keep the #482/#497 fork-by-copy CREATE. Fix a non-zero `summary.orphaned` with `sync pull`. **Since 0.91.0 (#686)** a manifest entry without `metadata.config_hash_version` (written by a pre-0.91.0 kbagent) is compared leniently: a stored hash equal to the pre-0.91.0 hash of the SAME remote config counts as in sync, so the phantom `codes changed` entries disappear immediately; every other field is still pinned by that hash, so real remote drift is unaffected. One `sync pull` per project stamps the version and ends the leniency. Ignored components (since 0.91.0) -- `keboola.sandboxes`, `keboola.mcp-server-tool`, and anything listed in the manifest's `ignoredComponents` -- are excluded from BOTH sides of the comparison, so a stale local directory for one of them never shows up as `DELETED`. Since 0.96.1 (#792) a config or row the manifest fetched from the target branch and that is missing on the remote is `remote_deleted` (human: `- REMOTE DELETED`, `summary.remote_deleted`): run `sync pull`, push never re-creates it. `DELETED` changes are applied only by `sync push --force`. **Since vNEXT (#745)** `--all-projects` exits **1** when `summary.failed > 0` (a project whose diff failed), like `sync pull --all-projects`. +- `sync pull --project ALIAS [--all-projects] [--force] [--theirs] [--dry-run] [--with-samples] [--no-storage] [--no-jobs] [--job-limit N] [--branch ID]` -- download configs to local files. **Auto-inits:** if the target directory has no `.keboola/manifest.json`, pull runs `init` first, so a separate `sync init` is not needed for a first checkout of a project. For large projects (>100 configs), automatically fetches jobs per-config when the grouped API limit is insufficient. `--force` is conflict-aware (since 0.53.0): a locally-modified config whose remote is unchanged is **preserved** (pending delta stays pushable, never silently re-stamped); a true merge conflict (local AND remote both changed since last pull) **aborts** the pull (exit 1, `SYNC_CONFLICT`; `--json` lists `details.conflicts`); local-untouched + remote-changed takes remote. `--theirs` (since v0.72.0) is the supported "discard local, take production" reconcile path: overwrites locally-modified configs/rows, restores deleted/missing files, resolves conflicts by taking remote (no abort, no manifest surgery). Since v0.72.0 plain pull also re-materializes a tracked config whose local dir was deleted (manifest<->disk invariant), so delete-dir-then-pull refetches. Config-level `isDisabled` is kept through pull and push (since v0.72.0) as sparse `is_disabled: true` in `_config.yml` -- absent key = enabled. `--branch` (0.47.0+) per-invocation dev-branch override, beats every other branch source. Ignored components (since 0.91.0): `keboola.sandboxes` + `keboola.mcp-server-tool` are always excluded (`keboola.sandboxes` shared SQL workspaces are synced when the manifest sets `syncWorkspaces`, *since 0.96.1*), unioned with the manifest's `ignoredComponents` list; a component newly ignored has its manifest entry dropped and local directory removed, reported with pull action `"ignored"` (distinct from `"removed"` = genuinely deleted on remote). Config-folder round-trip *(since 0.94.0)*: pull captures each config's UI folder (`KBC.configuration.folderName`, from the branch-only `search/component-configurations` endpoint) into the manifest, and reports `folder_lookup_failed` when that lookup fails, keeping the previously captured folder. **Since 0.98.0 (#745)** `--all-projects` exits **1** when `summary.failed > 0` (a project whose pull raised); `--dry-run` exits **1** on a failure too. +- `sync push --project ALIAS [--all-projects] [--dry-run] [--force] [--allow-plaintext-on-encrypt-failure] [--branch ID] [--no-name-drift-warnings]` -- push local changes (auto-encrypts secrets, fails if encryption fails). Workspace delete *(since 0.96.1)*: in a `syncWorkspaces` tree, `push --force` deleting a shared SQL workspace also deletes its SQL editor sessions (every user's, push branch) and their backend workspaces, which `config restore` does not bring back, so check `push --dry-run --force` (`warnings[]` `workspace_sessions`) first; `--force` is destructive-class (`sync.push --force`, blocked by `--deny-destructive`). Fresh-CREATE writeback updates placeholder manifest entries in place (since 0.47.0) and propagates any `KBC.configuration.*` metadata via `set_config_metadata`. Fresh-CREATE variable binding (since 0.47.2): when a `keboola.variables` config + its values row are created alongside a transformation in the same push, the transformation's `variables_id` / `variables_values_id` placeholders are rebound to the assigned ULIDs and the row's `values` are hoisted even without a `_keboola` block, so `job run` succeeds with no post-push `config variables-set` step (unresolvable/ambiguous links add a `variable_link` entry in `errors[]`, never a broken link). Since 0.96.1 push also remaps shared-code links, flow and orchestrator task `configId`s / `configRowIds` and schedule targets to the configs created in the same push. A link it cannot set is an `errors[]` entry with `change_type` `shared_code_link`, `flow_task_link` or `schedule_target_link` (error code `LINK_UNRESOLVED` for a row id without a new row, `API_ERROR` for a failed PUT). After a failed PUT the local files already hold the new ids and the manifest hash stays stale, so the next `sync push` sends them; this also applies to `variable_link`. The result carries `link_remaps` (`flow_tasks`, `orchestrator_tasks`, `schedule_targets`, `shared_code`, `config_row_ids`); `flow_task_remaps` counts flow tasks only. Never-fetched guard (since v0.72.0): a manifest entry with an empty `pull_hash` and no local files (pre-0.72 name-collision phantom) is **never** planned as a remote DELETE -- diff/push exclude it and report it under `never_fetched` with a warning (run `sync pull` to materialize); local deletion of a properly-pulled config deletes on `push --force`; since 0.96.1 (#792) a plain push deletes nothing and lists the deletion under `skipped_deletions` (also in `--dry-run`), and a config or row deleted on the remote since the last pull is `remote_deleted`, which push never re-creates. Adopted-by-id writeback (since v0.72.0): pushing an untracked file whose `_keboola.config_id` resolves on the branch also writes the manifest entry, so follow-up diffs are stable. `--branch` (0.47.0+) per-invocation override; when no `/` subtree exists on disk (since 0.47.2) the local default tree (`main/`) is promoted to the target branch (API writes still target the branch id); `--no-name-drift-warnings` (0.47.0+) drops the cosmetic warnings array. Branch-scoped since v0.89.0 (issue #649): push consumes the diff's changeset, so configs tracked on another branch's tree are never planned as creates -- they ride along on the result envelope under `orphaned` instead (see `sync diff`). **Since 0.91.0 (#686)** the manifest baseline `pull_config_hash` is stamped from the API response (or a read-back), not from the files on disk, so a pushed multi-statement SQL transformation -- or anything disabled in the UI whose local YAML lacks `is_disabled` -- no longer shows permanent phantom `REMOTE MODIFIED` drift; if the config cannot be read back after the write the baseline is left UNTOUCHED and a `warnings[]` entry says to run `sync pull` (never a disk-derived fallback). One legacy change is refused per-change with `SYNC_LEGACY_BOUNDARY`: a tree pulled before statement-boundary markers existed whose only difference from the remote is the lost boundaries (pushing it would collapse separate SQL statements into one) -- run `sync pull` for that project first. Ignored components (since 0.91.0) are filtered out on both sides of the diff push builds on, so a stale local directory for an ignored component (e.g. `keboola.mcp-server-tool`) is never classified as `DELETED` and can never be pushed as a remote deletion. **Since 0.98.0 (#745)** a push with a non-empty `errors[]` exits **1** and the human headline reads `Failed: Pushed: N created, ..., M failed` instead of a green `Success:`. `--all-projects` exits 1 when `summary.failed > 0`, and `summary.failed` now also counts a project whose push returned a non-empty `errors[]` (before, it counted as `success` and its line read `OK`; now the line is marked `x` and states the failed count). A `--dry-run` exits 1 on a failure too. `--json` output is still emitted before the non-zero exit. +- `sync clone --source DIR --target ALIAS --target-dir DIR [--bucket-map FILE] [--variable-values FILE] [--instance-rename FILE] [--no-create-buckets] [--dry-run] [--branch ID]` -- clone a reference synced project into a **fresh** target project and parameterize it. Copies the reference tree at `--source` into `--target-dir`, applies declarative overrides from JSON/YAML files (`--bucket-map` `{old_bucket_id: new_bucket_id}` rewrites storage input/output table refs; `--variable-values` `{var_name: value}` overrides `keboola.variables` rows; `--instance-rename` `{old_path_prefix: new_path_prefix}` renames config dirs + manifest paths), re-points the manifest at the target project, and pushes. Because the reference's config ids do not exist in the fresh target, every config is CREATEd fresh and **keboola.flow task `configId`s + transformation variable links are remapped reference->ULID** by push Phase C/D (the push result carries `flow_task_remaps`). Since 0.96.1 push also remaps shared-code links (`shared_code_id`, `shared_code_row_ids`, `{{}}` script placeholders), legacy `keboola.orchestrator` task `configId`s, task `configRowIds` and a schedule's `target.configurationId`; the clone result carries `link_remaps` per kind (see `sync push`). **`warnings[]` (since 0.96.1)**: the clone result (also `--dry-run`; human mode prints them) lists `missing_task_target` (a flow or orchestrator task runs a config that is not in the tree), `encrypted_values_copied` (the `KBC::` paths the target cannot decrypt, as `_config.yml` paths: `secret_keys` get a plaintext + `sync push`, `unencryptable_keys` need `kbagent encrypt values`, `oauth_keys` a new authorization), `data_app_not_deployed` (run `data-app deploy`) and `schedule_not_active` (clone never activates a schedule; `flow schedule` does, and the hint is left out when several schedules run one flow), plus the push warnings. Only the run that creates the configs reports them: a re-run returns `warnings: []`, so keep them from the first run. **Idempotent**: re-running with an existing `--target-dir` skips copy/overrides and just pushes, reporting `no_changes` / `created: 0`. Fails fast (`CONFIG_ERROR`) if the target already contains the reference's configs -- clone requires a fresh/empty target. `SyncService.clone_project(...)` returns a typed `CloneResult` for in-process SDK callers. Override files must be flat `{id: scalar}` mappings *(since v0.89.0)* -- a nested mapping, list, or null value is rejected with `CONFIG_ERROR` (exit 5) naming the key and its actual type. `--branch` is optional on a fresh clone *(since v0.93.1)*. It defaults to the target's production branch, resolved from the API the same way `sync init` does. Pass `--branch ` only to target a dev branch. The config folder (`KBC.configuration.folderName`) is recreated in the target *(since 0.94.0)* -- clone re-points its production configs onto the branch push resolves, so the create-path writeback carries the folder for a plain, `--branch`, and git-branching production clone. **Data-app runtime type (since 0.94.0)**: a `keboola.data-apps` config's type (`python-js` / `streamlit`) lives only on the Data Science `/apps` record, so `sync pull` records it in `_keboola.data_app_type` and clone sends it through `create_app`. A config with no recorded type is created as `python-js`, the default, with a `data_app_type_default` warning *(since 0.96.1)*. Re-pull a tree pulled by an older version before cloning a Streamlit app, or it is created as `python-js`. **Storage buckets (since 0.96.1)**: clone copies configs, not storage, so a cloned config's input/output mappings point at buckets a fresh target lacks. Clone reads the `storage/buckets.json` pull export and creates the missing buckets in the target **by default** (`--no-create-buckets` skips it) -- idempotent (an existing bucket is skipped, a per-bucket API failure is collected in `bucket_errors`), and the created id is `--bucket-map`-remapped so it matches the rewritten config refs. Each bucket is created on the backend the export recorded. A linked (shared) bucket is linked to the same source as in the reference, under the same id, and listed in `linked_buckets` with its source (an empty bucket in its place would stay empty). The source project's sharing settings decide whether the target may link it; a refused link lands in `bucket_errors`. A tree pulled by an older version does not record which buckets are linked, so clone creates no bucket from it and records one `bucket_errors` entry -- re-pull the reference, or pass `--no-create-buckets`. Only the buckets are created, never their tables or data (the export has no table data) -- populate tables by bucket sharing / `storage upload-table`, or by running the flows. **Since 0.98.0 (#745)** a clone with a non-empty `errors[]` or `bucket_errors[]` exits **1** and the human headline reads `Failed: ...` with the failed count -- a clone where every config failed used to print `Success ... 0 created` and exit 0. A re-run (`no_changes`) with a `bucket_errors[]` entry exits 1 too. +- `sync diff --project ALIAS [--all-projects] [--branch ID]` -- 3-way diff (local vs base vs remote), detects conflicts. `--branch` (0.47.0+) per-invocation dev-branch override. Branch-scoped since v0.89.0 (issue #649): the local side is read from exactly ONE tree (the target branch's subtree, or `main/` when the target has none). Manifest entries belonging to another branch's tree -- what `sync pull --branch ` leaves behind when it re-targets the manifest -- are excluded from the changeset and reported under `orphaned` (`summary.orphaned` + details with `component_id`, `config_id`, `path`, `branch_id`, `branch_path`, `exists_on_target`, `reason`, `hint`); human mode previews the first 10. An orphaned FILE whose `_keboola.config_id` still resolves on the target is adopted (diffed as `unchanged`/`modified`), never re-created; same-tree id claims keep the #482/#497 fork-by-copy CREATE. Fix a non-zero `summary.orphaned` with `sync pull`. **Since 0.91.0 (#686)** a manifest entry without `metadata.config_hash_version` (written by a pre-0.91.0 kbagent) is compared leniently: a stored hash equal to the pre-0.91.0 hash of the SAME remote config counts as in sync, so the phantom `codes changed` entries disappear immediately; every other field is still pinned by that hash, so real remote drift is unaffected. One `sync pull` per project stamps the version and ends the leniency. Ignored components (since 0.91.0) -- `keboola.sandboxes`, `keboola.mcp-server-tool`, and anything listed in the manifest's `ignoredComponents` -- are excluded from BOTH sides of the comparison, so a stale local directory for one of them never shows up as `DELETED`. Since 0.96.1 (#792) a config or row the manifest fetched from the target branch and that is missing on the remote is `remote_deleted` (human: `- REMOTE DELETED`, `summary.remote_deleted`): run `sync pull`, push never re-creates it. `DELETED` changes are applied only by `sync push --force`. **Since 0.98.0 (#745)** `--all-projects` exits **1** when `summary.failed > 0` (a project whose diff failed), like `sync pull --all-projects`. - `sync status [--directory DIR]` -- show locally modified/added/deleted configs. Also surfaces `plaintext_secret_warnings` (since 0.55.0): in-sync configs/rows whose `#`-secrets are still plaintext on the remote (a leftover from pre-0.54.0 writes; #378). Pending (un-pushed) edits are not flagged. Fix = re-push on >=0.54.0 + rotate (version history keeps the plaintext). - `sync branch-link --project ALIAS [--branch-id ID] [--branch-name NAME]` -- link git branch to Keboola dev branch - `sync branch-unlink [--directory DIR]` -- remove git-to-Keboola branch mapping @@ -423,8 +423,8 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer add dataset --project P [--model M] --name N --table-id TABLE_ID [--description D] [--grain G] [--primary-key COL ...] [--deep-fields] [--fqn FQN]` -- add a dataset. *(since 0.95.0, #761)* The `fqn` is the table's warehouse location read from Storage (the owning bucket's `backendPath`, i.e. `storage table-detail` -> `sql_path`), so the table must exist; a linked bucket's path names the SOURCE project's database and schema. `--fqn` stores the given value verbatim and skips the lookup. Before 0.95.0 the fqn hardcoded a `"KEBOOLA"` database that does not resolve. `--primary-key` is repeatable for composite PKs. `--deep-fields` fetches the storage schema and synthesises role-classified `fields[]`: PK_/FK_->`key`, *_DATE/*_DT->`timestamp`, numeric amount/value/rate->`measure`, else `dimension`. - `semantic-layer add relationship --project P [--model M] --name N --from TABLE_ID --to TABLE_ID --on EXPR [--type left|inner]` -- add a join relationship. `--type` defaults to `left`. - `semantic-layer add constraint --project P [--model M] --name N --constraint-type T --rule "EXPR" --metrics M1,M2 [--severity error|warning|info]` -- add a constraint. `--constraint-type` is the closed enum `inequality|equality|range|composition|exclusion|temporal|conditional`. `--rule` is a **STRING expression** (e.g. `"value >= 0"`), NEVER a `{bounds: {min, max}}` object (sl-builder docs are wrong -- see [gotchas.md](gotchas.md)). `--metrics` is a comma-separated list of metric names that must already exist in the model. `--severity` defaults to `warning`. Name regex `^[a-z][a-z0-9_]*$`; the 4-band health convention lives in the name suffix `_critical / _warning / _healthy / _review`, distinct from the 3-value API `severity`. -- `semantic-layer add glossary --project P [--model M] --term TERM --definition D` -- add a glossary term. `--definition` is required *(since vNEXT)*: every `semantic-glossary` schema requires it, and without it the metastore answered a bare 422 "Validation failed". -- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. Updates the metric in place (`PUT`: same id, scope and grants kept). Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. **Since vNEXT (#745)** a failed cascade entry exits **1** (was 0). +- `semantic-layer add glossary --project P [--model M] --term TERM --definition D` -- add a glossary term. `--definition` is required *(since 0.98.0)*: every `semantic-glossary` schema requires it, and without it the metastore answered a bare 422 "Validation failed". +- `semantic-layer edit metric --project P [--model M] --name N [--new-name N2] [--new-sql SQL] [--new-dataset TABLE_ID] [--new-description D] [--yes]` -- edit a metric. Updates the metric in place (`PUT`: same id, scope and grants kept). Rename CASCADES through every constraint whose `metrics[]` includes the old name (DELETE old constraint + POST new with updated `metrics[]`). Prints the old/new CODE_METRIC computed via `re.sub(r"[^A-Z0-9]+", "_", name.upper()).strip("_")`. `--yes` skips the confirm prompt. On POST failure the service re-POSTs `original_attrs` and reports rollback success/failure explicitly in the envelope. **Partial-state envelope**: when metric rename succeeds but one or more dependent constraints fail to repoint, the response sets `partial_state: true` and `recovery_hint: ""` at the envelope's top level (was previously buried in `cascaded_constraints[i].status == 'failed'`). Human-mode CLI prints a bright red `PARTIAL STATE` banner above the per-entry list. Recovery recipe: `kbagent semantic-layer validate` to surface the dangling refs, then re-run each failed cascade via `edit constraint --new-metrics ...`. **Since 0.98.0 (#745)** a failed cascade entry exits **1** (was 0). - `semantic-layer edit dataset --project P [--model M] --name N [--new-name N2] [--new-description D] [--new-grain G]` -- edit a dataset. No cascade -- metrics reference the dataset's tableId, not its name. - `semantic-layer edit constraint --project P [--model M] --name N [--new-name N2] [--new-rule "EXPR"] [--new-constraint-type T] [--new-severity error|warning|info] [--new-metrics M1,M2]` -- edit a constraint (in-place `PUT`). Local validators enforce the name regex, constraintType enum, severity enum, and that every entry in `--new-metrics` exists in the model. - `semantic-layer edit relationship --project P [--model M] --name N [--new-name N2] [--new-from TABLE_ID] [--new-to TABLE_ID] [--new-on EXPR] [--new-type left|inner]` -- edit a relationship (in-place `PUT`). No constraint cascade. Rollback on POST failure. @@ -434,14 +434,14 @@ Manage Keboola metastore models -- datasets, metrics, relationships, constraints - `semantic-layer remove constraint --project P [--model M] --name N [--yes]` -- destructive. Confirmation prompt unless `--yes`. - `semantic-layer remove relationship --project P [--model M] --name N [--yes]` -- destructive. Relationships aren't referenced by other entities; no orphan-check. - `semantic-layer remove glossary --project P [--model M] --term TERM [--yes]` -- destructive. Glossary entries aren't referenced by other entities; no orphan-check. -- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] [--scope project|organization|targeted] [--target-project ALIAS|ID ...]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into an in-place update of conflicting items (they keep their scope). `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. **Since vNEXT (#745)** a non-empty `failed[]` in any type exits **1** (was 0); `--dry-run` exits **1** on a failure too. -- `semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes] [--scope project|organization|targeted] [--target-project ALIAS|ID ...]` -- cross-project model copy with `modelUUID` rewrite to the target model's UUID. Classifies items NEW / IDENTICAL / CHANGED (deep-equality after stripping `modelUUID` + timestamps). Additive + overwrite only: NEVER deletes target items absent from source. Holds two MetastoreClients in try/finally. Response: per-type counts + `changes[]` with `diff_keys` and `failed[]`. **Since vNEXT (#745)** a non-empty `failed[]` in any type exits **1** (was 0); `--dry-run` exits **1** on a failure too. -- `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN = the table's Storage location, *(since 0.95.0)* no longer a hardcoded `"KEBOOLA"` database; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). **Rollback on push failure**: every successfully-POSTed child is DELETEd in reverse PUSH_ORDER, and the model itself is DELETEd if we created it during this call. The wrapped `KeboolaApiError` carries `details.rollback={attempted, posted_children, deleted, failed_deletes, model_created_here, model_deleted, model_uuid}` so operators get full diagnostics. Pass `--keep-on-failure` to preserve the partial state for forensic inspection (mirrors `data-app create --keep-on-failure`); the wrapped error then carries `details.rollback.attempted=False, reason='keep_on_failure'`. **Since vNEXT (#745)** a table whose schema could not be read is left out of the model and listed in `fetch_errors[]`: human mode now lists it under `Failed: N table(s) left out of the model`, and the exit is **1** (was 0, and the table was not shown); `--dry-run` exits **1** on a failure too. `type_resolution_errors[]` stays a warning (exit 0). +- `semantic-layer import --project P --file PATH [--model M] [--types T,T,...] [--dry-run] [--yes] [--overwrite] [--scope project|organization|targeted] [--target-project ALIAS|ID ...]` -- replay a snapshot. Default: SKIP on conflict (no surprise overwrites). `--overwrite` opts into an in-place update of conflicting items (they keep their scope). `--types` filters to a subset (`datasets,metrics,relationships,glossary,constraints`). Dependency-ordered push: datasets -> metrics -> relationships -> glossary -> constraints. Response: `imported: {: {created, skipped, overwritten, failed: [{name, reason}]}}`. **Since 0.98.0 (#745)** a non-empty `failed[]` in any type exits **1** (was 0); `--dry-run` exits **1** on a failure too. +- `semantic-layer promote --from-project A --to-project B [--from-model M] [--to-model M] [--types T,T,...] [--dry-run] [--yes] [--scope project|organization|targeted] [--target-project ALIAS|ID ...]` -- cross-project model copy with `modelUUID` rewrite to the target model's UUID. Classifies items NEW / IDENTICAL / CHANGED (deep-equality after stripping `modelUUID` + timestamps). Additive + overwrite only: NEVER deletes target items absent from source. Holds two MetastoreClients in try/finally. Response: per-type counts + `changes[]` with `diff_keys` and `failed[]`. **Since 0.98.0 (#745)** a non-empty `failed[]` in any type exits **1** (was 0); `--dry-run` exits **1** on a failure too. +- `semantic-layer build --project P [--model M] --tables T,T,... [--name N] [--dry-run] [--keep-on-failure] [--output PATH]` -- non-interactive heuristic builder. **AI caveat**: the existing `ai_client` has no arbitrary-JSON endpoint, so `build` falls back to a deterministic heuristic synthesising one dataset + one COUNT(*) metric + one glossary entry per table (FQN = the table's Storage location, *(since 0.95.0)* no longer a hardcoded `"KEBOOLA"` database; fields[] role-classified). Response carries `fallback_used: "heuristic"`. The push loop walks all 5 child types in dependency order -- this **fixes** the `sl-build` skill bug where `semantic-constraint` was silently dropped. `--model` omitted creates a new model (default name `kbagent_build_model` or `--name N`). **Rollback on push failure**: every successfully-POSTed child is DELETEd in reverse PUSH_ORDER, and the model itself is DELETEd if we created it during this call. The wrapped `KeboolaApiError` carries `details.rollback={attempted, posted_children, deleted, failed_deletes, model_created_here, model_deleted, model_uuid}` so operators get full diagnostics. Pass `--keep-on-failure` to preserve the partial state for forensic inspection (mirrors `data-app create --keep-on-failure`); the wrapped error then carries `details.rollback.attempted=False, reason='keep_on_failure'`. **Since 0.98.0 (#745)** a table whose schema could not be read is left out of the model and listed in `fetch_errors[]`: human mode now lists it under `Failed: N table(s) left out of the model`, and the exit is **1** (was 0, and the table was not shown); `--dry-run` exits **1** on a failure too. `type_resolution_errors[]` stays a warning (exit 0). - `semantic-layer token --encrypt --project P --component-id C` -- encrypt the project's storage token for a transformation's `user_properties`. Builds `{"#metastore_token": }` from the project's already-stored Storage API token and delegates to the existing EncryptService. `--encrypt` is currently required; other modes are refused with `USAGE_ERROR` (exit 2). Output (human): the raw envelope ready to paste. JSON: full `{encrypted, component_id, project}`. ### Scope / target-project grants / elevation (`scope` sub-app, PSGO-140) -*(since 0.97.0)* Every semantic-layer item carries a visibility `scope`: `project` (owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above. With `--scope` omitted, `model create` makes a `project` model and `add ` **inherits its model's scope and target projects**. *(since vNEXT)* `import`, `promote` and `build --model` create their new items at the target model's scope too (overwritten items keep theirs); `import` and `promote` also take `--scope` / `--target-project` to override it, like `add `. Before, they always created `project` items, so an import into a shared model was invisible to the other projects. `--target-project` takes a registered alias or a numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack) and needs `--scope targeted` (else exit 2). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other projects on the stack; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2). **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule; `--scope organization` is irreversible and gated as destructive. `edit`, `import --overwrite` and `promote` update the item in place (PUT), so an organization/targeted item keeps its scope, grants, id and pending elevation request. +*(since 0.97.0)* Every semantic-layer item carries a visibility `scope`: `project` (owner project only), `organization` (every project in the org), or `targeted` (owner + an explicit target-project grant list). `--scope`/`--target-project` are accepted by `model create` and every `add ` command above. With `--scope` omitted, `model create` makes a `project` model and `add ` **inherits its model's scope and target projects**. *(since 0.98.0)* `import`, `promote` and `build --model` create their new items at the target model's scope too (overwritten items keep theirs); `import` and `promote` also take `--scope` / `--target-project` to override it, like `add `. Before, they always created `project` items, so an import into a shared model was invisible to the other projects. `--target-project` takes a registered alias or a numeric project ID (repeatable or comma-separated; an alias must be on the owner's stack) and needs `--scope targeted` (else exit 2). With `--scope targeted` and no `--target-project`: on a real terminal an interactive checkbox picker runs over the other projects on the stack; in `--json`/non-interactive mode it fails fast with `INVALID_ARGUMENT` (exit 2). **Always ask the user which project(s) before passing `--scope organization|targeted`** -- see `keboola-expert.md`'s scope rule; `--scope organization` is irreversible and gated as destructive. `edit`, `import --overwrite` and `promote` update the item in place (PUT), so an organization/targeted item keeps its scope, grants, id and pending elevation request. - `semantic-layer scope get --project P --type model|dataset|metric|relationship|constraint|glossary --context-id ID` -- show an item's current `scope`, `target_project_ids`, and any pending `scope_elevation_requested_at`. - `semantic-layer scope add|remove --project P --type T --context-id ID --target-project ALIAS|ID [--target-project ...]` -- attach / detach target projects of a `--scope targeted` item (alias or numeric ID; repeatable or comma-separated). A client-side **merge** (read the grants, apply the delta, PUT) -- **not atomic** against a concurrent grant change. Refused (exit 2) from a project that does not own the item -- the server hides the grants from a non-owner, so use `scope set --target-project` there. diff --git a/plugins/kbagent/skills/kbagent/references/gotchas.md b/plugins/kbagent/skills/kbagent/references/gotchas.md index 34a5460d..c9149ba7 100644 --- a/plugins/kbagent/skills/kbagent/references/gotchas.md +++ b/plugins/kbagent/skills/kbagent/references/gotchas.md @@ -2597,7 +2597,7 @@ config, the retry fires, and the retry destroys it for good. The `rows[]` array in the result is in completion order, not CSV order. - Per-row parsing of `failed_rows` should match by `email`, not by index. - A failed row never aborts the run -- the executor accumulates results and - reports `failed > 0` in the JSON summary. *(since vNEXT)* The command then + reports `failed > 0` in the JSON summary. *(since 0.98.0)* The command then exits 1, like `org setup`; on 0.97.0 and older it exited 0 even then, so check `failed` there. The full JSON payload is emitted before the exit. @@ -3542,7 +3542,7 @@ write descriptive metadata onto storage objects. Three behaviors are easy to mis collected into `result.errors[]` but the batch keeps processing the remaining items. The CLI exits non-zero only if `error_count > 0`, so in scripts always inspect `errors[]` (or at least `error_count`) rather than relying solely on - the exit code. *(since vNEXT)* `--json` also exits 1 when `error_count > 0`; + the exit code. *(since 0.98.0)* `--json` also exits 1 when `error_count > 0`; on 0.97.0 and older it exited 0 even then, so there never trust a zero-exit as "everything applied." That tolerance covers **API** failures only: a `--from-file` whose shape is wrong (a `tables:` / `buckets:` / `columns:` @@ -5803,7 +5803,7 @@ examples. Surprises worth knowing before you touch this: a typed `--scope organization` (`--deny-destructive` blocks it), and a project-admin token that is not an org admin gets a 403 on it, with a hint to pass `--scope project`. -- **`import`, `promote` and `build --model` follow the same rule (since vNEXT).** +- **`import`, `promote` and `build --model` follow the same rule (since 0.98.0).** Their NEW items take the target model's scope and target projects, with the same permission gate. Before, they always created `project` items: an import into a `targeted` or `organization` model gave items that no other @@ -5811,22 +5811,22 @@ examples. Surprises worth knowing before you touch this: `promote` take `--scope` / `--target-project` to override the inherited scope, like `add `: a project-admin token that is not an org admin passes `--scope project` under an `organization` model, or every new item - fails with a 403 (listed under `failed`; since vNEXT the command exits 1, + fails with a 403 (listed under `failed`; since 0.98.0 the command exits 1, on 0.97.0 and older it exited 0). -- **Item names are unique per project, not per model (since vNEXT, message).** +- **Item names are unique per project, not per model (since 0.98.0, message).** The metastore keeps names unique per object type across ALL models of a project (and across the organization at `organization` scope). A second model in the same project cannot reuse an item name: `ALREADY_EXISTS` now says "in this project" (it used to say "in the target model"). To copy a model within a project, delete the old one first, or rename the items. -- **An item stored at schema version `1.0.0` cannot be elevated (since vNEXT, +- **An item stored at schema version `1.0.0` cannot be elevated (since 0.98.0, message).** Items created before 0.97.0 pinned `1.0.0`. `scope request-create` and `scope set --scope organization` on them fail with "scope not supported"; the error now names the stored version and the fix: `export`, `model delete`, `model create`, `import`. A `project` item also cannot become `targeted` (grants exist only on items created `targeted`), so the same steps apply to share it with chosen projects. -- **`add glossary` needs `--definition` (since vNEXT).** Every +- **`add glossary` needs `--definition` (since 0.98.0).** Every `semantic-glossary` schema requires `definition`. Without it the metastore answered a bare 422 "Validation failed"; now it is a usage error (exit 2). - **`--target-project` takes an alias or a numeric project ID** (repeatable or @@ -5879,7 +5879,7 @@ examples. Surprises worth knowing before you touch this: ## A partial failure is no longer reported as success -*(since vNEXT, closes #745)* A command that keeps going after one item fails +*(since 0.98.0, closes #745)* A command that keeps going after one item fails used to print a green `Success:` line and exit **0**, with the failures only as warnings below it. A `sync clone` where every config failed reported `Success ... 0 created` and exit 0. A script could not tell a clean run from a @@ -5958,7 +5958,7 @@ On 0.97.0 and older, exit 0 from these commands does not prove success. Parse ## Uploads above 64 MiB on AWS are multipart -- the 5 GiB ceiling is gone -*(since vNEXT, #834)* On an AWS stack (S3 file storage), `storage upload-table`, +*(since 0.98.0, #834)* On an AWS stack (S3 file storage), `storage upload-table`, `storage file-upload` and the SDK `Client.upload_table` used to read the whole file into memory and send it as ONE `PutObject`. That capped every upload at 5 GiB (the S3 single-PUT limit) and needed RAM the size of the file. @@ -5991,7 +5991,7 @@ What did NOT change, and the limits that remain: ## `upload-table` / `load-file --no-wait` and `storage job-detail`: never re-run an upload whose import is still running -*(since vNEXT, #834)* A table upload is two phases: the cloud upload, then an +*(since 0.98.0, #834)* A table upload is two phases: the cloud upload, then an asynchronous Storage **import job**. For a big file the import can outlast any reasonable wait. Both `storage upload-table` and `storage load-file` now take `--wait/--no-wait` (default wait) and `--timeout SECONDS` (import wait budget, @@ -6026,7 +6026,7 @@ on any of this. See `large-upload-workflow.md` for the full procedure. ## `--progress` on storage uploads and downloads: progress on stderr, also with `--json` and in CI -*(since vNEXT)* `storage upload-table`, `storage file-upload`, +*(since 0.98.0)* `storage upload-table`, `storage file-upload`, `storage download-table`, `storage file-download` and `storage unload-table --download` take `--progress`. A multi-hour transfer run by an agent or in CI used to be silent until it finished: the progress bar only diff --git a/plugins/kbagent/skills/kbagent/references/large-upload-workflow.md b/plugins/kbagent/skills/kbagent/references/large-upload-workflow.md index 6bd7aee3..169cd74c 100644 --- a/plugins/kbagent/skills/kbagent/references/large-upload-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/large-upload-workflow.md @@ -2,7 +2,7 @@ Loading one very large CSV (100+ GB, optionally `.csv.gz`) into ONE Storage table. Everything here needs the version that added S3 multipart upload, -`--no-wait` and `storage job-detail` *(since vNEXT, #834)* -- check +`--no-wait` and `storage job-detail` *(since 0.98.0, #834)* -- check `kbagent version` first. On older versions an AWS upload stops at 5 GiB and there is no way to follow the import job. @@ -87,7 +87,7 @@ FILE_ID=$(jq -r '.data.file_id' upload.json) handles for following and recovering the import. - In human mode (no `--json`) a progress bar on stderr shows the upload, and the command prints the job id plus the follow-up command. -- **Use `--progress` for long runs** *(since vNEXT)*. It reports progress on +- **Use `--progress` for long runs** *(since 0.98.0)*. It reports progress on stderr even with `--json` and without a terminal, so stdout stays clean JSON and `progress.log` gets one line every 10 s plus a final line: diff --git a/plugins/kbagent/skills/kbagent/references/member-workflow.md b/plugins/kbagent/skills/kbagent/references/member-workflow.md index af674824..ff55a312 100644 --- a/plugins/kbagent/skills/kbagent/references/member-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/member-workflow.md @@ -88,7 +88,7 @@ Result schema: The `rows[]` array is in **completion order**, not CSV order (parallel workers). Match by `email`, not by index. A run with `failed > 0` exits 1 -*(since vNEXT)*, like `org setup`; the JSON is emitted first. On 0.97.0 and +*(since 0.98.0)*, like `org setup`; the JSON is emitted first. On 0.97.0 and older it exited 0, so check `failed` there. `--dry-run` resolves every row and reports what *would* happen without diff --git a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md index a1a61300..c95ecf9a 100644 --- a/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/metastore-scope-workflow.md @@ -16,7 +16,7 @@ constraint, glossary term) carries a visibility **scope**: `add ` without `--scope` **inherits its model's scope** (and target projects), so an org-level model gets org-level children. Pass `--scope` to override (a project-admin token that is not an org admin must pass -`--scope project` under an org-level model, or it gets a 403). *(since vNEXT)* +`--scope project` under an org-level model, or it gets a 403). *(since 0.98.0)* `import`, `promote` and `build --model` create their new items at the target model's scope too; items they overwrite keep their own scope. `import` and `promote` take `--scope` / `--target-project` to override it, like `add`. diff --git a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md index afa88267..0665eccf 100644 --- a/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/semantic-layer-workflow.md @@ -255,7 +255,7 @@ updated independently (an in-place PUT, no rollback needed). If the metric rename succeeds but M of N dependent constraints fail to repoint, the envelope sets `partial_state: true` at the top level and a `recovery_hint` string. Human-mode CLI prints a bright red -`PARTIAL STATE` banner, and *(since vNEXT)* the command exits 1. Use the +`PARTIAL STATE` banner, and *(since 0.98.0)* the command exits 1. Use the recovery recipe: ```bash diff --git a/plugins/kbagent/skills/kbagent/references/storage-files-workflow.md b/plugins/kbagent/skills/kbagent/references/storage-files-workflow.md index f9b37615..453eb70c 100644 --- a/plugins/kbagent/skills/kbagent/references/storage-files-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/storage-files-workflow.md @@ -95,7 +95,7 @@ no local upload step needed. The table must already exist. `load-file` is also the recovery path after a failed `upload-table` import: the file is already uploaded, so re-import it by `--file-id` instead of uploading -again. *(since vNEXT)* Both `upload-table` and `load-file` take `--no-wait` +again. *(since 0.98.0)* Both `upload-table` and `load-file` take `--no-wait` (return the import `job_id` at once) and `--timeout SECONDS`; follow the job with `storage job-detail --job-id ID --wait`. For a 100+ GB CSV / `.csv.gz`, see [large-upload-workflow](large-upload-workflow.md). diff --git a/plugins/kbagent/skills/kbagent/references/sync-workflow.md b/plugins/kbagent/skills/kbagent/references/sync-workflow.md index 536bb9e6..41a5775f 100644 --- a/plugins/kbagent/skills/kbagent/references/sync-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/sync-workflow.md @@ -67,7 +67,7 @@ kbagent sync push --all-projects # apply Each project gets its own subdirectory (named by alias). Projects are processed in parallel. -A project that fails does not stop the others. *(since vNEXT)* `sync pull --all-projects` and `sync push --all-projects` then exit 1, and push counts a project whose push returned per-config `errors[]` as failed (its line is marked `x`, not `OK`). `sync diff --all-projects` exits 1 too when a project failed (`summary.failed`). A `--dry-run` exits 1 when it reports a failed project, like the real run. +A project that fails does not stop the others. *(since 0.98.0)* `sync pull --all-projects` and `sync push --all-projects` then exit 1, and push counts a project whose push returned per-config `errors[]` as failed (its line is marked `x`, not `OK`). `sync diff --all-projects` exits 1 too when a project failed (`summary.failed`). A `--dry-run` exits 1 when it reports a failed project, like the real run. ## Reconciling a drifted tree with production (since v0.72.0) diff --git a/plugins/kbagent/skills/kbagent/references/workspace-workflow.md b/plugins/kbagent/skills/kbagent/references/workspace-workflow.md index b731bbad..25e8b939 100644 --- a/plugins/kbagent/skills/kbagent/references/workspace-workflow.md +++ b/plugins/kbagent/skills/kbagent/references/workspace-workflow.md @@ -309,7 +309,7 @@ Behavior: - **`workspace gc`** deletes each orphaned workspace one by one. Per-workspace failures accumulate into `errors[]` without stopping the batch -- one locked sandbox does not prevent the rest from being cleaned up. - *(since vNEXT)* A run with a non-empty `errors[]` (a failed delete, or a + *(since 0.98.0)* A run with a non-empty `errors[]` (a failed delete, or a project that could not be listed) exits 1, also under `--dry-run`. - **`--dry-run`** surfaces the would-be-deleted list via `data.would_delete[]` in JSON mode and a Rich table in human mode. diff --git a/pyproject.toml b/pyproject.toml index 870b2e1d..6a6acda7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "keboola-cli" -version = "0.97.0" +version = "0.98.0" description = "AI-friendly CLI for managing Keboola projects" readme = "README.md" requires-python = ">=3.12" diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 3e7e0a53..04c89271 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -26,6 +26,44 @@ # Ordered newest-first. Each value is a list of brief one-line descriptions. CHANGELOG: dict[str, list[str]] = { + "0.98.0": [ + "New (#837): `storage upload-table` and `file-upload` now upload large files to AWS " + "stacks in parts, so files over 5 GB work and memory stays flat. Files above 64 MiB are " + "sent as an S3 multipart upload: 64 MiB parts, four in parallel, each part retried on " + "its own. Before, kbagent read the whole file into memory and S3 refused anything over " + "5 GB. In a test, a 10 GB file that 0.97.0 could not upload (it used 10 GB of memory and " + "then failed) uploaded in about four minutes with under 350 MB of memory. A `.csv.gz` " + "file is uploaded as it is. Azure and GCP stacks are unchanged.", + "New (#837): `storage upload-table` and `load-file` can queue the import without waiting " + "for it, and the new `storage job-detail` command follows the import. `--no-wait` " + "returns `job_id` and `file_id` as soon as the import is queued; `--timeout` sets how " + "long the default `--wait` waits. `storage job-detail --job-id ID [--wait]` shows the " + "status, the table and the imported rows. When the wait times out, the error says the " + "import keeps running, names the job, and is no longer marked retryable, so a long " + "upload is not repeated by mistake. `kbagent serve` (`GET /storage/jobs/{project}/" + "{job_id}`) and the SDK (`Client.storage_job()`) have the same functions.", + "New (#837): `--progress` shows percent, speed, elapsed time and the estimated time left " + "for `storage upload-table`, `file-upload`, `download-table`, `file-download` and " + "`unload-table --download`. It writes to stderr, also with `--json`: a progress bar in " + "a terminal, otherwise one line every 10 seconds.", + "Fix (#837): `storage download-table --keep-slices`, `file-download` and `unload-table` " + "now keep every slice file inside the output directory. A slice name from the export manifest that " + "could point outside it (`..`, a backslash, a drive letter) is replaced by " + "`part-NNNNN`.", + "Fix (#747): commands that process several items now exit 1 and print `Failed:` when " + "an item fails. Before, `sync push`, `sync clone`, `org setup`, `project refresh`, " + "`workspace gc`, several `semantic-layer` commands, `storage describe-batch --json`, " + "`flow schedule-remove` and others printed `Success:` and exited 0 even when items " + "failed. A `--dry-run` that reports a failed item exits 1 too. The `--json` output " + "keeps its keys, with a few additions such as `errors[]` in `flow schedule-remove`.", + "Fix (#836): `semantic-layer import`, `promote` and `build --model` now give new items " + "the scope of the target model. Before, they created every item at `project` scope, " + "also in a shared model, so the other projects could not see it. Items they overwrite " + "keep their own scope. Duplicate-name and elevation errors now say what is wrong and " + "how to fix it.", + "Change (#836): `semantic-layer add glossary` now requires `--definition`. Without it the " + "Metastore always rejected the item with a bare 422 error, so no working call breaks.", + ], "0.97.0": [ "New (#715): semantic-layer items can now be shared with chosen projects or with the " "whole organization, not only kept in the owning project (AI-3790). `model create` and " diff --git a/src/keboola_agent_cli/commands/context.py b/src/keboola_agent_cli/commands/context.py index f596c7f7..17fbca62 100644 --- a/src/keboola_agent_cli/commands/context.py +++ b/src/keboola_agent_cli/commands/context.py @@ -769,12 +769,12 @@ as STRING from the CSV header; read through gzip for .csv.gz). Use --no-auto-create to require the table to exist. Full load by default; --incremental to append rows. Branch-aware. Two phases: cloud upload, then an async Storage import job. - Cloud upload (since vNEXT): on AWS stacks a file above 64 MiB goes up as an S3 MULTIPART upload (64 MiB parts, + Cloud upload (since 0.98.0): on AWS stacks a file above 64 MiB goes up as an S3 MULTIPART upload (64 MiB parts, auto-scaled to fit 10,000 parts, 4 in parallel, per-part retry; peak memory ~4 x part size). The old 5 GiB single-PUT ceiling is gone. Azure/GCP stacks are unchanged. Human mode shows a progress bar on stderr. The S3 credentials last 12 h (a 200 GB file needs ~5 MB/s sustained); the source file must not change during the upload (error if size/mtime moves); a failed upload restarts from zero (no resume). - Import (since vNEXT): --wait (default) waits up to --timeout SECONDS (default 600) for the import job. + Import (since 0.98.0): --wait (default) waits up to --timeout SECONDS (default 600) for the import job. --no-wait still waits for the upload, then enqueues the import and returns at once with file_id, job_id, job_status (imported_rows is null while pending) -- poll with `storage job-detail --job-id ID --wait`. A wait timeout is STORAGE_JOB_TIMEOUT (exit 4, retryable=false): the import KEEPS RUNNING server-side; the @@ -782,7 +782,7 @@ re-run duplicates rows. If the import fails or its enqueue fails after the upload, re-import the already uploaded file with `storage load-file --file-id ID` instead of uploading again. For a 100+ GB file use --no-wait and poll (see the large-upload workflow in the kbagent skill). - --progress (since vNEXT): ALWAYS report progress on stderr, also with --json and without a terminal: a bar on + --progress (since 0.98.0): ALWAYS report progress on stderr, also with --json and without a terminal: a bar on a terminal, else one line every 10 s + a final line, e.g. `upload big.csv: 42.0% 4.20/10.00 GiB, 67.30 MiB/s, elapsed 0:01:03, ETA 0:01:27`. stdout stays clean JSON. For long runs: `kbagent --json storage upload-table ... --progress 2>progress.log`, then read the log. @@ -893,7 +893,7 @@ kbagent storage file-upload --project NAME --file PATH [--name NAME] [--tag TAG ...] [--permanent] [--branch ID] [--progress] Upload any file to Storage Files. --tag assigns tags (repeatable). --permanent prevents auto-deletion after 15 days. --name overrides the filename (default: local filename). Branch-aware. --progress as on upload-table. - Since vNEXT: same S3 multipart upload as upload-table on AWS stacks (above 64 MiB; no 5 GiB ceiling). + Since 0.98.0: same S3 multipart upload as upload-table on AWS stacks (above 64 MiB; no 5 GiB ceiling). kbagent storage file-download --project NAME [--file-id ID | --tag TAG ...] [--output FILE] [--progress] Download a Storage File. Either --file-id (by ID) or --tag (latest file matching all tags). @@ -912,11 +912,11 @@ kbagent storage load-file --project NAME --file-id ID --table-id TABLE_ID [--incremental] [--delimiter D] [--enclosure E] [--wait/--no-wait] [--timeout SECONDS] [--branch ID] Import an already-uploaded Storage File into a table. Useful for files uploaded by components or file-upload, and the recovery path after a failed upload-table import (no re-upload). --incremental to append rows. - Since vNEXT: --wait/--no-wait + --timeout SECONDS (default 600) behave exactly as on upload-table + Since 0.98.0: --wait/--no-wait + --timeout SECONDS (default 600) behave exactly as on upload-table (--no-wait returns job_id; a wait timeout is STORAGE_JOB_TIMEOUT, exit 4, the import keeps running). Branch-aware. kbagent storage job-detail --project NAME --job-id ID [--wait] [--timeout SECONDS] - (since vNEXT) Show one Storage API job: status, operation_name, table_id, file_id, created/start/end times, + (since 0.98.0) Show one Storage API job: status, operation_name, table_id, file_id, created/start/end times, imported_rows, warnings, results, error. --wait polls until the job finishes (--timeout budget). Exit 0 for success/waiting/processing; exit 1 STORAGE_JOB_FAILED when the job ended in error (details are still returned); exit 4 on a --wait timeout. Job ids are project-scoped (no --branch; the active branch does diff --git a/uv.lock b/uv.lock index 5062fd4e..052b3eb4 100644 --- a/uv.lock +++ b/uv.lock @@ -465,7 +465,7 @@ wheels = [ [[package]] name = "keboola-cli" -version = "0.97.0" +version = "0.98.0" source = { editable = "." } dependencies = [ { name = "croniter" }, From b96054b069281f507b98d3c22c7e1316036390c6 Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 9 Oct 2026 21:39:22 +0200 Subject: [PATCH 2/3] chore(release): 0.98.0 -- shorten a changelog headline --- src/keboola_agent_cli/changelog.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/keboola_agent_cli/changelog.py b/src/keboola_agent_cli/changelog.py index 04c89271..5edef70b 100644 --- a/src/keboola_agent_cli/changelog.py +++ b/src/keboola_agent_cli/changelog.py @@ -42,9 +42,9 @@ "import keeps running, names the job, and is no longer marked retryable, so a long " "upload is not repeated by mistake. `kbagent serve` (`GET /storage/jobs/{project}/" "{job_id}`) and the SDK (`Client.storage_job()`) have the same functions.", - "New (#837): `--progress` shows percent, speed, elapsed time and the estimated time left " - "for `storage upload-table`, `file-upload`, `download-table`, `file-download` and " - "`unload-table --download`. It writes to stderr, also with `--json`: a progress bar in " + "New (#837): `--progress` shows percent, speed, elapsed time and time left for storage " + "uploads and downloads. It works on `storage upload-table`, `file-upload`, " + "`download-table`, `file-download` and `unload-table --download`, and writes to stderr, also with `--json`: a progress bar in " "a terminal, otherwise one line every 10 seconds.", "Fix (#837): `storage download-table --keep-slices`, `file-download` and `unload-table` " "now keep every slice file inside the output directory. A slice name from the export manifest that " From e9d74dcbcbeb89f942ce66cf2c24c318ddab93fd Mon Sep 17 00:00:00 2001 From: Petr Date: Fri, 9 Oct 2026 21:42:36 +0200 Subject: [PATCH 3/3] chore(release): resolve backticked SDK version gates in docs/sdk.md --- docs/sdk.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/sdk.md b/docs/sdk.md index 47b81faf..14310fea 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -251,7 +251,7 @@ Escape hatch for endpoints the facade omits. See [§7](#7-clientraw-the-escape-h ## 5. Typed result models -`result_models.py` defines the **stable return shapes** (`JobResult`, `QueryResult`, `UploadTableResult`, `ConfigDetailResult`, `SyncPushResult`, `CloneResult`, the `0.66.0+` device-enrollment pair `ScopedTokenResult` / `StreamSourceResult`, `TokenListEntryResult` from `0.86.0+`, and `StorageJobResult` from `vNEXT+`), all re-exported from the package root. They exist so a downstream consumer types against a **semver-versioned contract** instead of an undocumented `dict[str, Any]` — a contract change then surfaces at *type-check* time, not at runtime against a customer build. +`result_models.py` defines the **stable return shapes** (`JobResult`, `QueryResult`, `UploadTableResult`, `ConfigDetailResult`, `SyncPushResult`, `CloneResult`, the `0.66.0+` device-enrollment pair `ScopedTokenResult` / `StreamSourceResult`, `TokenListEntryResult` from `0.86.0+`, and `StorageJobResult` from `0.98.0+`), all re-exported from the package root. They exist so a downstream consumer types against a **semver-versioned contract** instead of an undocumented `dict[str, Any]` — a contract change then surfaces at *type-check* time, not at runtime against a customer build. Two design rules every model follows (`_ApiResultModel` base): @@ -272,7 +272,7 @@ The two device-enrollment models (`0.66.0+`) commit these named fields: - **`ScopedTokenResult`** — `id`, `token` (the one-time secret, see the gotcha in §4), `description`, `expires` (`str | None`), `can_read_all_file_uploads` (alias `canReadAllFileUploads`). - **`StreamSourceResult`** — `id`, `source_id`, `name`, `type`, `description`, `branch_id` (default `"default"`), `otlp_url` (ingest URL, secret in the path — unmasked), `otlp_secret`, `base_endpoint`, `sink_bucket_id` (`str | None`; the `in.c-otlp-` bucket to grant a device token write on). -`UploadTableResult` also commits (`vNEXT+`) `file_id`, `job_id` (`int | None`) and `job_status` (`str | None`). `StorageJobResult` (`vNEXT+`) commits `job_id` (alias `id`), `status`, `operation_name` (alias `operationName`), `table_id` (alias `tableId`), `file_id`, `created_time` / `start_time` / `end_time` (aliases `createdTime` / `startTime` / `endTime`), `imported_rows`, `warnings`, `results`, `error`, plus the properties `finished` and `failed`. +`UploadTableResult` also commits (`0.98.0+`) `file_id`, `job_id` (`int | None`) and `job_status` (`str | None`). `StorageJobResult` (`0.98.0+`) commits `job_id` (alias `id`), `status`, `operation_name` (alias `operationName`), `table_id` (alias `tableId`), `file_id`, `created_time` / `start_time` / `end_time` (aliases `createdTime` / `startTime` / `endTime`), `imported_rows`, `warnings`, `results`, `error`, plus the properties `finished` and `failed`. `TokenListEntryResult` (`0.86.0+`) commits `id`, `description`, `created` (`str | None`), `expires` (`str | None`), `is_expired` (alias `isExpired`), `is_master_token` (alias `isMasterToken`), and (`0.88.0+`) `last_used` (alias `lastUsed`), `last_used_event` (alias `lastUsedEvent`) and `last_used_status` (alias `lastUsedStatus`) — all three `str | None`, `None` unless `list_tokens(with_last_used=True)` was used. It has **no secret field by design** — see `list_tokens` above.