diff --git a/.github/labeler.yml b/.github/labeler.yml index 513dc7b31..14693687c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -77,6 +77,10 @@ converter/ontology: - changed-files: - any-glob-to-any-file: 'converters/ontology/**' +converter/holistics: + - changed-files: + - any-glob-to-any-file: 'converters/holistics/**' + # Other areas cli: - changed-files: diff --git a/.github/workflows/converter-holistics-ci.yml b/.github/workflows/converter-holistics-ci.yml new file mode 100644 index 000000000..4831ab8c6 --- /dev/null +++ b/.github/workflows/converter-holistics-ci.yml @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Holistics CI +on: + push: + branches: [ "main" ] + paths: + - 'converters/holistics/**' + # core-spec/** is the convention every sibling workflow adopted in #396. + # examples/** is load-bearing here rather than merely conventional: the + # reverse-path snapshot converts examples/tpcds_semantic_model.yaml, so an + # edit there fails this converter's tests without touching a converter file. + - 'core-spec/**' + - 'examples/**' + - '.github/workflows/converter-holistics-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/holistics/**' + - 'core-spec/**' + - 'examples/**' + - '.github/workflows/converter-holistics-ci.yml' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.19" + + - name: Set up Node + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: "22" + + - name: Install the Holistics CLI + # The tests that run `holistics aml compile` and `holistics aml validate` + # skip without the CLI. @holistics/cli-core is the CLI's logic as one + # bundled JavaScript file, with no native binary and no platform + # restriction, so it runs on the runner as it does on a developer's + # machine. @commander-js/extra-typings is imported by its `bin` script + # and missing from its dependencies, so it is installed alongside. + # + # Pinned: 2.3.32 is the first version whose compiler accepts the + # composite relationship in the ecommerce fixture. + run: | + npm install --no-package-lock --prefix "$RUNNER_TEMP/holistics" \ + @holistics/cli-core@2.3.32 @commander-js/extra-typings@14.0.0 + echo "$RUNNER_TEMP/holistics/node_modules/.bin" >> "$GITHUB_PATH" + + - name: Sync dependencies + working-directory: converters/holistics + run: | + uv sync + + - name: Unit Tests + working-directory: converters/holistics + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 62cdf72cc..4456ce416 100644 --- a/converters/README.md +++ b/converters/README.md @@ -90,6 +90,7 @@ The Ossie specification currently defines extensions for the following vendors: | `THOUGHTSPOT` | ThoughtSpot TML (Model + Table/SQL View) | | `CUBE` | Cube data model | | `HEX` | Hex semantic project | +| `HOLISTICS` | Holistics AML dataset | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py b/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py index 122d138b8..0557a4fa6 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py @@ -36,6 +36,7 @@ OssieDialect.SIGMA: None, OssieDialect.THOUGHTSPOT: None, OssieDialect.DAX: None, + OssieDialect.HOLISTICS_AQL: None, } diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py index 18b521267..65f33d404 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py @@ -111,6 +111,7 @@ def _to_sqlglot_dialect(ossie_dialect: OssieDialect) -> SQLGlotDialect | None: OssieDialect.THOUGHTSPOT, OssieDialect.SIGMA, OssieDialect.DAX, + OssieDialect.HOLISTICS_AQL, ): # There's no parallel for these read = None diff --git a/converters/hex/tests/cli/test_cli_export.py b/converters/hex/tests/cli/test_cli_export.py index dbcc425f0..4f37b1c08 100644 --- a/converters/hex/tests/cli/test_cli_export.py +++ b/converters/hex/tests/cli/test_cli_export.py @@ -225,5 +225,5 @@ def test_invalid_dialect(capsys: pytest.CaptureFixture[str], tmp_path: Path) -> message = message + separator + choices.replace("'", "") assert message == snapshot("""\ usage: ossie-hex export [-h] -i INPUT [-o OUTPUT] [-d DIALECT] [-v] -ossie-hex export: error: argument -d/--dialect: invalid choice: 'invalid' (choose from ansi_sql, snowflake, mdx, maql, tableau, databricks, bigquery, sigma, thoughtspot, dax, ossie_sql_2026) +ossie-hex export: error: argument -d/--dialect: invalid choice: 'invalid' (choose from ansi_sql, snowflake, mdx, maql, tableau, databricks, bigquery, sigma, thoughtspot, dax, holistics_aql, ossie_sql_2026) """) diff --git a/converters/holistics/.gitignore b/converters/holistics/.gitignore new file mode 100644 index 000000000..80f74e214 --- /dev/null +++ b/converters/holistics/.gitignore @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__pycache__/ +*.py[cod] +.pytest_cache/ +*.egg-info/ +dist/ +build/ +.venv/ +venv/ + +# Rendered previews +*.html diff --git a/converters/holistics/README.md b/converters/holistics/README.md new file mode 100644 index 000000000..2f2da9403 --- /dev/null +++ b/converters/holistics/README.md @@ -0,0 +1,177 @@ + + +# Apache Ossie Holistics converter + +Converts between a [Holistics](https://docs.holistics.io) AML dataset and an +[Apache Ossie](https://github.com/apache/ossie) semantic model. + +- `to-ossie` reads the JSON `holistics aml compile` produces, and writes one Ossie semantic model. +- `to-aml` reads one Ossie semantic model, and writes AML source files. + +One AML `Dataset` becomes one Ossie semantic model. + +| AML | Ossie | +|---|---| +| the `Dataset` | the document root | +| each model | a `datasets[]` entry | +| each model `dimension` | a `fields[]` entry on that dataset | +| each model `measure` | a `metrics[]` entry named `__` | +| each dataset-level `dimension` | a `fields[]` entry on the model it names | +| each dataset-level `metric` | a `metrics[]` entry | +| each `relationship` | a `relationships[]` entry | + +[Mapping](docs/mapping.md) covers data types, relationship kinds and the rest. + +AQL travels under the `HOLISTICS_AQL` dialect, registered upstream the way `THOUGHTSPOT` was in +[#351](https://github.com/apache/ossie/pull/351). The converter does not translate AQL into SQL, and does not +rewrite warehouse SQL from one dialect into another. [Limitations](docs/limitations.md) lists what that costs. + +## The two SQLs + +Both sides write SQL, and the two are not the same text. + +- **Holistics SQL** is an AML `@sql` body. It interpolates through `{{ ... }}`, and it resolves inside the one + model it sits in. +- **Ossie SQL** is the `expression` of a `dialects[]` entry. It writes a reference as `dataset.field`, and its + dialect tag names the warehouse it was written for. + +**Warehouse SQL** is what Holistics generates and sends to the database. This converter reads and writes the +first two, and Holistics generates the third. + +## Installation + +```bash +pip install -e . # from a checkout of this directory +``` + +Python 3.10+. The runtime dependencies are `PyYAML` and `sqlglot`. `sqlglot` is here for `to-aml`, which uses +it to find the column references in an Ossie SQL expression. [Ossie to AML](docs/reverse.md#expressions) says +why that is needed. + +## Usage + +AML to Ossie. `to-ossie` takes compiled JSON, so run the Holistics CLI first: + +```bash +holistics aml compile ecommerce.dataset.aml -r . -o build/ +ossie-holistics to-ossie build/ecommerce.dataset.aml.json -o ecommerce.yaml --sql-dialect BIGQUERY +``` + +Ossie to AML. `holistics aml validate` checks the result and needs no account: + +```bash +ossie-holistics to-aml ecommerce.yaml -o aml/ --sql-dialect BIGQUERY --data-source-name my_warehouse +holistics aml validate aml/ecommerce.dataset.aml -r aml/ +``` + +Options: + +- `-o`, required in both directions. A file for `to-ossie`, a directory for `to-aml`. +- `--sql-dialect DIALECT`, required in both directions. The choices are `ANSI_SQL`, `OSSIE_SQL_2026`, + `BIGQUERY`, `DATABRICKS` and `SNOWFLAKE`, the Ossie dialects sqlglot can parse. + - On `to-ossie` it tags every `@sql` body. AML does not record which warehouse a body was written for. + - On `to-aml` it names the warehouse the Holistics connection reads. An expression already in that dialect + is written as it stands. One in `ANSI_SQL` or `OSSIE_SQL_2026` is rendered into it, with a WARNING + showing both texts. An expression offering neither gets an ERROR, and its body is written unchanged. +- `--data-source-name NAME`, on `to-aml`. The Holistics connection the models read from. AML requires it on a + dataset and Ossie has no field for it. Required when the Ossie document carries no `HOLISTICS` entry in + `custom_extensions`. +- `--issues FILE`. Where to write the issue log, a YAML sequence. Defaults to stderr. Never mixed into the + document output. A message runs to a paragraph, so YAML folds it rather than putting it on one line. +- `--force`. Overwrite existing output files. Without it, an existing target stops the run before anything is + written. + +The process exits `1` when the issue log holds an ERROR-severity issue, and `0` otherwise. + +`to-aml` writes one dataset file plus one model file per Ossie dataset. An Ossie dataset named +`reporting__calendar` goes to `modules/reporting/calendar.model.aml`: + +``` +aml/ + ecommerce.dataset.aml + ecommerce_orders.model.aml + modules/reporting/calendar.model.aml +``` + +## Documents + +| Document | What it covers | +|---|---| +| [The fixture](docs/fixture.md) | The AML source, how to recompile it, and the shape of the compiled JSON | +| [Mapping](docs/mapping.md) | Which AML construct becomes which Ossie construct, including relationships and data types | +| [Expression translation](docs/expressions.md) | What the converter emits for each definition body, and what it leaves untranslated | +| [Ossie to AML](docs/reverse.md) | The generated file layout, and what AML cannot express | +| [Ossie SQL to AQL](docs/aql.md) | How a cross-dataset metric's Ossie SQL becomes AQL, and what is not translated | +| [Measures](docs/measures.md) | How `aggregation_type` and a definition body compose into one Ossie metric expression | +| [The HOLISTICS payload](docs/vendor-payload.md) | Everything carried in `custom_extensions` | +| [Limitations](docs/limitations.md) | The coverage matrix, and what is known not to work | + +## Fixtures + +`tests/fixtures/ecommerce/` holds the AML source of the `ecommerce` dataset, written for this converter, and +that source compiled by the Holistics CLI. The `expected*` files are what the converter produces from it in +both directions. + +`tests/fixtures/tpcds/expected_aml/` is `examples/tpcds_semantic_model.yaml` converted to AML. That document +carries no `HOLISTICS` stash, so every AML property with no Ossie field is derived or supplied rather than +read back. + +`tests/fixtures/aql/` is written for the Ossie SQL to AQL translation. Every metric in it spans two datasets, +which is what makes AML take an `@aql` body, and each covers a different path: a function AQL has, a function +it does not, a column no dataset declares, a field with no `datatype`, and two that are dropped with an +ERROR. + +`tests/fixtures/metric_refs/` is written for a metric that names another metric. AML gives a metric two homes +that take different bodies, so the home the named metric landed in decides where the naming one can go. Each +metric covers one way that lands. + +## Development + +```bash +uv run pytest +python3 tools/check_links.py # every cross-document link and anchor resolves +python3 tools/update_snapshots.py # rewrite the snapshots after a deliberate change +``` + +Most tests compare a committed snapshot against what the converter produces now. The rest check the output +against something other than itself. + +These run the Holistics CLI: + +- the generated AML through `holistics aml validate`, for all three fixtures +- the committed compiled JSON against what `holistics aml compile` produces from the committed AML source +- a round trip: compile the generated AML, convert that back to Ossie, and compare it with the document + `to-aml` started from. For `ecommerce` the two are byte-identical. + +These need nothing beyond the package: + +- the converted document against `core-spec/ossie-schema.json` +- every qualified `.` reference against the fields the document declares +- the issue log read back from its folded YAML, which a fold inside a URL would corrupt +- each path through the Ossie SQL to AQL translation, pinned by metric name so a snapshot update cannot quietly + stop covering one +- the issue-code table in `docs/limitations.md` against the codes the source raises, in both directions. See + `tests/test_issue_codes.py` +- a malformed document reaching `to-aml`, which must report rather than raise. See + `tests/test_malformed_input.py`. + +CI installs the CLI, so the first group runs there too: + +```bash +npm install --no-package-lock --prefix /tmp/holistics \ + @holistics/cli-core@2.3.32 @commander-js/extra-typings@14.0.0 +export PATH="/tmp/holistics/node_modules/.bin:$PATH" +``` + +`@holistics/cli-core` is the CLI's logic as one bundled JavaScript file, so it needs only Node. The +[native CLI](https://docs.holistics.io/docs/cli) works the same way for these tests. With neither installed, +that first group skips and the rest of the suite still runs. diff --git a/converters/holistics/docs/aql.md b/converters/holistics/docs/aql.md new file mode 100644 index 000000000..339d1541f --- /dev/null +++ b/converters/holistics/docs/aql.md @@ -0,0 +1,179 @@ + + +# Ossie SQL to AQL + +AML gives a metric that spans datasets one body type, `@aql`. So `to-aml` renders its Ossie SQL as AQL, one +construct at a time. Along with rendering a portable body into `--sql-dialect`, this is the only place the +converter rewrites an expression rather than carrying it. + +## Why a metric is translated + +An AQL expression can name fields from several models. A `@sql` expression can name only fields of the model +it sits in. Naming another model's field from a `@sql` body fails with +`Cannot refer to external field in TableModel/QueryModel`. + +A metric whose expression names two datasets therefore has to be AQL, in either place AML allows one. + +| Where it is written | Body it accepts | +|---|---| +| a dataset-level `metric` | `@aql` only. The compiler rejects `@sql` there with `Incompatible property 'definition' of 'Metric'` | +| a model-level `measure` | `@sql` or `@aql`, and a `@sql` one names only this model's fields | + +`customer_lifetime_value` in `examples/tpcds_semantic_model.yaml` is +`SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)` and becomes +`sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk)`. + +## What translation changes in the models + +AQL resolves a reference differently from Ossie SQL, so rendering a metric can need two changes to the models it +names. The converter reports each one. + +| What the Ossie SQL does | What the converter writes | Why | +|---|---|---| +| reads a column no dataset declares as a field | a hidden `dimension` on that model over the column of that name | AQL resolves against declared fields only, answering ``Field `subtotal` not found in model `orders` `` where Ossie SQL resolves against the table | +| sums or averages a field with no Ossie `datatype` | `type: 'number'` on that dimension | the fallback is `text`, and AQL then answers ``The `sum` function expects `Dimension(Number)`, but got `Text` `` | + +In the second row the metric is the only evidence the field is a number, because the Ossie document +never said so. + +## A name that is another metric + +An Ossie SQL body may name a metric where a column would go. AQL resolves both, and spells them differently. + +| The metric named | The AQL written | +|---|---| +| a dataset-level `metric` | `quantity_per_customer`, the bare name | +| a `measure` on a model | `items.items_quantity`, qualified by the model holding it | + +[Ossie to AML](reverse.md#a-metric-that-names-another-metric) covers which home a metric lands in, which is +what picks the row. + +## A window function depends on how the metric is queried + +A window function computes over a set of rows, and the query decides which rows those are. The two AML +homes handle that differently. + +**On a model, as a `@sql` measure.** AML carries the body as written, and it compiles. The warehouse SQL it +generates is valid only for a query that groups by the columns in the frame. +`SUM(SUM(q)) OVER (ORDER BY oid ...)` queried alongside `oid` gives: + +```sql +SELECT oid, SUM(SUM(quantity)) OVER (ORDER BY oid ...) FROM t GROUP BY 1 +``` + +Queried on its own, as a grand total, it gives the same expression with no `GROUP BY`, which the warehouse +rejects. Holistics checks none of this at conversion time, so the converter raises a WARNING naming the +columns the frame reads. + +**On the dataset, as a `metric`.** The body has to be AQL, and the next section is why that cannot be +written mechanically. + +## A window function cannot be translated + +Even though AQL has window functions, this converter cannot output one. + +The two languages take the frame from different places. Ossie SQL carries the partition and the ordering in the +expression, inside the `OVER` clause. AQL takes both from the grain of the query being run. + +Translating one into the other means deciding which query grain a given `OVER` clause corresponds to. The +Ossie specification defines no semantics for how a window function behaves when queried, so nothing says +which grain that is. Picking one changes what the metric computes. + +The ERROR therefore records a limit of this converter. AQL can express the metric, so write the AQL by hand +to carry one of these over. + +`brand_rank_in_store` in `examples/tpcds_semantic_model.yaml` is one of these: +`RANK() OVER (PARTITION BY store.s_store_sk ORDER BY SUM(store_sales.ss_ext_sales_price) DESC)`. + +AQL's window functions are listed under Window Function in the +[AQL function reference](https://docs.holistics.io/reference/aql/functions), and +[`rank`](https://docs.holistics.io/reference/aql/rank) shows one in full. + +## The function table + +Most Ossie SQL functions keep their name in AQL, lowercased. `SUM(x)` becomes `sum(x)`, `ROUND(x, 2)` becomes +`round(x, 2)`, `SAFE_DIVIDE(a, b)` becomes `safe_divide(a, b)`. 45 of the table's 51 entries work that way, +so the table is worth reading only for the ones that do not. + +| Ossie SQL | AQL | +|---|---| +| `LENGTH` | `len` | +| `POWER` | `pow` | +| `STDDEV` | `stdev` | +| `STDDEV_POP` | `stdevp` | +| `VARIANCE` | `var` | +| `VARIANCE_POP` | `varp` | +| `COUNT(DISTINCT x)` | `count_distinct(x)` | + +The last is not a rename but a reshape. Ossie SQL puts `DISTINCT` inside the argument list and AQL puts it in the +function name, so `COUNT` reads as either `count` or `count_distinct` depending on what it wraps. + +The table itself comes from parsing every name AQL registers and reading which node sqlglot produced, rather +than from matching names by eye. + +The table leaves out some names. What happens to one depends on whether the original call can be read back +out of the source text, which [the passthrough](#the-passthrough) explains. + +**A name that is one sqlglot node with several spellings.** `LPAD` and `RPAD` are both `Pad`, `LTRIM` and +`RTRIM` are both `Trim`, `LOG` and `LOG10` are both `Log`. The node says which shape was parsed, not which +name was written, so rendering from it would turn `LPAD(s, 5)` into `PAD(s, 5)`. The node's offset points at +the name in the source, so the passthrough recovers it. + +**A name whose arguments sqlglot rewrote.** `LOG10(x)` gains a base argument the source never had, and +`DATE_TRUNC('month', d)` has its unit normalized to `'MONTH'`. Either way the argument carries no offset. +Dropping the invented base is right and dropping the normalized unit is not, and an absent offset does not +say which case it is, so both raise an ERROR. + +**A name whose arguments mean something else in AQL.** Ossie SQL `SUBSTRING(s, start, len)` and `STRPOS(s, sub)` +do not line up with AQL's `mid` and `find`, so neither is in the table. The passthrough carries `STRPOS` +under its own name with the warehouse's own meaning. `SUBSTRING`'s node carries no offset, so the converter +raises an ERROR for it. + +## The passthrough + +A function with no AQL name becomes `sql_number('NAME', arg, ...)`, AQL's raw-SQL call, which hands the +warehouse the original call with AQL-resolved arguments. + +``` +MY_WAREHOUSE_UDF(COUNT(users.id), 3) -> sql_number('MY_WAREHOUSE_UDF', count(users.id), 3) +LPAD(users.full_name, 5) -> sql_number('LPAD', users.full_name, 5) +``` + +The name and the argument order come from the source text rather than from sqlglot, which names a node +after the shape it parsed. A function sqlglot does not model keeps its name and arguments in an `Anonymous` +node already. For one it does model, the node carries the offset of the name token and each argument carries +its own, so the converter reads both back out of the source. + +Every argument has to carry an offset for that to work. An argument sqlglot synthesized or normalized has +none, and the converter raises an ERROR rather than rebuild the call without it. + +The family is typed by return value: `sql_number`, `sql_text`, `sql_date`, `sql_datetime` and +`sql_truefalse`. A metric body is numeric, so the converter uses `sql_number`. + +## Reading it back + +`tests/fixtures/aql/translation.ossie.yaml` holds one metric for each path through this document. + +| Metric | Path | +|---|---| +| `quantity_per_customer` | aggregates with AQL names | +| `rounded_ratio` | scalar functions with AQL names | +| `untyped_total` | a field with no Ossie `datatype` | +| `borrowed_column_total` | a column no dataset declares as a field | +| `passthrough_call` | a function with no AQL name | +| `windowed_rank` | a window function, dropped with an ERROR | +| `unmapped_function` | a call sqlglot rewrote, dropped with an ERROR | + +`tests/test_snapshots.py` pins each result by metric name, and runs the generated AML through +`holistics aml validate`, which type-checks the AQL rather than only parsing the file. + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/expressions.md b/converters/holistics/docs/expressions.md new file mode 100644 index 000000000..8ea49d377 --- /dev/null +++ b/converters/holistics/docs/expressions.md @@ -0,0 +1,114 @@ + + +# Expression translation + +AML writes a field body in one of two languages, named by the heredoc. + +| Heredoc | What it holds | Ossie dialect | +|---|---|---| +| `@sql` | Holistics SQL: warehouse SQL with `{{ ... }}` interpolation | the `--sql-dialect` argument | +| `@aql` | AQL, the Holistics analytical query language | `HOLISTICS_AQL` | + +The converter substitutes the interpolation and passes the rest of each body through as written. It does not +translate AQL into SQL, and it does not re-render warehouse SQL from one dialect into another. + +## Why AQL travels untranslated + +AQL is not SQL. `count(ecommerce_orders.id) | where(...) | of_all()` is relationship-aware, so no SQL +rendering of it is correct without the dataset's whole relationship graph. + +The specification defines `dialects[]` as `{dialect, expression}` pairs, so a value a converter cannot +translate still travels, tagged with the language it is valid in. `HOLISTICS_AQL` is a member of the +`Dialect` enumeration, registered the way `THOUGHTSPOT` was in +[#351](https://github.com/apache/ossie/pull/351). An AQL body is therefore carried rather than stashed, and +every field keeps the `expression` that Ossie requires. + +## Interpolation + +A `@sql` body interpolates through `{{ ... }}`. + +| Form | Names | Becomes | +|---|---|---| +| `{{ #SOURCE.column }}` | a column of this model's relation | the column name | +| `{{ field }}` | another dimension of the same model | `.` | +| `{{ field }}` where `field` is a measure | a measure of the same model | the metric's own name | + +A measure is the one reference that is not dataset-qualified, because a measure becomes a root-level Ossie +metric rather than a field inside a dataset. See [measures](measures.md#naming). + +References chain. `cities.downstream_measure` is `{{ upstream_measure }} + 1`, and `upstream_measure` is +`COUNT({{ id }})`. + +## What the converter emits + +| AML definition | Ossie expression | Example | +|---|---|---| +| absent | the column sharing the field's name | `ecommerce_orders.id` becomes `id` | +| exactly one interpolation | what that interpolation resolves to | `{{ #SOURCE.quantity }}` becomes `quantity` | +| anything else | the body, with every interpolation resolved | `CAST({{ #SOURCE.created_at }} AS DATE)` becomes `CAST(created_at AS DATE)` | +| `@aql` | the body, unchanged | | + +Every `@sql` row takes the `--sql-dialect` tag, and the `@aql` row takes `HOLISTICS_AQL`. Holistics SQL is +warehouse SQL throughout, so one dialect covers all of them and a reader needs no rule for which field +carries which tag. + +A bare reference runs on any warehouse. A body with SQL around it runs on the one it was written for, and +`ecommerce_users.signup_month` shows why: it is `DATE_TRUNC({{ #SOURCE.created_at }}, month)`, which is +BigQuery argument order. + +### An omitted definition + +An omitted `definition` is a default rather than a missing value. AML fills it with +`{{ #SOURCE. }}`, the source column sharing the field's name. + +- Only a dimension may omit it. A measure must carry one, and the compiler says so rather than defaulting: + `Property 'definition' is missing on element but required on type 'Measure'`. +- The default is the same in a `TableModel` and a `QueryModel`. A `QueryModel` resolves it against the + columns its query returns rather than against a table. +- Omitting it is common. Most dimensions in the Holistics AMQL test corpus do. + +### Why a sibling reference is qualified + +A `{{ field }}` names another AML field in the same model, and that field becomes an Ossie field in the same +dataset, so the reference travels as `dataset.field` in a dimension body and a measure body alike. + +- `ecommerce_users.age` is `extract(year from age({{ birth_date }}))` and becomes + `extract(year from age(ecommerce_users.birth_date))`. +- `ecommerce_orders.last_created_at` is `{{ created_at }}` under `aggregation_type: max` and becomes + `MAX(ecommerce_orders.created_at)`. + +A bare name would be right only by accident. An AML field usually shares its name with the column it reads, +which makes `birth_date` look like it would resolve on its own, and AML does not require the two to match. +`dimension created_at { definition: @sql {{ #SOURCE.sign_up_at }} }` is a field named `created_at` over a +column named `sign_up_at`, so a bare `created_at` would name a column that does not exist. + +The shipped examples read the same way. `customer.customer_full_name` in +`examples/tpcds_semantic_model.yaml` is `c_first_name || ' ' || c_last_name`, where the field `c_first_name` +has the expression `c_first_name`, so the bare name could be either. The metric expressions settle it, and +they qualify: `SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)`. + +Referencing also beats inlining. Inlining would copy the referenced field's body into every dependent +expression, so one warehouse-specific body spreads across many and an edit to the referenced field stops +reaching them. The specification points the same way: its "Not Supported in Expressions" table answers both +subqueries and CTEs with "use field references instead". + +## Where the dialect comes from + +A `@sql` body is the warehouse's SQL, and AML records nothing that says which warehouse. `data_source_name` +is the closest thing to it, and it is an instance-local connection name chosen by whoever created the +connection. + +`to-ossie` therefore requires `--sql-dialect`. Its value is one of `ANSI_SQL`, `OSSIE_SQL_2026`, `BIGQUERY`, +`DATABRICKS` or `SNOWFLAKE`, the Ossie dialects sqlglot can parse. The reverse path parses the expression to +restore its field references, so a dialect outside that set would convert in one direction only. + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/fixture.md b/converters/holistics/docs/fixture.md new file mode 100644 index 000000000..4878e1abf --- /dev/null +++ b/converters/holistics/docs/fixture.md @@ -0,0 +1,43 @@ + + +# Produce the fixture + +The forward path reads compiled JSON, not AML source. The [Holistics CLI](https://docs.holistics.io/docs/cli) does the compilation. The reverse path writes AML source directly, and never reads this fixture. See [Ossie to AML](reverse.md). + +```bash +holistics aml compile ecommerce.dataset.aml -r . -o +``` + +`-r` names the root of the AML repository. The compiler needs it to resolve module paths such as `reporting::calendar`. Run the +command from the directory that holds the dataset file, or pass the file path and set `-r` to the repository root. + +The AML source sits in `tests/fixtures/ecommerce/aml/`. Regenerate the compiled JSON from it with Holistics CLI 2.3.32 or later. The composite relationship needs that version: + +```bash +cd tests/fixtures/ecommerce/aml +holistics aml compile ecommerce.dataset.aml -r . -o .. +``` + +The source is small. It models a marketplace in eight models, and each one covers a row of the mapping, so trimming it further drops a construct the mapping describes. + +## What the compiled JSON looks like + +The CLI emits one self-contained JSON document per dataset. The document inlines every model it references, so +it needs no sibling files. It also inlines a full copy of a model under each dataset-level dimension that names +that model, and under each `QueryModel` for every model its query references, so most of the file is +duplication. Read `models` and `relationships` as the single source of truth, and treat the copies under +`dimension[].model` and `models[].models` as redundant. + +Each node carries a `__type__` tag. The converter dispatches on that tag rather than on the shape of the object. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/limitations.md b/converters/holistics/docs/limitations.md new file mode 100644 index 000000000..02be4779f --- /dev/null +++ b/converters/holistics/docs/limitations.md @@ -0,0 +1,128 @@ + + +# Limitations + +## Coverage matrix + +Every construct the converter does not carry, with its consequence. A construct it drops or changes raises a +structured issue at conversion time, listed under [Issue codes](#issue-codes). + +| Construct | Limitation | Consequence | +|---|---|---| +| AQL field and metric bodies | Carried under `HOLISTICS_AQL`, never translated to SQL | A consumer that does not speak AQL cannot evaluate these, though it can read and round-trip them | +| Inactive relationships | `active: false` has no Ossie marker. **WARNING severity**, one issue per relationship | A consumer joins across a relationship that Holistics leaves out by default | +| Params | Runtime inputs with no Ossie equivalent | A `@sql` body that reads a param cannot be resolved statically | +| `number` precision | AML does not distinguish exact from approximate | A consumer reading `Decimal` may infer exactness AML never stated. The stash holds the original AML type | +| Warehouse SQL portability | Emitted under `--sql-dialect`, never rewritten | A document converted for one warehouse does not run on another | +| Dashboards, reports and canvases | Out of scope. Ossie models semantics, not presentation | No loss to the semantic model | +| A query body using Holistics template syntax | Detected and carried unresolved, never resolved. **ERROR severity** | The dataset's `source` is not runnable SQL. A consumer must resolve it through Holistics or drop the dataset | +| `RangeRelationship` and `ManyToManyRelationship` | Neither has an Ossie form. **WARNING severity**, one issue per relationship | Both carry an AQL `match` rather than column pairs, which is why no column mapping exists. The payload is stashed on the document and the reverse path rebuilds it, so a consumer reading Ossie sees the two datasets as unrelated while the round trip keeps the join | +| A relationship `where` filter | A `RelationshipFilter` restricts the join condition. No Ossie equivalent | The join widens to every matching row, so a consumer counts rows Holistics excludes. **WARNING severity** | +| A `QueryModel` becoming a query in `source` | `source` is one string, so a consumer reads the text to tell a query from a table reference. **WARNING severity** | Whether a given target reads it as a query depends on that target. See below | + +These are the reverse path's, writing AML from Ossie. + +| Construct | Limitation | Consequence | +|---|---|---| +| A metric spanning datasets, carrying Ossie SQL | Rendered as AQL, which is the only body a dataset-level `metric` accepts. **WARNING severity** | The AQL is a construct-for-construct map of the Ossie SQL, so it is worth reading once. See [Ossie SQL to AQL](aql.md) | +| A metric whose Ossie SQL this converter does not write as AQL | A window function, or a construct outside the map. **ERROR severity** | The metric is dropped. AQL may well express it; the converter does not translate it. See [Ossie SQL to AQL](aql.md) | +| A `@sql` body with a window function | The warehouse SQL it generates is valid only for a query that groups by the frame's columns. **WARNING severity** | A grand total over it produces warehouse SQL the database rejects. See [Ossie SQL to AQL](aql.md#a-window-function-depends-on-how-the-metric-is-queried) | +| An expression sqlglot cannot parse, or tagged with a dialect that is not SQL | Its column references cannot be located, so they cannot be written as interpolation. **ERROR severity** | The body is written out unchanged. It compiles, and AQL cannot see which fields it reads, so query rewriting and optimisation lose that link | +| An expression in a warehouse dialect that is not `--sql-dialect` | Translating Ossie SQL between warehouses is out of scope. **ERROR severity** | The body is written unchanged and will not run on the connection. An `ANSI_SQL` or `OSSIE_SQL_2026` body is rendered into the target instead, with a WARNING | +| A qualifier naming no dataset in the document | A table alias from a subquery has no AML form. **ERROR severity** | The reference is left as written, and the body compiles without the field linkage | +| A `@sql` body naming a field of another dataset | A `@sql` expression names only fields of the model it sits in. **ERROR severity** | Holistics rejects it at SQL generation with "Cannot refer to external field in TableModel/QueryModel". An expression that names two models belongs in AQL | +| A field with no `datatype` | AML requires `type` on every dimension. **WARNING severity** | `text` is written. `date_dim.d_quarter_name` in `examples/tpcds_semantic_model.yaml` is one | +| A name AML cannot spell | An Ossie name is any non-empty string. An AML identifier is letters, digits and underscore, with no leading digit. **WARNING severity** | Each illegal character becomes an underscore, so `Flights semantic model` in `examples/flights.semantic_model.yaml` is written `Flights_semantic_model`. Two names that collapse onto one spelling stop the run | +| A semantic model sharing a name with one of its datasets | Ossie keeps the two in separate namespaces and AML has one namespace for `Dataset` and `Model`. **WARNING severity** | The AML dataset takes a `_dataset` suffix. Models and relationships are unaffected, since nothing references it. `converters/cube/tests/fixtures/databricks_ossie.yaml` names both `orders` | +| File layout | The compiled JSON records no file provenance | `AML to Ossie to AML` returns the same models, not the same files. See [Ossie to AML](reverse.md) | +| Comments and property order | Neither survives the forward path | A diff against the original AML shows both, even where nothing semantic changed | + + +## Issue codes + +Every code the converter writes to its issue log, with the severity it carries. The sections above give the +reasoning behind the ones that need it. + +`to-ossie`: + +| Code | Severity | Raised when | +|---|---|---| +| `HOLISTICS_AQL_NOT_TRANSLATED` | INFO | A body is AQL, carried under `HOLISTICS_AQL` | +| `HOLISTICS_INACTIVE_RELATIONSHIP` | WARNING | `active: false`, which Ossie cannot express | +| `HOLISTICS_PARAM_DROPPED` | WARNING | A model declares params, stashed rather than carried | +| `HOLISTICS_QUERY_SOURCE` | WARNING | A `QueryModel` becomes a query in `source` | +| `HOLISTICS_RELATIONSHIP_FILTER` | WARNING | A relationship carries a `where` filter | +| `HOLISTICS_UNSUPPORTED_RELATIONSHIP` | WARNING | A range or many-to-many relationship, stashed on the document | +| `HOLISTICS_UNKNOWN_DATATYPE` | WARNING | An AML type outside the data type table | +| `HOLISTICS_UNRESOLVED_QUERY_TEMPLATE` | ERROR | A query body uses Holistics template syntax | +| `HOLISTICS_UNKNOWN_AGGREGATION` | ERROR | An `aggregation_type` outside the aggregation table | +| `HOLISTICS_UNRESOLVED_REFERENCE` | ERROR | An interpolation naming nothing in the dataset | + +`to-aml`: + +| Code | Severity | Raised when | +|---|---|---| +| `HOLISTICS_ASSUMED_TYPE` | WARNING | A field has no `datatype`, so `text` is written | +| `HOLISTICS_NAME_NOT_SPELLABLE_IN_AML` | WARNING | A name holds characters AML identifiers do not | +| `HOLISTICS_DATASET_NAME_COLLIDES_WITH_MODEL` | WARNING | The model name is also a dataset name | +| `HOLISTICS_EXPRESSION_RENDERED_INTO_TARGET_DIALECT` | WARNING | A portable body was rendered into `--sql-dialect` | +| `HOLISTICS_AGGREGATION_NOT_STRIPPED` | WARNING | A stashed aggregation does not wrap the whole expression | +| `HOLISTICS_METRIC_RENDERED_AS_AQL` | WARNING | A cross-dataset metric's Ossie SQL became AQL | +| `HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION` | WARNING | An AQL metric reads a column no dataset declares | +| `HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING` | WARNING | A `@sql` body has a window function | +| `HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL` | ERROR | A construct the converter does not write as AQL | +| `HOLISTICS_CROSS_MODEL_SQL_BODY` | ERROR | A `@sql` body names a field of another dataset, or a metric it cannot spell | +| `HOLISTICS_METRIC_REFERENCE_CYCLE` | ERROR | A metric references itself through another metric | +| `HOLISTICS_UNKNOWN_QUALIFIER` | ERROR | A qualifier naming no dataset in the document | +| `HOLISTICS_UNPARSEABLE_SQL` | ERROR | sqlglot cannot read the body at its declared dialect | +| `HOLISTICS_NO_USABLE_DIALECT` | ERROR | No body in the target dialect and none portable | + +## Known limitations + +### Name flattening can collide + +The converter replaces `::` with `__` to turn an AML module path into a flat Ossie dataset name, so +`reporting::calendar` becomes `reporting__calendar`. A doubled separator is used rather than a single `_` because a +single one collides with any model name that already contains an underscore, which most do. A root model named +`reporting_calendar` would take the name `reporting::calendar` wants. + +A collision is still reachable, just rarer. A root model named `reporting__calendar` still takes it, and the +converter fails naming both rather than letting one dataset overwrite the other. The unflattened `__fqn__` +goes to the stash either way, so the reverse path reads the module boundaries from there. + +### A query model becomes a query in `source` + +A `QueryModel` has no table name, only a Holistics SQL body, so its Ossie `source` is that query. The specification +provides for this. `core-spec/spec.md` describes `source` as a "Reference to underlying physical table/view +(e.g., `database.schema.table`) or query". + +`source` is a single string, `{"type": "string", "minLength": 1}`, so a consumer tells a query from a table +reference by reading the text. Several converters do, each with its own test, and they do not all draw the +line in the same place. `(SELECT * FROM public.users)` and `EXEC sp_get_orders` are two spellings where the +answers differ. + +So the converter emits a `QueryModel` as a query in `source` and raises a WARNING naming the model. Whether a +given target reads it as a query depends on that target, and a model worth sharing widely is better +materialised as a table in Holistics first. + +### Warehouse SQL has no dialect of its own + +`Dialect` in `core-spec/ossie-schema.json` is a closed enumeration. Its members +today are `ANSI_SQL`, `SNOWFLAKE`, `MDX`, `TABLEAU`, `DATABRICKS`, `MAQL`, `BIGQUERY`, `SIGMA`, `THOUGHTSPOT`, `DAX`, `HOLISTICS_AQL`, and +`OSSIE_SQL_2026`. + +`HOLISTICS_AQL` covers AQL. It does not cover a `@sql` body, because that body is the warehouse's SQL rather than a +Holistics language, and AML does not record which warehouse. A computed `@sql` body takes its dialect from +`--sql-dialect`, which must name an existing member. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md) and [Ossie SQL to AQL](aql.md). diff --git a/converters/holistics/docs/mapping.md b/converters/holistics/docs/mapping.md new file mode 100644 index 000000000..f86b0990b --- /dev/null +++ b/converters/holistics/docs/mapping.md @@ -0,0 +1,134 @@ + + +# Mapping + +| AML | Ossie | Notes | +|---|---|---| +| `Dataset` | the semantic model (the document root) | One dataset per document, its fields at the root beside `version` | +| `Dataset.description` | `description` | Falling back to `label` when the dataset declares no description. The unused one goes to the stash | +| `TableModel` | `dataset` with `source` from `table_name` | `table_name` arrives already quoted for the warehouse, for example `` `ecommerce`.`order_items` `` | +| `QueryModel` | `dataset` with `source` derived from the `query` body | The body is not plain SQL and cannot be copied verbatim. See [Query model bodies](#query-model-bodies) | +| model `__fqn__` | `dataset.name` | AML joins module paths with `::`. Ossie names are flat, so each `::` becomes `__` | +| `dimension` | `dataset.fields[]` entry | See [Expression translation](expressions.md) | +| `measure` | `metrics[]` entry named `__` | See [Measures](measures.md#naming) | +| dimension `primary_key: true` | `dataset.primary_key` | AML marks the key on the dimension. Ossie carries it on the dataset | +| `dimension.type` | `field.datatype` | See [Data types](#data-types) | +| `relationship(a.x > b.y, true)` | `relationships[]` entry | `>` is many-to-one. `a` becomes `from`, `b` becomes `to` | +| `relationship(a.x - b.y, true)` | `relationships[]` entry | `-` is one-to-one. Ossie has no one-to-one marker, so the kind goes to the stash | +| dataset-level `dimension` | `dataset.fields[]` entry on the model it names | The AML dimension names its owning model in `model` | +| dataset-level `metric` | `metrics[]` entry | Always AQL in practice, so always tagged `HOLISTICS_AQL` | +| everything else | `custom_extensions[HOLISTICS]` | See [The HOLISTICS payload](vendor-payload.md) | + +## Relationships + +AML writes a relationship as a pair of field references plus an active flag. Ossie needs a name, and AML gives +none, so the converter synthesizes `_to_`, the spelling Cube, GoodData, NVIDIA and Orion Belt all use +in their Ossie fixtures. A second relationship between the same pair takes `_2`, then `_3`. The reverse path +reads the endpoints from `from`, `to`, `from_columns` and `to_columns`, so the name it was given does not +change the AML it writes. + +AML has three relationship types, and only one of them has an Ossie form. + +| AML type | Ossie | +|---|---| +| `EqualityRelationship` (`many_to_one`, `one_to_one`) | a `relationships[]` entry | +| `RangeRelationship` (`range`) | none. Its `match` is an AQL condition, not a column pair | +| `ManyToManyRelationship` (`many_to_many`) | none. Ossie relationships run many to one | + +An `EqualityRelationship` carries its first column pair in `from` and `to`, and each further pair in a +repeated `on` block. Blocks combine with AND, which is exactly Ossie's positional `from_columns` and +`to_columns`: + +``` +from: FieldRef { model: 'merchants', field: 'city_id' } from_columns: [city_id, country_code] +to: FieldRef { model: 'cities', field: 'id' } to_columns: [id, country_code] +on { from: merchants.country_code to: cities.country_code } +``` + +The `relationship(a.x > b.y, true)` shorthand carries one pair only. A composite needs the long +`RelationshipConfig` form, which is what the fixture uses for `merchants` to `cities`. + +A relationship endpoint and a model spell the same name two different ways. The endpoint uses dots, as in +`reporting.calendar`, while the model's own `__fqn__` uses colons, as in `reporting::calendar`. The compiler +fills a `__refFqn__` shortcut on some endpoints and leaves it `null` on others, and in the fixture the one it +leaves null names a model inside a module. The converter therefore ignores `__refFqn__` and resolves an +endpoint by replacing each `.` with `::` and looking the result up in `models`. + +A relationship may set `active: false`. Holistics does not join across an inactive relationship unless a query asks +for it. Ossie has no inactive marker. Emitting an inactive relationship as an ordinary Ossie relationship would change +which joins a consumer makes, so the converter records the flag in the stash and raises a WARNING for each one. + +## Query model bodies + +A `TableModel` names a table, so its `source` is that name. A `QueryModel` carries Holistics SQL in `query` instead, +and that body may use Holistics template syntax. The compiler does not resolve it. The body is passed +through, and each referenced model is inlined under `models[]`. + +`delivered_orders` in the fixture compiles to this: + +``` +query: select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} +models[]: [ecommerce_orders, inlined whole, table_name `ecommerce`.`orders`] +``` + +| Form | Means | +|---|---| +| `{{ #model }}` | the referenced model's relation | +| `{{ #model.* }}` | every column of the referenced model | +| `{% filter(param) %} ... {% end %}` | a predicate, once a param value arrives | + +### What the converter does + +A query body that matches none of those forms is plain SQL. It becomes the dataset's `source` unchanged. + +A query body that matches any of them is carried unresolved. The converter puts the verbatim body in +`source`, records the forms it found in the stash, and raises an ERROR naming the dataset. + +### Why it resolves nothing, including what it could + +`models[]` inlines the referenced model with its `table_name`, so `{{ #ecommerce_orders }}` could be resolved +to `` `ecommerce`.`orders` ``. The converter does not do it. + +A filter block cannot be resolved at conversion time, because the param value arrives at query time. So a +body can carry both a form the converter could resolve and a form it cannot. Resolving the first leaves a +`source` that reads as SQL and is not, and a reader would have to know which forms this converter handles to +tell. Leaving every form in place keeps the body uniformly unresolved. + +Resolving them would also mean a second implementation of Holistics' own resolution rules, kept in step by +hand. That is the same reason the converter does not translate AQL and does not rewrite warehouse SQL. See +[expression translation](expressions.md). + +A dbt semantic manifest carries `node_relation.relation_name`, which dbt resolved during its own +compilation, so `ossie_to_msi.py` reads a finished string and the question does not arise there. The +Holistics CLI passes the template through instead. + +## Data types + +| AML `type` | Ossie `datatype` | `dimension.is_time` | +|---|---|---| +| `text` | `String` | defaults to false | +| `truefalse` | `Boolean` | defaults to false | +| `date` | `Date` | defaults to true | +| `datetime` | `DateTime` | defaults to true | +| `number` | `Decimal` | defaults to false | + +`number` is the one lossy row. AML does not say whether a `number` is exact or approximate, so no Ossie member matches it +exactly. `Decimal` is the closest, because the Ossie specification leaves its precision and scale unspecified. The +converter writes the original AML `type` into the stash, so an Ossie document converted back to AML recovers the AML type +regardless of this choice. + +`is_time` follows the Ossie default rule for each mapped `datatype`, so the converter writes no explicit `is_time`. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/measures.md b/converters/holistics/docs/measures.md new file mode 100644 index 000000000..4ab4c4b6d --- /dev/null +++ b/converters/holistics/docs/measures.md @@ -0,0 +1,63 @@ + + +# Measures + +A measure is the one AML construct where two properties compose into one Ossie expression. `aggregation_type` names the +aggregate, and `definition` gives the body. + +## Naming + +A measure belongs to a model. An Ossie metric sits at the document root, where one namespace covers the whole +document, so two models with a `total_quantity` each would collide. The converter therefore names a metric +`__`: `order_items.total_quantity` becomes the metric `order_items__total_quantity`. A +dataset-level AML metric keeps its own name, because it was already document-wide. + +The fixture needs this. `ecommerce_orders` has a measure `count_orders` and the dataset has a metric +`count_orders`, and the two become `ecommerce_orders__count_orders` and `count_orders`. + +`__` rather than `.` because a metric reference inside an expression is the metric's own name, and the +sibling converters that face the same collision spell it the same way: Cube writes `orders__count`, and Omni +writes `__`. A field reference stays dotted, as in `cities.id`, because fields are namespaced +per dataset and the shipped examples qualify them that way. + +| `aggregation_type` | Meaning | Ossie expression | +|---|---|---| +| `sum`, `count`, `count distinct`, `min`, `max`, `average`, `median` | The aggregate wraps the body | `SUM()` and so on | +| `custom` | The body already aggregates | the resolved body, unchanged | + +Worked examples from the fixture. + +`order_items.total_quantity` has `aggregation_type: sum` and the body `{{ #SOURCE.quantity }}`. The body is a bare source +column, so it resolves to `quantity`, and the metric expression is `SUM(quantity)` under `--sql-dialect`. + +`order_items.custom_total_quantity` has `aggregation_type: custom` and the body `sum({{ #SOURCE.quantity }})`. The aggregate +is already in the body, so the converter wraps nothing and emits `sum(quantity)`. + +`ecommerce_orders.last_created_at` has `aggregation_type: max` and the body `{{ created_at }}`. That is a sibling reference, +not a source column. `created_at` is a dimension in the same model, so it becomes a field in the same dataset and the metric expression is `MAX(ecommerce_orders.created_at)`. + +`cities.downstream_measure` has `aggregation_type: custom` and the body `{{ upstream_measure }} + 1`. `upstream_measure` is a measure in the same model, so it becomes a metric and the expression is `cities__upstream_measure + 1`. `upstream_measure` itself is `COUNT({{ id }})`, whose sibling `id` is `{{ #SOURCE.id }}`, so it becomes `COUNT(cities.id)`. +Referencing rather than inlining means the converter never walks that chain, so a reference cycle in AML cannot make it recurse. + +Referencing handles a case inlining could not. A measure whose body names another measure, +under a named `aggregation_type`, would inline to `SUM(COUNT(x))`, which no warehouse accepts. As a reference it is +`SUM(cities__upstream_measure)`, a metric naming a metric, which the semantic layer resolves rather than the warehouse. + +`core-spec/expression_language.md` lists "Column and Metric references" among the supported constructs. The +converter emits the dataset-qualified form the shipped examples use for a field, and the metric's own name for +a metric, and raises a WARNING naming the metric so a consumer can check the spelling against its own +resolver. + +A measure written in `@aql` travels under `HOLISTICS_AQL` whatever its `aggregation_type` says, because the body already contains its own aggregation. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/reverse.md b/converters/holistics/docs/reverse.md new file mode 100644 index 000000000..a7a18b09d --- /dev/null +++ b/converters/holistics/docs/reverse.md @@ -0,0 +1,213 @@ + + +# Ossie to AML + +The reverse path writes AML source text. + +``` +AML source --(holistics aml compile)--> JSON --(this converter)--> Ossie YAML +AML source <---------------------(this converter)------------------ Ossie YAML +``` + +`holistics aml compile` reads AML source and writes JSON, and it is the only tool that moves between the +two. So AML source is the format a person edits, a repository holds, and the CLI reads back, and generated +JSON would be a file with no reader. + +## Output layout + +One Ossie document becomes one dataset file plus one model file per dataset: + +``` +/ + .dataset.aml + orders.model.aml a dataset named orders + modules/reporting/calendar.model.aml a dataset named reporting__calendar +``` + +A dataset name splits on `__` back into a module path, so `reporting__calendar` becomes +`modules/reporting/calendar.model.aml` holding `Model calendar`. When the stash carries the original `__fqn__`, +that wins over splitting, because it records where the module boundaries actually were. + +The input file layout is lost on the way in. The compiled JSON records which models a dataset holds and not +which file each came from, so by the time a document reaches Ossie the layout is already gone. AML accepts any +arrangement, so the generator picks one. `AML to Ossie to AML` returns the same models in the generator's +layout. + +## Mapping + +| Ossie | AML | +|---|---| +| the document root | `Dataset { ... }` | +| `description` | the dataset's `description` | +| `dataset` whose `source` is a dotted reference | `Model { type: 'table' table_name: '' }` | +| `dataset` whose `source` is a query | `Model { type: 'query' query: @sql ;; }` | +| `dataset.name` | the model name, `__` split back into a module path | +| `dataset.fields[]` | `dimension { ... }` | +| `dataset.primary_key` | `primary_key: true` on each named dimension | +| `metrics[]` | a `measure` on one model, or a dataset-level `metric` | +| `relationships[]`, one column pair | `relationship(from.col > to.col, true)` in the dataset block | +| `relationships[]`, composite | the long `RelationshipConfig` form, extra pairs in repeated `on` blocks | +| `custom_extensions[HOLISTICS]` | restores `hidden`, `format`, `type`, `aggregation_type`, `param`, `persistence`, `owner`, `table_name`, the relationship kind and `active` | + +## Properties AML requires and Ossie has no field for + +Four properties are mandatory in AML. `label` and `type` on every dimension and measure, and +`data_source_name` and `relationships` on the dataset. The compiler says so rather than defaulting: +`Property 'label' is missing on element but required on type 'Dimension'`. + +| Property | Where it comes from | +|---|---| +| `label` | the stash, else the name titleized the way AML itself does it, so `created_at` becomes `Created At` | +| `type` | the stash, else `datatype` through the reverse of the type table, else `text` with a WARNING | +| `data_source_name` | the stash, else the `--data-source-name` argument. Without either, the run fails | +| a legal identifier | each illegal character in an Ossie name becomes an underscore, and the rename is reported | +| a dataset name no model holds | AML has one namespace for `Dataset` and `Model`, so a collision moves the dataset to `_dataset` | +| `relationships` | the Ossie `relationships[]`, written as `[]` when there are none | + +## Expressions + +An Ossie `expression` may carry several dialects. `--sql-dialect` names the warehouse the Holistics +connection reads, and the converter picks against it: + +| The expression carries | Written as | Issue | +|---|---|---| +| `HOLISTICS_AQL` | `definition: @aql ;;`, unchanged | | +| the `--sql-dialect` dialect | `definition: @sql ;;`, unchanged | | +| `ANSI_SQL` or `OSSIE_SQL_2026` | `@sql`, rendered into `--sql-dialect` by sqlglot | WARNING, showing both texts | +| none of those | `@sql`, written unchanged | ERROR. The body will not run on this connection | + +AQL comes first and is exact, because the body came out of AML unchanged and goes back unchanged. + +A warehouse dialect that is not the target gets an ERROR rather than a rendering. Turning a `SNOWFLAKE` body +into a BigQuery one is a translation between warehouses, which this converter does not do. `ANSI_SQL` and +`OSSIE_SQL_2026` are the exception, because they are the portable forms the specification provides, and +every sibling converter reads `OSSIE_SQL_2026` as an `ANSI_SQL` equivalent. + +A `@sql` body needs every reference written back as Holistics interpolation. A bare column name does compile, +and it is not equivalent: AQL reads the interpolation to learn which field a body depends on, and a body +without it loses the query rewriting and the optimisation that link drives. + +| Ossie expression | AML | +|---|---| +| a bare column name | `{{ #SOURCE.column }}` | +| `.` | `{{ field }}` | +| a metric name owned by this model | `{{ }}` | +| a reference to a field of another dataset | no `@sql` form, so it is written model-qualified and an ERROR names it. See [Ossie SQL to AQL](aql.md) | + +Restoring those references means telling a column apart from a keyword, a function name and a date part. The +converter parses the expression with sqlglot at the dialect the expression declares, takes the character +offsets sqlglot reports for each column it found, and splices the replacement text into the original string. +In BigQuery's `DATE_TRUNC(created_at, month)` sqlglot reads `created_at` as a column and `month` as a date +part, so the result is `DATE_TRUNC({{ #SOURCE.created_at }}, month)`. + +The Ossie SQL is never written back out through sqlglot. `DATE_TRUNC(created_at, month)` rendered through the +default dialect returns `DATE_TRUNC(created_at, MONTH)`, which that same default dialect then re-reads with +the unit and the expression exchanged. Splicing on offsets keeps every character outside a replaced span +exactly as it was, so no round trip through the writer can happen. + +An expression the parser cannot read, or one tagged with a dialect that is not SQL, raises an ERROR naming +the field. The body is written out unchanged, which compiles and loses the field linkage, and the error says +so. + +## Metrics have two possible homes + +An Ossie metric sits at the document root and may span datasets. AML offers two places to put one. A stashed +`model` decides it, because it records which model the measure came from. Without one the expression +decides: + +- An expression naming exactly one dataset becomes a `measure` on that model. +- An expression naming several, or none, becomes a dataset-level `metric`. +- An expression naming another metric counts the datasets that metric resolves to as its own. + +### A metric that names another metric + +An Ossie body may name a metric where a column would go. `cities__upstream_measure + 1` in +`tests/fixtures/ecommerce` is one, and the forward path writes that shape every time one AML measure +references another. + +Which home the named metric landed in decides where the naming one can go. + +| The named metric is | The naming metric becomes | +|---|---| +| a `measure` on the same model | a `measure` on that model, written `{{ items_quantity }}` | +| a `measure` on another model | a dataset-level `metric`, written `items.items_quantity` in AQL | +| a dataset-level `metric` | a dataset-level `metric`, written as the bare name in AQL | + +AQL names a measure in any model, and a `@sql` body has no spelling for a dataset-level metric. So the last +two rows move the naming metric onto the dataset, where AQL resolves the reference. + +The third bullet above is what puts a metric on the right model. `items_quantity + 1` names no dataset of +its own. Counting only its columns sends it to the dataset, and `items_quantity` is an Ossie SQL name that AQL does +not resolve there, so the metric is dropped. + +Two shapes are still an ERROR, and a stash or a qualifier that disagrees with the body produces both: + +- A `@sql` body naming a dataset-level metric. A qualifier gives this: `orders.spanning` reads as a field of + `orders`, which has no such field. +- A measure stashed onto one model, naming a measure of another. Holistics rejects it at SQL generation with + `Cannot refer to external field in TableModel/QueryModel`. + +A metric that names itself through another metric is an ERROR as well. `holistics aml validate` accepts a +cycle, and Holistics inlines one metric into the body of the next, so the pair has no warehouse SQL. + +`tests/fixtures/metric_refs` covers each row of the table, and `holistics aml validate` runs over the AML it +generates. + +### A cross-dataset metric becomes AQL + +A `@sql` expression names only fields of the model it sits in, so a metric naming two datasets is rendered +into AQL whichever of the two places AML allows it. The converter raises a WARNING showing the Ossie SQL and +the AQL side by side. A metric that came from Holistics arrives as `HOLISTICS_AQL` already and is untouched. + +A construct the converter does not write as AQL drops the metric with an ERROR. A window function is the one +to expect. [Ossie SQL to AQL](aql.md) covers the function table, the raw-SQL call for a function AQL has no name +for, and the two changes rendering can make to the models a metric names. + +`aggregation_type` and the body compose into one Ossie expression on the way out, so the reverse path splits +them apart again. + +| Stash | AML written | +|---|---| +| a named `aggregation_type` wrapping the whole expression | that aggregation, and the body from inside the wrapper. `sum` over `SUM(quantity)` gives `aggregation_type: 'sum'` and `{{ #SOURCE.quantity }}` | +| no `aggregation_type` | `custom`, with the expression kept whole. The aggregate already sits inside it | +| a named `aggregation_type` that does not wrap the whole expression | `custom`, with the expression kept whole, and a WARNING. The expression was edited after the forward path wrote it, and aggregating it again would double the aggregate | + +TPC-DS takes the second row throughout, because it carries no stash. Its `total_sales` is +`SUM(store_sales.ss_ext_sales_price)` and becomes `definition: @sql SUM({{ ss_ext_sales_price }});;` under +`custom`, where `store_sales` is the model being written so the reference is a sibling. An unqualified +`SUM(ss_ext_sales_price)` would have become `SUM({{ #SOURCE.ss_ext_sales_price }})` instead. + +## Verifying the output + +The generated AML is checkable without a Holistics account: + +```bash +holistics aml validate /.dataset.aml -r +``` + +That parses every generated file and type-checks the AQL. `tests/test_snapshots.py` runs it over every +fixture and asserts a clean exit, so "the output is valid AML" is measured rather than asserted. CI installs +the CLI, so those tests run there too. + +Compiling goes further than validating. `test_the_round_trip_returns_the_same_ossie_document` runs +`holistics aml compile` over the generated AML and converts the result back to Ossie, then compares it with +the document the reverse path started from. For the `ecommerce` fixture the two are byte-identical. + +## What does not survive + +Beyond the file layout above: + +- Comments. The compiled JSON keeps `__doc__` fields but the forward path does not carry them into Ossie. +- Property order inside a block, which the generator chooses. +- Anything the forward path already declared lost. See [limitations](limitations.md). + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/vendor-payload.md b/converters/holistics/docs/vendor-payload.md new file mode 100644 index 000000000..90e10b8a1 --- /dev/null +++ b/converters/holistics/docs/vendor-payload.md @@ -0,0 +1,58 @@ + + +# The HOLISTICS payload + +AML carries properties that the Ossie core specification has no field for. The converter stashes each one under a single +`custom_extensions` entry with `vendor_name: HOLISTICS`, attached to the Ossie object it came from. + +```yaml +custom_extensions: +- vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo"}' +``` + +`data` is always a single JSON-encoded string, never a nested object, because the core specification requires that. It +carries `_v`, a shape version, currently 1. The converter fails on an unrecognised version rather than misreading a +shape it has never seen. Keys are sorted, so a change to the stash shows up as a diff on the keys that changed. + +What goes into the stash: + +| AML property | Scope | Why Ossie has no field for it | +|---|---|---| +| `aggregation_type` | metric | Ossie carries the whole aggregate in one expression | +| `hidden` | field, metric | Ossie models semantics, not visibility | +| `format` | field, metric | A display format, not semantics | +| `type` | field, metric | The AML type, kept verbatim so the return trip recovers it | +| query template forms | dataset | Which of the three forms the body uses, so a consumer knows why `source` is not runnable | +| `param` | dataset | A runtime input with no Ossie equivalent | +| `persistence` | dataset | A materialization directive, not semantics | +| `data_source_name` | model, dataset | An instance-local connection name | +| `owner` | model, dataset | An instance-local email address | +| `table_name` | dataset | Kept verbatim, because `source` normalizes the quoting | +| relationship kind | relationship | Ossie has no one-to-one marker | +| `active` | relationship | See [Relationships](mapping.md#relationships) | +| `where` | relationship | A `RelationshipFilter` narrowing the join condition. No Ossie equivalent | +| `direction`, `nullable`, `rlp_propagation` | relationship | No Ossie equivalent | +| `__fqn__` | dataset | The unflattened module path, kept so the return trip restores it | +| `aml_type` | model, dataset | `TableModel`, `QueryModel` or `Dataset`, so the return trip does not have to guess a query from the `source` text | +| `label` | field, metric, dataset | AML requires a label on every dimension and measure, and Ossie's `label` means categorization rather than a display name | +| `model` | metric | The model a measure came from. The reverse path reads this rather than splitting the metric name, which `__` makes ambiguous | +| `aml_name` | metric | The measure's own name, since the metric carries `__` | +| `scope` | field | `dataset` when the field came from a dataset-level AML `dimension`. Without the marker the reverse path would write it back inside a model, where a body spanning two models does not resolve | + +A foreign vendor's `custom_extensions` entry on the same object passes through untouched. The `HOLISTICS` +entry appears only on an object that has something to carry. + +`distinct_rows` is absent from the table above. It is deprecated in AML, so the converter neither reads it nor carries it. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/pyproject.toml b/converters/holistics/pyproject.toml new file mode 100644 index 000000000..13f6fdbff --- /dev/null +++ b/converters/holistics/pyproject.toml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + # Dev/test-only: tests/test_forward.py validates the converted document + # against core-spec/ossie-schema.json. Guarded by pytest.importorskip, so + # the package's only runtime dependency stays PyYAML. Same dev-group + # placement as converters/thoughtspot's pyproject.toml. + "jsonschema>=4.26.0", +] + +[project] +name = "apache-ossie-holistics" +version = "0.1.0" +description = "Convert between Holistics AML and the Apache Ossie semantic model" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.10" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Holistics", + "AML", +] +dependencies = [ + "PyYAML>=6.0", + # Runtime, not dev-only, unlike the sibling converters that use sqlglot for + # validation. ossie_to_aml needs it to tell a column apart from a keyword or a + # date part, which is what restores the AML interpolation AQL reads to learn + # a body's field dependencies. Without it the reverse path cannot produce + # usable AML at all. See src/ossie_holistics/sqlrefs.py. + "sqlglot>=30.12.0", +] + +[project.scripts] +ossie-holistics = "ossie_holistics.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_holistics"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] diff --git a/converters/holistics/src/ossie_holistics/__init__.py b/converters/holistics/src/ossie_holistics/__init__.py new file mode 100644 index 000000000..f00db3d2a --- /dev/null +++ b/converters/holistics/src/ossie_holistics/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert between Holistics AML and the Apache Ossie semantic model.""" + +from . import aml, aml_to_ossie, ossie_to_aml + +__all__ = ["aml", "aml_to_ossie", "ossie_to_aml"] diff --git a/converters/holistics/src/ossie_holistics/_yaml.py b/converters/holistics/src/ossie_holistics/_yaml.py new file mode 100644 index 000000000..60910f148 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/_yaml.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""YAML codec with the boolean resolver narrowed to YAML 1.2. + +PyYAML implements YAML 1.1, in which `on`, `off`, `yes`, `no`, `y` and `n` +resolve to booleans. An AML identifier may be any of them, so a bare +`yaml.safe_load` corrupts them silently. Only the boolean resolver is +narrowed here. No other YAML 1.1/1.2 divergence (e.g. octal/sexagesimal +number parsing) is addressed. + +Both directions matter. The loader stops 1.1 bool tokens becoming booleans. The +dumper quotes them on the way out, so the next reader cannot re-resolve them +even if it implements 1.1. +""" +import re + +import yaml + +from .errors import ConversionError + +#: YAML 1.2 core schema: only these spellings are booleans. +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") + +#: Bare scalars YAML 1.1 resolves as booleans and YAML 1.2 does not. +_YAML11_ONLY_BOOLS = frozenset( + {"y", "Y", "yes", "Yes", "YES", "n", "N", "no", "No", "NO", + "on", "On", "ON", "off", "Off", "OFF"} +) + + +class Yaml12Loader(yaml.SafeLoader): + """SafeLoader with the YAML 1.1 boolean resolver narrowed to the 1.2 set.""" + + +# Drop the inherited bool resolver outright, then reinstate the 1.2-only one. +# Mutating in place would affect SafeLoader itself, so rebuild the mapping. +Yaml12Loader.yaml_implicit_resolvers = { + key: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:bool"] + for key, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} +Yaml12Loader.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +class Yaml12Dumper(yaml.SafeDumper): + """SafeDumper that quotes strings a YAML 1.1 reader would take for booleans.""" + + +def _represent_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + style = "'" if data in _YAML11_ONLY_BOOLS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +Yaml12Dumper.add_representer(str, _represent_str) + + +def load(text: str) -> object: + """Parse YAML text under YAML 1.2 boolean rules. + + A malformed document raises `ConversionError` naming the failure, never a + bare `yaml.YAMLError` traceback. `stash.py` holds the same contract for + malformed `custom_extensions` JSON. + """ + try: + return yaml.load(text, Loader=Yaml12Loader) + except yaml.YAMLError as exc: + raise ConversionError(f"malformed YAML: {exc}") from exc + + +def dump(data: object) -> str: + """Serialise to YAML, preserving insertion order and quoting 1.1 bool tokens. + + `allow_unicode=True` so a non-ASCII value (e.g. a display label) emits as + a literal character rather than a `\\xXX`/`\\uXXXX` escape. Ossie + documents are read by people. + """ + return yaml.dump( + data, + Dumper=Yaml12Dumper, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + ) + + +#: Width to fold an issue message at. Wide enough to keep a line meaningful, +#: narrow enough to read in a side-by-side diff. +ISSUE_WIDTH = 96 + + +class IssueDumper(Yaml12Dumper): + """Dumper that folds a long message instead of running it off the screen.""" + + +def _represent_issue_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + """Fold a long single-line string, quote a YAML 1.1 bool token, else plain. + + Folded style (`>-`) breaks only at spaces, so a URL stays on one line, and a + reader joins the lines back with spaces. `load` returns the original string, + which `tests/test_snapshots.py` asserts. + """ + if data in _YAML11_ONLY_BOOLS: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") + style = ">" if len(data) > ISSUE_WIDTH and "\n" not in data else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +IssueDumper.add_representer(str, _represent_issue_str) + + +def dump_issues(issues: list[dict[str, object]]) -> str: + """The issue log as YAML, one block per issue, long messages folded. + + JSON put each message on one unbroken line, which a reviewer had to scroll + sideways to read. The same log as folded YAML fits a diff. + """ + if not issues: + return "[]\n" + return yaml.dump( + issues, + Dumper=IssueDumper, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + width=ISSUE_WIDTH, + ) diff --git a/converters/holistics/src/ossie_holistics/aml.py b/converters/holistics/src/ossie_holistics/aml.py new file mode 100644 index 000000000..fb4b18562 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aml.py @@ -0,0 +1,316 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A typed reading of `holistics aml compile` output. + +The compiled JSON is the boundary. Everything past this module works on the +dataclasses below rather than on raw dictionaries, so a shape the compiler +changes breaks here with a named error instead of silently reaching the +converted document. + +Three things in the payload are deliberately not read: + +`distinct_rows` is deprecated in AML. + +`models[]` on a `QueryModel` is a full inlined copy of every model the query +references. The same models are already in the dataset's own `models` list, so +reading the copy would duplicate every dataset. + +`__doc__` holds the AML comment text. The forward path has nowhere to put it, +and `reverse.md` records comments as not surviving the round trip. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from .errors import ConversionError + + +class ModelKind(str, Enum): + TABLE = "table" + QUERY = "query" + + +class RelationshipKind(str, Enum): + MANY_TO_ONE = "many_to_one" + ONE_TO_ONE = "one_to_one" + MANY_TO_MANY = "many_to_many" + RANGE = "range" + + +#: Only an equality relationship has an Ossie form. The other two carry an AQL +#: `match` condition rather than column pairs, so no column mapping exists. +EQUALITY_KINDS = frozenset({RelationshipKind.MANY_TO_ONE, RelationshipKind.ONE_TO_ONE}) + + +@dataclass(frozen=True) +class Heredoc: + """An AML `@sql` or `@aql` body. `language` is the heredoc's own name.""" + + language: str + content: str + + @property + def is_aql(self) -> bool: + return self.language == "aql" + + +@dataclass(frozen=True) +class Field: + """One `dimension` or `measure`. + + `aggregation_type` is None on a dimension and set on a measure, which is the + only structural difference between the two in the compiled payload. + """ + + name: str + label: str | None + description: str | None + hidden: bool + aml_type: str | None + format: str | None + definition: Heredoc | None + primary_key: bool + aggregation_type: str | None + #: Set on a dataset-level dimension, naming the model it attaches to. + owner_model_fqn: str | None = None + + +@dataclass(frozen=True) +class Param: + name: str + label: str | None + aml_type: str | None + + +@dataclass(frozen=True) +class Model: + name: str + fqn: str + label: str | None + description: str | None + owner: str | None + data_source_name: str | None + kind: ModelKind + table_name: str | None + query: Heredoc | None + dimensions: list[Field] + measures: list[Field] + params: list[Param] + persistence: dict[str, Any] | None + aml_type: str + + +@dataclass(frozen=True) +class Endpoint: + """One side of a relationship. `model` is spelled with dots, not `::`.""" + + model: str + field: str + + +@dataclass(frozen=True) +class Relationship: + kind: RelationshipKind + #: Column pairs, `from` first then each `on` block. They combine with AND. + pairs: list[tuple[Endpoint, Endpoint]] + active: bool + direction: str | None + nullable: bool | None + rlp_propagation: str | None + where: Any | None + aml_type: str + #: The payload as the compiler wrote it. A kind with no Ossie form is + #: carried in the stash verbatim, so the reverse path can rebuild it. + raw: dict[str, Any] + + +@dataclass(frozen=True) +class Dataset: + name: str + fqn: str + label: str | None + description: str | None + owner: str | None + data_source_name: str | None + models: list[Model] + relationships: list[Relationship] + #: Dataset-level `dimension` entries, each naming its owning model. + dimensions: list[Field] + #: Dataset-level `metric` entries, always AQL in practice. + metrics: list[Field] + + def model_by_fqn(self) -> dict[str, Model]: + return {model.fqn: model for model in self.models} + + +def _require(payload: Any, key: str, where: str) -> Any: + if not isinstance(payload, dict) or key not in payload: + raise ConversionError(f"{where}: compiled AML is missing {key!r}") + return payload[key] + + +def _optional_text(payload: dict[str, Any], key: str) -> str | None: + """A string property, with the compiler's empty string read as absent. + + `holistics aml compile` writes `""` for a property the source omits, and an + empty description is not a description. + """ + value = payload.get(key) + return value if isinstance(value, str) and value else None + + +def _heredoc(payload: Any, where: str) -> Heredoc | None: + if payload is None: + return None + if not isinstance(payload, dict): + raise ConversionError(f"{where}: expected a heredoc object, got {type(payload).__name__}") + return Heredoc( + language=_require(payload, "name", where), + content=_require(payload, "content", where), + ) + + +def _ordered(container: Any, where: str) -> list[dict[str, Any]]: + """Members of a `dimension`/`measure`/`param` block in AML source order. + + The compiler writes these as an object keyed by name, and a dataset-level + `dimension` as an array. `__childIdx__` is the source position, used as the + sort key so the converted document lists fields in the order the AML author + wrote them rather than in whatever order the payload happens to hold. + """ + if container is None: + return [] + if isinstance(container, dict): + members = list(container.values()) + elif isinstance(container, list): + members = list(container) + else: + raise ConversionError(f"{where}: expected an object or array") + return sorted(members, key=lambda m: m.get("__childIdx__", 0)) + + +def _field(payload: dict[str, Any], where: str) -> Field: + name = _require(payload, "name", where) + return Field( + name=name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + hidden=bool(payload.get("hidden", False)), + aml_type=payload.get("type"), + format=_optional_text(payload, "format"), + definition=_heredoc(payload.get("definition"), f"{where}.{name}"), + primary_key=bool(payload.get("primary_key", False)), + aggregation_type=payload.get("aggregation_type"), + owner_model_fqn=(payload.get("model") or {}).get("__fqn__"), + ) + + +def _param(payload: dict[str, Any], where: str) -> Param: + return Param( + name=_require(payload, "name", where), + label=_optional_text(payload, "label"), + aml_type=payload.get("type"), + ) + + +def _model(payload: dict[str, Any]) -> Model: + name = _require(payload, "name", "model") + where = f"model {name!r}" + raw_kind = _require(payload, "type", where) + try: + kind = ModelKind(raw_kind) + except ValueError as exc: + raise ConversionError(f"{where}: unknown model type {raw_kind!r}") from exc + return Model( + name=name, + fqn=payload.get("__fqn__") or name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + owner=_optional_text(payload, "owner"), + data_source_name=_optional_text(payload, "data_source_name"), + kind=kind, + table_name=_optional_text(payload, "table_name"), + query=_heredoc(payload.get("query"), where), + dimensions=[_field(d, where) for d in _ordered(payload.get("dimension"), where)], + measures=[_field(m, where) for m in _ordered(payload.get("measure"), where)], + params=[_param(p, where) for p in _ordered(payload.get("param"), where)], + persistence=payload.get("persistence"), + aml_type=payload.get("__type__", ""), + ) + + +def _endpoint(payload: dict[str, Any], where: str) -> Endpoint: + return Endpoint( + model=_require(payload, "model", where), + field=_require(payload, "field", where), + ) + + +def _relationship(payload: dict[str, Any], index: int) -> Relationship: + where = f"relationship {index}" + rel = _require(payload, "rel", where) + raw_kind = _require(rel, "type", where) + try: + kind = RelationshipKind(raw_kind) + except ValueError as exc: + raise ConversionError(f"{where}: unknown relationship type {raw_kind!r}") from exc + + pairs: list[tuple[Endpoint, Endpoint]] = [] + if "from" in rel and "to" in rel: + pairs.append((_endpoint(rel["from"], where), _endpoint(rel["to"], where))) + for extra in _ordered(rel.get("on"), where): + pairs.append((_endpoint(extra["from"], where), _endpoint(extra["to"], where))) + + return Relationship( + kind=kind, + pairs=pairs, + active=bool(payload.get("active", True)), + direction=payload.get("direction"), + nullable=payload.get("nullable"), + rlp_propagation=payload.get("rlp_propagation"), + where=rel.get("where"), + aml_type=rel.get("__type__", ""), + raw=payload, + ) + + +def parse(payload: Any) -> Dataset: + """Read one compiled `*.dataset.aml.json` document.""" + if not isinstance(payload, dict): + raise ConversionError("compiled AML is not an object") + if payload.get("__type__") != "Dataset": + raise ConversionError( + f"expected a compiled Dataset, got __type__={payload.get('__type__')!r}" + ) + name = _require(payload, "name", "dataset") + return Dataset( + name=name, + fqn=payload.get("__fqn__") or name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + owner=_optional_text(payload, "owner"), + data_source_name=_optional_text(payload, "data_source_name"), + models=[_model(m) for m in payload.get("models") or []], + relationships=[ + _relationship(r, i) for i, r in enumerate(payload.get("relationships") or []) + ], + dimensions=[_field(d, "dataset") for d in _ordered(payload.get("dimension"), "dataset")], + metrics=[_field(m, "dataset") for m in _ordered(payload.get("metric"), "dataset")], + ) diff --git a/converters/holistics/src/ossie_holistics/aml_to_ossie.py b/converters/holistics/src/ossie_holistics/aml_to_ossie.py new file mode 100644 index 000000000..07f780f4a --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aml_to_ossie.py @@ -0,0 +1,611 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Compiled AML to one Apache Ossie semantic model. + +`convert` takes the parsed `holistics aml compile` output and returns the Ossie +document plus the issues the conversion raised. A construct it drops or changes +raises an `ISSUE_*` code, and `docs/limitations.md` lists every one. + +Naming is the decision that shapes the rest. An AML module path joins with +`::`, so `reporting::calendar` flattens to the Ossie dataset name +`reporting__calendar`. A model measure becomes a root-level Ossie metric, and +root level has one namespace for the whole document, so the metric takes the +name `__`. A reference to a field inside an expression is +dataset-qualified (`ecommerce_orders.created_at`), and a reference to a metric +is the metric's own name, because that is what each one resolves against. +""" +from __future__ import annotations + +from collections import Counter +from typing import Any + +from . import aml, interpolation, stash +from .datatypes import AGGREGATIONS, AML_TO_OSSIE_DATATYPE, CUSTOM_AGGREGATION, apply_aggregation +from .errors import ConversionError +from .issues import IssueLog, Severity + +SPEC_VERSION = "0.2.0.dev0" + +AQL_DIALECT = "HOLISTICS_AQL" + +ISSUE_AQL_NOT_TRANSLATED = "HOLISTICS_AQL_NOT_TRANSLATED" +ISSUE_INACTIVE_RELATIONSHIP = "HOLISTICS_INACTIVE_RELATIONSHIP" +ISSUE_UNSUPPORTED_RELATIONSHIP = "HOLISTICS_UNSUPPORTED_RELATIONSHIP" +ISSUE_RELATIONSHIP_FILTER = "HOLISTICS_RELATIONSHIP_FILTER" +ISSUE_QUERY_SOURCE = "HOLISTICS_QUERY_SOURCE" +ISSUE_UNRESOLVED_QUERY = "HOLISTICS_UNRESOLVED_QUERY_TEMPLATE" +ISSUE_UNKNOWN_DATATYPE = "HOLISTICS_UNKNOWN_DATATYPE" +ISSUE_UNKNOWN_AGGREGATION = "HOLISTICS_UNKNOWN_AGGREGATION" +ISSUE_UNRESOLVED_REFERENCE = "HOLISTICS_UNRESOLVED_REFERENCE" +ISSUE_PARAM_DROPPED = "HOLISTICS_PARAM_DROPPED" + + +class Result: + def __init__(self, model: dict[str, Any], issues: IssueLog): + self.model = model + self.issues = issues + + +def flatten(fqn: str) -> str: + """An AML module path as a flat Ossie name. + + `__` rather than `_` because a single underscore collides with any model + name that already contains one, which most do. `reporting::calendar` and a + root model named `reporting_calendar` would otherwise take the same name. + """ + return fqn.replace("::", "__") + + +def _expression(expression: str, dialect: str) -> dict[str, Any]: + return {"dialects": [{"dialect": dialect, "expression": expression}]} + + +class _Converter: + """One conversion. Holds the name tables every expression resolves against.""" + + def __init__(self, dataset: aml.Dataset, sql_dialect: str): + self.dataset = dataset + self.sql_dialect = sql_dialect + self.issues = IssueLog() + + #: Relationships whose condition is a `match` predicate. Ossie encodes a + #: relationship as column pairs, so these travel in the document stash. + self.match_relationships: list[dict[str, Any]] = [] + self.models_by_fqn = dataset.model_by_fqn() + self.dataset_name: dict[str, str] = {} + claimed_by: dict[str, str] = {} + for model in dataset.models: + flat = flatten(model.fqn) + clash = claimed_by.get(flat) + if clash is not None: + raise ConversionError( + f"models {clash!r} and {model.fqn!r} both flatten to the Ossie " + f"dataset name {flat!r}; rename one of them in AML" + ) + claimed_by[flat] = model.fqn + self.dataset_name[model.fqn] = flat + + #: Dataset-level dimensions, grouped by the model they attach to. + self.extra_dimensions: dict[str, list[aml.Field]] = {} + for field in dataset.dimensions: + owner = field.owner_model_fqn + if owner is None or owner not in self.models_by_fqn: + raise ConversionError( + f"dataset dimension {field.name!r} names model {owner!r}, " + f"which is not in this dataset" + ) + self.extra_dimensions.setdefault(owner, []).append(field) + + #: Every field name reachable on a model, for resolving `{{ field }}`. + self.model_fields: dict[str, set[str]] = {} + self.model_measures: dict[str, set[str]] = {} + for model in dataset.models: + extra = self.extra_dimensions.get(model.fqn, []) + self.model_fields[model.fqn] = {d.name for d in model.dimensions} | { + d.name for d in extra + } + self.model_measures[model.fqn] = {m.name for m in model.measures} + + # -- names ------------------------------------------------------------ + + def metric_name(self, model_fqn: str, measure_name: str) -> str: + return f"{self.dataset_name[model_fqn]}__{measure_name}" + + def resolve_model(self, reference: str) -> str | None: + """An AML model reference to its `__fqn__`. + + A relationship endpoint and a model spell the same name differently. The + endpoint uses dots (`reporting.calendar`) and the model's own `__fqn__` + uses colons (`reporting::calendar`). The compiler fills a `__refFqn__` + shortcut on some endpoints and leaves it null on others, so this + rewrites the separator and looks the result up instead. + """ + for candidate in (reference, reference.replace(".", "::")): + if candidate in self.models_by_fqn: + return candidate + return None + + # -- expressions ------------------------------------------------------ + + def resolve_reference(self, model_fqn: str, name: str, scope: str) -> str: + """`{{ field }}` as the Ossie name it resolves to. + + A dimension becomes a dataset-qualified field reference. A measure + becomes a metric reference, which is the metric's own name because + metrics live in one root namespace. + """ + if name in self.model_fields.get(model_fqn, ()): + return f"{self.dataset_name[model_fqn]}.{name}" + if name in self.model_measures.get(model_fqn, ()): + return self.metric_name(model_fqn, name) + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"expression references {name!r}, which is not a field or measure of this model", + object_ref=scope, + remedy="fix the reference in AML, or drop the field", + ) + return name + + def resolve_model_field( + self, model_fqn: str, reference: str, field_name: str, scope: str + ) -> str: + """`{{ #model.field }}` as an Ossie reference. + + Naming this model is not the same as `{{ field }}`. `{{ field }}` inlines + the sibling field's own definition, while `{{ #model.field }}` reads the + column of that model's relation, exactly as `{{ #SOURCE.field }}` does. + `ecommerce_users.age_bucket` is `FLOOR({{ #ecommerce_users.age }} / 10)` + and Holistics renders it `FLOOR(`ecommerce_users`.`age` / 10)`, where + `{{ age }}` would have rendered the whole `extract(...)` body instead. + Collapsing the two changes the warehouse SQL, so a self-naming reference becomes a + column and only a measure, which is never a relation column, stays a + metric reference. + """ + target = self.resolve_model(reference) + if target is None: + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"expression references model {reference!r}, which is not in this dataset", + object_ref=scope, + remedy="add the model to the dataset, or drop the field", + ) + return f"{flatten(reference)}.{field_name}" + if target == model_fqn and field_name not in self.model_measures.get(target, ()): + return field_name + return self.resolve_reference(target, field_name, scope) + + def resolve_interpolation( + self, model_fqn: str, part: interpolation.Interpolation, scope: str + ) -> str: + """One interpolation as the Ossie text that replaces it.""" + if isinstance(part, interpolation.Source): + return part.column + if isinstance(part, interpolation.Sibling): + return self.resolve_reference(model_fqn, part.field, scope) + if isinstance(part, interpolation.ModelField): + return self.resolve_model_field(model_fqn, part.model, part.field, scope) + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"unrecognised AML interpolation {part.to_aml()!r}, carried verbatim", + object_ref=scope, + remedy="rewrite the body in AML, or drop the field", + ) + return part.to_aml() + + def translate_sql(self, model_fqn: str, body: str, scope: str) -> str: + """A `@sql` body with its interpolations replaced by Ossie references.""" + return "".join( + part if isinstance(part, str) else self.resolve_interpolation(model_fqn, part, scope) + for part in interpolation.parse_sql_body(body) + ) + + def sql_body(self, model_fqn: str, body: str, scope: str) -> str: + """A `@sql` body as an Ossie expression, with its references resolved. + + Every `@sql` body is tagged `--sql-dialect`, including one that is a + bare reference. Holistics SQL is warehouse SQL throughout, so + one dialect describes the whole document and a reader needs no rule for + which fields carry which tag. + """ + sole = interpolation.sole_reference(body) + if sole is not None: + return self.resolve_interpolation(model_fqn, sole, scope) + return self.translate_sql(model_fqn, body, scope) + + def field_expression(self, model_fqn: str, field: aml.Field, scope: str) -> dict[str, Any]: + """The `expression` for one dimension. + + An omitted `definition` is a default rather than a missing value: AML + fills it with `{{ #SOURCE. }}`, so it resolves exactly like + a bare source column written out in full. + """ + definition = field.definition + if definition is None: + return _expression(field.name, self.sql_dialect) + if definition.is_aql: + self._note_aql(scope) + return _expression(definition.content, AQL_DIALECT) + return _expression(self.sql_body(model_fqn, definition.content, scope), self.sql_dialect) + + def measure_expression(self, model_fqn: str, measure: aml.Field, scope: str) -> dict[str, Any]: + """The `expression` for one measure, aggregation and body composed. + + An AQL body carries its own aggregation, so `aggregation_type` is not + applied on top of it whatever it says. + """ + definition = measure.definition + if definition is None: + raise ConversionError( + f"{scope}: a measure must carry a definition, and this one has none" + ) + if definition.is_aql: + self._note_aql(scope) + return _expression(definition.content, AQL_DIALECT) + + body = self.sql_body(model_fqn, definition.content, scope) + + aggregation = measure.aggregation_type + if aggregation in (None, CUSTOM_AGGREGATION): + return _expression(body, self.sql_dialect) + if aggregation not in AGGREGATIONS: + self.issues.add( + code=ISSUE_UNKNOWN_AGGREGATION, + severity=Severity.ERROR, + message=f"unknown aggregation_type {aggregation!r}, body emitted unaggregated", + object_ref=scope, + remedy="add the aggregation to datatypes.AGGREGATIONS", + ) + return _expression(body, self.sql_dialect) + return _expression(apply_aggregation(aggregation, body), self.sql_dialect) + + def _note_aql(self, scope: str) -> None: + self.issues.add( + code=ISSUE_AQL_NOT_TRANSLATED, + severity=Severity.INFO, + message=( + "body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does " + "not speak AQL can read and round-trip it but cannot evaluate it" + ), + object_ref=scope, + remedy="evaluate through Holistics, or rewrite the body as Holistics SQL", + ) + + # -- objects ---------------------------------------------------------- + + def datatype(self, field: aml.Field, scope: str) -> str | None: + if field.aml_type is None: + return None + mapped = AML_TO_OSSIE_DATATYPE.get(field.aml_type) + if mapped is None: + self.issues.add( + code=ISSUE_UNKNOWN_DATATYPE, + severity=Severity.WARNING, + message=f"unknown AML type {field.aml_type!r}, datatype omitted", + object_ref=scope, + remedy="add the type to datatypes.AML_TO_OSSIE_DATATYPE", + ) + return mapped + + def field_stash(self, field: aml.Field) -> dict[str, Any]: + data: dict[str, Any] = {} + if field.hidden: + data["hidden"] = True + if field.format: + data["format"] = field.format + if field.aml_type: + data["type"] = field.aml_type + if field.label: + data["label"] = field.label + return data + + def convert_field( + self, model_fqn: str, field: aml.Field, dataset_level: bool = False + ) -> dict[str, Any]: + """One dimension as an Ossie field. + + A dataset-level AML dimension becomes a field on the model it names, and + the stash records where it was declared. Without that marker the reverse + path would write it back as a model dimension, and an AQL body spanning + two models does not resolve from inside one model. + """ + scope = f"{model_fqn}.{field.name}" + out: dict[str, Any] = { + "name": field.name, + "expression": self.field_expression(model_fqn, field, scope), + } + if field.description: + out["description"] = field.description + datatype = self.datatype(field, scope) + if datatype: + out["datatype"] = datatype + data = self.field_stash(field) + if dataset_level: + data["scope"] = "dataset" + stash.attach(out, data) + return out + + def convert_measure(self, model_fqn: str, measure: aml.Field) -> dict[str, Any]: + scope = f"{model_fqn}.{measure.name}" + out: dict[str, Any] = { + "name": self.metric_name(model_fqn, measure.name), + "expression": self.measure_expression(model_fqn, measure, scope), + } + if measure.description: + out["description"] = measure.description + datatype = self.datatype(measure, scope) + if datatype: + out["datatype"] = datatype + data = self.field_stash(measure) + data["model"] = model_fqn + data["aml_name"] = measure.name + if measure.aggregation_type: + data["aggregation_type"] = measure.aggregation_type + stash.attach(out, data) + return out + + def convert_dataset_metric(self, metric: aml.Field) -> dict[str, Any]: + scope = f"{self.dataset.name}.{metric.name}" + definition = metric.definition + if definition is None: + raise ConversionError(f"{scope}: a dataset metric must carry a definition") + if not definition.is_aql: + raise ConversionError( + f"{scope}: a dataset-level metric spans models, so its body has no single " + f"model to resolve `@sql` against" + ) + self._note_aql(scope) + out: dict[str, Any] = { + "name": metric.name, + "expression": _expression(definition.content, AQL_DIALECT), + } + if metric.description: + out["description"] = metric.description + datatype = self.datatype(metric, scope) + if datatype: + out["datatype"] = datatype + stash.attach(out, self.field_stash(metric)) + return out + + def model_source(self, model: aml.Model) -> str: + scope = model.fqn + if model.kind is aml.ModelKind.TABLE: + if not model.table_name: + raise ConversionError(f"{scope}: a table model has no table_name") + return model.table_name + if model.query is None: + raise ConversionError(f"{scope}: a query model has no query") + body = model.query.content + self.issues.add( + code=ISSUE_QUERY_SOURCE, + severity=Severity.WARNING, + message=( + "a query model becomes a query in `source`, which is one string, so a " + "consumer reads the text to tell it from a table reference" + ), + object_ref=scope, + remedy="materialise the query as a table in Holistics before converting", + ) + forms = interpolation.query_forms_used(body) + if forms: + self.issues.add( + code=ISSUE_UNRESOLVED_QUERY, + severity=Severity.ERROR, + message=( + f"the query body uses Holistics template syntax ({', '.join(forms)}) " + f"and is carried unresolved. `source` is not runnable SQL" + ), + object_ref=scope, + remedy="resolve the query through Holistics, or drop the model", + ) + return body + + def model_stash(self, model: aml.Model) -> dict[str, Any]: + data: dict[str, Any] = {"aml_type": model.aml_type} + if model.fqn != model.name: + data["fqn"] = model.fqn + if model.label: + data["label"] = model.label + if model.owner: + data["owner"] = model.owner + if model.data_source_name: + data["data_source_name"] = model.data_source_name + if model.table_name: + data["table_name"] = model.table_name + if model.persistence: + data["persistence"] = model.persistence + if model.params: + data["param"] = [ + {"name": p.name, "label": p.label, "type": p.aml_type} for p in model.params + ] + self.issues.add( + code=ISSUE_PARAM_DROPPED, + severity=Severity.WARNING, + message=( + f"{len(model.params)} param(s) have no Ossie equivalent and are stashed. " + f"A body that reads one cannot be resolved statically" + ), + object_ref=model.fqn, + remedy="supply the param value in Holistics before converting", + ) + if model.kind is aml.ModelKind.QUERY and model.query is not None: + forms = interpolation.query_forms_used(model.query.content) + if forms: + data["query_forms"] = forms + return data + + def convert_model(self, model: aml.Model) -> dict[str, Any]: + out: dict[str, Any] = { + "name": self.dataset_name[model.fqn], + "source": self.model_source(model), + } + primary_key = [d.name for d in model.dimensions if d.primary_key] + if primary_key: + out["primary_key"] = primary_key + description = model.description or model.label + if description: + out["description"] = description + out["fields"] = [self.convert_field(model.fqn, f) for f in model.dimensions] + [ + self.convert_field(model.fqn, f, dataset_level=True) + for f in self.extra_dimensions.get(model.fqn, []) + ] + stash.attach(out, self.model_stash(model)) + return out + + def convert_relationship( + self, relationship: aml.Relationship, index: int, taken: set[str] + ) -> dict[str, Any] | None: + scope = f"relationship {index}" + if relationship.kind not in aml.EQUALITY_KINDS: + self.issues.add( + code=ISSUE_UNSUPPORTED_RELATIONSHIP, + severity=Severity.WARNING, + message=( + f"a {relationship.kind.value} relationship carries an AQL match condition " + f"rather than column pairs, so it has no `relationships[]` entry and is " + f"stashed on the document instead. A consumer reading Ossie sees the two " + f"datasets as unrelated" + ), + object_ref=scope, + remedy="model the join as an equality relationship in AML", + ) + self.match_relationships.append(relationship.raw) + return None + + from_fqn = self.resolve_model(relationship.pairs[0][0].model) + to_fqn = self.resolve_model(relationship.pairs[0][1].model) + if from_fqn is None or to_fqn is None: + missing = relationship.pairs[0][0 if from_fqn is None else 1].model + raise ConversionError(f"{scope}: endpoint names model {missing!r}, not in this dataset") + + from_name = self.dataset_name[from_fqn] + to_name = self.dataset_name[to_fqn] + name = f"{from_name}_to_{to_name}" + base, suffix = name, 2 + while name in taken: + name, suffix = f"{base}_{suffix}", suffix + 1 + taken.add(name) + scope = name + + out: dict[str, Any] = { + "name": name, + "from": from_name, + "to": to_name, + "from_columns": [pair[0].field for pair in relationship.pairs], + "to_columns": [pair[1].field for pair in relationship.pairs], + } + + if not relationship.active: + self.issues.add( + code=ISSUE_INACTIVE_RELATIONSHIP, + severity=Severity.WARNING, + message=( + "the relationship is inactive in AML, and Ossie has no inactive marker. " + "A consumer joins across it where Holistics leaves it out by default" + ), + object_ref=scope, + remedy="drop the relationship, or activate it in AML", + ) + if relationship.where is not None: + self.issues.add( + code=ISSUE_RELATIONSHIP_FILTER, + severity=Severity.WARNING, + message=( + "the relationship carries a filter that narrows the join, and Ossie has " + "no equivalent. The join widens to every matching row" + ), + object_ref=scope, + remedy="move the condition into a dimension in AML", + ) + + data: dict[str, Any] = {"type": relationship.kind.value, "active": relationship.active} + for key, value in ( + ("direction", relationship.direction), + ("nullable", relationship.nullable), + ("rlp_propagation", relationship.rlp_propagation), + ("where", relationship.where), + ): + if value is not None: + data[key] = value + stash.attach(out, data) + return out + + # -- entry point ------------------------------------------------------ + + def run(self) -> Result: + datasets = [self.convert_model(m) for m in self.dataset.models] + + metrics: list[dict[str, Any]] = [] + for model in self.dataset.models: + metrics.extend(self.convert_measure(model.fqn, m) for m in model.measures) + metrics.extend(self.convert_dataset_metric(m) for m in self.dataset.metrics) + counts = Counter(m["name"] for m in metrics) + duplicates = sorted(name for name, count in counts.items() if count > 1) + if duplicates: + raise ConversionError( + f"metric name(s) {', '.join(duplicates)} derived twice; " + f"rename the colliding measures in AML" + ) + + taken: set[str] = set() + relationships = [ + converted + for index, relationship in enumerate(self.dataset.relationships) + if (converted := self.convert_relationship(relationship, index, taken)) is not None + ] + + model: dict[str, Any] = { + "version": SPEC_VERSION, + "name": flatten(self.dataset.fqn), + } + description = self.dataset.description or self.dataset.label + if description: + model["description"] = description + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + data: dict[str, Any] = {"aml_type": "Dataset"} + if self.match_relationships: + data["match_relationships"] = self.match_relationships + if self.dataset.fqn != self.dataset.name: + data["fqn"] = self.dataset.fqn + if self.dataset.label: + data["label"] = self.dataset.label + if self.dataset.owner: + data["owner"] = self.dataset.owner + if self.dataset.data_source_name: + data["data_source_name"] = self.dataset.data_source_name + stash.attach(model, data) + return Result(model, self.issues) + + +def convert(payload: Any, sql_dialect: str) -> Result: + """One compiled AML dataset to one Ossie semantic model. + + `sql_dialect` names the warehouse a computed `@sql` body is written for. AML + does not record it, because `data_source_name` is an instance-local + connection name rather than a dialect declaration. It is required even for a + dataset whose bodies are all bare references, so that adding one computed + body later does not change the command a caller has to run. + """ + return _Converter(aml.parse(payload), sql_dialect).run() diff --git a/converters/holistics/src/ossie_holistics/amlgen.py b/converters/holistics/src/ossie_holistics/amlgen.py new file mode 100644 index 000000000..d29a0eb86 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/amlgen.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Writing AML source text. + +The reverse path produces AML source rather than compiled JSON, because +`holistics aml compile` only runs one way and nothing reads its output back. +AML source is what a person edits, a repository holds, and the CLI accepts. + +Everything here is about the text: quoting, indentation, identifiers and +heredocs. What to write is `ossie_to_aml`'s decision. +""" +from __future__ import annotations + +import re + +from .errors import ConversionError + +def sanitize(name: str) -> str: + """An Ossie name as a legal AML identifier. + + An Ossie `name` is any non-empty string, so a valid document can carry one + AML cannot spell. `examples/flights.semantic_model.yaml` is named + `Flights semantic model`. Each illegal character becomes an underscore, and + a name starting with a digit gains a leading one. + + The caller reports the rename and checks for a collision. This function only + does the spelling. + """ + if not name: + raise ConversionError("an Ossie name is empty, and AML has no spelling for that") + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", name) + return cleaned if cleaned[0].isalpha() or cleaned[0] == "_" else f"_{cleaned}" + + +def quote(value: str) -> str: + """An AML single-quoted string. A backslash escapes a quote or a backslash.""" + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def titleize(name: str) -> str: + """The label Holistics generates for a field that declares none. + + `label` is required on a dimension and on a measure, so a document with no + stashed label still needs one, and this is the spelling AML itself uses: + `created_at` becomes `Created At`. + """ + return " ".join(word.capitalize() for word in name.split("_") if word) or name + + +class Block: + """One `Name header { ... }` block, rendered with its children.""" + + def __init__(self, header: str): + self.header = header + self.lines: list[str | Block] = [] + + def property(self, key: str, value: str) -> "Block": + """A `key: value` line. `value` is already rendered AML.""" + self.lines.append(f"{key}: {value}") + return self + + def text(self, key: str, value: str | None) -> "Block": + """A `key: 'value'` line, skipped when `value` is None or empty.""" + if value: + self.property(key, quote(value)) + return self + + def flag(self, key: str, value: bool) -> "Block": + """A `key: true` line, written only when true. + + AML defaults every one of these to false, so writing `false` would add + noise without adding meaning. + """ + if value: + self.property(key, "true") + return self + + def heredoc(self, key: str, language: str, body: str) -> "Block": + """A `key: @sql ... ;;` property. + + A single-line body stays on one line, which is how AML is normally + written. A multi-line body is indented under the opener with the + terminator on its own line, because `;;` on the last content line would + read as part of that line. + """ + if "\n" not in body: + return self.property(key, f"@{language} {body};;") + self.lines.append(_Heredoc(key, language, body)) + return self + + def block(self, header: str) -> "Block": + child = Block(header) + self.lines.append(child) + return child + + def blank(self) -> "Block": + self.lines.append("") + return self + + def render(self, indent: int = 0) -> str: + pad = " " * indent + out = [f"{pad}{self.header} {{"] + for line in self.lines: + if isinstance(line, Block): + out.append(line.render(indent + 1)) + elif isinstance(line, _Heredoc): + out.append(line.render(indent + 1)) + elif line == "": + out.append("") + else: + out.append(f"{pad} {line}") + out.append(f"{pad}}}") + return "\n".join(out) + + +class _Heredoc: + def __init__(self, key: str, language: str, body: str): + self.key = key + self.language = language + self.body = body + + def render(self, indent: int) -> str: + pad = " " * indent + body = "\n".join(f"{pad} {line}" if line.strip() else "" for line in self.body.split("\n")) + return f"{pad}{self.key}: @{self.language}\n{body}\n{pad};;" + + +def array(items: list[str], indent: int = 1) -> str: + """A multi-line AML array with a trailing comma, as AML files are written.""" + if not items: + return "[]" + pad = " " * (indent + 1) + closing = " " * indent + body = "\n".join(f"{pad}{item}," for item in items) + return f"[\n{body}\n{closing}]" + + +LICENSE_HEADER = """\ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +""" + + +def document(block: Block, license_header: bool) -> str: + head = f"{LICENSE_HEADER}\n" if license_header else "" + return f"{head}{block.render()}\n" diff --git a/converters/holistics/src/ossie_holistics/aql.py b/converters/holistics/src/ossie_holistics/aql.py new file mode 100644 index 000000000..c57fb9572 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aql.py @@ -0,0 +1,360 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Rendering an Ossie SQL expression as AQL. + +An AQL expression can name fields from several models. A `@sql` expression can +name only fields of the model it sits in, and a dataset-level AML `metric` +accepts only `@aql`. So an Ossie metric naming two datasets has no AML home +unless its Ossie SQL becomes AQL. + +What this translates is a name map and nothing else. `FUNCTIONS` pairs a sqlglot +node with the AQL function of the same meaning, references are already +`dataset.field` on both sides, and the operators match. Anything outside the +table raises `Untranslatable`, so the caller reports it rather than guessing. + +Window functions are the deliberate omission. Even though AQL has `rank`, +`previous`, `window_sum` and the rest, this converter cannot output one. + +Ossie SQL carries the partition and the ordering in the `OVER` clause, while AQL takes +both from the grain of the query being run. Translating one into the other means +deciding which query grain a given `OVER` clause corresponds to. The Ossie +specification says nothing about how a window function should behave when +queried, so nothing says which grain that is, and picking one would change what +the metric computes. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from . import sqlrefs + +if sqlrefs.AVAILABLE: # pragma: no branch - exercised by the packaging test + import sqlglot + from sqlglot import exp + + +#: The AQL cheatsheet, whose Window Function section lists `rank`, `previous`, +#: `next`, `window_sum` and the rest. +WINDOW_FUNCTION_DOCS = "https://docs.holistics.io/reference/aql/functions" + +#: One window function in full, showing that the grain comes from the query. +RANK_DOCS = "https://docs.holistics.io/reference/aql/rank" + +#: AQL's raw-SQL escape hatch, `sql_number(func_name, arg1, arg2, ...)`. The +#: family is typed by return value, and a metric body is numeric. +SQL_PASSTHROUGH = "sql_number" + + +@dataclass(frozen=True) +class _Context: + """What every `_render` call needs: the source text and the caller's hooks.""" + + source: str + functions: dict + reference: object + numeric: set + + +class Untranslatable(Exception): + """Raised when this converter will not write an expression as AQL. + + The expression may well have an AQL form. A window function does. This says + the converter did not write one, not that AQL cannot express it. + """ + + +def _functions(): + """sqlglot node to the AQL function of the same meaning and argument order. + + Built lazily because the keys are sqlglot classes, and sqlglot is imported + at module scope only when it is installed. + + Membership was derived by parsing each name AQL registers in + `src/checker/row_level.ts` and reading which node sqlglot produced, rather + than by matching names by eye. A name outside it falls to `_from_source`, + which rebuilds the call from the source text when the offsets allow it: + + A name that is one sqlglot node with several spellings. `LPAD` and `RPAD` + are both `Pad`, `LTRIM` and `RTRIM` are both `Trim`. The node's offset + points at the name in the source, so the passthrough recovers it. + + A name whose arguments sqlglot rewrote. `LOG10(x)` gains a base the source + never had, and `DATE_TRUNC('month', d)` has its unit normalized. Neither + argument carries an offset, so neither call can be rebuilt faithfully. + + A name whose arguments mean something else in AQL. Ossie SQL `SUBSTRING(s, start, + len)` and `STRPOS(s, sub)` do not line up with AQL's `mid` and `find`, so + the passthrough carries them under their own names instead. + """ + return { + # Aggregates. + exp.Sum: "sum", + exp.Avg: "avg", + exp.Min: "min", + exp.Max: "max", + exp.Stddev: "stdev", + exp.StddevPop: "stdevp", + exp.Variance: "var", + exp.VariancePop: "varp", + # Numeric. + exp.Abs: "abs", + exp.Ceil: "ceil", + exp.Floor: "floor", + exp.Round: "round", + exp.Trunc: "trunc", + exp.Sqrt: "sqrt", + exp.Pow: "pow", + exp.Exp: "exp", + exp.Ln: "ln", + exp.Mod: "mod", + exp.Sign: "sign", + exp.SafeDivide: "safe_divide", + # Trigonometric. + exp.Acos: "acos", + exp.Asin: "asin", + exp.Atan: "atan", + exp.Atan2: "atan2", + exp.Cos: "cos", + exp.Cot: "cot", + exp.Sin: "sin", + exp.Tan: "tan", + exp.Degrees: "degrees", + exp.Radians: "radians", + # Text. + exp.Concat: "concat", + exp.Lower: "lower", + exp.Upper: "upper", + exp.Length: "len", + exp.Left: "left", + exp.Right: "right", + exp.Replace: "replace", + exp.SplitPart: "split_part", + exp.RegexpExtract: "regexp_extract", + exp.RegexpLike: "regexp_like", + exp.RegexpReplace: "regexp_replace", + # Date parts, which take one date argument and so carry no order risk. + exp.Year: "year", + exp.Quarter: "quarter", + exp.Month: "month", + exp.Week: "week", + exp.Day: "day", + exp.Hour: "hour", + exp.Minute: "minute", + exp.LastDay: "last_day", + # Conditional. + exp.Nullif: "nullif", + exp.Coalesce: "coalesce", + } + + +#: Aggregates whose argument must be numeric. A reference inside one of these +#: tells the caller the field's type, which matters when the Ossie document +#: declares no `datatype`: AQL answers `sum` over a text dimension with +#: "The `sum` function expects `Dimension(Number)`, but got `Text`". +def _numeric_aggregates(): + return (exp.Sum, exp.Avg, exp.Stddev, exp.StddevPop, exp.Variance, exp.VariancePop) + + +#: Ossie SQL binary operators that are spelled the same in AQL. +OPERATORS = { + "Add": "+", + "Sub": "-", + "Mul": "*", + "Div": "/", + "EQ": "==", + "NEQ": "!=", + "GT": ">", + "GTE": ">=", + "LT": "<", + "LTE": "<=", + "And": "and", + "Or": "or", +} + + +def translate(expression: str, dialect: str, reference) -> tuple[str, set[tuple[str, str]]]: + """`expression` as AQL, with the columns it aggregates numerically. + + `reference(table, column)` returns the AQL spelling of one column, so the + caller owns the mapping from Ossie dataset names to AML model names. Raises + `Untranslatable` naming what stopped it. + """ + if not sqlrefs.AVAILABLE: + raise Untranslatable("sqlglot is not installed") + try: + tree = sqlglot.parse_one(expression, read=sqlrefs.SQLGLOT_DIALECT.get(dialect)) + except Exception as exc: # noqa: BLE001 - sqlglot raises several error types + raise Untranslatable(f"sqlglot cannot read it as {dialect}. {exc}") from exc + numeric: set[tuple[str, str]] = set() + context = _Context(source=expression, functions=_functions(), reference=reference, numeric=numeric) + return _render(tree, context), numeric + + +def _render(node, ctx) -> str: + if isinstance(node, exp.Window): + raise Untranslatable( + "It is a window function. Even though AQL has window functions, this converter " + "cannot output one. Ossie SQL carries the partition and the ordering in the OVER " + "clause, " + "while AQL takes both from the grain of the query being run. Translating one " + "into the other means deciding which query grain this OVER clause corresponds " + "to, and the Ossie specification defines no semantics for how a window function " + "behaves when queried, so nothing says which grain that is. See " + f"{WINDOW_FUNCTION_DOCS} for AQL's window functions and {RANK_DOCS} for one in " + "full, then write the AQL by hand." + ) + if isinstance(node, exp.Column): + return ctx.reference(node.table, node.name) + if isinstance(node, exp.Literal): + return node.sql() + if isinstance(node, exp.Boolean): + return "true" if node.this else "false" + if isinstance(node, exp.Null): + return "null" + if isinstance(node, exp.Paren): + return f"({_render(node.this, ctx)})" + if isinstance(node, exp.Neg): + return f"-{_render(node.this, ctx)}" + if isinstance(node, exp.Count): + # sqlglot holds the DISTINCT as a child node rather than a flag, so + # `COUNT(DISTINCT x)` parses to Count(this=Distinct(expressions=[x])). + name = "count_distinct" if isinstance(node.this, exp.Distinct) else "count" + return f"{name}({', '.join(_arguments(node, ctx))})" + if isinstance(node, exp.Distinct): + inner = node.expressions + if len(inner) != 1: + raise Untranslatable( + "It applies DISTINCT to more than one expression, which this converter does " + "not write as AQL." + ) + return _render(inner[0], ctx) + + for kind, aql_name in ctx.functions.items(): + if isinstance(node, kind): + if isinstance(node, _numeric_aggregates()): + ctx.numeric.update( + (c.table, c.name) for c in node.find_all(exp.Column) + ) + return f"{aql_name}({', '.join(_arguments(node, ctx))})" + + operator = OPERATORS.get(type(node).__name__) + if operator and node.this is not None and node.args.get("expression") is not None: + left = _render(node.this, ctx) + right = _render(node.args["expression"], ctx) + return f"{left} {operator} {right}" + + if isinstance(node, exp.Anonymous): + # AQL's escape hatch for a warehouse function it has no name of its own + # for: `sql_number('APPROX_TOP_COUNT', x, 3)` calls it with AQL-resolved + # arguments. The family is typed by return value, and a metric body is + # numeric, so `sql_number` is the member to use. + # + # Only `Anonymous` qualifies, because only there do the name and the + # argument order come from the source text. A function sqlglot models + # carries sqlglot's own name and slot order instead, which produced + # `sql_number('PAD', s, 5)` for `LPAD(s, 5)` and + # `sql_number('LOG', 10, x)` for `LOG10(x)`. Neither is a call any + # warehouse answers. + arguments = [_render(a, ctx) for a in node.expressions] + return f"{SQL_PASSTHROUGH}('{node.name}'{''.join(', ' + a for a in arguments)})" + + if isinstance(node, exp.Func): + rebuilt = _from_source(node, ctx) + if rebuilt is not None: + return rebuilt + raise Untranslatable( + f"{node.sql_name()} is not in this converter's AQL function table, and its call " + f"cannot be read back from the source text, so there is no faithful way to pass " + f"it through." + ) + + raise Untranslatable( + f"It contains a {type(node).__name__}, which this converter does not write as AQL." + ) + + +def _span(node) -> int | None: + """Where `node` starts in the source, or None when sqlglot made it up. + + sqlglot records an offset for a token it read. An argument it synthesized, + such as the `10` in `LOG10(x)`, or normalized, such as `'month'` becoming + `'MONTH'` in `DATE_TRUNC`, carries none. + """ + meta = node.meta + if not meta and isinstance(getattr(node, "this", None), exp.Expression): + meta = node.this.meta + return meta.get("start") + + +def _from_source(node, ctx) -> str | None: + """A function sqlglot models, passed through under the name the source used. + + sqlglot names a node after the shape it parsed, not after the text. `LPAD` + and `RPAD` are both `Pad`, `LTRIM` and `RTRIM` are both `Trim`, so rendering + from the node loses which one was written. The node's own offset points at + the name token in the source, so the original name is still there to read. + + Returns None when the call cannot be rebuilt faithfully, which happens when + sqlglot synthesized or normalized an argument and so left it with no offset. + `LOG10(x)` gains a base argument the source never had, and + `DATE_TRUNC('month', d)` has its unit rewritten to `'MONTH'`. Dropping the + first is right and dropping the second is not, and an absent offset does not + say which case it is, so neither is passed through. + """ + start = node.meta.get("start") + if start is None: + return None + opening = ctx.source.find("(", start) + if opening < 0: + return None + name = ctx.source[start:opening].strip() + if not name.replace("_", "").isalnum(): + return None + + arguments = [ + value + for slot in node.args.values() + for value in (slot if isinstance(slot, list) else [slot]) + if isinstance(value, exp.Expression) + ] + offsets = [_span(argument) for argument in arguments] + if any(offset is None for offset in offsets): + return None + + in_source_order = [a for _, a in sorted(zip(offsets, arguments), key=lambda pair: pair[0])] + rendered = [_render(argument, ctx) for argument in in_source_order] + return f"{SQL_PASSTHROUGH}('{name}'{''.join(', ' + r for r in rendered)})" + + +def _arguments(node, ctx) -> list[str]: + """Every argument of a function node, in the order sqlglot declares them. + + Each node names its own slots, and they are not the same from one function + to the next: `NULLIF` uses `this` and `expression`, `COALESCE` uses `this` + and `expressions`, `ROUND` uses `this` and `decimals`. Reading a fixed set + of slot names drops the rest without saying so, which is how + `ROUND(SUM(x), 2)` first rendered as `sql_number('ROUND', sum(x))`. + `arg_types` gives the declared order, so no slot is missed. + """ + rendered = [] + for slot in getattr(node, "arg_types", {"this": True}): + value = node.args.get(slot) + for item in value if isinstance(value, list) else [value]: + if isinstance(item, exp.Expression): + rendered.append(_render(item, ctx)) + return rendered diff --git a/converters/holistics/src/ossie_holistics/cli.py b/converters/holistics/src/ossie_holistics/cli.py new file mode 100644 index 000000000..c7e8ef890 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/cli.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Holistics AML converter. + + ossie-holistics to-ossie -o --sql-dialect BIGQUERY + ossie-holistics to-aml -o --sql-dialect BIGQUERY + +`to-ossie` reads the JSON `holistics aml compile` produces. `to-aml` writes AML +source text, which is the format that CLI reads back: + + holistics aml compile .dataset.aml -r -o + holistics aml validate /.dataset.aml -r + +Every declared loss is written as a YAML sequence of issues, to `--issues` when +given and to stderr otherwise, never mixed into the document output. The process +exits 1 when that log holds an ERROR-severity issue and 0 otherwise. A +conversion that only warned is still a successful conversion, and failing on a +warning would teach scripts to ignore it. + +Neither subcommand overwrites an existing output file unless `--force` is given. +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +from . import _yaml, aml_to_ossie, ossie_to_aml, sqlrefs +from .errors import ConversionError +from .issues import IssueLog + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="ossie-holistics", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True + + to_ossie = sub.add_parser( + "to-ossie", help="compiled Holistics AML JSON -> one Apache Ossie semantic model" + ) + to_ossie.add_argument( + "input", metavar="COMPILED_JSON", help="output of `holistics aml compile`" + ) + to_ossie.add_argument( + "-o", "--output", required=True, metavar="FILE", help="output Apache Ossie YAML file" + ) + to_ossie.add_argument( + "--sql-dialect", + required=True, + metavar="DIALECT", + choices=sorted(sqlrefs.SQLGLOT_DIALECT), + help="the Ossie dialect a computed `@sql` body is written for. AML does not " + "record the warehouse. The choices are the dialects sqlglot can parse, because " + "`to-aml` has to read the expression back to restore its field references", + ) + to_ossie.add_argument( + "--issues", metavar="FILE", help="write the issue log as YAML here (default: stderr)" + ) + to_ossie.add_argument( + "--force", action="store_true", help="overwrite an existing output file" + ) + + to_aml = sub.add_parser("to-aml", help="one Apache Ossie semantic model -> AML source files") + to_aml.add_argument("input", metavar="OSSIE_YAML", help="Apache Ossie YAML document") + to_aml.add_argument( + "-o", + "--output", + required=True, + metavar="DIR", + help="output directory. One dataset file plus one model file per dataset is " + "written here, created if it does not already exist", + ) + to_aml.add_argument( + "--sql-dialect", + required=True, + metavar="DIALECT", + choices=sorted(sqlrefs.SQLGLOT_DIALECT), + help="the warehouse the Holistics connection reads. Every `@sql` body is " + "written in this dialect. An expression already in it is written as it stands, " + "and an ANSI_SQL or OSSIE_SQL_2026 one is rendered into it", + ) + to_aml.add_argument( + "--data-source-name", + metavar="NAME", + help="the Holistics connection the models read from. AML requires it and Ossie " + "has no field for it, so a document this converter did not write needs it", + ) + to_aml.add_argument( + "--issues", metavar="FILE", help="write the issue log as YAML here (default: stderr)" + ) + to_aml.add_argument( + "--force", action="store_true", help="overwrite existing output files" + ) + return parser + + +def _safe_target(directory: Path, relative: str) -> Path: + """`directory / relative`, which has to resolve inside `directory`. + + A model name comes from user-controlled document content, and the reverse + path turns it into a file path. Both sides are resolved before comparing, + because a string-prefix check is wrong on a filesystem with symlinks, as on + macOS where `/tmp` links to `/private/tmp`. + """ + root = directory.resolve() + target = (root / relative).resolve() + if target == root or root not in target.parents: + raise ConversionError( + f"{relative!r} resolves outside the output directory {root}, so nothing is written" + ) + return target + + +def _colliding(paths: list[Path]) -> list[Path]: + """Any path this run would write more than once. + + Compared after `resolve()` and case-folded, because two spellings of one + file are the same file on disk, and the default filesystem on macOS and + Windows folds case. Checked before any write, so `--issues` pointing at `-o` + cannot replace the converted document with the issue log. + """ + seen: dict[str, Path] = {} + counts: Counter[str] = Counter() + for path in paths: + resolved = path.resolve() + key = str(resolved).casefold() + seen.setdefault(key, resolved) + counts[key] += 1 + return sorted(seen[key] for key, count in counts.items() if count > 1) + + +def _blocked(paths: list[Path], reason: str) -> str: + return f"{reason}: {', '.join(str(p) for p in paths)}" + + +def _write_issues(issues: IssueLog, path: Path | None) -> None: + """Issues as a YAML sequence, always written even when empty. + + A caller scripting against this output relies on the shape, not on whether + anything was said. YAML rather than JSON because a message runs to a + paragraph, and folded YAML wraps it instead of putting it on one long line. + """ + text = _yaml.dump_issues(issues.as_dicts()) + if path is None: + print(text, end="", file=sys.stderr) + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _guard(targets: list[Path], issues: Path | None, force: bool) -> str | None: + collisions = _colliding(targets) + if collisions: + issue_path = str(issues.resolve()).casefold() if issues else None + if issue_path and any(str(p).casefold() == issue_path for p in collisions): + reason = ( + "this run would write the same path twice. --issues must name a " + "different file from the converted output" + ) + else: + reason = ( + "this run would write the same path twice. Two datasets produce " + "this filename, and the filesystem may not tell them apart by case. " + "Rename one of them in Ossie" + ) + return _blocked(collisions, reason) + if not force: + existing = [p for p in targets if p.exists()] + if existing: + return _blocked(existing, "file(s) already exist; pass --force to overwrite") + return None + + +def _cmd_to_ossie(args: argparse.Namespace) -> int: + output = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + try: + payload = json.loads(Path(args.input).read_text(encoding="utf-8")) + result = aml_to_ossie.convert(payload, sql_dialect=args.sql_dialect) + except (ConversionError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + blocked = _guard( + [output] + ([issues_path] if issues_path else []), issues_path, args.force + ) + if blocked: + print(f"Error: {blocked}", file=sys.stderr) + return 1 + + try: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(_yaml.dump(result.model), encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 1 if result.issues.has_errors() else 0 + + +def _cmd_to_aml(args: argparse.Namespace) -> int: + output_dir = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + if output_dir.exists() and not output_dir.is_dir(): + print(f"Error: output path {output_dir} exists and is not a directory", file=sys.stderr) + return 1 + + try: + document = _yaml.load(Path(args.input).read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, args.sql_dialect, data_source_name=args.data_source_name + ) + targets = [(_safe_target(output_dir, name), text) for name, text in result.files] + except (ConversionError, OSError, UnicodeDecodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + paths = [path for path, _ in targets] + ([issues_path] if issues_path else []) + blocked = _guard(paths, issues_path, args.force) + if blocked: + print(f"Error: {blocked}", file=sys.stderr) + return 1 + + try: + for path, text in targets: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 1 if result.issues.has_errors() else 0 + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + if args.command == "to-ossie": + return _cmd_to_ossie(args) + return _cmd_to_aml(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/holistics/src/ossie_holistics/datatypes.py b/converters/holistics/src/ossie_holistics/datatypes.py new file mode 100644 index 000000000..5dee0d0b5 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/datatypes.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tables mapping AML vocabularies onto Ossie ones. + +Every mapping here is a dict rather than a branch, so adding an AML type or an +aggregation is one row and the reverse direction reads the same table backwards. +""" +from __future__ import annotations + +#: AML `type` to Ossie `datatype`. `number` is the lossy row: AML does not say +#: whether a number is exact or approximate, and `Decimal` is the closest Ossie +#: member because the specification leaves its precision and scale unspecified. +#: The original AML type goes to the stash either way, so the reverse path reads +#: it from there rather than from this table. +AML_TO_OSSIE_DATATYPE: dict[str, str] = { + "text": "String", + "truefalse": "Boolean", + "date": "Date", + "datetime": "DateTime", + "number": "Decimal", +} + +#: Ossie `datatype` to AML `type`, for a document with no stashed AML type. +#: `Decimal`, `Float` and `Integer` all collapse onto `number`, which is why the +#: forward path stashes the original rather than relying on this table. +OSSIE_TO_AML_TYPE: dict[str, str] = { + "String": "text", + "Boolean": "truefalse", + "Date": "date", + "Time": "datetime", + "DateTime": "datetime", + "DateTimeTz": "datetime", + "Decimal": "number", + "Float": "number", + "Integer": "number", + "Opaque": "text", +} + +#: AML `aggregation_type` to the `(prefix, suffix)` that wraps the measure body. +#: Stored as a pair rather than a function name so the reverse path strips what +#: the forward path added, instead of re-deriving it from the Ossie SQL. +#: `custom` is absent on purpose: its body already aggregates, so the converter +#: wraps nothing. A key missing from this table is an aggregation this converter +#: has never seen, and `aml_to_ossie` reports it rather than guessing. +AGGREGATIONS: dict[str, tuple[str, str]] = { + "sum": ("SUM(", ")"), + "count": ("COUNT(", ")"), + "count distinct": ("COUNT(DISTINCT ", ")"), + "min": ("MIN(", ")"), + "max": ("MAX(", ")"), + "average": ("AVG(", ")"), + "median": ("MEDIAN(", ")"), +} + +CUSTOM_AGGREGATION = "custom" + +#: Characters that open a quoted region in a SQL expression. A string literal +#: uses `'`, and a delimited identifier uses `"` or a backtick. +QUOTES = frozenset("'\"`") + + +def apply_aggregation(aggregation_type: str, body: str) -> str: + prefix, suffix = AGGREGATIONS[aggregation_type] + return f"{prefix}{body}{suffix}" + + +def strip_aggregation(aggregation_type: str, expression: str) -> str | None: + """The body inside the aggregate, or None when `expression` is not wrapped. + + The reverse path calls this to recover a measure `definition` from an Ossie + metric expression. A None result means the expression was edited after the + forward path wrote it, so the reverse path writes `aggregation_type: + 'custom'` and keeps the expression whole rather than producing a measure + that aggregates twice. + """ + pair = AGGREGATIONS.get(aggregation_type) + if pair is None: + return None + prefix, suffix = pair + text = expression.strip() + if not (text.startswith(prefix) and text.endswith(suffix)): + return None + if not _closes_at_end(text, prefix.index("(")): + return None + return text[len(prefix) : len(text) - len(suffix)] + + +def _closes_at_end(text: str, open_index: int) -> bool: + """Whether the parenthesis at `open_index` is closed by the last character. + + Without this, `SUM(a) + SUM(b)` matches the `sum` prefix and suffix and + strips to `a) + SUM(b`, which is not an expression. + + Quoted regions are skipped, so a parenthesis inside one does not move the + depth. All three quoting characters count: a string literal uses `'`, and a + delimited identifier uses `"` or, on BigQuery and Databricks, a backtick. + """ + depth = 0 + index = open_index + while index < len(text): + character = text[index] + if character in QUOTES: + closer = character + index += 1 + while index < len(text) and text[index] != closer: + index += 2 if text[index] == "\\" else 1 + elif character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth == 0: + return index == len(text) - 1 + index += 1 + return False diff --git a/converters/holistics/src/ossie_holistics/errors.py b/converters/holistics/src/ossie_holistics/errors.py new file mode 100644 index 000000000..d81421963 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/errors.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Hard failures. Distinct from ConverterIssue, which records a survivable loss.""" + + +class ConversionError(Exception): + """Raised when the converter cannot proceed, e.g. a malformed stash.""" diff --git a/converters/holistics/src/ossie_holistics/interpolation.py b/converters/holistics/src/ossie_holistics/interpolation.py new file mode 100644 index 000000000..ab889ddb6 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/interpolation.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `{{ ... }}` and `{% ... %}` syntax AML embeds in a `@sql` body. + +One `{{ ... }}` spelling covers three unrelated things, and telling them apart +decides what the forward path may emit: + + {{ #SOURCE.created_at }} a physical column of this model's own source + {{ created_at }} another AML field of the same model + {{ #orders.created_at }} a field reached through a named model + +`parse_sql_body` returns the body as an alternating list of literal text and +`Interpolation` values, so a caller rewrites the references and leaves every +other character untouched. Both directions use it: the forward path turns an +interpolation into an Ossie reference, and the reverse path turns an Ossie +reference back into an interpolation. + +A query body uses a second, disjoint set of forms, listed in `QUERY_FORMS`. +Those are never resolved. `query_forms_used` only reports which ones appear, so +the forward path can say why the dataset's `source` is not runnable SQL. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +#: The sentinel model name meaning "this model's own source relation". +SOURCE = "SOURCE" + +#: An AML identifier. `::` is allowed because a model reference inside a body +#: may name a model that lives in a module, as in `{{ #reporting::calendar.date }}`. +_IDENT = r"[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*" + +_MOUSTACHE = re.compile(r"\{\{(.*?)\}\}", re.DOTALL) +_SOURCE_REF = re.compile(rf"^#{SOURCE}\.({_IDENT})$") +_MODEL_REF = re.compile(rf"^#({_IDENT})\.({_IDENT})$") +_SIBLING_REF = re.compile(rf"^({_IDENT})$") + + +@dataclass(frozen=True) +class Source: + """`{{ #SOURCE.column }}`. A physical column, so it carries no semantics.""" + + column: str + + def to_aml(self) -> str: + return f"{{{{ #{SOURCE}.{self.column} }}}}" + + +@dataclass(frozen=True) +class Sibling: + """`{{ field }}`. Another AML field of the same model.""" + + field: str + + def to_aml(self) -> str: + return f"{{{{ {self.field} }}}}" + + +@dataclass(frozen=True) +class ModelField: + """`{{ #model.field }}`. A field reached through a named model.""" + + model: str + field: str + + def to_aml(self) -> str: + return f"{{{{ #{self.model}.{self.field} }}}}" + + +@dataclass(frozen=True) +class Unrecognised: + """A `{{ ... }}` matching none of the three forms, kept verbatim.""" + + text: str + + def to_aml(self) -> str: + return f"{{{{{self.text}}}}}" + + +Interpolation = Source | Sibling | ModelField | Unrecognised + + +def classify(inner: str) -> Interpolation: + """Classify the text between one pair of braces.""" + stripped = inner.strip() + match = _SOURCE_REF.match(stripped) + if match: + return Source(match.group(1)) + match = _MODEL_REF.match(stripped) + if match: + return ModelField(match.group(1), match.group(2)) + match = _SIBLING_REF.match(stripped) + if match: + return Sibling(match.group(1)) + return Unrecognised(inner) + + +def parse_sql_body(body: str) -> list[str | Interpolation]: + """Split a `@sql` body into literal text and interpolations, in order. + + Literal segments are kept even when empty, so a body that is exactly one + interpolation returns `["", Source(...), ""]`. `render` rebuilds the input + from the output, so a caller that changes nothing changes nothing. + """ + parts: list[str | Interpolation] = [] + position = 0 + for match in _MOUSTACHE.finditer(body): + parts.append(body[position : match.start()]) + parts.append(classify(match.group(1))) + position = match.end() + parts.append(body[position:]) + return parts + + +def render(parts: list[str | Interpolation]) -> str: + """Rebuild an AML body from `parse_sql_body` output.""" + return "".join(p if isinstance(p, str) else p.to_aml() for p in parts) + + +def interpolations(body: str) -> list[Interpolation]: + return [p for p in parse_sql_body(body) if not isinstance(p, str)] + + +def sole_reference(body: str) -> Interpolation | None: + """The interpolation when `body` is exactly one, surrounded by whitespace. + + This is the test for "carries no Holistics SQL of its own". `{{ #SOURCE.quantity }}` + and `{{ created_at }}` both pass, and each becomes a plain Ossie reference + under `ANSI_SQL`. `CAST({{ #SOURCE.created_at }} AS DATE)` fails, because + the literal text around the interpolation is warehouse SQL that this + converter will not rewrite. + + An `Unrecognised` interpolation returns None. It is not a reference this + converter can resolve, so it must not be treated as one. + """ + parts = parse_sql_body(body) + if len(parts) != 3: + return None + before, middle, after = parts + if before.strip() or after.strip(): + return None + return None if isinstance(middle, Unrecognised) else middle + + +#: Every form that makes a query body something other than plain SQL. The names +#: are stashed verbatim, so a consumer reading the stash learns which Holistics +#: feature it must resolve before the `source` will run. +QUERY_FORMS: dict[str, re.Pattern[str]] = { + "model_columns": re.compile(rf"\{{\{{\s*#{_IDENT}\.\*\s*\}}\}}"), + "model_relation": re.compile(rf"\{{\{{\s*#{_IDENT}\s*\}}\}}"), + "filter_block": re.compile(r"\{%\s*filter\s*\(.*?\)\s*%\}", re.DOTALL), +} + + +def query_forms_used(body: str) -> list[str]: + """Which `QUERY_FORMS` appear in `body`, in declaration order. + + `model_columns` is tested before `model_relation` and the two never both + match one site, because `{{ #orders.* }}` ends in `.*` and the + `model_relation` pattern requires the closing braces straight after the + identifier. + """ + return [name for name, pattern in QUERY_FORMS.items() if pattern.search(body)] diff --git a/converters/holistics/src/ossie_holistics/issues.py b/converters/holistics/src/ossie_holistics/issues.py new file mode 100644 index 000000000..adb08ab0d --- /dev/null +++ b/converters/holistics/src/ossie_holistics/issues.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured, never-silent loss reporting. + +Discussion apache/ossie#325 treats a silently dropped field as a contract +violation rather than a documentation gap, so every declared loss produces an +issue here at conversion time. The same discussion treats a warning storm as a +defect, which is why severity is first-class and why callers can summarise via +count_by_severity() instead of printing every line. +""" +from collections import Counter +from dataclasses import dataclass, field +from enum import Enum + + +class Severity(str, Enum): + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + + +@dataclass(frozen=True) +class ConverterIssue: + """One declared loss or degradation, traceable to a specific object. + + object_ref is mandatory: an issue a reader cannot trace to an object cannot + be acted on, which is the complaint #325 raises about warning noise. + """ + + code: str + severity: Severity + message: str + object_ref: str + remedy: str | None = None + + def as_dict(self) -> dict[str, str | None]: + return { + "code": self.code, + "severity": self.severity.value, + "message": self.message, + "object_ref": self.object_ref, + "remedy": self.remedy, + } + + +@dataclass +class IssueLog: + """Ordered collection of issues raised during one conversion.""" + + issues: list[ConverterIssue] = field(default_factory=list) + + def add( + self, + *, + code: str, + severity: Severity, + message: str, + object_ref: str, + remedy: str | None = None, + ) -> None: + self.issues.append( + ConverterIssue( + code=code, + severity=severity, + message=message, + object_ref=object_ref, + remedy=remedy, + ) + ) + + def extend(self, other: "IssueLog") -> None: + self.issues.extend(other.issues) + + def as_dicts(self) -> list[dict[str, str | None]]: + return [i.as_dict() for i in self.issues] + + def has_errors(self) -> bool: + return any(i.severity is Severity.ERROR for i in self.issues) + + def count_by_severity(self) -> dict[str, int]: + return dict(Counter(i.severity.value for i in self.issues)) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py new file mode 100644 index 000000000..37f9465e5 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -0,0 +1,1269 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One Apache Ossie semantic model to a set of AML source files. + +`convert` returns `(relative path, text)` pairs rather than writing them, so the +CLI owns every filesystem decision and a test can read the output without a +temporary directory. + +The layout is one dataset file plus one model file per Ossie dataset: + + .dataset.aml + orders.model.aml an Ossie dataset named orders + modules/reporting/calendar.model.aml an Ossie dataset named reporting__calendar + +The input file layout is lost on the way in. The compiled JSON the forward path +reads records which models a dataset holds and not which file each came from, so +the layout is already gone by the time a document reaches Ossie. AML accepts any +arrangement, so the generator picks the one above. + +Four AML properties are required and have no Ossie field: `label` and `type` on +every dimension and measure, and `data_source_name` and `relationships` on the +dataset. A stashed value wins. Otherwise `label` is derived from the name the +way AML itself derives it, `type` from `datatype`, and `data_source_name` from +the `--data-source-name` argument. +""" +from __future__ import annotations + +from dataclasses import dataclass, field as dataclass_field +from typing import Any + +from . import amlgen, aql, interpolation, sqlrefs, stash +from .amlgen import Block, quote, sanitize, titleize +from .datatypes import AGGREGATIONS, CUSTOM_AGGREGATION, OSSIE_TO_AML_TYPE, strip_aggregation +from .errors import ConversionError +from .issues import IssueLog, Severity + +AQL_DIALECT = "HOLISTICS_AQL" + +ISSUE_CROSS_MODEL_SQL = "HOLISTICS_CROSS_MODEL_SQL_BODY" +ISSUE_UNPARSEABLE_SQL = "HOLISTICS_UNPARSEABLE_SQL" +ISSUE_UNKNOWN_QUALIFIER = "HOLISTICS_UNKNOWN_QUALIFIER" +ISSUE_METRIC_NOT_TRANSLATED = "HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL" +ISSUE_ASSUMED_TYPE = "HOLISTICS_ASSUMED_TYPE" +ISSUE_AGGREGATION_NOT_STRIPPED = "HOLISTICS_AGGREGATION_NOT_STRIPPED" +ISSUE_RENAMED = "HOLISTICS_NAME_NOT_SPELLABLE_IN_AML" +ISSUE_DATASET_RENAMED = "HOLISTICS_DATASET_NAME_COLLIDES_WITH_MODEL" +ISSUE_TRANSPILED = "HOLISTICS_EXPRESSION_RENDERED_INTO_TARGET_DIALECT" +ISSUE_NO_USABLE_DIALECT = "HOLISTICS_NO_USABLE_DIALECT" +ISSUE_RENDERED_AS_AQL = "HOLISTICS_METRIC_RENDERED_AS_AQL" +ISSUE_BORROWED_COLUMN = "HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION" +ISSUE_WINDOW_NEEDS_GROUPING = "HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING" +ISSUE_METRIC_CYCLE = "HOLISTICS_METRIC_REFERENCE_CYCLE" + +#: Placement markers for `_datasets_named`. A metric that finds one still marked +#: `_RESOLVING` closed a reference cycle, which no single model can hold. +_UNVISITED = object() +_RESOLVING = object() + +#: The AML type written for a field whose Ossie `datatype` is absent. +FALLBACK_TYPE = "text" + +#: The AML type for a column borrowed into a model by an AQL metric. Ossie +#: declares nothing about it, and every borrowed column so far is an aggregate's +#: argument, so `number` is the reading that lets the metric compile. +FALLBACK_TYPE_FOR_BORROWED = "number" + + +class Result: + def __init__(self, files: list[tuple[str, str]], issues: IssueLog): + self.files = files + self.issues = issues + + +@dataclass +class Model: + """One Ossie dataset, as the AML model it will be written as.""" + + ossie_name: str + fqn: str + payload: dict[str, Any] + stash: dict[str, Any] + measures: list[dict[str, Any]] = dataclass_field(default_factory=list) + #: Columns an AQL metric needed that the dataset does not declare as a + #: field. Each becomes a hidden dimension so the AQL reference resolves. + borrowed: list[str] = dataclass_field(default_factory=list) + #: Fields an AQL metric aggregates numerically. The metric is evidence of + #: the type when the Ossie document declares no `datatype`. + numeric: set[str] = dataclass_field(default_factory=set) + + @property + def segments(self) -> list[str]: + return [sanitize(part) for part in self.fqn.split("::")] + + @property + def aml_fqn(self) -> str: + return "::".join(self.segments) + + @property + def aml_name(self) -> str: + return self.segments[-1] + + @property + def path(self) -> str: + parts = self.segments + if len(parts) == 1: + return f"{parts[0]}.model.aml" + return "/".join(["modules", *parts[:-1], f"{parts[-1]}.model.aml"]) + + @property + def reference(self) -> str: + """How the dataset block names this model. Dots, not `::`.""" + return ".".join(self.segments) + + +def unflatten(name: str) -> str: + """An Ossie dataset name as an AML module path. + + The forward path writes `reporting::calendar` as `reporting__calendar`, so + this reverses that. A stashed `fqn` wins over this rule, because it records + where the module boundaries actually were rather than inferring them. + """ + return name.replace("__", "::") + + +#: The dialects a portable body may be written in. Every sibling converter that +#: reads Ossie treats `OSSIE_SQL_2026` as an `ANSI_SQL` equivalent, including +#: Snowflake's `_extract_expression` and Cube's `pick_expression`. +PORTABLE_DIALECTS = ("ANSI_SQL", "OSSIE_SQL_2026") + +#: What the compiler fills in for a relationship that declares none of them. +#: Every compiled payload carries all three, so writing them back +#: unconditionally would turn every short `relationship(...)` into a +#: `RelationshipConfig` block that means exactly the same thing. +RELATIONSHIP_DEFAULTS: dict[str, Any] = { + "direction": "two_way", + "nullable": True, + "rlp_propagation": "inherit", +} + + +def _named_objects(container: Any, where: str) -> list[dict[str, Any]]: + """`container` as a list of named mappings, or a `ConversionError` saying why. + + A document this converter did not write is parsed at the boundary rather + than indexed into. The CLI catches `ConversionError` alone, so any other + exception reaches the user as a traceback. apache/ossie#297 and + apache/ossie#407 both asked a sibling converter for the same guard. + """ + if container is None: + return [] + if not isinstance(container, list): + raise ConversionError( + f"{where} is {type(container).__name__}, and the specification defines it as a list" + ) + for index, member in enumerate(container): + if not isinstance(member, dict): + raise ConversionError( + f"{where}[{index}] is {type(member).__name__}, and must be a mapping" + ) + name = member.get("name") + if not isinstance(name, str) or not name: + raise ConversionError(f"{where}[{index}] has no name") + return container + + +def dialects_of(expression: Any) -> dict[str, str]: + """Every `{dialect: expression}` pair, validated at the boundary. + + A document this converter did not write arrives unchecked, and the CLI + catches `ConversionError` alone, so the shape is tested rather than assumed. + """ + if expression is None: + raise ConversionError("expression is missing, and the specification requires one") + if not isinstance(expression, dict): + raise ConversionError( + f"expression is {type(expression).__name__}, and the specification defines it " + f"as a mapping holding a 'dialects' list" + ) + dialects = expression.get("dialects") or [] + if not dialects: + raise ConversionError("expression has no dialects") + if not isinstance(dialects, list): + raise ConversionError( + f"expression.dialects is {type(dialects).__name__}, and must be a list" + ) + available: dict[str, str] = {} + for index, entry in enumerate(dialects): + if not isinstance(entry, dict): + raise ConversionError( + f"expression.dialects[{index}] is {type(entry).__name__}, and must be a " + f"mapping holding 'dialect' and 'expression'" + ) + available[entry.get("dialect", "ANSI_SQL")] = entry.get("expression", "") + return available + + +class _Reverse: + def __init__( + self, document: dict[str, Any], data_source_name: str | None, sql_dialect: str + ): + self.document = document + self.sql_dialect = sql_dialect + self.issues = IssueLog() + self.root_stash = stash.read(document) + self.data_source_name = self.root_stash.get("data_source_name") or data_source_name + if not self.data_source_name: + raise ConversionError( + "the document carries no HOLISTICS data_source_name, so one must be " + "supplied: pass --data-source-name naming the Holistics connection" + ) + + #: AML identifier to the Ossie name it was spelled from, so two names + #: that collapse onto one spelling fail rather than overwrite. + self._renamed: dict[str, str] = {} + + datasets = _named_objects(document.get("datasets"), "datasets") + if not datasets: + raise ConversionError( + "the document declares no datasets, and the specification requires at least one" + ) + for dataset in datasets: + _named_objects(dataset.get("fields"), f"datasets[{dataset['name']}].fields") + _named_objects(document.get("metrics"), "metrics") + + self.models: list[Model] = [] + for dataset in datasets: + name = dataset["name"] + data = stash.read(dataset) + self.models.append( + Model( + ossie_name=name, + fqn=data.get("fqn") or unflatten(name), + payload=dataset, + stash=data, + ) + ) + self.by_ossie_name = {m.ossie_name: m for m in self.models} + + taken: dict[str, str] = {} + for model in self.models: + clash = taken.setdefault(model.aml_fqn, model.ossie_name) + if clash != model.ossie_name: + raise ConversionError( + f"datasets {clash!r} and {model.ossie_name!r} both spell as the AML " + f"model {model.aml_fqn!r}; rename one of them in Ossie" + ) + + self.dataset_name = self._dataset_name() + + #: Every metric that belongs on a model, indexed by Ossie metric name, + #: alongside the AML measure name it is written as. + self.metric_owner: dict[str, Model] = {} + self.metric_aml_name: dict[str, str] = {} + self.dataset_metrics: list[dict[str, Any]] = [] + #: Relationships that carry a `where` filter, declared above the + #: Dataset block because only a named relationship can hold one. + self.relationship_declarations: list[str] = [] + self._metric_payloads: dict[str, dict[str, Any]] = {} + self._metric_datasets: dict[str, Any] = {} + self._cyclic_metrics: set[str] = set() + self._place_metrics() + + # -- placement -------------------------------------------------------- + + def _place_metrics(self) -> None: + """Decide where each Ossie metric goes: on a model, or on the dataset. + + A stashed `model` settles it. Without one, the expression decides: a + body naming exactly one dataset becomes a measure on that model, and a + body naming several becomes a dataset-level metric, which is the only + AML construct that spans models. A body may also name another metric, + and `_scan_for_datasets` covers what that counts as. + """ + metrics = self.document.get("metrics") or [] + for metric in metrics: + name = metric.get("name") + self.metric_aml_name[name] = stash.read(metric).get("aml_name") or name + self._metric_payloads[name] = metric + for metric in metrics: + name = metric.get("name") + named = self._datasets_named(name) or frozenset() + owner = self.by_ossie_name[next(iter(named))] if len(named) == 1 else None + if owner is None: + self.dataset_metrics.append(metric) + else: + owner.measures.append(metric) + self.metric_owner[name] = owner + self._report_metric_cycles() + + def _datasets_named(self, name: str) -> frozenset[str] | None: + """The Ossie datasets metric `name` resolves to, or None when that is unknown. + + None covers every reading that cannot put the metric on a single model: + an AQL body, Ossie SQL this converter cannot read, and a reference cycle. + """ + resolved = self._metric_datasets.get(name, _UNVISITED) + if resolved is _RESOLVING: + self._cyclic_metrics.add(name) + return None + if resolved is not _UNVISITED: + return resolved + self._metric_datasets[name] = _RESOLVING + result = self._scan_for_datasets(name) + self._metric_datasets[name] = result + return result + + def _scan_for_datasets(self, name: str) -> frozenset[str] | None: + """Which datasets one metric body names, directly or through a metric. + + An Ossie metric body may name another metric instead of a column, as + `cities__upstream_measure + 1` in `tests/fixtures/ecommerce` does. That + body names no dataset of its own, and the datasets the metric it + references resolves to stand in for it. Without that substitution a + metric built only from other metrics lands on the dataset, where AML + takes an AQL body, and the Ossie SQL name it was written with does not + resolve. + """ + metric = self._metric_payloads[name] + data = stash.read(metric) + owner_fqn = data.get("model") + if owner_fqn: + owner = next((m for m in self.models if m.fqn == owner_fqn), None) + if owner is None: + raise ConversionError( + f"metric {name!r} is stashed against model {owner_fqn!r}, " + f"which is not a dataset in this document" + ) + return frozenset({owner.ossie_name}) + available = dialects_of(metric.get("expression")) + if AQL_DIALECT in available: + return None + dialect = next( + (d for d in (self.sql_dialect, *PORTABLE_DIALECTS) if d in available), None + ) + if dialect is None: + return None + try: + references = sqlrefs.columns(available[dialect], dialect) + except sqlrefs.UnparseableSQL: + return None + named = {c.table for c in references if c.table in self.by_ossie_name} + for reference in references: + if reference.table or reference.name not in self._metric_payloads: + continue + through = self._datasets_named(reference.name) + if through is None: + return None + named |= through + return frozenset(named) + + def _report_metric_cycles(self) -> None: + for name in sorted(self._cyclic_metrics): + self.issues.add( + code=ISSUE_METRIC_CYCLE, + severity=Severity.ERROR, + message=( + f"metric {name!r} references itself through another metric. Holistics " + f"inlines one metric into the body of the next, so a cycle has no warehouse SQL, " + f"and `holistics aml validate` accepts the file without catching it" + ), + object_ref=name, + remedy="break the cycle in Ossie", + ) + + # -- names ------------------------------------------------------------ + + def _dataset_name(self) -> str: + """The AML `Dataset` name, moved aside when a model already has it. + + Ossie keeps the semantic model's name and its dataset names in separate + namespaces, so a document may use one name for both. Three fixtures in + this repository do, including + `converters/cube/tests/fixtures/databricks_ossie.yaml`, whose root name + and first dataset are both `orders`. AML has one namespace for `Dataset` + and `Model`, and rejects the pair with "Duplicated name 'orders'". + + Nothing references the AML dataset name. Models reference each other and + relationships reference models, so the dataset is the safe side to move. + """ + name = self.root_stash.get("fqn") or self.document.get("name") + if not name: + raise ConversionError("the document has no name") + spelled = self.name_in_aml(name, str(name)) + taken = {model.aml_fqn for model in self.models} | { + model.aml_name for model in self.models + } + if spelled not in taken: + return spelled + candidate, suffix = f"{spelled}_dataset", 2 + while candidate in taken: + candidate, suffix = f"{spelled}_dataset_{suffix}", suffix + 1 + self.issues.add( + code=ISSUE_DATASET_RENAMED, + severity=Severity.WARNING, + message=( + f"a model is already named {spelled!r}, and AML holds Dataset and Model " + f"in one namespace, so the dataset is written as {candidate!r}" + ), + object_ref=str(name), + remedy="rename the semantic model or the dataset in Ossie", + ) + return candidate + + def name_in_aml(self, name: str, scope: str) -> str: + """An Ossie name as the AML identifier it is written as. + + Lookups elsewhere stay keyed on the Ossie name, because that is what an + expression references. Only the output spelling changes, and every site + that writes a name goes through here, so a declaration and a reference + to it cannot disagree. + """ + spelled = sanitize(name) + if spelled != name: + previous = self._renamed.setdefault(spelled, name) + if previous != name: + raise ConversionError( + f"{scope}: {name!r} and {previous!r} both spell as the AML " + f"identifier {spelled!r}; rename one of them in Ossie" + ) + self.issues.add( + code=ISSUE_RENAMED, + severity=Severity.WARNING, + message=( + f"{name!r} is not a legal AML identifier and is written as " + f"{spelled!r}. AML allows letters, digits and underscore, and no " + f"leading digit" + ), + object_ref=scope, + remedy=f"rename it to {spelled!r} in Ossie to keep the two in step", + ) + return spelled + + # -- expressions ------------------------------------------------------ + + def pick_body(self, expression: Any, scope: str) -> tuple[str, str]: + """The body to write and the dialect it is in, after any rendering. + + AQL wins outright. It came out of AML unchanged and goes back unchanged. + + For Ossie SQL the target is `--sql-dialect`, the warehouse the Holistics + connection reads. A body already in that dialect is written as it + stands. Otherwise a portable body is rendered into the target with + sqlglot, because the alternative is AML that will not run. An expression + offering neither has no body this connection can execute, and the + converter says so rather than writing Holistics SQL for the wrong warehouse. + """ + available = dialects_of(expression) + if AQL_DIALECT in available: + return AQL_DIALECT, available[AQL_DIALECT] + if self.sql_dialect in available: + return self.sql_dialect, available[self.sql_dialect] + + source = next((d for d in PORTABLE_DIALECTS if d in available), None) + if source is not None: + try: + rendered = sqlrefs.transpile(available[source], source, self.sql_dialect) + except sqlrefs.UnparseableSQL as exc: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"the {source} body cannot be rendered as {self.sql_dialect} ({exc}), " + f"so it is written unchanged and may not run" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} expression, or fix the {source} one", + ) + return source, available[source] + self.issues.add( + code=ISSUE_TRANSPILED, + severity=Severity.WARNING, + message=( + f"no {self.sql_dialect} expression, so the {source} one was rendered " + f"into it: {available[source]!r} became {rendered!r}" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} expression to keep the body exact", + ) + return self.sql_dialect, rendered + + fallback = next(iter(available)) + self.issues.add( + code=ISSUE_NO_USABLE_DIALECT, + severity=Severity.ERROR, + message=( + f"the expression offers {', '.join(sorted(available))} and the connection " + f"reads {self.sql_dialect}, with no portable body to render from, so the " + f"{fallback} body is written unchanged and will not run" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} or ANSI_SQL expression", + ) + return fallback, available[fallback] + + def definition(self, block: Block, expression: Any, model: Model, scope: str) -> None: + """Write a `definition` property for a field or a measure.""" + dialect, text = self.pick_body(expression, scope) + if dialect == AQL_DIALECT: + block.heredoc("definition", "aql", text) + return + block.heredoc("definition", "sql", self.to_aml_sql(text, dialect, model, scope)) + + def to_aml_sql(self, expression: str, dialect: str, model: Model, scope: str) -> str: + """An Ossie SQL expression with its references written as AML interpolation. + + A bare column name does compile as AML, but AQL reads the interpolation + to learn which field the body depends on. A body without it loses the + query rewriting and the optimisation that link drives, so this restores + every reference rather than passing the SQL through. + """ + if dialect in sqlrefs.NOT_SQL or not sqlrefs.AVAILABLE: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"the body is tagged {dialect!r}, which is not Ossie SQL this converter can " + f"read, so its field references cannot be restored and AQL cannot see them" + ), + object_ref=scope, + remedy="supply an ANSI_SQL or warehouse-SQL dialect for this expression", + ) + return expression + grouping = sqlrefs.window_grouping(expression, dialect) + if grouping: + self.issues.add( + code=ISSUE_WINDOW_NEEDS_GROUPING, + severity=Severity.WARNING, + message=( + f"the body has a window function whose frame reads " + f"{', '.join(grouping)}. AML carries it as written, and the warehouse SQL " + f"it generates is valid only for a query that groups by those. A query " + f"that does not, a grand total for one, produces warehouse SQL the " + f"database rejects, and Holistics does not check this at conversion time" + ), + object_ref=scope, + remedy="query it only alongside those fields, or rewrite the body in AQL", + ) + try: + return sqlrefs.rewrite( + expression, dialect, lambda column: self.to_interpolation(column, model, scope) + ) + except sqlrefs.UnparseableSQL as exc: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"sqlglot cannot read the body as {dialect} ({exc}), so its field " + f"references cannot be restored and AQL cannot see them" + ), + object_ref=scope, + remedy="fix the expression, or convert it by hand", + ) + return expression + + def to_interpolation(self, column: sqlrefs.ColumnReference, model: Model, scope: str) -> str: + """One Ossie column reference as the AML interpolation it came from.""" + if not column.table: + if column.name in self.metric_aml_name: + return self._measure_reference(column.name, model, scope) + return interpolation.Source(column.name).to_aml() + + target = self.by_ossie_name.get(column.table) + if target is None: + self.issues.add( + code=ISSUE_UNKNOWN_QUALIFIER, + severity=Severity.ERROR, + message=( + f"the body qualifies {column.name!r} with {column.table!r}, which is not " + f"a dataset in this document, so it is left unresolved" + ), + object_ref=scope, + remedy="qualify the reference with a dataset name, or drop the field", + ) + return f"{column.table}.{column.name}" + if target is model: + # A qualifier naming this model usually names one of its fields. It + # can also name a metric this converter put somewhere else, and a + # sibling reference to that spells a field the model does not have. + if column.name in self.metric_aml_name and self.metric_owner.get(column.name) is not model: + return self._measure_reference(column.name, model, scope) + return interpolation.Sibling(self.name_in_aml(column.name, scope)).to_aml() + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body names {column.table}.{column.name}, a field of another dataset. A `@sql` " + f"body resolves inside one model, and Holistics rejects this at SQL generation " + f"with \"Cannot refer to external field in TableModel/QueryModel\" even though " + f"`holistics aml validate` accepts the file" + ), + object_ref=scope, + remedy="move the reference into a dimension on its own model, or rewrite the body in AQL", + ) + return interpolation.ModelField(target.aml_fqn, self.name_in_aml(column.name, scope)).to_aml() + + def _measure_reference(self, metric_name: str, model: Model, scope: str) -> str: + owner = self.metric_owner.get(metric_name) + aml_name = self.metric_aml_name[metric_name] + if owner is model: + return interpolation.Sibling(self.name_in_aml(aml_name, scope)).to_aml() + if owner is None: + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body references the dataset-level metric {metric_name!r}. A `@sql` " + f"body resolves inside one model, so there is no AML spelling for it" + ), + object_ref=scope, + remedy="rewrite the expression in AQL, which does span models", + ) + return metric_name + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body references {metric_name!r}, a measure of model {owner.aml_fqn!r}. A " + f"`@sql` body resolves inside one model, and Holistics rejects this at SQL " + f"generation with \"Cannot refer to external field in TableModel/QueryModel\" " + f"even though `holistics aml validate` accepts the file" + ), + object_ref=scope, + remedy="rewrite the expression in AQL, which does span models", + ) + return interpolation.ModelField(owner.aml_fqn, self.name_in_aml(aml_name, scope)).to_aml() + + # -- fields ----------------------------------------------------------- + + def aml_type( + self, payload: dict[str, Any], data: dict[str, Any], scope: str, model: Model | None = None + ) -> str: + stashed = data.get("type") + if stashed: + return stashed + datatype = payload.get("datatype") + mapped = OSSIE_TO_AML_TYPE.get(datatype) if datatype else None + if mapped: + return mapped + if model is not None and payload["name"] in model.numeric: + # A metric aggregates this field with `sum` or `avg`, so it is a + # number whatever the document failed to say. + return "number" + self.issues.add( + code=ISSUE_ASSUMED_TYPE, + severity=Severity.WARNING, + message=( + f"AML requires a type and the Ossie datatype is {datatype!r}, " + f"so {FALLBACK_TYPE!r} is written" + ), + object_ref=scope, + remedy="set a datatype on the field, or edit the AML type after conversion", + ) + return FALLBACK_TYPE + + def common_properties( + self, + block: Block, + payload: dict[str, Any], + data: dict[str, Any], + scope: str, + model: Model | None = None, + ) -> None: + block.text("label", data.get("label") or titleize(payload["name"])) + block.property("type", quote(self.aml_type(payload, data, scope, model))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + + def dimension(self, model: Model, payload: dict[str, Any], primary_key: set[str]) -> Block: + name = payload["name"] + scope = f"{model.ossie_name}.{name}" + data = stash.read(payload) + block = Block(f"dimension {self.name_in_aml(name, scope)}") + self.common_properties(block, payload, data, scope, model) + self.definition(block, payload.get("expression"), model, scope) + block.text("format", data.get("format")) + block.flag("primary_key", name in primary_key) + return block + + def dataset_dimension(self, model: Model, payload: dict[str, Any]) -> Block: + """A field the forward path took from a dataset-level AML dimension.""" + name = payload["name"] + scope = f"{model.ossie_name}.{name}" + data = stash.read(payload) + block = Block(f"dimension {self.name_in_aml(name, scope)}") + block.property("model", model.reference) + self.common_properties(block, payload, data, scope, model) + self.definition(block, payload.get("expression"), model, scope) + block.text("format", data.get("format")) + return block + + def measure(self, model: Model, payload: dict[str, Any]) -> Block: + """One Ossie metric as a `measure` on its owning model. + + `aggregation_type` and the body compose into one Ossie expression on the + way out, so this splits them back apart. When the stashed aggregation + does not wrap the whole expression, the expression was edited after the + forward path wrote it, and `custom` keeps it whole rather than letting + the measure aggregate twice. + """ + metric_name = payload["name"] + aml_name = self.metric_aml_name[metric_name] + scope = f"{model.ossie_name}.{aml_name}" + data = stash.read(payload) + block = Block(f"measure {self.name_in_aml(aml_name, scope)}") + block.text("label", data.get("label") or titleize(aml_name)) + block.property("type", quote(self.aml_type(payload, data, scope))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + + dialect, text = self.pick_body(payload.get("expression"), scope) + aggregation = data.get("aggregation_type") or CUSTOM_AGGREGATION + if dialect == AQL_DIALECT: + block.heredoc("definition", "aql", text) + else: + if aggregation in AGGREGATIONS: + inner = strip_aggregation(aggregation, text) + if inner is None: + self.issues.add( + code=ISSUE_AGGREGATION_NOT_STRIPPED, + severity=Severity.WARNING, + message=( + f"the stashed aggregation {aggregation!r} does not wrap the whole " + f"expression, so the measure is written as 'custom' with the " + f"expression kept whole" + ), + object_ref=scope, + remedy="check the expression against the aggregation", + ) + aggregation = CUSTOM_AGGREGATION + else: + text = inner + block.heredoc("definition", "sql", self.to_aml_sql(text, dialect, model, scope)) + block.property("aggregation_type", quote(aggregation)) + block.text("format", data.get("format")) + return block + + def as_aql(self, text: str, dialect: str, scope: str) -> str | None: + """A dataset-level metric's Ossie SQL body rendered as AQL, or None if it cannot be. + + AML accepts only `@aql` here, so an Ossie SQL body either becomes AQL or has no + AML form at all. `aql.translate` is a name map over functions this + converter has paired with an AQL equivalent. Anything outside that table + raises `Untranslatable`, and the metric is dropped with an ERROR. + """ + try: + rendered, numeric = aql.translate( + text, dialect, lambda table, column: self.aql_reference(table, column, scope) + ) + except aql.Untranslatable as exc: + self.issues.add( + code=ISSUE_METRIC_NOT_TRANSLATED, + severity=Severity.ERROR, + message=( + f"the metric sits on the dataset, where AML accepts only an AQL body. The " + f"converter did not translate this {dialect} body, and the metric is " + f"dropped. {exc}" + ), + object_ref=scope, + remedy="rewrite the metric in AQL, or scope it to a single dataset", + ) + return None + self.issues.add( + code=ISSUE_RENDERED_AS_AQL, + severity=Severity.WARNING, + message=( + f"the metric sits on the dataset, where AML accepts only an AQL body. The " + f"{dialect} expression was rendered as AQL: {text!r} became {rendered!r}" + ), + object_ref=scope, + remedy="check the AQL against the Ossie SQL it came from", + ) + for table, column in numeric: + target = self.by_ossie_name.get(table) + if target is not None: + target.numeric.add(self.name_in_aml(column, scope)) + return rendered + + def aql_reference(self, table: str, column: str, scope: str) -> str: + """One column of an Ossie SQL metric body as the AQL reference it becomes. + + AQL names a field `model.field`, which is the shape the Ossie expression + already uses, so this maps the Ossie dataset name onto the AML model + name and keeps the field name. + + An Ossie SQL body may name a warehouse column the document does not declare as + a field. `SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id)` + in `converters/nvidia/tests/fixtures/sales.ossie.yaml` is one: `orders` + declares no `subtotal`. Ossie SQL resolves that against the table, and AQL + does not, answering `Field `subtotal` not found in model `orders``. So + the column becomes a hidden dimension on that model, and the AQL + reference points at it. + """ + if not table: + # An unqualified name that is a metric is a metric reference, which + # AQL takes: a dataset-level one by its bare name, and a measure by + # the model that holds it. + if column in self.metric_aml_name: + referenced = self.name_in_aml(self.metric_aml_name[column], scope) + owner = self.metric_owner.get(column) + return referenced if owner is None else f"{owner.aml_fqn}.{referenced}" + raise aql.Untranslatable( + f"{column!r} names no dataset, and a metric spanning models has no single " + f"model to resolve it against." + ) + target = self.by_ossie_name.get(table) + if target is None: + raise aql.Untranslatable(f"{table!r} is not a dataset in this document.") + name = self.name_in_aml(column, scope) + # Compared on the AML spelling, because `name` is already sanitized. + # `sanitize` rather than `name_in_aml`, which logs a rename per call. + declared = { + sanitize(field["name"]) for field in target.payload.get("fields") or [] + } | {sanitize(self.metric_aml_name[m["name"]]) for m in target.measures} + if name not in declared and name not in target.borrowed: + target.borrowed.append(name) + self.issues.add( + code=ISSUE_BORROWED_COLUMN, + severity=Severity.WARNING, + message=( + f"the metric reads {table}.{column}, which the dataset does not declare " + f"as a field. AQL resolves a reference against declared fields only, so " + f"{column!r} is added to model {target.aml_fqn!r} as a hidden dimension " + f"over the column of that name" + ), + object_ref=scope, + remedy=f"declare {column!r} as a field of {table!r} in Ossie", + ) + return f"{target.aml_fqn}.{name}" + + def dataset_metric(self, payload: dict[str, Any]) -> Block | None: + """One Ossie metric as a dataset-level AML `metric`. + + AML accepts only an `@aql` body here. The compiler rejects `@sql` with + "Incompatible property 'definition' of 'Metric'", so a cross-model + metric carrying Ossie SQL has no AML form at all. + """ + name = payload["name"] + scope = name + data = stash.read(payload) + dialect, text = self.pick_body(payload.get("expression"), scope) + if dialect != AQL_DIALECT: + rendered = self.as_aql(text, dialect, scope) + if rendered is None: + return None + text = rendered + block = Block(f"metric {self.name_in_aml(name, scope)}") + block.text("label", data.get("label") or titleize(name)) + block.property("type", quote(self.aml_type(payload, data, scope))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + block.heredoc("definition", "aql", text) + block.text("format", data.get("format")) + return block + + # -- models ----------------------------------------------------------- + + def model_file(self, model: Model) -> str: + payload = model.payload + data = model.stash + scope = model.ossie_name + is_query = data.get("aml_type") == "QueryModel" or ( + "aml_type" not in data and bool(interpolation.query_forms_used(payload.get("source", ""))) + ) + source = payload.get("source", "") + if "aml_type" not in data and not is_query: + is_query = _looks_like_query(source) + + block = Block(f"Model {model.aml_name}") + block.property("type", quote("query" if is_query else "table")) + block.text("label", data.get("label") or titleize(model.aml_name)) + block.text("description", payload.get("description")) + block.text("data_source_name", data.get("data_source_name") or self.data_source_name) + if is_query: + referenced = _models_referenced(source) + if referenced: + block.property("models", f"[{', '.join(sorted(referenced))}]") + block.blank() + block.heredoc("query", "sql", source) + else: + block.text("table_name", data.get("table_name") or source) + + for param in data.get("param") or []: + block.blank() + param_block = block.block(f"param {self.name_in_aml(param['name'], scope)}") + param_block.text("label", param.get("label") or titleize(param["name"])) + param_block.property("type", quote(param.get("type") or FALLBACK_TYPE)) + + persistence = data.get("persistence") + if persistence: + block.blank() + _write_persistence(block, persistence, scope) + + primary_key = set(payload.get("primary_key") or []) + for field in payload.get("fields") or []: + if stash.read(field).get("scope") == "dataset": + continue + block.blank() + block.lines.append(self.dimension(model, field, primary_key)) + + for metric in model.measures: + block.blank() + block.lines.append(self.measure(model, metric)) + + for column in model.borrowed: + block.blank() + borrowed = block.block(f"dimension {column}") + borrowed.text("label", titleize(column)) + borrowed.property("type", quote(FALLBACK_TYPE_FOR_BORROWED)) + borrowed.flag("hidden", True) + borrowed.heredoc("definition", "sql", interpolation.Source(column).to_aml()) + + owner = data.get("owner") or self.root_stash.get("owner") + if owner: + block.blank() + block.text("owner", owner) + return amlgen.document(block, license_header=True) + + # -- dataset ---------------------------------------------------------- + + def relationship(self, payload: dict[str, Any]) -> str: + data = stash.read(payload) + kind = data.get("type", "many_to_one") + active = "true" if data.get("active", True) else "false" + from_model = self.by_ossie_name.get(payload.get("from")) + to_model = self.by_ossie_name.get(payload.get("to")) + if from_model is None or to_model is None: + missing = payload.get("from") if from_model is None else payload.get("to") + raise ConversionError( + f"relationship {payload.get('name')!r} names dataset {missing!r}, " + f"which is not in this document" + ) + name = payload.get("name") + from_columns = payload.get("from_columns") or [] + to_columns = payload.get("to_columns") or [] + if not from_columns or len(from_columns) != len(to_columns): + raise ConversionError( + f"relationship {name!r} has {len(from_columns)} from_columns and " + f"{len(to_columns)} to_columns. AML joins a column to a column, so the " + f"two lists must be the same length and must not be empty" + ) + + extra = self._relationship_properties(data, name) + where = data.get("where") + if where is not None: + return self._named_relationship( + payload, kind, active, from_model, to_model, + from_columns, to_columns, extra, where, + ) + if len(from_columns) == 1 and kind in ("many_to_one", "one_to_one") and not extra: + operator = ">" if kind == "many_to_one" else "-" + return ( + f"relationship({from_model.reference}.{from_columns[0]} {operator} " + f"{to_model.reference}.{to_columns[0]}, {active})" + ) + + config = Block("RelationshipConfig") + rel = config.block("EqualityRelationship") + rel.header = "rel: EqualityRelationship" + rel.property("type", quote(kind)) + rel.property("from", _field_ref(from_model.reference, from_columns[0])) + rel.property("to", _field_ref(to_model.reference, to_columns[0])) + for from_column, to_column in zip(from_columns[1:], to_columns[1:]): + on = rel.block("on") + on.property("from", _field_ref(from_model.reference, from_column)) + on.property("to", _field_ref(to_model.reference, to_column)) + config.property("active", active) + for key, value in extra: + config.property(key, value) + return config.render(2).lstrip() + + def _named_relationship( + self, + payload: dict[str, Any], + kind: str, + active: str, + from_model: Model, + to_model: Model, + from_columns: list[str], + to_columns: list[str], + extra: list[tuple[str, str]], + where: Any, + ) -> str: + """A relationship carrying a `where` filter, as a named declaration. + + A `where` block belongs to the relationship, and one written inline in + `relationships:` has nowhere to put it. So the relationship is declared + above the `Dataset` block under a name, and the list holds + `RelationshipConfig { rel: , ... }`. + + `Relationship` rather than `EqualityRelationship`, because it is the + supertype and the compiler narrows it from `type`. The compiled payload + comes back tagged `EqualityRelationship` either way. + """ + scope = f"relationship {payload.get('name')}" + rel_name = self.name_in_aml(str(payload.get("name")), scope) + declaration = Block(f"Relationship {rel_name}") + declaration.property("type", quote(kind)) + declaration.property("from", _field_ref(from_model.reference, from_columns[0])) + declaration.property("to", _field_ref(to_model.reference, to_columns[0])) + for from_column, to_column in zip(from_columns[1:], to_columns[1:]): + on = declaration.block("on") + on.property("from", _field_ref(from_model.reference, from_column)) + on.property("to", _field_ref(to_model.reference, to_column)) + self._write_where(declaration, where, scope) + self.relationship_declarations.append(amlgen.document(declaration, license_header=False)) + + config = Block("RelationshipConfig") + config.property("rel", rel_name) + config.property("active", active) + for key, value in extra: + config.property(key, value) + return config.render(2).lstrip() + + def _write_where(self, block: Block, where: Any, scope: str) -> None: + """The stashed `RelationshipFilter` as a `where { ... }` block. + + Each side is an AQL predicate narrowing the join on that end, and + either may be absent. `comment_photo` in the Holistics AMQL test corpus + carries only `from`. + """ + if not isinstance(where, dict): + raise ConversionError( + f"{scope}: the stashed `where` is {type(where).__name__}, and AML writes " + f"it as a block holding a `from` and a `to` predicate" + ) + child = block.block("where") + written = False + for side in ("from", "to"): + heredoc = where.get(side) + if heredoc is None: + continue + if not isinstance(heredoc, dict) or "content" not in heredoc: + raise ConversionError( + f"{scope}: the stashed `where.{side}` is not a heredoc object, so " + f"there is no predicate to write" + ) + # Verbatim, including trailing space: the round trip compares the + # compiled payload, and the compiler keeps what the heredoc held. + child.heredoc(side, heredoc.get("name") or "aql", heredoc["content"]) + written = True + if not written: + raise ConversionError( + f"{scope}: the stashed `where` names neither a `from` nor a `to` " + f"predicate, so it narrows nothing" + ) + + def match_relationship(self, payload: dict[str, Any]) -> str: + """A stashed `match` relationship, as its declaration and config entry. + + A range or many-to-many join states its whole condition as one AQL + predicate over both models, which Ossie's column pairs cannot hold, so + the forward path carries the compiler's payload verbatim. Rebuilding it + needs no interpretation: the predicate, the endpoints and the kind all + travel as written. + """ + rel = payload.get("rel") + if not isinstance(rel, dict): + raise ConversionError( + "a stashed match relationship has no `rel` object to rebuild from" + ) + name = rel.get("name") or rel.get("__fqn__") + if not name: + raise ConversionError( + "a stashed match relationship has no name, and AML can only carry a " + "`match` predicate on a named relationship" + ) + scope = f"relationship {name}" + aml_name = self.name_in_aml(str(name), scope) + declaration = Block(f"{rel.get('__type__') or 'Relationship'} {aml_name}") + declaration.property("type", quote(rel.get("type") or "many_to_many")) + for side in ("from", "to"): + endpoint = rel.get(side) + if not isinstance(endpoint, dict) or "model" not in endpoint: + raise ConversionError(f"{scope}: the stashed `{side}` is not a field reference") + declaration.property(side, _field_ref(endpoint["model"], endpoint.get("field", ""))) + match = rel.get("match") + if not isinstance(match, dict) or "content" not in match: + raise ConversionError( + f"{scope}: the stashed `match` is not a heredoc, so there is no predicate " + f"to write and the join has no condition" + ) + declaration.heredoc("match", match.get("name") or "aql", match["content"]) + self.relationship_declarations.append( + amlgen.document(declaration, license_header=False) + ) + + config = Block("RelationshipConfig") + config.property("rel", aml_name) + config.property("active", "true" if payload.get("active", True) else "false") + for key, value in self._relationship_properties(payload, name): + config.property(key, value) + return config.render(2).lstrip() + + def _relationship_properties(self, data: dict[str, Any], name: Any) -> list[tuple[str, str]]: + """The stashed `RelationshipConfig` properties worth writing back. + + `direction`, `nullable` and `rlp_propagation` are all settable on a + `RelationshipConfig`. Each carries a compiler default on every payload, + so only a value that differs from it is written. + """ + extra: list[tuple[str, str]] = [] + for key, default in RELATIONSHIP_DEFAULTS.items(): + value = data.get(key) + if value is not None and value != default: + extra.append((key, _aml_value(value, f"relationship {name}"))) + return extra + + def dataset_file(self) -> tuple[str, str]: + name = self.dataset_name + block = Block(f"Dataset {name}") + block.text( + "label", + self.root_stash.get("label") or titleize(self.document.get("name") or name), + ) + block.text("description", self.document.get("description")) + block.text("data_source_name", self.data_source_name) + block.blank() + block.property("models", amlgen.array([m.reference for m in self.models])) + block.blank() + entries = [self.relationship(r) for r in self.document.get("relationships") or []] + entries.extend(self.match_relationship(r) for r in self.root_stash.get("match_relationships") or []) + block.property("relationships", amlgen.array(entries)) + + for model in self.models: + for field in model.payload.get("fields") or []: + if stash.read(field).get("scope") == "dataset": + block.blank() + block.lines.append(self.dataset_dimension(model, field)) + + for metric in self.dataset_metrics: + written = self.dataset_metric(metric) + if written is not None: + block.blank() + block.lines.append(written) + + owner = self.root_stash.get("owner") + if owner: + block.blank() + block.text("owner", owner) + body = amlgen.document(block, license_header=True) + if self.relationship_declarations: + head, separator, rest = body.partition(f"Dataset {name} {{") + body = head + "".join(self.relationship_declarations) + "\n" + separator + rest + return f"{name}.dataset.aml", body + + def run(self) -> Result: + # The dataset file is rendered first because its metrics decide which + # columns each model has to borrow, and a model file has to carry those + # as dimensions. The output order is unchanged: models, then dataset. + dataset = self.dataset_file() + files = [(m.path, self.model_file(m)) for m in self.models] + files.append(dataset) + return Result(files, self.issues) + + +def _field_ref(model_name: str, column: str) -> str: + """`ref()` is aml-std's constructor for a FieldRef, and is how AML spells one.""" + return f"ref({quote(model_name)}, {quote(column)})" + + +def _aml_value(value: Any, where: str) -> str: + """One stashed value as the AML source text that spells it. + + A type with no spelling here stops the run, because a body AML cannot parse + fails later and further from its cause. + """ + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return quote(value) + if isinstance(value, list): + return "[" + ", ".join(_aml_value(member, where) for member in value) + "]" + raise ConversionError( + f"{where}: the stash holds {type(value).__name__}, which this converter cannot " + f"write as AML. Drop the key from custom_extensions, or add a spelling for it" + ) + + +def _write_stashed_object(block: Block, key: str, payload: dict[str, Any], where: str) -> None: + """One stashed object as `key: { ... }`. + + `__type__` names the AML type, and every key the compiler added for its own + bookkeeping starts and ends with `__`, so those are skipped. A nested object + carries its own `__type__` and becomes a nested block under its own key. + """ + kind = payload.get("__type__") + if not kind: + raise ConversionError( + f"{where}: {key} is an object with no __type__, so there is no AML type " + f"name to write it under" + ) + child = block.block(f"{key}: {kind}") + for name, value in payload.items(): + if name.startswith("__"): + continue + if isinstance(value, dict): + _write_stashed_object(child, name, value, f"{where}.{name}") + continue + child.property(name, _aml_value(value, f"{where}.{name}")) + + +def _write_persistence(block: Block, persistence: dict[str, Any], where: str) -> None: + """The stashed persistence object as its AML block.""" + payload = dict(persistence) + payload.setdefault("__type__", "FullPersistence") + _write_stashed_object(block, "persistence", payload, f"{where}.persistence") + + +def _models_referenced(query: str) -> set[str]: + """Every model a query body names, so the AML `models:` list can be rebuilt. + + A `QueryModel` must declare the models its body references. The forward path + carries the body verbatim, so the references are still in the text. + """ + found = set() + for pattern in ("model_relation", "model_columns"): + for match in interpolation.QUERY_FORMS[pattern].finditer(query): + inner = match.group(0).strip("{} ").lstrip("#") + found.add(inner[:-2] if inner.endswith(".*") else inner) + return found + + +def _looks_like_query(source: str) -> bool: + """Whether an Ossie `source` holds a query rather than a table reference. + + Only reached for a document with no stashed `aml_type`, meaning one this + converter did not write. `source` is a single string, so the shape is read + from the text. This takes the permissive reading, counting a leading + `SELECT` or `WITH`, or any whitespace outside a quoted identifier. + """ + text = source.strip() + if text[:6].upper() in ("SELECT",) or text[:4].upper() == "WITH": + return True + return any(character.isspace() for character in text.strip("`\"[]")) + + +def convert( + document: Any, sql_dialect: str, data_source_name: str | None = None +) -> Result: + """One Ossie semantic model to AML source files. + + `sql_dialect` names the warehouse the Holistics connection reads. Every + `@sql` body written is in that dialect, rendered from a portable expression + where the document carries no body in it already. + + `data_source_name` names the connection itself. AML requires it on a dataset + and Ossie has no field for it, so a document this converter did not write + needs it supplied. + """ + if not isinstance(document, dict): + raise ConversionError("the Ossie document is not a mapping") + return _Reverse(document, data_source_name, sql_dialect).run() diff --git a/converters/holistics/src/ossie_holistics/sqlrefs.py b/converters/holistics/src/ossie_holistics/sqlrefs.py new file mode 100644 index 000000000..d03917a74 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/sqlrefs.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Finding the column references in an Ossie SQL expression. + +The reverse path has to turn `CAST(created_at AS DATE)` back into +`CAST({{ #SOURCE.created_at }} AS DATE)`. A bare column name does survive +`holistics aml validate`, but it is not equivalent: AQL reads the interpolation +to learn which field a body depends on, and a body with none loses the query +rewriting and the optimisation that depend on that link. So the reference has to +be restored, which means telling a column apart from a keyword, a function name +and a date part. + +sqlglot already makes that distinction, and it reports the character offsets of +the identifiers it parsed. In BigQuery's `DATE_TRUNC(created_at, month)` it +reads `created_at` as a column and `month` as a date part, and in +`CASE WHEN status = 'created_at' THEN created_at END` it reads the second +`created_at` and not the one inside the string literal. + +`rewrite` uses those offsets to splice replacement text into the original string. +The SQL is never re-rendered through sqlglot. `DATE_TRUNC(created_at, month)` +written back out through sqlglot's default dialect returns +`DATE_TRUNC(created_at, MONTH)`, which that same default dialect then re-reads +with the unit and the expression exchanged, so a round trip through the writer +silently changes the meaning. +""" +from __future__ import annotations + +from dataclasses import dataclass + +try: + import sqlglot + from sqlglot import exp + from sqlglot.errors import ParseError, TokenError + + AVAILABLE = True +except ImportError: # pragma: no cover - exercised by the packaging test + AVAILABLE = False + +#: Ossie dialect to the sqlglot dialect that parses it. Mirrors `DIALECT_MAP` in +#: `validation/validate.py`. None means sqlglot's default. +SQLGLOT_DIALECT: dict[str, str | None] = { + "ANSI_SQL": None, + "OSSIE_SQL_2026": None, + "SNOWFLAKE": "snowflake", + "DATABRICKS": "databricks", + "BIGQUERY": "bigquery", +} + +#: Dialects sqlglot cannot parse. A body in one of these is not SQL. +NOT_SQL = frozenset({"MDX", "TABLEAU", "MAQL", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL"}) + +#: Every expression is parsed as `SELECT `, because a bare column +#: name is not a statement on its own. Offsets shift back by this much. +_PREFIX = "SELECT " + + +class UnparseableSQL(Exception): + """Raised when sqlglot cannot read an expression at the dialect given.""" + + +@dataclass(frozen=True) +class ColumnReference: + """One column in the expression, with the span it occupies in the text.""" + + table: str + name: str + start: int + #: Exclusive, so `text[start:end]` is the reference as written. + end: int + + +def columns(expression: str, dialect: str) -> list[ColumnReference]: + """Every column reference in `expression`, in the order it appears.""" + if not AVAILABLE: + raise UnparseableSQL("sqlglot is not installed") + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(dialect)) + except (ParseError, TokenError, RecursionError) as exc: + raise UnparseableSQL(str(exc).split("\n")[0]) from exc + + found = [] + for column in tree.find_all(exp.Column): + name_meta = column.this.meta + table = column.args.get("table") + if "start" not in name_meta: + continue + start = table.meta["start"] if table is not None and "start" in table.meta else name_meta["start"] + found.append( + ColumnReference( + table=column.table, + name=column.name, + start=start - len(_PREFIX), + end=name_meta["end"] + 1 - len(_PREFIX), + ) + ) + return sorted(found, key=lambda c: c.start) + + +def transpile(expression: str, read: str, write: str) -> str: + """`expression` read as `read` and written as `write`. + + Used only to render a portable body into the warehouse the AML output is + aimed at. Both dialects are named members of `SQLGLOT_DIALECT`. + """ + if not AVAILABLE: + raise UnparseableSQL("sqlglot is not installed") + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(read)) + except (ParseError, TokenError, RecursionError) as exc: + raise UnparseableSQL(str(exc).split("\n")[0]) from exc + rendered = tree.sql(dialect=SQLGLOT_DIALECT.get(write)) + if not rendered.upper().startswith(_PREFIX.strip()): + raise UnparseableSQL(f"sqlglot wrote an unexpected shape: {rendered!r}") + return rendered[len(_PREFIX) :] + + +def window_grouping(expression: str, dialect: str) -> list[str]: + """The columns an OVER clause partitions or orders by, in order. + + A window function survives into an AML `@sql` body and compiles, and the + warehouse SQL it produces is only valid when the query groups by these columns. Holistics + does not check that, so the caller reports it. + """ + if not AVAILABLE: + return [] + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(dialect)) + except (ParseError, TokenError, RecursionError): + return [] + seen: list[str] = [] + for window in tree.find_all(exp.Window): + # `window.this` is the function being windowed, whose columns the + # aggregate consumes rather than groups by. Only the frame decides the + # grouping, so only PARTITION BY and ORDER BY are read. + frame = [window.args.get("partition_by"), window.args.get("order")] + for part in frame: + for item in part if isinstance(part, list) else [part]: + if item is None: + continue + # A column inside an aggregate within the frame is consumed by + # that aggregate, not grouped by. `ORDER BY SUM(t.v)` orders on + # the aggregate and puts no requirement on `t.v`. + aggregated = { + id(column) + for aggregate in item.find_all(exp.AggFunc) + for column in aggregate.find_all(exp.Column) + } + for column in item.find_all(exp.Column): + if id(column) in aggregated: + continue + name = f"{column.table}.{column.name}" if column.table else column.name + if name not in seen: + seen.append(name) + return seen + + +def rewrite(expression: str, dialect: str, replace) -> str: + """`expression` with each column replaced by `replace(column_reference)`. + + Every character outside a replaced span is kept exactly as it was. + """ + out = expression + for column in reversed(columns(expression, dialect)): + out = out[: column.start] + replace(column) + out[column.end :] + return out diff --git a/converters/holistics/src/ossie_holistics/stash.py b/converters/holistics/src/ossie_holistics/stash.py new file mode 100644 index 000000000..e05a2d710 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/stash.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `HOLISTICS` entry in `custom_extensions`. + +Every AML property the core specification has no field for travels here. The +entry's `data` is a single JSON-encoded string, never a nested object, because +the specification requires that: + + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "table_name": "`ecommerce`.`orders`"}' + +`_v` is the shape version. `read` fails on a version it does not know rather +than misreading a shape it has never seen. + +A foreign vendor's entry on the same object passes through untouched, and an +object with nothing to stash gets no `HOLISTICS` entry at all. +""" +from __future__ import annotations + +import json +from typing import Any + +from .errors import ConversionError + +VENDOR = "HOLISTICS" + +#: The only shape this converter writes, and the only one it reads. +VERSION = 1 + + +def entry(data: dict[str, Any]) -> dict[str, str] | None: + """One `custom_extensions` entry, or None when there is nothing to carry.""" + if not data: + return None + payload = {"_v": VERSION} + payload.update(data) + return {"vendor_name": VENDOR, "data": json.dumps(payload, sort_keys=True)} + + +def attach(target: dict[str, Any], data: dict[str, Any]) -> None: + """Append the `HOLISTICS` entry to `target['custom_extensions']`, if any.""" + stashed = entry(data) + if stashed is None: + return + target.setdefault("custom_extensions", []).append(stashed) + + +def read(obj: Any) -> dict[str, Any]: + """The stashed payload on an Ossie object, `{}` when there is none. + + `_v` is stripped from the result, so a caller reads AML properties by their + own names and never sees the bookkeeping key. + """ + if not isinstance(obj, dict): + return {} + extensions = obj.get("custom_extensions") + if not isinstance(extensions, list): + return {} + for extension in extensions: + if not isinstance(extension, dict) or extension.get("vendor_name") != VENDOR: + continue + raw = extension.get("data") + if not isinstance(raw, str): + raise ConversionError(f"{VENDOR} custom_extensions data is not a string") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise ConversionError(f"{VENDOR} custom_extensions data is not valid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ConversionError(f"{VENDOR} custom_extensions data is not a JSON object") + version = payload.pop("_v", None) + if version != VERSION: + raise ConversionError( + f"{VENDOR} custom_extensions data has _v={version!r}, " + f"and this converter writes and reads _v={VERSION}" + ) + return payload + return {} diff --git a/converters/holistics/tests/conftest.py b/converters/holistics/tests/conftest.py new file mode 100644 index 000000000..7151f0d8c --- /dev/null +++ b/converters/holistics/tests/conftest.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tests")) diff --git a/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml new file mode 100644 index 000000000..68fffe803 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) / COUNT(DISTINCT users.id)' became 'sum(items.quantity) + / count_distinct(users.id)' + object_ref: quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'ROUND(SAFE_DIVIDE(SUM(items.quantity), ABS(COUNT(users.id))), 2)' became 'round(safe_divide(sum(items.quantity), + abs(count(users.id))), 2)' + object_ref: rounded_ratio + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.product_id) + COUNT(users.id)' became 'sum(items.product_id) + count(users.id)' + object_ref: untyped_total + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION + severity: WARNING + message: >- + the metric reads items.sale_price, which the dataset does not declare as a field. AQL resolves + a reference against declared fields only, so 'sale_price' is added to model 'items' as a hidden + dimension over the column of that name + object_ref: borrowed_column_total + remedy: declare 'sale_price' as a field of 'items' in Ossie +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.sale_price) + COUNT(users.id)' became 'sum(items.sale_price) + count(users.id)' + object_ref: borrowed_column_total + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) + MY_WAREHOUSE_UDF(COUNT(users.id), 3)' became "sum(items.quantity) + + sql_number('MY_WAREHOUSE_UDF', count(users.id), 3)" + object_ref: passthrough_call + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: windowed_rank + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. LOG is not in this converter's AQL function table, + and its call cannot be read back from the source text, so there is no faithful way to pass it + through. + object_ref: unmapped_function + remedy: rewrite the metric in AQL, or scope it to a single dataset diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml b/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml new file mode 100644 index 000000000..163297527 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset aql_translation { + label: 'Aql Translation' + description: 'Metrics that exercise each path through the SQL to AQL translation' + data_source_name: 'bigquery_demo' + + models: [ + orders, + items, + users, + ] + + relationships: [ + relationship(items.order_id > orders.id, true), + relationship(orders.user_id > users.id, true), + ] + + metric quantity_per_customer { + label: 'Quantity Per Customer' + type: 'number' + description: 'The plain path. SUM and COUNT DISTINCT both have AQL names' + definition: @aql sum(items.quantity) / count_distinct(users.id);; + } + + metric rounded_ratio { + label: 'Rounded Ratio' + type: 'number' + description: 'Functions AQL has. ROUND, SAFE_DIVIDE and ABS are written natively' + definition: @aql round(safe_divide(sum(items.quantity), abs(count(users.id))), 2);; + } + + metric untyped_total { + label: 'Untyped Total' + type: 'number' + description: 'A field with no datatype, summed, so the metric is what types it' + definition: @aql sum(items.product_id) + count(users.id);; + } + + metric borrowed_column_total { + label: 'Borrowed Column Total' + type: 'number' + description: 'A column no dataset declares as a field, so one is added for it' + definition: @aql sum(items.sale_price) + count(users.id);; + } + + metric passthrough_call { + label: 'Passthrough Call' + type: 'number' + description: 'A function sqlglot does not model, carried by AQL\'s raw-SQL call' + definition: @aql sum(items.quantity) + sql_number('MY_WAREHOUSE_UDF', count(users.id), 3);; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml new file mode 100644 index 000000000..17e988fbf --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model items { + type: 'table' + label: 'Items' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension sale_price { + label: 'Sale Price' + type: 'number' + hidden: true + definition: @sql {{ #SOURCE.sale_price }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml new file mode 100644 index 000000000..83bc83752 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model orders { + type: 'table' + label: 'Orders' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml new file mode 100644 index 000000000..a20a34bfa --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model users { + type: 'table' + label: 'Users' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`users`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/translation.ossie.yaml b/converters/holistics/tests/fixtures/aql/translation.ossie.yaml new file mode 100644 index 000000000..f8b9ffdca --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/translation.ossie.yaml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie document written to exercise `to-aml`'s SQL to AQL translation. +# +# Every metric here spans two datasets, which is what forces AML to take an +# `@aql` body and so forces the translation. Each one covers a different path, +# named in its description. The ecommerce tables are the ones the Holistics +# BigQuery demo connection carries, so the generated AML compiles against a real +# schema rather than an invented one. + +version: "0.2.0.dev0" +name: aql_translation +description: Metrics that exercise each path through the SQL to AQL translation + +datasets: + - name: orders + source: "`ecommerce`.`orders`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: user_id + expression: {dialects: [{dialect: ANSI_SQL, expression: user_id}]} + datatype: Decimal + - name: status + expression: {dialects: [{dialect: ANSI_SQL, expression: status}]} + datatype: String + + - name: items + source: "`ecommerce`.`order_items`" + fields: + - name: order_id + expression: {dialects: [{dialect: ANSI_SQL, expression: order_id}]} + datatype: Decimal + - name: quantity + expression: {dialects: [{dialect: ANSI_SQL, expression: quantity}]} + datatype: Decimal + # No datatype. A metric sums it, which is the only evidence that it is a + # number, and `text` would make AQL refuse the aggregate. + - name: product_id + expression: {dialects: [{dialect: ANSI_SQL, expression: product_id}]} + + - name: users + source: "`ecommerce`.`users`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: full_name + expression: {dialects: [{dialect: ANSI_SQL, expression: full_name}]} + datatype: String + +relationships: + - name: items_to_orders + from: items + to: orders + from_columns: [order_id] + to_columns: [id] + - name: orders_to_users + from: orders + to: users + from_columns: [user_id] + to_columns: [id] + +metrics: + - name: quantity_per_customer + description: The plain path. SUM and COUNT DISTINCT both have AQL names + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) / COUNT(DISTINCT users.id) + + - name: rounded_ratio + description: Functions AQL has. ROUND, SAFE_DIVIDE and ABS are written natively + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: ROUND(SAFE_DIVIDE(SUM(items.quantity), ABS(COUNT(users.id))), 2) + + - name: untyped_total + description: A field with no datatype, summed, so the metric is what types it + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.product_id) + COUNT(users.id) + + - name: borrowed_column_total + description: A column no dataset declares as a field, so one is added for it + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.sale_price) + COUNT(users.id) + + - name: passthrough_call + description: A function sqlglot does not model, carried by AQL's raw-SQL call + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + MY_WAREHOUSE_UDF(COUNT(users.id), 3) + + - name: windowed_rank + description: A window function, which the converter does not translate + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: RANK() OVER (PARTITION BY users.id ORDER BY SUM(items.quantity) DESC) + + - name: unmapped_function + description: A function sqlglot models but the AQL table omits + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + LOG10(COUNT(users.id)) diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml new file mode 100644 index 000000000..a3701fb56 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model cities { + type: 'table' + label: 'Cities' + description: 'One row per city a merchant can operate in.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`cities`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + dimension in_vietnam { + label: 'In Vietnam' + type: 'truefalse' + definition: @aql cities.country_code == 'VN';; + } + + // upstream_measure and downstream_measure form a measure-to-measure chain. + // Resolving downstream_measure yields COUNT(id) + 1, so a converter has to + // follow the chain and detect a cycle rather than recurse forever. + measure upstream_measure { + label: 'Upstream Measure' + type: 'number' + definition: @sql COUNT({{ id }});; + } + + measure downstream_measure { + label: 'Downstream Measure' + type: 'number' + definition: @sql {{ upstream_measure }} + 1;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml new file mode 100644 index 000000000..7fb44ff20 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model delivered_orders { + type: 'query' + label: 'Delivered Orders' + description: 'Orders that reached the customer. Derived from ecommerce_orders rather than from a table.' + data_source_name: 'bigquery_demo' + + // The query body is not plain SQL. It names another model, and the compiler + // does not resolve the reference: it passes the body through and inlines the + // referenced model here instead. `{{ #ecommerce_orders }}` is this file's + // equivalent of dbt's `ref()`. + models: [ecommerce_orders] + + query: @sql + select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} + ;; + + // A filter block expands to a predicate only when a value is supplied at + // query time, so no conversion-time rewrite of the body above is correct. + param min_delivery_attempts { + label: 'Minimum Delivery Attempts' + type: 'number' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension delivery_attempts { + label: 'Delivery Attempts' + type: 'number' + definition: @sql {{ #SOURCE.delivery_attempts }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml new file mode 100644 index 000000000..30bd2068b --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml @@ -0,0 +1,132 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// A join filter narrowing each end. `where` belongs to the relationship, so a +// relationship carrying one is declared here and the dataset references it by +// name. `Relationship` is the supertype; the compiler narrows it from `type`. +Relationship delivered_to_users { + type: 'many_to_one' + from: ref('delivered_orders', 'user_id') + to: ref('ecommerce_users', 'id') + where { + from: @aql delivered_orders.delivery_attempts > 0 ;; + to: @aql ecommerce_users.age > 18 ;; + } +} + +// A many-to-many join. Its whole condition is a `match` expression rather than +// column pairs, so Ossie has no form for it and the converter stashes it. +ManyToManyRelationship orders_in_year { + type: 'many_to_many' + from: ref('ecommerce_orders', 'id') + to: ref('reporting.calendar', 'date') + match: @aql ecommerce_orders.created_date == reporting.calendar.date ;; +} + +Dataset ecommerce { + label: 'Ecommerce' + description: 'A marketplace model: orders, line items, products, merchants and cities.' + data_source_name: 'bigquery_demo' + + models: [ + ecommerce_orders, + ecommerce_users, + order_items, + products, + merchants, + cities, + reporting.calendar, + delivered_orders, + ] + + relationships: [ + relationship(order_items.order_id > ecommerce_orders.id, true), + relationship(order_items.product_id > products.id, true), + relationship(ecommerce_orders.user_id > ecommerce_users.id, true), + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: ref('products', 'merchant_id') + to: ref('merchants', 'id') + } + active: true + direction: 'one_way' + nullable: false + rlp_propagation: 'one_way' + }, + // Composite. The `relationship(...)` shorthand carries one column pair, so a + // second pair needs the long form with an `on` block. Blocks combine with AND. + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: FieldRef { model: 'merchants', field: 'city_id' } + to: FieldRef { model: 'cities', field: 'id' } + on { + from: FieldRef { model: 'merchants', field: 'country_code' } + to: FieldRef { model: 'cities', field: 'country_code' } + } + } + active: true + }, + RelationshipConfig { rel: delivered_to_users, active: true }, + + // One-to-one. Ossie has no marker for this, so the kind goes to the stash. + relationship(merchants.admin_id - ecommerce_users.id, true), + + // Inactive, and its target sits in a module. Holistics does not join across + // this edge by default, and the compiled endpoint carries no __refFqn__. + relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + RelationshipConfig { rel: orders_in_year, active: true }, + ] + + // A dataset-level dimension names the model it attaches to. The body spans + // two models, so it needs the relationship graph to resolve. + dimension item_value { + model: order_items + + label: 'Item Value' + type: 'number' + definition: @aql order_items.quantity * products.price;; + } + + metric total_revenue { + label: 'Total Revenue' + type: 'number' + definition: @aql sum(order_items.item_value);; + format: '#,###[$$]' + } + + metric count_orders { + label: 'Count Orders' + type: 'number' + definition: @aql count(order_items.order_id);; + } + + // of_all() rewrites the grain of the whole query. No SQL rendering of this + // body is correct without the dataset's relationship graph, which is why the + // converter carries AQL under HOLISTICS_AQL instead of translating it. + metric revenue_share_of_all_merchants { + label: 'Revenue Share Of All Merchants' + type: 'number' + definition: @aql + sum(order_items.item_value) + / (sum(order_items.item_value) | of_all(merchants)) + ;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml new file mode 100644 index 000000000..7f981f167 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model merchants { + type: 'table' + label: 'Merchants' + description: 'One row per seller on the marketplace.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`merchants`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + // Each merchant has at most one admin user, so this side is one-to-one. + dimension admin_id { + label: 'Admin Id' + type: 'number' + definition: @sql {{ #SOURCE.admin_id }};; + } + + dimension city_id { + label: 'City Id' + type: 'number' + definition: @sql {{ #SOURCE.city_id }};; + } + + // City ids are unique within a country, so the join to cities needs both + // columns. That makes it a composite relationship. + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml new file mode 100644 index 000000000..93c70b537 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model calendar { + type: 'table' + label: 'Calendar' + description: 'A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`calendar`' + + dimension date { + label: 'Date' + type: 'date' + definition: @sql {{ #SOURCE.date }};; + primary_key: true + } + + dimension year { + label: 'Year' + type: 'number' + definition: @sql {{ #SOURCE.year }};; + } + + dimension is_weekend { + label: 'Is Weekend' + type: 'truefalse' + definition: @sql {{ #SOURCE.is_weekend }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml new file mode 100644 index 000000000..696dbb357 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model order_items { + type: 'table' + label: 'Order Items' + description: 'One row per line item on an order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + // A named aggregation over a bare source column. This is the one measure + // shape that converts to portable ANSI SQL: SUM(quantity). + measure total_quantity { + label: 'Total Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + aggregation_type: 'sum' + } + + // The aggregate sits inside the body instead, so aggregation_type is 'custom'. + measure custom_total_quantity { + label: 'Custom Total Quantity' + type: 'number' + definition: @sql sum({{ #SOURCE.quantity }});; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml new file mode 100644 index 000000000..959c74361 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_orders { + type: 'table' + label: 'Orders' + description: 'One row per customer order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + // No definition: the dimension name is the column name. + dimension id { + label: 'Id' + type: 'number' + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension created_at { + label: 'Created At' + type: 'datetime' + definition: @sql {{ #SOURCE.created_at }};; + } + + // Warehouse SQL, not portable ANSI. + dimension created_date { + label: 'Created Date' + type: 'date' + definition: @sql CAST({{ #SOURCE.created_at }} AS DATE);; + } + + // BigQuery argument order; a converter must not rewrite this. + dimension created_month { + label: 'Created Month' + type: 'date' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + format: 'dd LLL, yyyy' + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } + + dimension delivered { + label: 'Delivered' + type: 'truefalse' + definition: @aql ecommerce_orders.status == 'delivered';; + } + + // Named aggregation over a sibling field, not a source column. + measure last_created_at { + label: 'Last Created At' + type: 'datetime' + definition: @sql {{ created_at }};; + aggregation_type: 'max' + } + + measure count_orders { + label: 'Count Orders' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count' + } + + // AQL: the filter resolves through the dataset relationship graph. + measure delivered_orders { + label: 'Delivered Orders' + type: 'number' + hidden: true + definition: @aql count(ecommerce_orders.id) | where(ecommerce_orders.delivered);; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml new file mode 100644 index 000000000..8fdbabf81 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model products { + type: 'table' + label: 'Products' + description: 'One row per sellable product.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`products`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension merchant_id { + label: 'Merchant Id' + type: 'number' + definition: @sql {{ #SOURCE.merchant_id }};; + } + + dimension price { + label: 'Price' + type: 'number' + definition: @sql {{ #SOURCE.price }};; + format: '#,###[$$]' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml new file mode 100644 index 000000000..7f2cbfa01 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_users { + type: 'query' + label: 'Users' + description: 'One row per registered customer.' + data_source_name: 'bigquery_demo' + + query: @sql + select * from `ecommerce`.`users` + ;; + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } + + // No definition. AML defaults the body to `{{ #SOURCE.gender }}`, the source + // column that shares the dimension's name. Only a dimension may omit it. + dimension gender { + label: 'Gender' + type: 'text' + } + + dimension birth_date { + label: 'Birth Date' + type: 'date' + definition: @sql {{ #SOURCE.birth_date }};; + } + + // Sibling reference inside warehouse SQL. + dimension age { + label: 'Age' + type: 'number' + definition: @sql extract(year from age({{ birth_date }}));; + } + + // A sibling reference to a field that is itself computed, so resolution + // follows a chain: age_bucket reads age, which reads birth_date. + dimension age_bucket { + label: 'Age Bucket' + type: 'number' + hidden: true + definition: @sql FLOOR({{ age }} / 10);; + } + + dimension signup_month { + label: 'Signup Month' + type: 'datetime' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + } + + measure count_users { + label: 'Count Users' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count distinct' + } + + param reporting_currency { + label: 'Reporting Currency' + type: 'text' + } + + param minimum_age { + label: 'Minimum Age' + type: 'number' + } + + persistence: FullPersistence { + schema: 'persisted' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json new file mode 100644 index 000000000..32878f5a2 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json @@ -0,0 +1,1488 @@ +{ + "name": "ecommerce", + "__fqn__": "ecommerce", + "label": "Ecommerce", + "description": "A marketplace model: orders, line items, products, merchants and cities.", + "owner": "dev@ossie.apache.org", + "distinct_rows": true, + "dimension": [ + { + "name": "item_value", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "order_items.quantity * products.price" + }, + "model": { + "__fqn__": "order_items", + "name": "order_items", + "__doc__": "", + "label": "Order Items", + "description": "One row per line item on an order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "order_id": { + "name": "order_id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.order_id }}" + }, + "primary_key": false, + "label": "Order Id", + "type": "number", + "__type__": "Dimension" + }, + "product_id": { + "name": "product_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.product_id }}" + }, + "primary_key": false, + "label": "Product Id", + "type": "number", + "__type__": "Dimension" + }, + "quantity": { + "name": "quantity", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "primary_key": false, + "label": "Quantity", + "type": "number", + "__type__": "Dimension" + } + }, + "measure": { + "total_quantity": { + "name": "total_quantity", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "aggregation_type": "sum", + "label": "Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "__type__": "Measure" + }, + "custom_total_quantity": { + "name": "custom_total_quantity", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Custom Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "sum({{ #SOURCE.quantity }})" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`order_items`", + "__type__": "TableModel" + }, + "label": "Item Value", + "type": "number", + "__type__": "DatasetDimension" + } + ], + "metric": { + "total_revenue": { + "name": "total_revenue", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "format": "#,###[$$]", + "label": "Total Revenue", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "sum(order_items.item_value)" + }, + "__type__": "Metric" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(order_items.order_id)" + }, + "__type__": "Metric" + }, + "revenue_share_of_all_merchants": { + "name": "revenue_share_of_all_merchants", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "label": "Revenue Share Of All Merchants", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "sum(order_items.item_value)\n/ (sum(order_items.item_value) | of_all(merchants))" + }, + "__type__": "Metric" + } + }, + "data_source_name": "bigquery_demo", + "models": [ + { + "name": "ecommerce_orders", + "__fqn__": "ecommerce_orders", + "label": "Orders", + "description": "One row per customer order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "created_at": { + "name": "created_at", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.created_at }}" + }, + "primary_key": false, + "label": "Created At", + "type": "datetime", + "__type__": "Dimension" + }, + "created_date": { + "name": "created_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "CAST({{ #SOURCE.created_at }} AS DATE)" + }, + "primary_key": false, + "label": "Created Date", + "type": "date", + "__type__": "Dimension" + }, + "created_month": { + "name": "created_month", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "format": "dd LLL, yyyy", + "primary_key": false, + "label": "Created Month", + "type": "date", + "__type__": "Dimension" + }, + "status": { + "name": "status", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.status }}" + }, + "primary_key": false, + "label": "Status", + "type": "text", + "__type__": "Dimension" + }, + "delivered": { + "name": "delivered", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.status == 'delivered'" + }, + "primary_key": false, + "label": "Delivered", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "last_created_at": { + "name": "last_created_at", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "max", + "label": "Last Created At", + "type": "datetime", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ created_at }}" + }, + "__type__": "Measure" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "aggregation_type": "count", + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + }, + "delivered_orders": { + "name": "delivered_orders", + "__doc__": "", + "__childIdx__": 14, + "hidden": true, + "aggregation_type": "custom", + "label": "Delivered Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(ecommerce_orders.id) | where(ecommerce_orders.delivered)" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`orders`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "ecommerce_users", + "__fqn__": "ecommerce_users", + "label": "Users", + "description": "One row per registered customer.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "models": [], + "persistence": { + "__doc__": "", + "on_cascade": "rebuild", + "schema": "persisted", + "__type__": "FullPersistence" + }, + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "full_name": { + "name": "full_name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.full_name }}" + }, + "primary_key": false, + "label": "Full Name", + "type": "text", + "__type__": "Dimension" + }, + "gender": { + "name": "gender", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "primary_key": false, + "label": "Gender", + "type": "text", + "__type__": "Dimension" + }, + "birth_date": { + "name": "birth_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.birth_date }}" + }, + "primary_key": false, + "label": "Birth Date", + "type": "date", + "__type__": "Dimension" + }, + "age": { + "name": "age", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "extract(year from age({{ birth_date }}))" + }, + "primary_key": false, + "label": "Age", + "type": "number", + "__type__": "Dimension" + }, + "age_bucket": { + "name": "age_bucket", + "__doc__": "", + "__childIdx__": 10, + "hidden": true, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "FLOOR({{ age }} / 10)" + }, + "primary_key": false, + "label": "Age Bucket", + "type": "number", + "__type__": "Dimension" + }, + "signup_month": { + "name": "signup_month", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "primary_key": false, + "label": "Signup Month", + "type": "datetime", + "__type__": "Dimension" + } + }, + "measure": { + "count_users": { + "name": "count_users", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "count distinct", + "label": "Count Users", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + } + }, + "param": { + "reporting_currency": { + "name": "reporting_currency", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "label": "Reporting Currency", + "type": "text", + "__type__": "Param" + }, + "minimum_age": { + "name": "minimum_age", + "__doc__": "", + "__childIdx__": 14, + "hidden": false, + "label": "Minimum Age", + "type": "number", + "__type__": "Param" + } + }, + "type": "query", + "query": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "select * from `ecommerce`.`users`" + }, + "__doc__": "", + "__type__": "QueryModel" + }, + { + "name": "order_items", + "__fqn__": "order_items", + "label": "Order Items", + "description": "One row per line item on an order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "order_id": { + "name": "order_id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.order_id }}" + }, + "primary_key": false, + "label": "Order Id", + "type": "number", + "__type__": "Dimension" + }, + "product_id": { + "name": "product_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.product_id }}" + }, + "primary_key": false, + "label": "Product Id", + "type": "number", + "__type__": "Dimension" + }, + "quantity": { + "name": "quantity", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "primary_key": false, + "label": "Quantity", + "type": "number", + "__type__": "Dimension" + } + }, + "measure": { + "total_quantity": { + "name": "total_quantity", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "aggregation_type": "sum", + "label": "Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "__type__": "Measure" + }, + "custom_total_quantity": { + "name": "custom_total_quantity", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Custom Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "sum({{ #SOURCE.quantity }})" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`order_items`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "products", + "__fqn__": "products", + "label": "Products", + "description": "One row per sellable product.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "merchant_id": { + "name": "merchant_id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.merchant_id }}" + }, + "primary_key": false, + "label": "Merchant Id", + "type": "number", + "__type__": "Dimension" + }, + "price": { + "name": "price", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.price }}" + }, + "format": "#,###[$$]", + "primary_key": false, + "label": "Price", + "type": "number", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`products`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "merchants", + "__fqn__": "merchants", + "label": "Merchants", + "description": "One row per seller on the marketplace.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "admin_id": { + "name": "admin_id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.admin_id }}" + }, + "primary_key": false, + "label": "Admin Id", + "type": "number", + "__type__": "Dimension" + }, + "city_id": { + "name": "city_id", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.city_id }}" + }, + "primary_key": false, + "label": "City Id", + "type": "number", + "__type__": "Dimension" + }, + "country_code": { + "name": "country_code", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.country_code }}" + }, + "primary_key": false, + "label": "Country Code", + "type": "text", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`merchants`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "cities", + "__fqn__": "cities", + "label": "Cities", + "description": "One row per city a merchant can operate in.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "country_code": { + "name": "country_code", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.country_code }}" + }, + "primary_key": false, + "label": "Country Code", + "type": "text", + "__type__": "Dimension" + }, + "in_vietnam": { + "name": "in_vietnam", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "cities.country_code == 'VN'" + }, + "primary_key": false, + "label": "In Vietnam", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "upstream_measure": { + "name": "upstream_measure", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Upstream Measure", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "COUNT({{ id }})" + }, + "__type__": "Measure" + }, + "downstream_measure": { + "name": "downstream_measure", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "aggregation_type": "custom", + "label": "Downstream Measure", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ upstream_measure }} + 1" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`cities`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "calendar", + "__fqn__": "reporting::calendar", + "label": "Calendar", + "description": "A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "date": { + "name": "date", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.date }}" + }, + "primary_key": true, + "label": "Date", + "type": "date", + "__type__": "Dimension" + }, + "year": { + "name": "year", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.year }}" + }, + "primary_key": false, + "label": "Year", + "type": "number", + "__type__": "Dimension" + }, + "is_weekend": { + "name": "is_weekend", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.is_weekend }}" + }, + "primary_key": false, + "label": "Is Weekend", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`calendar`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "delivered_orders", + "__fqn__": "delivered_orders", + "label": "Delivered Orders", + "description": "Orders that reached the customer. Derived from ecommerce_orders rather than from a table.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "models": [ + { + "name": "ecommerce_orders", + "__fqn__": "ecommerce_orders", + "label": "Orders", + "description": "One row per customer order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "created_at": { + "name": "created_at", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.created_at }}" + }, + "primary_key": false, + "label": "Created At", + "type": "datetime", + "__type__": "Dimension" + }, + "created_date": { + "name": "created_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "CAST({{ #SOURCE.created_at }} AS DATE)" + }, + "primary_key": false, + "label": "Created Date", + "type": "date", + "__type__": "Dimension" + }, + "created_month": { + "name": "created_month", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "format": "dd LLL, yyyy", + "primary_key": false, + "label": "Created Month", + "type": "date", + "__type__": "Dimension" + }, + "status": { + "name": "status", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.status }}" + }, + "primary_key": false, + "label": "Status", + "type": "text", + "__type__": "Dimension" + }, + "delivered": { + "name": "delivered", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.status == 'delivered'" + }, + "primary_key": false, + "label": "Delivered", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "last_created_at": { + "name": "last_created_at", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "max", + "label": "Last Created At", + "type": "datetime", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ created_at }}" + }, + "__type__": "Measure" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "aggregation_type": "count", + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + }, + "delivered_orders": { + "name": "delivered_orders", + "__doc__": "", + "__childIdx__": 14, + "hidden": true, + "aggregation_type": "custom", + "label": "Delivered Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(ecommerce_orders.id) | where(ecommerce_orders.delivered)" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`orders`", + "__doc__": "", + "__type__": "TableModel" + } + ], + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "delivery_attempts": { + "name": "delivery_attempts", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.delivery_attempts }}" + }, + "primary_key": false, + "label": "Delivery Attempts", + "type": "number", + "__type__": "Dimension" + } + }, + "param": { + "min_delivery_attempts": { + "name": "min_delivery_attempts", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "label": "Minimum Delivery Attempts", + "type": "number", + "__type__": "Param" + } + }, + "type": "query", + "query": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "select {{ #ecommerce_orders.* }}\nfrom {{ #ecommerce_orders }}\nwhere {% filter(min_delivery_attempts) %} delivery_attempts {% end %}" + }, + "__doc__": "", + "__type__": "QueryModel" + } + ], + "relationships": [ + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "order_items", + "field": "order_id", + "__refFqn__": "order_items", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "order_items", + "field": "product_id", + "__refFqn__": "order_items", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "products", + "field": "id", + "__refFqn__": "products", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "user_id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "one_way", + "rlp_propagation": "one_way", + "nullable": false, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "products", + "field": "merchant_id", + "__refFqn__": "products", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "merchants", + "field": "id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "on": [ + { + "__doc__": "", + "__childIdx__": 3, + "from": { + "__doc__": "", + "model": "merchants", + "field": "country_code", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "cities", + "field": "country_code", + "__type__": "FieldRef" + }, + "__type__": "RelationshipKey" + } + ], + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "merchants", + "field": "city_id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "cities", + "field": "id", + "__refFqn__": "cities", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__fqn__": "delivered_to_users", + "name": "delivered_to_users", + "__doc__": "", + "where": { + "__doc__": "", + "from": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "delivered_orders.delivery_attempts > 0 " + }, + "to": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_users.age > 18 " + }, + "__type__": "RelationshipFilter" + }, + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "delivered_orders", + "field": "user_id", + "__refFqn__": "delivered_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "one_to_one", + "from": { + "__doc__": "", + "model": "merchants", + "field": "admin_id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "created_date", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "reporting.calendar", + "field": "date", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": false, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__fqn__": "orders_in_year", + "name": "orders_in_year", + "__doc__": "", + "type": "many_to_many", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "reporting.calendar", + "field": "date", + "__type__": "FieldRef" + }, + "match": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.created_date == reporting.calendar.date " + }, + "__type__": "ManyToManyRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + } + ], + "__doc__": "", + "__type__": "Dataset" +} \ No newline at end of file diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml new file mode 100644 index 000000000..496f697bf --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce_orders.delivered + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_QUERY_SOURCE + severity: WARNING + message: >- + a query model becomes a query in `source`, which is one string, so a consumer reads the text to + tell it from a table reference + object_ref: ecommerce_users + remedy: materialise the query as a table in Holistics before converting +- code: HOLISTICS_PARAM_DROPPED + severity: WARNING + message: >- + 2 param(s) have no Ossie equivalent and are stashed. A body that reads one cannot be resolved + statically + object_ref: ecommerce_users + remedy: supply the param value in Holistics before converting +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: order_items.item_value + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: cities.in_vietnam + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_QUERY_SOURCE + severity: WARNING + message: >- + a query model becomes a query in `source`, which is one string, so a consumer reads the text to + tell it from a table reference + object_ref: delivered_orders + remedy: materialise the query as a table in Holistics before converting +- code: HOLISTICS_UNRESOLVED_QUERY_TEMPLATE + severity: ERROR + message: >- + the query body uses Holistics template syntax (model_columns, model_relation, filter_block) and + is carried unresolved. `source` is not runnable SQL + object_ref: delivered_orders + remedy: resolve the query through Holistics, or drop the model +- code: HOLISTICS_PARAM_DROPPED + severity: WARNING + message: >- + 1 param(s) have no Ossie equivalent and are stashed. A body that reads one cannot be resolved + statically + object_ref: delivered_orders + remedy: supply the param value in Holistics before converting +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce_orders.delivered_orders + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.total_revenue + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.count_orders + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.revenue_share_of_all_merchants + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_RELATIONSHIP_FILTER + severity: WARNING + message: >- + the relationship carries a filter that narrows the join, and Ossie has no equivalent. The join + widens to every matching row + object_ref: delivered_orders_to_ecommerce_users + remedy: move the condition into a dimension in AML +- code: HOLISTICS_INACTIVE_RELATIONSHIP + severity: WARNING + message: >- + the relationship is inactive in AML, and Ossie has no inactive marker. A consumer joins across + it where Holistics leaves it out by default + object_ref: ecommerce_orders_to_reporting__calendar + remedy: drop the relationship, or activate it in AML +- code: HOLISTICS_UNSUPPORTED_RELATIONSHIP + severity: WARNING + message: >- + a many_to_many relationship carries an AQL match condition rather than column pairs, so it has + no `relationships[]` entry and is stashed on the document instead. A consumer reading Ossie sees + the two datasets as unrelated + object_ref: relationship 8 + remedy: model the join as an equality relationship in AML diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml new file mode 100644 index 000000000..6748f8b65 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml @@ -0,0 +1,662 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: ecommerce +description: 'A marketplace model: orders, line items, products, merchants and cities.' +datasets: +- name: ecommerce_orders + source: '`ecommerce`.`orders`' + primary_key: + - id + description: One row per customer order. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: BIGQUERY + expression: user_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "User Id", "type": "number"}' + - name: created_at + expression: + dialects: + - dialect: BIGQUERY + expression: created_at + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Created At", "type": "datetime"}' + - name: created_date + expression: + dialects: + - dialect: BIGQUERY + expression: CAST(created_at AS DATE) + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Created Date", "type": "date"}' + - name: created_month + expression: + dialects: + - dialect: BIGQUERY + expression: DATE_TRUNC(created_at, month) + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "dd LLL, yyyy", "label": "Created Month", "type": + "date"}' + - name: status + expression: + dialects: + - dialect: BIGQUERY + expression: status + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Status", "type": "text"}' + - name: delivered + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: ecommerce_orders.status == 'delivered' + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Delivered", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Orders", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`orders`"}' +- name: ecommerce_users + source: select * from `ecommerce`.`users` + primary_key: + - id + description: One row per registered customer. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: full_name + expression: + dialects: + - dialect: BIGQUERY + expression: full_name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Full Name", "type": "text"}' + - name: gender + expression: + dialects: + - dialect: BIGQUERY + expression: gender + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Gender", "type": "text"}' + - name: birth_date + expression: + dialects: + - dialect: BIGQUERY + expression: birth_date + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Birth Date", "type": "date"}' + - name: age + expression: + dialects: + - dialect: BIGQUERY + expression: extract(year from age(ecommerce_users.birth_date)) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Age", "type": "number"}' + - name: age_bucket + expression: + dialects: + - dialect: BIGQUERY + expression: FLOOR(ecommerce_users.age / 10) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "hidden": true, "label": "Age Bucket", "type": "number"}' + - name: signup_month + expression: + dialects: + - dialect: BIGQUERY + expression: DATE_TRUNC(created_at, month) + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Signup Month", "type": "datetime"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "QueryModel", "data_source_name": "bigquery_demo", + "label": "Users", "owner": "dev@ossie.apache.org", "param": [{"label": "Reporting + Currency", "name": "reporting_currency", "type": "text"}, {"label": "Minimum + Age", "name": "minimum_age", "type": "number"}], "persistence": {"__doc__": + "", "__type__": "FullPersistence", "on_cascade": "rebuild", "schema": "persisted"}}' +- name: order_items + source: '`ecommerce`.`order_items`' + description: One row per line item on an order. + fields: + - name: order_id + expression: + dialects: + - dialect: BIGQUERY + expression: order_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Order Id", "type": "number"}' + - name: product_id + expression: + dialects: + - dialect: BIGQUERY + expression: product_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Product Id", "type": "number"}' + - name: quantity + expression: + dialects: + - dialect: BIGQUERY + expression: quantity + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Quantity", "type": "number"}' + - name: item_value + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: order_items.quantity * products.price + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Item Value", "scope": "dataset", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Order Items", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`order_items`"}' +- name: products + source: '`ecommerce`.`products`' + primary_key: + - id + description: One row per sellable product. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: merchant_id + expression: + dialects: + - dialect: BIGQUERY + expression: merchant_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Merchant Id", "type": "number"}' + - name: price + expression: + dialects: + - dialect: BIGQUERY + expression: price + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "#,###[$$]", "label": "Price", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Products", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`products`"}' +- name: merchants + source: '`ecommerce`.`merchants`' + primary_key: + - id + description: One row per seller on the marketplace. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: admin_id + expression: + dialects: + - dialect: BIGQUERY + expression: admin_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Admin Id", "type": "number"}' + - name: city_id + expression: + dialects: + - dialect: BIGQUERY + expression: city_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "City Id", "type": "number"}' + - name: country_code + expression: + dialects: + - dialect: BIGQUERY + expression: country_code + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Country Code", "type": "text"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Merchants", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`merchants`"}' +- name: cities + source: '`ecommerce`.`cities`' + primary_key: + - id + description: One row per city a merchant can operate in. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: country_code + expression: + dialects: + - dialect: BIGQUERY + expression: country_code + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Country Code", "type": "text"}' + - name: in_vietnam + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: cities.country_code == 'VN' + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "In Vietnam", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Cities", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`cities`"}' +- name: reporting__calendar + source: '`ecommerce`.`calendar`' + primary_key: + - date + description: A date spine. Lives in the reporting module, so its compiled name is + reporting::calendar. + fields: + - name: date + expression: + dialects: + - dialect: BIGQUERY + expression: date + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Date", "type": "date"}' + - name: year + expression: + dialects: + - dialect: BIGQUERY + expression: year + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Year", "type": "number"}' + - name: is_weekend + expression: + dialects: + - dialect: BIGQUERY + expression: is_weekend + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Is Weekend", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "fqn": "reporting::calendar", "label": "Calendar", "owner": "dev@ossie.apache.org", + "table_name": "`ecommerce`.`calendar`"}' +- name: delivered_orders + source: 'select {{ #ecommerce_orders.* }} + + from {{ #ecommerce_orders }} + + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %}' + primary_key: + - id + description: Orders that reached the customer. Derived from ecommerce_orders rather + than from a table. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: BIGQUERY + expression: user_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "User Id", "type": "number"}' + - name: delivery_attempts + expression: + dialects: + - dialect: BIGQUERY + expression: delivery_attempts + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Delivery Attempts", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "QueryModel", "data_source_name": "bigquery_demo", + "label": "Delivered Orders", "owner": "dev@ossie.apache.org", "param": [{"label": + "Minimum Delivery Attempts", "name": "min_delivery_attempts", "type": "number"}], + "query_forms": ["model_columns", "model_relation", "filter_block"]}' +relationships: +- name: order_items_to_ecommerce_orders + from: order_items + to: ecommerce_orders + from_columns: + - order_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: order_items_to_products + from: order_items + to: products + from_columns: + - product_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: ecommerce_orders_to_ecommerce_users + from: ecommerce_orders + to: ecommerce_users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: products_to_merchants + from: products + to: merchants + from_columns: + - merchant_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "one_way", "nullable": false, "rlp_propagation": + "one_way", "type": "many_to_one"}' +- name: merchants_to_cities + from: merchants + to: cities + from_columns: + - city_id + - country_code + to_columns: + - id + - country_code + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: delivered_orders_to_ecommerce_users + from: delivered_orders + to: ecommerce_users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one", "where": {"__doc__": "", "__type__": "RelationshipFilter", + "from": {"__doc__": "", "__type__": "Heredoc", "content": "delivered_orders.delivery_attempts + > 0 ", "name": "aql"}, "to": {"__doc__": "", "__type__": "Heredoc", "content": + "ecommerce_users.age > 18 ", "name": "aql"}}}' +- name: merchants_to_ecommerce_users + from: merchants + to: ecommerce_users + from_columns: + - admin_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "one_to_one"}' +- name: ecommerce_orders_to_reporting__calendar + from: ecommerce_orders + to: reporting__calendar + from_columns: + - created_date + to_columns: + - date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": false, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +metrics: +- name: ecommerce_orders__last_created_at + expression: + dialects: + - dialect: BIGQUERY + expression: MAX(ecommerce_orders.created_at) + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "max", "aml_name": "last_created_at", "label": + "Last Created At", "model": "ecommerce_orders", "type": "datetime"}' +- name: ecommerce_orders__count_orders + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "count", "aml_name": "count_orders", "label": + "Count Orders", "model": "ecommerce_orders", "type": "number"}' +- name: ecommerce_orders__delivered_orders + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: count(ecommerce_orders.id) | where(ecommerce_orders.delivered) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "delivered_orders", + "hidden": true, "label": "Delivered Orders", "model": "ecommerce_orders", "type": + "number"}' +- name: ecommerce_users__count_users + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(DISTINCT id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "count distinct", "aml_name": "count_users", + "label": "Count Users", "model": "ecommerce_users", "type": "number"}' +- name: order_items__total_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(quantity) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "sum", "aml_name": "total_quantity", "label": + "Total Quantity", "model": "order_items", "type": "number"}' +- name: order_items__custom_total_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: sum(quantity) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "custom_total_quantity", + "label": "Custom Total Quantity", "model": "order_items", "type": "number"}' +- name: cities__upstream_measure + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(cities.id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "upstream_measure", + "label": "Upstream Measure", "model": "cities", "type": "number"}' +- name: cities__downstream_measure + expression: + dialects: + - dialect: BIGQUERY + expression: cities__upstream_measure + 1 + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "downstream_measure", + "label": "Downstream Measure", "model": "cities", "type": "number"}' +- name: total_revenue + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: sum(order_items.item_value) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "#,###[$$]", "label": "Total Revenue", "type": "number"}' +- name: count_orders + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: count(order_items.order_id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Count Orders", "type": "number"}' +- name: revenue_share_of_all_merchants + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: 'sum(order_items.item_value) + + / (sum(order_items.item_value) | of_all(merchants))' + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Revenue Share Of All Merchants", "type": "number"}' +custom_extensions: +- vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "Dataset", "data_source_name": "bigquery_demo", "label": + "Ecommerce", "match_relationships": [{"__type__": "RelationshipConfig", "active": + true, "direction": "two_way", "nullable": true, "rel": {"__doc__": "", "__fqn__": + "orders_in_year", "__type__": "ManyToManyRelationship", "from": {"__doc__": "", + "__refFqn__": "ecommerce_orders", "__type__": "FieldRef", "field": "id", "model": + "ecommerce_orders"}, "match": {"__doc__": "", "__type__": "Heredoc", "content": + "ecommerce_orders.created_date == reporting.calendar.date ", "name": "aql"}, "name": + "orders_in_year", "to": {"__doc__": "", "__type__": "FieldRef", "field": "date", + "model": "reporting.calendar"}, "type": "many_to_many"}, "rlp_propagation": "inherit"}], + "owner": "dev@ossie.apache.org"}' diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml new file mode 100644 index 000000000..505757ed9 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[] diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml new file mode 100644 index 000000000..ea3906566 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model cities { + type: 'table' + label: 'Cities' + description: 'One row per city a merchant can operate in.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`cities`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + dimension in_vietnam { + label: 'In Vietnam' + type: 'truefalse' + definition: @aql cities.country_code == 'VN';; + } + + measure upstream_measure { + label: 'Upstream Measure' + type: 'number' + definition: @sql COUNT({{ id }});; + aggregation_type: 'custom' + } + + measure downstream_measure { + label: 'Downstream Measure' + type: 'number' + definition: @sql {{ upstream_measure }} + 1;; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml new file mode 100644 index 000000000..765d76fac --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model delivered_orders { + type: 'query' + label: 'Delivered Orders' + description: 'Orders that reached the customer. Derived from ecommerce_orders rather than from a table.' + data_source_name: 'bigquery_demo' + models: [ecommerce_orders] + + query: @sql + select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} + ;; + + param min_delivery_attempts { + label: 'Minimum Delivery Attempts' + type: 'number' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension delivery_attempts { + label: 'Delivery Attempts' + type: 'number' + definition: @sql {{ #SOURCE.delivery_attempts }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml new file mode 100644 index 000000000..45e669bf9 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Relationship delivered_orders_to_ecommerce_users { + type: 'many_to_one' + from: ref('delivered_orders', 'user_id') + to: ref('ecommerce_users', 'id') + where { + from: @aql delivered_orders.delivery_attempts > 0 ;; + to: @aql ecommerce_users.age > 18 ;; + } +} +ManyToManyRelationship orders_in_year { + type: 'many_to_many' + from: ref('ecommerce_orders', 'id') + to: ref('reporting.calendar', 'date') + match: @aql ecommerce_orders.created_date == reporting.calendar.date ;; +} + +Dataset ecommerce { + label: 'Ecommerce' + description: 'A marketplace model: orders, line items, products, merchants and cities.' + data_source_name: 'bigquery_demo' + + models: [ + ecommerce_orders, + ecommerce_users, + order_items, + products, + merchants, + cities, + reporting.calendar, + delivered_orders, + ] + + relationships: [ + relationship(order_items.order_id > ecommerce_orders.id, true), + relationship(order_items.product_id > products.id, true), + relationship(ecommerce_orders.user_id > ecommerce_users.id, true), + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: ref('products', 'merchant_id') + to: ref('merchants', 'id') + } + active: true + direction: 'one_way' + nullable: false + rlp_propagation: 'one_way' + }, + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: ref('merchants', 'city_id') + to: ref('cities', 'id') + on { + from: ref('merchants', 'country_code') + to: ref('cities', 'country_code') + } + } + active: true + }, + RelationshipConfig { + rel: delivered_orders_to_ecommerce_users + active: true + }, + relationship(merchants.admin_id - ecommerce_users.id, true), + relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + RelationshipConfig { + rel: orders_in_year + active: true + }, + ] + + dimension item_value { + model: order_items + label: 'Item Value' + type: 'number' + definition: @aql order_items.quantity * products.price;; + } + + metric total_revenue { + label: 'Total Revenue' + type: 'number' + definition: @aql sum(order_items.item_value);; + format: '#,###[$$]' + } + + metric count_orders { + label: 'Count Orders' + type: 'number' + definition: @aql count(order_items.order_id);; + } + + metric revenue_share_of_all_merchants { + label: 'Revenue Share Of All Merchants' + type: 'number' + definition: @aql + sum(order_items.item_value) + / (sum(order_items.item_value) | of_all(merchants)) + ;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml new file mode 100644 index 000000000..edb11d78f --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_orders { + type: 'table' + label: 'Orders' + description: 'One row per customer order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension created_at { + label: 'Created At' + type: 'datetime' + definition: @sql {{ #SOURCE.created_at }};; + } + + dimension created_date { + label: 'Created Date' + type: 'date' + definition: @sql CAST({{ #SOURCE.created_at }} AS DATE);; + } + + dimension created_month { + label: 'Created Month' + type: 'date' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + format: 'dd LLL, yyyy' + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } + + dimension delivered { + label: 'Delivered' + type: 'truefalse' + definition: @aql ecommerce_orders.status == 'delivered';; + } + + measure last_created_at { + label: 'Last Created At' + type: 'datetime' + definition: @sql {{ created_at }};; + aggregation_type: 'max' + } + + measure count_orders { + label: 'Count Orders' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count' + } + + measure delivered_orders { + label: 'Delivered Orders' + type: 'number' + hidden: true + definition: @aql count(ecommerce_orders.id) | where(ecommerce_orders.delivered);; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml new file mode 100644 index 000000000..f104518dc --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_users { + type: 'query' + label: 'Users' + description: 'One row per registered customer.' + data_source_name: 'bigquery_demo' + + query: @sql select * from `ecommerce`.`users`;; + + param reporting_currency { + label: 'Reporting Currency' + type: 'text' + } + + param minimum_age { + label: 'Minimum Age' + type: 'number' + } + + persistence: FullPersistence { + on_cascade: 'rebuild' + schema: 'persisted' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } + + dimension gender { + label: 'Gender' + type: 'text' + definition: @sql {{ #SOURCE.gender }};; + } + + dimension birth_date { + label: 'Birth Date' + type: 'date' + definition: @sql {{ #SOURCE.birth_date }};; + } + + dimension age { + label: 'Age' + type: 'number' + definition: @sql extract(year from age({{ birth_date }}));; + } + + dimension age_bucket { + label: 'Age Bucket' + type: 'number' + hidden: true + definition: @sql FLOOR({{ age }} / 10);; + } + + dimension signup_month { + label: 'Signup Month' + type: 'datetime' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + } + + measure count_users { + label: 'Count Users' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count distinct' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml new file mode 100644 index 000000000..6132530f6 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model merchants { + type: 'table' + label: 'Merchants' + description: 'One row per seller on the marketplace.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`merchants`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension admin_id { + label: 'Admin Id' + type: 'number' + definition: @sql {{ #SOURCE.admin_id }};; + } + + dimension city_id { + label: 'City Id' + type: 'number' + definition: @sql {{ #SOURCE.city_id }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml new file mode 100644 index 000000000..93c70b537 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model calendar { + type: 'table' + label: 'Calendar' + description: 'A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`calendar`' + + dimension date { + label: 'Date' + type: 'date' + definition: @sql {{ #SOURCE.date }};; + primary_key: true + } + + dimension year { + label: 'Year' + type: 'number' + definition: @sql {{ #SOURCE.year }};; + } + + dimension is_weekend { + label: 'Is Weekend' + type: 'truefalse' + definition: @sql {{ #SOURCE.is_weekend }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml new file mode 100644 index 000000000..e78339343 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model order_items { + type: 'table' + label: 'Order Items' + description: 'One row per line item on an order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + measure total_quantity { + label: 'Total Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + aggregation_type: 'sum' + } + + measure custom_total_quantity { + label: 'Custom Total Quantity' + type: 'number' + definition: @sql sum({{ #SOURCE.quantity }});; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml new file mode 100644 index 000000000..8fdbabf81 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model products { + type: 'table' + label: 'Products' + description: 'One row per sellable product.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`products`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension merchant_id { + label: 'Merchant Id' + type: 'number' + definition: @sql {{ #SOURCE.merchant_id }};; + } + + dimension price { + label: 'Price' + type: 'number' + definition: @sql {{ #SOURCE.price }};; + format: '#,###[$$]' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml new file mode 100644 index 000000000..6f50c40ae --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) / COUNT(users.id)' became 'sum(items.quantity) / count(users.id)' + object_ref: quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'quantity_per_customer / 2' became 'quantity_per_customer / 2' + object_ref: half_quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'items_quantity + COUNT(users.id)' became 'items.items_quantity + count(users.id)' + object_ref: quantity_and_customers + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) * quantity_per_customer' became 'sum(items.quantity) * quantity_per_customer' + object_ref: scaled_quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml new file mode 100644 index 000000000..1ec2a360c --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model items { + type: 'table' + label: 'Items' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + measure items_quantity { + label: 'Items Quantity' + type: 'number' + description: 'A plain measure on one model, which the metrics below reference' + definition: @sql SUM({{ quantity }});; + aggregation_type: 'custom' + } + + measure quantity_plus_one { + label: 'Quantity Plus One' + type: 'number' + description: 'A body naming only another metric, so placement follows the reference' + definition: @sql {{ items_quantity }} + 1;; + aggregation_type: 'custom' + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml new file mode 100644 index 000000000..9105caf8a --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset metric_references { + label: 'Metric References' + description: 'Metrics that name other metrics, across both AML metric homes' + data_source_name: 'bigquery_demo' + + models: [ + orders, + items, + users, + ] + + relationships: [ + relationship(items.order_id > orders.id, true), + relationship(orders.user_id > users.id, true), + ] + + metric quantity_per_customer { + label: 'Quantity Per Customer' + type: 'number' + description: 'A body spanning two datasets, so AML takes it only as AQL' + definition: @aql sum(items.quantity) / count(users.id);; + } + + metric half_quantity_per_customer { + label: 'Half Quantity Per Customer' + type: 'number' + description: 'An AQL metric naming another AQL metric, which AQL takes by bare name' + definition: @aql quantity_per_customer / 2;; + } + + metric quantity_and_customers { + label: 'Quantity And Customers' + type: 'number' + description: 'An AQL metric naming a measure, which AQL takes qualified by its model' + definition: @aql items.items_quantity + count(users.id);; + } + + metric scaled_quantity_per_customer { + label: 'Scaled Quantity Per Customer' + type: 'number' + description: 'A SQL body naming an AQL metric, so the body becomes AQL as well' + definition: @aql sum(items.quantity) * quantity_per_customer;; + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml new file mode 100644 index 000000000..d2c854294 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model orders { + type: 'table' + label: 'Orders' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml new file mode 100644 index 000000000..0a5d0d9b3 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model users { + type: 'table' + label: 'Users' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`users`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml b/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml new file mode 100644 index 000000000..8f0d7c9b4 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie document written to exercise a metric that names another metric. +# +# AML gives a metric two homes, and the two take different bodies. A model-level +# `measure` takes `@sql`, which resolves inside one model. A dataset-level +# `metric` takes `@aql`, which spans models. A reference across that line only +# works one way: AQL names a measure, and a `@sql` body has no spelling for a +# dataset-level metric. +# +# So the referenced metric decides where the referencing one can go, and these +# metrics cover each way that lands. The ecommerce tables are the ones the +# Holistics BigQuery demo connection carries, so the generated AML compiles +# against a real schema rather than an invented one. + +version: "0.2.0.dev0" +name: metric_references +description: Metrics that name other metrics, across both AML metric homes + +datasets: + - name: orders + source: "`ecommerce`.`orders`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: user_id + expression: {dialects: [{dialect: ANSI_SQL, expression: user_id}]} + datatype: Decimal + + - name: items + source: "`ecommerce`.`order_items`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: order_id + expression: {dialects: [{dialect: ANSI_SQL, expression: order_id}]} + datatype: Decimal + - name: quantity + expression: {dialects: [{dialect: ANSI_SQL, expression: quantity}]} + datatype: Decimal + + - name: users + source: "`ecommerce`.`users`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + +relationships: + - name: items_to_orders + from: items + to: orders + from_columns: [order_id] + to_columns: [id] + - name: orders_to_users + from: orders + to: users + from_columns: [user_id] + to_columns: [id] + +metrics: + - name: items_quantity + description: A plain measure on one model, which the metrics below reference + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + + - name: quantity_plus_one + description: A body naming only another metric, so placement follows the reference + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: items_quantity + 1 + + - name: quantity_per_customer + description: A body spanning two datasets, so AML takes it only as AQL + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) / COUNT(users.id) + + - name: half_quantity_per_customer + description: An AQL metric naming another AQL metric, which AQL takes by bare name + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: quantity_per_customer / 2 + + - name: quantity_and_customers + description: An AQL metric naming a measure, which AQL takes qualified by its model + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: items_quantity + COUNT(users.id) + + - name: scaled_quantity_per_customer + description: A SQL body naming an AQL metric, so the body becomes AQL as well + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) * quantity_per_customer diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml new file mode 100644 index 000000000..7140f485e --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)' + became 'sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk)' + object_ref: customer_lifetime_value + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), + 0)' became 'sum(store_sales.ss_ext_sales_price) / nullif(sum(store.s_number_employees), 0)' + object_ref: store_productivity + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: cumulative_sales + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: brand_rank_in_store + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: monthly_sales_change + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_ASSUMED_TYPE + severity: WARNING + message: AML requires a type and the Ossie datatype is None, so 'text' is written + object_ref: date_dim.d_quarter_name + remedy: set a datatype on the field, or edit the AML type after conversion +- code: HOLISTICS_ASSUMED_TYPE + severity: WARNING + message: AML requires a type and the Ossie datatype is None, so 'text' is written + object_ref: date_dim.d_moy + remedy: set a datatype on the field, or edit the AML type after conversion diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml new file mode 100644 index 000000000..d1da6bbbc --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model customer { + type: 'table' + label: 'Customer' + description: 'Customer dimension with demographic information' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.customer' + + dimension c_customer_sk { + label: 'C Customer Sk' + type: 'number' + description: 'Surrogate key for customer' + definition: @sql {{ #SOURCE.c_customer_sk }};; + primary_key: true + } + + dimension c_customer_id { + label: 'C Customer Id' + type: 'text' + description: 'Business key for customer' + definition: @sql {{ #SOURCE.c_customer_id }};; + } + + dimension c_first_name { + label: 'C First Name' + type: 'text' + description: 'Customer first name' + definition: @sql {{ #SOURCE.c_first_name }};; + } + + dimension c_last_name { + label: 'C Last Name' + type: 'text' + description: 'Customer last name' + definition: @sql {{ #SOURCE.c_last_name }};; + } + + dimension customer_full_name { + label: 'Customer Full Name' + type: 'text' + description: 'Customer full name (computed field)' + definition: @sql {{ #SOURCE.c_first_name }} || ' ' || {{ #SOURCE.c_last_name }};; + } + + dimension c_email_address { + label: 'C Email Address' + type: 'text' + description: 'Customer email address' + definition: @sql {{ #SOURCE.c_email_address }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml new file mode 100644 index 000000000..74a67672c --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model date_dim { + type: 'table' + label: 'Date Dim' + description: 'Date dimension with calendar attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.date_dim' + + dimension d_date_sk { + label: 'D Date Sk' + type: 'number' + description: 'Surrogate key for date' + definition: @sql {{ #SOURCE.d_date_sk }};; + primary_key: true + } + + dimension d_date { + label: 'D Date' + type: 'date' + description: 'Actual date value' + definition: @sql {{ #SOURCE.d_date }};; + } + + dimension d_year { + label: 'D Year' + type: 'number' + description: 'Year' + definition: @sql {{ #SOURCE.d_year }};; + } + + dimension d_quarter_name { + label: 'D Quarter Name' + type: 'text' + description: 'Quarter name (e.g., 2024Q1)' + definition: @sql {{ #SOURCE.d_quarter_name }};; + } + + dimension d_moy { + label: 'D Moy' + type: 'text' + description: 'Month of year (1-12)' + definition: @sql {{ #SOURCE.d_moy }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml new file mode 100644 index 000000000..1b21bf48e --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model item { + type: 'table' + label: 'Item' + description: 'Item/Product dimension with product attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.item' + + dimension i_item_sk { + label: 'I Item Sk' + type: 'number' + description: 'Surrogate key for item' + definition: @sql {{ #SOURCE.i_item_sk }};; + primary_key: true + } + + dimension i_item_id { + label: 'I Item Id' + type: 'text' + description: 'Business key for item' + definition: @sql {{ #SOURCE.i_item_id }};; + } + + dimension i_item_desc { + label: 'I Item Desc' + type: 'text' + description: 'Item description' + definition: @sql {{ #SOURCE.i_item_desc }};; + } + + dimension i_brand { + label: 'I Brand' + type: 'text' + description: 'Brand name' + definition: @sql {{ #SOURCE.i_brand }};; + } + + dimension i_category { + label: 'I Category' + type: 'text' + description: 'Item category' + definition: @sql {{ #SOURCE.i_category }};; + } + + dimension i_current_price { + label: 'I Current Price' + type: 'number' + description: 'Current price of the item' + definition: @sql {{ #SOURCE.i_current_price }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml new file mode 100644 index 000000000..a51a24997 --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model store { + type: 'table' + label: 'Store' + description: 'Store dimension with location and store attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.store' + + dimension s_store_sk { + label: 'S Store Sk' + type: 'number' + description: 'Surrogate key for store' + definition: @sql {{ #SOURCE.s_store_sk }};; + primary_key: true + } + + dimension s_store_id { + label: 'S Store Id' + type: 'text' + description: 'Business key for store' + definition: @sql {{ #SOURCE.s_store_id }};; + } + + dimension s_store_name { + label: 'S Store Name' + type: 'text' + description: 'Store name' + definition: @sql {{ #SOURCE.s_store_name }};; + } + + dimension s_city { + label: 'S City' + type: 'text' + description: 'City where store is located' + definition: @sql {{ #SOURCE.s_city }};; + } + + dimension s_state { + label: 'S State' + type: 'text' + description: 'State where store is located' + definition: @sql {{ #SOURCE.s_state }};; + } + + dimension s_number_employees { + label: 'S Number Employees' + type: 'number' + description: 'Number of employees at the store' + definition: @sql {{ #SOURCE.s_number_employees }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml new file mode 100644 index 000000000..8b8c6265a --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model store_sales { + type: 'table' + label: 'Store Sales' + description: 'Fact table containing all store sales transactions' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.store_sales' + + dimension ss_sold_date_sk { + label: 'Ss Sold Date Sk' + type: 'number' + description: 'Foreign key to date dimension' + definition: @sql {{ #SOURCE.ss_sold_date_sk }};; + } + + dimension ss_item_sk { + label: 'Ss Item Sk' + type: 'number' + description: 'Foreign key to item dimension' + definition: @sql {{ #SOURCE.ss_item_sk }};; + primary_key: true + } + + dimension ss_customer_sk { + label: 'Ss Customer Sk' + type: 'number' + description: 'Foreign key to customer dimension' + definition: @sql {{ #SOURCE.ss_customer_sk }};; + } + + dimension ss_store_sk { + label: 'Ss Store Sk' + type: 'number' + description: 'Foreign key to store dimension' + definition: @sql {{ #SOURCE.ss_store_sk }};; + } + + dimension ss_quantity { + label: 'Ss Quantity' + type: 'number' + description: 'Quantity of items sold' + definition: @sql {{ #SOURCE.ss_quantity }};; + } + + dimension ss_sales_price { + label: 'Ss Sales Price' + type: 'number' + description: 'Sales price per unit' + definition: @sql {{ #SOURCE.ss_sales_price }};; + } + + dimension ss_ext_sales_price { + label: 'Ss Ext Sales Price' + type: 'number' + description: 'Extended sales price (quantity * price)' + definition: @sql {{ #SOURCE.ss_ext_sales_price }};; + } + + dimension ss_net_profit { + label: 'Ss Net Profit' + type: 'number' + description: 'Net profit from the sale' + definition: @sql {{ #SOURCE.ss_net_profit }};; + } + + measure total_sales { + label: 'Total Sales' + type: 'number' + description: 'Total sales revenue across all transactions' + definition: @sql SUM({{ ss_ext_sales_price }});; + aggregation_type: 'custom' + } + + measure total_profit { + label: 'Total Profit' + type: 'number' + description: 'Total net profit from store sales' + definition: @sql SUM({{ ss_net_profit }});; + aggregation_type: 'custom' + } + + measure sales_by_brand { + label: 'Sales By Brand' + type: 'number' + description: 'Total sales by brand (requires grouping by item.i_brand)' + definition: @sql SUM({{ ss_ext_sales_price }});; + aggregation_type: 'custom' + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml new file mode 100644 index 000000000..f1eed69c9 --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset tpcds_retail_model { + label: 'Tpcds Retail Model' + description: 'TPC-DS retail semantic model for sales and customer analytics' + data_source_name: 'tpcds_warehouse' + + models: [ + store_sales, + date_dim, + customer, + item, + store, + ] + + relationships: [ + relationship(store_sales.ss_sold_date_sk > date_dim.d_date_sk, true), + relationship(store_sales.ss_customer_sk > customer.c_customer_sk, true), + relationship(store_sales.ss_item_sk > item.i_item_sk, true), + relationship(store_sales.ss_store_sk > store.s_store_sk, true), + ] + + metric customer_lifetime_value { + label: 'Customer Lifetime Value' + type: 'number' + description: 'Average lifetime sales value per customer' + definition: @aql sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk);; + } + + metric store_productivity { + label: 'Store Productivity' + type: 'number' + description: 'Sales per employee across stores' + definition: @aql sum(store_sales.ss_ext_sales_price) / nullif(sum(store.s_number_employees), 0);; + } +} diff --git a/converters/holistics/tests/snapshots.py b/converters/holistics/tests/snapshots.py new file mode 100644 index 000000000..3a53b133f --- /dev/null +++ b/converters/holistics/tests/snapshots.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""What every snapshot holds, computed once and shared. + +`tests/test_snapshots.py` compares these against the committed files, and +`tools/update_snapshots.py` writes them. One definition, so a snapshot can never +disagree with the test that checks it. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from ossie_holistics import _yaml, aml_to_ossie, ossie_to_aml +from ossie_holistics.issues import IssueLog + +REPO_ROOT = Path(__file__).resolve().parents[3] +CONVERTER_ROOT = Path(__file__).resolve().parents[1] +FIXTURES = CONVERTER_ROOT / "tests" / "fixtures" + +ECOMMERCE_COMPILED = FIXTURES / "ecommerce" / "ecommerce.dataset.aml.json" +TPCDS_OSSIE = REPO_ROOT / "examples" / "tpcds_semantic_model.yaml" +AQL_TRANSLATION = FIXTURES / "aql" / "translation.ossie.yaml" +METRIC_REFERENCES = FIXTURES / "metric_refs" / "references.ossie.yaml" + +#: The fixture is a BigQuery dataset: `DATE_TRUNC(created_at, month)` is BigQuery +#: argument order, not ANSI SQL. +SQL_DIALECT = "BIGQUERY" + +#: TPC-DS carries no HOLISTICS stash, so the AML-required connection name has to +#: come from the caller. Its expressions are all `ANSI_SQL`, so naming that as +#: the target warehouse writes every body as it stands. +TPCDS_DATA_SOURCE = "tpcds_warehouse" +TPCDS_SQL_DIALECT = "ANSI_SQL" + +LICENSE_HEADER = """\ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" + + +def _issues(log: IssueLog) -> str: + return LICENSE_HEADER + _yaml.dump_issues(log.as_dicts()) + + +def forward() -> tuple[dict[str, str], aml_to_ossie.Result]: + """Compiled AML to Ossie: one YAML document and its issue log.""" + payload = json.loads(ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + result = aml_to_ossie.convert(payload, sql_dialect=SQL_DIALECT) + return { + "ecommerce/expected.ossie.yaml": LICENSE_HEADER + _yaml.dump(result.model), + "ecommerce/expected.issues.yaml": _issues(result.issues), + }, result + + +def reverse_ecommerce() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie back to AML, starting from this converter's own forward output. + + Reading the forward result rather than the committed snapshot keeps the two + directions from drifting apart between snapshot updates. + """ + _, forward_result = forward() + result = ossie_to_aml.convert(forward_result.model, SQL_DIALECT) + files = {f"ecommerce/expected_aml/{name}": text for name, text in result.files} + files["ecommerce/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_tpcds() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML, starting from a document this converter did not write. + + `examples/tpcds_semantic_model.yaml` carries no HOLISTICS stash, so every + AML property the core specification has no field for is derived or supplied. + """ + document = _yaml.load(TPCDS_OSSIE.read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, TPCDS_SQL_DIALECT, data_source_name=TPCDS_DATA_SOURCE + ) + files = {f"tpcds/expected_aml/{name}": text for name, text in result.files} + files["tpcds/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_aql() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML for a document written to exercise the SQL to AQL translation. + + Every metric in it spans two datasets, which is what makes AML take an + `@aql` body, and each covers a different path through `aql.translate`. + """ + document = _yaml.load(AQL_TRANSLATION.read_text(encoding="utf-8")) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="bigquery_demo") + files = {f"aql/expected_aml/{name}": text for name, text in result.files} + files["aql/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_metric_references() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML for metrics that name other metrics. + + Which AML home a metric lands in decides how it can name another one, so + these cover a reference into each home, and the placement that a body + naming only another metric depends on. + """ + document = _yaml.load(METRIC_REFERENCES.read_text(encoding="utf-8")) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="bigquery_demo") + files = {f"metric_refs/expected_aml/{name}": text for name, text in result.files} + files["metric_refs/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def all_snapshots() -> dict[str, str]: + """Every committed snapshot, keyed by its path under `tests/fixtures/`.""" + files: dict[str, str] = {} + for produce in ( + forward, + reverse_ecommerce, + reverse_tpcds, + reverse_aql, + reverse_metric_references, + ): + files.update(produce()[0]) + return files diff --git a/converters/holistics/tests/test_issue_codes.py b/converters/holistics/tests/test_issue_codes.py new file mode 100644 index 000000000..3b7d64176 --- /dev/null +++ b/converters/holistics/tests/test_issue_codes.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The issue-code table in the documentation matches the source. + +A reader who sees `HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING` in an issue log +looks it up in `docs/limitations.md`. A code the converter raises and the table +omits leaves them with nothing to find, and a code the table lists and the +converter never raises sends them looking for a cause that cannot occur. +""" +from __future__ import annotations + +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LIMITATIONS = (ROOT / "docs" / "limitations.md").read_text(encoding="utf-8") + + +def raised_codes() -> dict[str, set[str]]: + """Each issue code the converter can raise, and the severities it uses.""" + found: dict[str, set[str]] = {} + for module in ("aml_to_ossie", "ossie_to_aml"): + text = (ROOT / "src" / "ossie_holistics" / f"{module}.py").read_text(encoding="utf-8") + constants = dict(re.findall(r'^(ISSUE_\w+) = "(\w+)"', text, re.M)) + for node in ast.walk(ast.parse(text)): + if not (isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add"): + continue + keywords = {k.arg: k.value for k in node.keywords} + code, severity = keywords.get("code"), keywords.get("severity") + if isinstance(code, ast.Name) and isinstance(severity, ast.Attribute): + found.setdefault(constants[code.id], set()).add(severity.attr) + return found + + +def documented_codes() -> dict[str, str]: + return dict(re.findall(r"^\| `(HOLISTICS_\w+)` \| (\w+) \|", LIMITATIONS, re.M)) + + +def test_every_raised_code_is_documented(): + assert set(raised_codes()) - set(documented_codes()) == set() + + +def test_every_documented_code_is_raised(): + assert set(documented_codes()) - set(raised_codes()) == set() + + +def test_the_documented_severity_matches_the_source(): + documented = documented_codes() + mismatched = { + code: (documented[code], sorted(severities)) + for code, severities in raised_codes().items() + if {documented[code]} != severities + } + assert not mismatched diff --git a/converters/holistics/tests/test_malformed_input.py b/converters/holistics/tests/test_malformed_input.py new file mode 100644 index 000000000..3752c5b78 --- /dev/null +++ b/converters/holistics/tests/test_malformed_input.py @@ -0,0 +1,182 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A malformed Ossie document is reported, not crashed on. + +`to-aml` reads documents this converter did not write, so its input is a +boundary rather than a known shape. The CLI catches `ConversionError`, `OSError` +and `UnicodeDecodeError`, so anything else reaches the user as a traceback. + +Two sibling reviews asked for exactly this. apache/ossie#407 found a converter +that checked `datasets` was present but not that it was a list of mappings, and +apache/ossie#297 found several direct `element["id"]` reads that raised +`KeyError` on a hand-edited document. +""" +from __future__ import annotations + +import pytest + +from ossie_holistics import ossie_to_aml +from ossie_holistics.errors import ConversionError + +VERSION = "0.2.0.dev0" +EXPRESSION = {"dialects": [{"dialect": "ANSI_SQL", "expression": "x"}]} + + +def document(**overrides): + base = {"version": VERSION, "name": "m", "datasets": [{"name": "d", "source": "s"}]} + base.update(overrides) + return base + + +MALFORMED = { + "datasets is a mapping": document(datasets={"a": {}}), + "datasets is a string": document(datasets="nope"), + "datasets is empty": document(datasets=[]), + "datasets is absent": {"version": VERSION, "name": "m"}, + "a dataset is a string": document(datasets=["nope"]), + "a dataset has no name": document(datasets=[{"source": "s"}]), + "a dataset name is not a string": document(datasets=[{"name": 7, "source": "s"}]), + "fields is a mapping": document(datasets=[{"name": "d", "source": "s", "fields": {}}]), + "a field has no name": document( + datasets=[{"name": "d", "source": "s", "fields": [{"expression": EXPRESSION}]}] + ), + "a metric has no name": document(metrics=[{"expression": EXPRESSION}]), + "a metric is a string": document(metrics=["nope"]), + "the document is a list": [], + "the document has no name": {"version": VERSION, "datasets": [{"name": "d", "source": "s"}]}, + "an expression is a string": document( + datasets=[{"name": "d", "source": "s", "fields": [{"name": "x", "expression": "SUM(x)"}]}] + ), + "a dialect entry is a string": document( + datasets=[ + { + "name": "d", + "source": "s", + "fields": [{"name": "x", "expression": {"dialects": ["ANSI_SQL"]}}], + } + ] + ), + "dialects is a string": document( + datasets=[ + { + "name": "d", + "source": "s", + "fields": [{"name": "x", "expression": {"dialects": "ANSI_SQL"}}], + } + ] + ), +} + + +def joined(from_columns, to_columns): + """Two datasets and one relationship pairing the column lists given.""" + column = {"name": "id", "expression": EXPRESSION} + return document( + datasets=[ + {"name": "orders", "source": "db.orders", "fields": [column]}, + {"name": "users", "source": "db.users", "fields": [column]}, + ], + relationships=[ + { + "name": "r", + "from": "orders", + "to": "users", + "from_columns": from_columns, + "to_columns": to_columns, + } + ], + ) + + +@pytest.mark.parametrize("payload", MALFORMED.values(), ids=list(MALFORMED)) +def test_a_malformed_document_raises_conversion_error(payload): + with pytest.raises(ConversionError): + ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + +@pytest.mark.parametrize( + "from_columns,to_columns", + [([], []), (["id"], []), ([], ["id"]), (["a", "b", "c"], ["a", "b"])], + ids=["both empty", "no to_columns", "no from_columns", "unequal lengths"], +) +def test_relationship_columns_must_pair_up(from_columns, to_columns): + """AML joins a column to a column, so the two lists have to line up. + + An unequal pair has no reading: a three-column key against two target + columns would join on a condition nobody wrote. + """ + with pytest.raises(ConversionError, match="from_columns"): + ossie_to_aml.convert( + joined(from_columns, to_columns), "ANSI_SQL", data_source_name="probe" + ) + + +def test_a_dataset_sharing_the_model_name_is_moved_aside(): + """Ossie keeps the two in separate namespaces and AML has one. + + A document naming both the semantic model and a dataset `orders` would + emit `Dataset orders` beside `Model orders`, which AML rejects as a + duplicate. The dataset is the safe side to move, since nothing references + it by name. + """ + payload = document( + name="orders", + datasets=[{"name": "orders", "source": "db.orders", "fields": [ + {"name": "id", "expression": EXPRESSION} + ]}], + ) + result = ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_DATASET_RENAMED] + assert len(raised) == 1 + text = next(t for name, t in result.files if name.endswith(".dataset.aml")) + assert "Dataset orders_dataset {" in text + assert "Model orders {" in next(t for name, t in result.files if name.endswith("orders.model.aml")) + + +def test_an_aggregation_that_does_not_wrap_the_whole_expression_is_kept_whole(): + """`SUM(a) + SUM(b)` starts with `SUM(` and ends with `)` and is not a sum. + + Stripping the wrapper would leave `a) + SUM(b` and re-aggregating it would + double the aggregate, so the expression is carried whole under `custom`. + """ + payload = document( + datasets=[{"name": "orders", "source": "db.orders", "fields": [ + {"name": "a", "expression": EXPRESSION}, + {"name": "b", "expression": EXPRESSION}, + ]}], + metrics=[{ + "name": "total", + "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(orders.a) + SUM(orders.b)"} + ]}, + "custom_extensions": [{ + "vendor_name": "HOLISTICS", + "data": '{"_v": 1, "model": "orders", "aml_name": "total", "aggregation_type": "sum"}', + }], + }], + ) + result = ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + raised = [ + i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_AGGREGATION_NOT_STRIPPED + ] + assert len(raised) == 1 + text = next(t for name, t in result.files if name.endswith("orders.model.aml")) + assert "aggregation_type: 'custom'" in text + assert "SUM({{ a }}) + SUM({{ b }})" in text diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py new file mode 100644 index 000000000..bb728a695 --- /dev/null +++ b/converters/holistics/tests/test_snapshots.py @@ -0,0 +1,713 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Snapshot tests over the two fixtures, plus the external checks that confirm +the snapshots are not merely stable but correct. + +A snapshot test proves the converter still produces what it produced. It cannot +prove the output is right. Three checks here do that, each against the real +artifact rather than a proxy: + +`test_the_converted_document_passes_the_ossie_schema` runs the forward output +through `core-spec/ossie-schema.json`. + +`test_the_generated_aml_compiles` runs `holistics aml validate` over every +generated file, which parses the AML and type-checks its AQL. + +`test_the_round_trip_returns_the_same_ossie_document` compiles the generated AML +back to JSON, converts that to Ossie, and compares it with the document the +reverse path started from. That closes the loop AML to Ossie to AML to Ossie. + +Those that need the Holistics CLI skip when it is absent, so the suite runs +without it. CI installs it, so they do not skip there. +""" +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +import snapshots +from ossie_holistics import _yaml, aml_to_ossie, aql, ossie_to_aml +from ossie_holistics.errors import ConversionError +from ossie_holistics.issues import Severity + +HOLISTICS_CLI = shutil.which("holistics") or str(Path.home() / ".holistics" / "bin" / "holistics") + +needs_cli = pytest.mark.skipif( + not Path(HOLISTICS_CLI).exists(), + reason="the Holistics CLI is not installed (https://docs.holistics.io/docs/cli)", +) + + +@pytest.mark.parametrize("relative", sorted(snapshots.all_snapshots())) +def test_the_snapshot_matches_what_the_converter_produces(relative): + expected = snapshots.FIXTURES / relative + assert expected.exists(), ( + f"{relative} is missing. Run python3 tools/update_snapshots.py" + ) + assert snapshots.all_snapshots()[relative] == expected.read_text(encoding="utf-8"), ( + f"{relative} is stale. Read the diff, then run python3 tools/update_snapshots.py" + ) + + +def test_no_committed_snapshot_is_orphaned(): + """Every file under `tests/fixtures/expected*` is one a snapshot produces. + + Renaming a model leaves its old file behind, and a stale file that no test + reads looks like coverage it is not. + """ + produced = {snapshots.FIXTURES / name for name in snapshots.all_snapshots()} + committed = { + path + for path in snapshots.FIXTURES.rglob("expected*") + if path.is_file() + } | { + path + for path in snapshots.FIXTURES.rglob("expected_aml/**/*") + if path.is_file() + } + assert committed - produced == set() + + +def test_the_converted_document_passes_the_ossie_schema(): + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads( + (snapshots.REPO_ROOT / "core-spec" / "ossie-schema.json").read_text(encoding="utf-8") + ) + _, result = snapshots.forward() + jsonschema.validate(result.model, schema) + + +def test_the_forward_direction_reports_the_losses_it_declares(): + """Every limitation `docs/limitations.md` lists for this fixture is raised. + + Asserted on the codes rather than the count, so adding an unrelated issue + does not fail this and removing a declared one does. + """ + _, result = snapshots.forward() + raised = {issue.code for issue in result.issues.issues} + assert raised == { + aml_to_ossie.ISSUE_AQL_NOT_TRANSLATED, + aml_to_ossie.ISSUE_INACTIVE_RELATIONSHIP, + aml_to_ossie.ISSUE_PARAM_DROPPED, + aml_to_ossie.ISSUE_QUERY_SOURCE, + aml_to_ossie.ISSUE_UNRESOLVED_QUERY, + aml_to_ossie.ISSUE_RELATIONSHIP_FILTER, + aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP, + } + + +def test_every_qualified_reference_resolves(): + """A `.` reference names a field the document declares. + + apache/ossie#459 found 211 of 612 references in one converter's output where + the qualifier was the Ossie dataset name and the other half was the + warehouse column, so the pair resolved in neither namespace. Upstream's + validator cannot see it, because `Dim_Customer.Customer_Name` parses as SQL. + """ + _, result = snapshots.forward() + model = result.model + declared = { + dataset["name"]: {field["name"] for field in dataset.get("fields") or []} + for dataset in model["datasets"] + } + metrics = {metric["name"] for metric in model.get("metrics") or []} + + def sql_expressions(): + for dataset in model["datasets"]: + for field in dataset.get("fields") or []: + for entry in field["expression"]["dialects"]: + yield f"field {dataset['name']}.{field['name']}", entry + for metric in model.get("metrics") or []: + for entry in metric["expression"]["dialects"]: + yield f"metric {metric['name']}", entry + + unresolved = [] + for where, entry in sql_expressions(): + if entry["dialect"] == aml_to_ossie.AQL_DIALECT: + continue + for qualifier, name in re.findall(r"\b([A-Za-z_]\w*)\.(\w+)", entry["expression"]): + if name not in declared.get(qualifier, ()): + unresolved.append(f"{where}: {qualifier}.{name}") + for bare in re.findall(r"(? dict: + """Two joined datasets and the metrics given, as one Ossie document.""" + + def column(name: str) -> dict: + return { + "name": name, + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}, + } + + return { + "version": aml_to_ossie.SPEC_VERSION, + "name": "m", + "datasets": [ + { + "name": "orders", + "source": "db.orders", + "primary_key": ["id"], + "fields": [column("id"), column("amount")], + }, + { + "name": "users", + "source": "db.users", + "primary_key": ["id"], + "fields": [column("id"), column("order_id")], + }, + ], + "relationships": [ + { + "name": "users_to_orders", + "from": "users", + "to": "orders", + "from_columns": ["order_id"], + "to_columns": ["id"], + } + ], + "metrics": metrics, + } + + +def _metric(name: str, sql: str, **stashed) -> dict: + payload = { + "name": name, + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql}]}, + } + if stashed: + payload["custom_extensions"] = [ + {"vendor_name": "HOLISTICS", "data": json.dumps({"_v": 1, **stashed})} + ] + return payload + + +def test_a_sql_measure_naming_a_dataset_level_metric_is_an_error(): + """A `@sql` body has no spelling for a metric that lives on the dataset. + + `orders.spanning` qualifies a dataset-level metric with a model, and the + sibling reference that reads like names a field `orders` does not have. + Holistics accepts the file and fails at query time, so the converter has to + catch it. + """ + document = _two_dataset_document( + [ + _metric("spanning", "SUM(orders.amount) / COUNT(users.id)"), + _metric("doubled", "SUM(orders.amount) * orders.spanning"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_CROSS_MODEL_SQL] + assert [i.object_ref for i in raised] == ["orders.doubled"] + assert raised[0].severity is Severity.ERROR + assert "dataset-level metric 'spanning'" in raised[0].message + + +def test_a_sql_measure_naming_another_models_measure_is_an_error(): + """A measure stashed onto one model, naming a measure of another. + + Without the stash the body's two models send the metric to the dataset, + where AQL resolves it. The stash pins it to one model, and the `@sql` body + that leaves cannot name the other, failing SQL generation with + `Cannot refer to external field in TableModel/QueryModel`. + """ + document = _two_dataset_document( + [ + _metric("revenue", "SUM(orders.amount)", model="orders", aml_name="revenue"), + _metric("per_user", "COUNT(users.id) + revenue", model="users", aml_name="per_user"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_CROSS_MODEL_SQL] + assert [i.object_ref for i in raised] == ["users.per_user"] + assert raised[0].severity is Severity.ERROR + assert "a measure of model 'orders'" in raised[0].message + + +def test_a_metric_reference_cycle_is_an_error(): + """Placement follows metric references, so a cycle has to end the walk. + + `holistics aml validate` accepts a cycle, and Holistics inlines one metric + into the next, so the pair has no SQL. + """ + document = _two_dataset_document( + [ + _metric("ping", "pong + 1"), + _metric("pong", "ping * 2"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_METRIC_CYCLE] + assert [i.object_ref for i in raised] == ["ping"] + assert raised[0].severity is Severity.ERROR + assert result.issues.has_errors() + + +#: sqlglot names a node after the shape it parsed, so several SQL spellings +#: share one node. The passthrough reads the original name back out of the +#: source using the offset sqlglot recorded for the name token. +SOURCE_NAMES = { + "LPAD(a.s, 5)": "sql_number('LPAD', a.s, 5)", + "RPAD(a.s, 5)": "sql_number('RPAD', a.s, 5)", + "LTRIM(a.s)": "sql_number('LTRIM', a.s)", + "RTRIM(a.s)": "sql_number('RTRIM', a.s)", + "STRPOS(a.s, 'z')": "sql_number('STRPOS', a.s, 'z')", + "MY_UDF(a.x)": "sql_number('MY_UDF', a.x)", +} + +#: Calls sqlglot rewrote, so an argument carries no offset and the original +#: cannot be rebuilt. `LOG10` gains a base the source never had, and +#: `DATE_TRUNC` has its unit normalised, and an absent offset does not say which. +REWRITTEN_CALLS = ["LOG10(a.x)", "DATE_TRUNC('month', a.d)", "SUBSTRING(a.s, 1, 3)"] + + +@pytest.mark.parametrize("sql,expected", SOURCE_NAMES.items(), ids=list(SOURCE_NAMES)) +def test_the_passthrough_uses_the_name_the_source_wrote(sql, expected): + rendered, _ = aql.translate(sql, "ANSI_SQL", lambda table, column: f"{table}.{column}") + assert rendered == expected + + +@pytest.mark.parametrize("sql", REWRITTEN_CALLS) +def test_a_call_sqlglot_rewrote_raises_untranslatable(sql): + with pytest.raises(aql.Untranslatable): + aql.translate(sql, "ANSI_SQL", lambda table, column: f"{table}.{column}") + + +def _write(files: dict[str, str], directory: Path) -> None: + for name, text in files.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _validate_aml(directory: Path, dataset_file: str) -> subprocess.CompletedProcess: + return subprocess.run( + [HOLISTICS_CLI, "aml", "validate", dataset_file, "-r", "."], + cwd=directory, + capture_output=True, + text=True, + ) + + +@needs_cli +@pytest.mark.parametrize( + "produce,dataset_file", + [ + (snapshots.reverse_ecommerce, "ecommerce.dataset.aml"), + (snapshots.reverse_tpcds, "tpcds_retail_model.dataset.aml"), + (snapshots.reverse_aql, "aql_translation.dataset.aml"), + (snapshots.reverse_metric_references, "metric_references.dataset.aml"), + ], + ids=["ecommerce", "tpcds", "aql", "metric_refs"], +) +def test_the_generated_aml_compiles(produce, dataset_file, tmp_path): + _, result = produce() + _write(dict(result.files), tmp_path) + completed = _validate_aml(tmp_path, dataset_file) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "No validation errors found" in completed.stdout + + +@needs_cli +def test_the_source_fixture_still_compiles_to_the_committed_json(tmp_path): + """The committed JSON is what the CLI produces from the committed AML. + + The fixture ships both, and an edit to the AML that forgets the recompile + would leave every other test reading the old payload. + """ + source = snapshots.FIXTURES / "ecommerce" / "aml" + completed = subprocess.run( + [HOLISTICS_CLI, "aml", "compile", "ecommerce.dataset.aml", "-r", ".", "-o", str(tmp_path)], + cwd=source, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + produced = json.loads((tmp_path / "ecommerce.dataset.aml.json").read_text(encoding="utf-8")) + committed = json.loads(snapshots.ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + assert produced == committed + + +@needs_cli +def test_the_round_trip_returns_the_same_ossie_document(tmp_path): + """AML to Ossie to AML to Ossie returns the document it started from. + + Compiling the generated AML is the only way to check the reverse path + against the thing that actually reads AML, rather than against this + converter's own idea of it. + """ + _, forward_result = snapshots.forward() + _, reverse_result = snapshots.reverse_ecommerce() + + generated = tmp_path / "aml" + _write(dict(reverse_result.files), generated) + compiled = tmp_path / "compiled" + completed = subprocess.run( + [HOLISTICS_CLI, "aml", "compile", "ecommerce.dataset.aml", "-r", ".", "-o", str(compiled)], + cwd=generated, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + payload = json.loads((compiled / "ecommerce.dataset.aml.json").read_text(encoding="utf-8")) + again = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) + assert _yaml.dump(again.model) == _yaml.dump(forward_result.model) + + +STASH_VALUES = { + "a list": (["a", "b"], "opt: ['a', 'b']"), + "a bool": (True, "opt: true"), + "a number": (3, "opt: 3"), + "a string": ("x", "opt: 'x'"), +} + + +def _with_persistence(value): + document = _two_dataset_document([]) + document["datasets"][0]["custom_extensions"] = [ + { + "vendor_name": "HOLISTICS", + "data": json.dumps( + {"_v": 1, "persistence": {"__type__": "FullPersistence", "opt": value}} + ), + } + ] + return document + + +@pytest.mark.parametrize("value,expected", STASH_VALUES.values(), ids=list(STASH_VALUES)) +def test_a_stashed_persistence_value_is_written_as_aml(value, expected): + """Each type has its own spelling, so none arrives as Python repr text.""" + result = ossie_to_aml.convert(_with_persistence(value), "ANSI_SQL", data_source_name="probe") + orders = next(text for name, text in result.files if name.endswith("orders.model.aml")) + assert expected in orders + + +@pytest.mark.parametrize("value", [None, {"no": "type"}], ids=["none", "untyped mapping"]) +def test_a_stashed_value_with_no_aml_spelling_stops_the_run(value): + """Writing a body AML cannot parse would fail later and further away.""" + with pytest.raises(ConversionError): + ossie_to_aml.convert(_with_persistence(value), "ANSI_SQL", data_source_name="probe") + + +def test_a_field_name_aml_cannot_spell_is_still_declared(): + """`declared` holds the AML spelling, which is what a reference carries. + + An Ossie field named `my field` is written `my_field`. Matched on the Ossie + name it reads as undeclared, and the metric borrows a hidden dimension of a + name the model already has, which Holistics rejects as a duplicate. + """ + document = _two_dataset_document( + [_metric("spanning", 'SUM(orders."my field") + COUNT(users.id)')] + ) + document["datasets"][0]["fields"].append( + { + "name": "my field", + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "my field"}]}, + } + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + + borrowed = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_BORROWED_COLUMN] + assert borrowed == [] + orders = next(text for name, text in result.files if name.endswith("orders.model.aml")) + assert orders.count("dimension my_field {") == 1 + + +def _dataset_text(document): + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + return next(text for name, text in result.files if name.endswith(".dataset.aml")) + + +def _with_relationship_stash(**stashed): + document = _two_dataset_document([]) + document["relationships"][0]["custom_extensions"] = [ + { + "vendor_name": "HOLISTICS", + "data": json.dumps({"_v": 1, "type": "many_to_one", "active": True, **stashed}), + } + ] + return document + + +def test_a_stashed_relationship_filter_is_written_back(): + """A `where` filter belongs to the relationship, not to the config. + + One written inline in `relationships:` has nowhere to put it, so a + relationship carrying a filter is declared above the `Dataset` block under + a name and the list holds `RelationshipConfig { rel: , ... }`. + """ + text = _dataset_text( + _with_relationship_stash( + where={ + "__type__": "RelationshipFilter", + "from": {"__type__": "Heredoc", "name": "aql", "content": "users.id > 0"}, + } + ) + ) + assert "Relationship users_to_orders {" in text + assert "from: ref('users', 'order_id')" in text + assert "to: ref('orders', 'id')" in text + assert "where {" in text + assert "from: @aql users.id > 0;;" in text + assert "rel: users_to_orders" in text + assert text.index("Relationship users_to_orders {") < text.index("Dataset m {") + + +def test_a_non_default_relationship_property_is_written_back(): + """Only a value that differs from the compiler's default is written. + + Every compiled payload carries all three, so writing them unconditionally + would turn every short `relationship(...)` into a block meaning the same. + """ + assert "relationship(users.order_id > orders.id, true)" in _dataset_text( + _two_dataset_document([]) + ) + + text = _dataset_text( + _with_relationship_stash(direction="one_way", nullable=False, rlp_propagation="one_way") + ) + assert "direction: 'one_way'" in text + assert "nullable: false" in text + assert "rlp_propagation: 'one_way'" in text + + +MATCH_RELATIONSHIPS = { + "many_to_many": "ManyToManyRelationship", + "range": "RangeRelationship", +} + + +@pytest.mark.parametrize("kind,aml_type", MATCH_RELATIONSHIPS.items(), ids=list(MATCH_RELATIONSHIPS)) +def test_a_match_relationship_is_stashed_and_written_back(kind, aml_type): + """A range or many-to-many join states its condition as one AQL predicate. + + Ossie encodes a relationship as `from_columns` and `to_columns`, which + cannot hold that, so the payload travels in the document stash and the + reverse path rebuilds the declaration from it. The ecommerce fixture covers + many-to-many end to end; this reaches `range` as well. + """ + payload = json.loads(snapshots.ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + payload["relationships"] = [ + { + "__type__": "RelationshipConfig", + "active": True, + "rel": { + "__type__": aml_type, + "__fqn__": "items_near_products", + "name": "items_near_products", + "type": kind, + "from": {"__type__": "FieldRef", "model": "order_items", "field": "id"}, + "to": {"__type__": "FieldRef", "model": "products", "field": "id"}, + "match": { + "__type__": "Heredoc", + "name": "aql", + "content": "order_items.id >= products.id", + }, + }, + } + ] + forward = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) + + raised = [ + i for i in forward.issues.issues if i.code == aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP + ] + assert len(raised) == 1 + assert kind in raised[0].message + assert raised[0].severity is Severity.WARNING + assert not forward.model.get("relationships") + + reverse = ossie_to_aml.convert(forward.model, snapshots.SQL_DIALECT) + text = next(t for name, t in reverse.files if name.endswith(".dataset.aml")) + assert f"{aml_type} items_near_products {{" in text + assert f"type: '{kind}'" in text + assert "match: @aql order_items.id >= products.id;;" in text + assert "rel: items_near_products" in text diff --git a/converters/holistics/tools/check_links.py b/converters/holistics/tools/check_links.py new file mode 100644 index 000000000..d41f1ba35 --- /dev/null +++ b/converters/holistics/tools/check_links.py @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Resolve every relative link and heading anchor across the documents. + +The mapping is split over several files, so a link that was a same-page anchor +can quietly become a dangling cross-file reference. This resolves each one +against the files and headings that actually exist. External URLs are not +fetched; only their shape is checked. + +Dev tooling, not part of the wheel. Run it with ``python3 tools/check_links.py``. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent.parent + +LINK = re.compile(r"\[(?P[^\]]+)\]\((?P[^)]+)\)") +HEADING = re.compile(r"(?m)^#{1,6}\s+(?P.+?)\s*$") + + +def slug(title: str) -> str: + """GitHub's heading anchor: lowercase, punctuation dropped, spaces hyphenated.""" + s = title.strip().lower() + s = re.sub(r"`([^`]*)`", r"\1", s) + s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s) + s = re.sub(r"[^\w\s-]", "", s) + return re.sub(r"\s+", "-", s).strip("-") + + +def anchors(path: Path) -> set[str]: + return {slug(m.group("title")) for m in HEADING.finditer(path.read_text())} + + +def main() -> int: + docs = [HERE / "README.md"] + sorted((HERE / "docs").glob("*.md")) + failures: list[str] = [] + checked = 0 + + for doc in docs: + text = doc.read_text() + for match in LINK.finditer(text): + target = match.group("target").strip() + where = f"{doc.relative_to(HERE).as_posix()}: [{match.group('text')}]({target})" + checked += 1 + + if target.startswith(("http://", "https://")): + if " " in target: + failures.append(f"{where} -> malformed URL") + continue + if target.startswith("mailto:"): + continue + + file_part, _, anchor = target.partition("#") + resolved = doc.parent / file_part if file_part else doc + if not resolved.exists(): + failures.append(f"{where} -> no such file {resolved.relative_to(HERE).as_posix()}") + continue + if anchor and anchor not in anchors(resolved): + failures.append( + f"{where} -> {resolved.name} has no heading anchored '{anchor}'" + ) + + if failures: + print(f"{len(failures)} of {checked} links do not resolve:\n", file=sys.stderr) + for f in failures: + print(f" {f}", file=sys.stderr) + return 1 + print(f"all {checked} links across {len(docs)} documents resolve") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/holistics/tools/update_snapshots.py b/converters/holistics/tools/update_snapshots.py new file mode 100644 index 000000000..9e5dda508 --- /dev/null +++ b/converters/holistics/tools/update_snapshots.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Rewrite every committed snapshot under `tests/fixtures/`. + +Run after a deliberate change to the converter, then read the diff. The same +`tests/snapshots.py` definitions drive `tests/test_snapshots.py`, so what this +writes is exactly what that compares. + + python3 tools/update_snapshots.py +""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tests")) + +from snapshots import FIXTURES, all_snapshots # noqa: E402 + + +def main() -> int: + written = 0 + for relative, text in sorted(all_snapshots().items()): + path = FIXTURES / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_text(encoding="utf-8") == text: + continue + path.write_text(text, encoding="utf-8") + print(f"wrote {path.relative_to(ROOT)}") + written += 1 + print(f"{written} snapshot(s) changed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/holistics/uv.lock b/converters/holistics/uv.lock new file mode 100644 index 000000000..d28986e83 --- /dev/null +++ b/converters/holistics/uv.lock @@ -0,0 +1,560 @@ +version = 1 +revision = 5 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version < '3.11'", +] + +[[package]] +name = "apache-ossie-holistics" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.dev-dependencies] +dev = [ + { name = "jsonschema" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=30.12.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/4ba3d0fb7297ebae71171822554abe48d7cab29c28b8f9f2c04b79988c05/rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00", size = 359751, upload-time = "2025-11-30T20:21:34.591Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7c/e4933565ef7f7a0818985d87c15d9d273f1a649afa6a52ea35ad011195ea/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6", size = 389696, upload-time = "2025-11-30T20:21:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/01/6271a2511ad0815f00f7ed4390cf2567bec1d4b1da39e2c27a41e6e3b4de/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7", size = 403136, upload-time = "2025-11-30T20:21:37.728Z" }, + { url = "https://files.pythonhosted.org/packages/55/64/c857eb7cd7541e9b4eee9d49c196e833128a55b89a9850a9c9ac33ccf897/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324", size = 524699, upload-time = "2025-11-30T20:21:38.92Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ed/94816543404078af9ab26159c44f9e98e20fe47e2126d5d32c9d9948d10a/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df", size = 412022, upload-time = "2025-11-30T20:21:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/707f6cf0066a6412aacc11d17920ea2e19e5b2f04081c64526eb35b5c6e7/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3", size = 390522, upload-time = "2025-11-30T20:21:42.17Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/57a85fda37a229ff4226f8cbcf09f2a455d1ed20e802ce5b2b4a7f5ed053/rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221", size = 404579, upload-time = "2025-11-30T20:21:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/c9339293513ec680a721e0e16bf2bac3db6e5d7e922488de471308349bba/rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7", size = 421305, upload-time = "2025-11-30T20:21:44.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/be/522cb84751114f4ad9d822ff5a1aa3c98006341895d5f084779b99596e5c/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff", size = 572503, upload-time = "2025-11-30T20:21:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/de879f7e7ceddc973ea6e4629e9b380213a6938a249e94b0cdbcc325bb66/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7", size = 598322, upload-time = "2025-11-30T20:21:48.709Z" }, + { url = "https://files.pythonhosted.org/packages/48/ac/f01fc22efec3f37d8a914fc1b2fb9bcafd56a299edbe96406f3053edea5a/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139", size = 560792, upload-time = "2025-11-30T20:21:50.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/da/4e2b19d0f131f35b6146425f846563d0ce036763e38913d917187307a671/rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464", size = 221901, upload-time = "2025-11-30T20:21:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/96/cb/156d7a5cf4f78a7cc571465d8aec7a3c447c94f6749c5123f08438bcf7bc/rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169", size = 235823, upload-time = "2025-11-30T20:21:52.505Z" }, + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/6d/ddb8dfcf41892ba9c672d27988592c038d30db61708230313831b6780a1b/rpds_py-2026.9.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2711d29b653b3bce48a63d18b9c6b53274669e6d6c4094dddeb4d9a0e45128b2", size = 344947, upload-time = "2026-10-04T16:28:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b2/632c3d034961149eff606e3e357547620a9ab37b849ae5815f67838753fd/rpds_py-2026.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3231c4c0e521dafa5be0c9f114ee2c2ad46650836f2d72caa86801950c3e7044", size = 346770, upload-time = "2026-10-04T16:28:56.884Z" }, + { url = "https://files.pythonhosted.org/packages/6d/45/4df346410bb873d03ec60c3b891d1fceb8cabde1971d29844bbb625cc296/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e01b3c878c8641913e688edd1b3f08658c6783d29cf6b826bd3c0d1ae7a1ffaa", size = 377795, upload-time = "2026-10-04T16:28:58.193Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b8/a70f302207e4f29f873c01a986fcac5def49b9c8224328cef1ad7da6b7a9/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3e524c7874ac72884d28e16dd5b8d839fd09e0fe76b020d3fbca23212a7b8c52", size = 383137, upload-time = "2026-10-04T16:28:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/13/e3/3aa28a48a01169d0e20728bf0e5ebbea3ae2774285b40853ae49b3ef332a/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761fdae6728ceb99ab182fad2f0cc1e262f610834dc891aea1d1a2a2e634776f", size = 496789, upload-time = "2026-10-04T16:29:01.011Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a7/414e8d088602d66ad6f47c65bf07fe5090992a8574cdd4571c1a83b647ed/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b723eb406dec5bc9ec516c73ab9c3239a3284e017f7eb89ee2b3258bd504fb7", size = 395929, upload-time = "2026-10-04T16:29:02.465Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5c/aac3b960fd55d37b49f77c0eebbe043eaf36762ebd48e7b46951e59c1c05/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff", size = 375426, upload-time = "2026-10-04T16:29:03.944Z" }, + { url = "https://files.pythonhosted.org/packages/d3/80/77987a41c40be7202a78d9aed4a650bfb05792237cfd3b9e22638d990fe5/rpds_py-2026.9.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:839dde845559254f34885267c6878f60d61d5205180226d976fe488d45fa128e", size = 381497, upload-time = "2026-10-04T16:29:05.308Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/640b02eb1ecd71a913c1178b6235287b31006266bca9a26862e80e497b54/rpds_py-2026.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56cd8b3f77d7b6812f533b662186a1f28316931166ddc00fb893b1b0db7e9888", size = 405740, upload-time = "2026-10-04T16:29:06.578Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8c/d034195d45d215625a23f2ae85dbbdb86a49a93a51272061c97c80b9d316/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:074a4d198bc34d9a8ea425114fc3ded6d11ec01f6a314a8db67454a5152d8834", size = 555850, upload-time = "2026-10-04T16:29:08.115Z" }, + { url = "https://files.pythonhosted.org/packages/de/f8/532f1addfc86207bc3cf23749638bb0d457bce891595012ed8a08d0543b8/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6beb738155fe8ab8091afdfa5a3226b21c2b1593f1e50ebb90eb25b44dbc0391", size = 618661, upload-time = "2026-10-04T16:29:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/50/5b/b0f2c9833a36a5b1f6ac5cb0e635fabacf713266f46ceefba2102da0bf51/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:57492a550a1d88d29d003247e5f78dd8cf04a701fac0e4c8db8745a6d2504e0a", size = 583530, upload-time = "2026-10-04T16:29:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/464ed4b98d3c8c6a334f3b5e24d3886de04b05658a95fc40a37e0e6e4775/rpds_py-2026.9.1-cp311-cp311-win32.whl", hash = "sha256:d95a354e02393eada6d7351184671aced9d4cce109dabf927cb7aa99624352a1", size = 208125, upload-time = "2026-10-04T16:29:12.408Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/384a9b62f8cd89c8e3e609b8c9052c140e8c01d807261e0517c904b8bf29/rpds_py-2026.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4d4f52e2a4324396caddbd45a97d8d7be5f42edd25d2355282a9c34f9b2f7f", size = 226045, upload-time = "2026-10-04T16:29:13.751Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/0a691f0dcb2301fa9c736346da2d0d4991778373283301c5c12b8f952f3d/rpds_py-2026.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:fdcd198979b4ecffcc1beba366a7fbcf4eb41243691a82fe52ceb0b902f09c12", size = 222636, upload-time = "2026-10-04T16:29:14.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/83/ea/ee88fd9e756ff93fb6b1182a47ec09504a242620e33ce1d20679efefe841/rpds_py-2026.9.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a36b70596407634ca82d4b989a3729074a008537a0522e4c8046a67c729103e9", size = 346229, upload-time = "2026-10-04T16:29:38.82Z" }, + { url = "https://files.pythonhosted.org/packages/57/71/a097d6552f837500fc36e6b23d09cfb9890c3cc47531f9ca64e149799615/rpds_py-2026.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eba5d173f7d5708b22a93815017a4611873ed54db9f268077c0dd1ed99cfc858", size = 340514, upload-time = "2026-10-04T16:29:40.405Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b7/497e85768bf4e0d8ddbaa096a4cac31d1509251dee2728a8490aa367e0b5/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:457866b85daf5034296666168b84a69e0b2e89dc4f1af102b46f6448a60b9063", size = 372611, upload-time = "2026-10-04T16:29:41.778Z" }, + { url = "https://files.pythonhosted.org/packages/52/4b/74ab4108916250b6e198e0d3af05bc6835eb046315f22f7a0ceb49667c5a/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a3a52a3ba86436ab3aef510fbe21512abc2ddd1993005dfe50514bd2284ef025", size = 377712, upload-time = "2026-10-04T16:29:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8e/067e77d9d7b3cc793c9d909b7e97e7aadbbb1fb094093b6876902cc96d38/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7841166b7fa64c9c56404617ae4341448847482d45933b13135d26c130519e5", size = 488106, upload-time = "2026-10-04T16:29:44.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b4/c5aae6c2dde269bf955f6b7d35065c655a57e47750d9668052ad67e74dda/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:926bdd3e3b5998ddf70cc64bc8cf57209571f9044542913afb673799fec77dd0", size = 390795, upload-time = "2026-10-04T16:29:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b", size = 370794, upload-time = "2026-10-04T16:29:47.603Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/cd2a80e6d7b871937a60e935c5d507aa390d143f4ff3636f640b9733d5df/rpds_py-2026.9.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3cd182d7291d29b92c521a0069d9c01ba6193628a9a105531d11b40a6d731a33", size = 375673, upload-time = "2026-10-04T16:29:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/7464a9953724e55a3b3206062fa0ffdeaa519584c6aabf65d3956d94f131/rpds_py-2026.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e6ea1cda8d8c688278430e4268a42f5e5da3bdd74578dfadc0820c3f1766ce83", size = 398758, upload-time = "2026-10-04T16:29:50.601Z" }, + { url = "https://files.pythonhosted.org/packages/c0/86/1534b436700fd49ff411063b7c4d7e938adfabf90895b6cf1622d5a7d1f6/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5943980471829f6de242a20b109de3111ba6b77e3af0ffc587028ac854b05e6c", size = 550370, upload-time = "2026-10-04T16:29:52.002Z" }, + { url = "https://files.pythonhosted.org/packages/57/1c/e1fa82a8a01e3c5820f3ba98a8b2673f642128eb368fa88871b01dd2c909/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:76d3af9732d2dab69f28179b40ba2d87e2f1d5824b4a694780aa787d685e8f36", size = 613106, upload-time = "2026-10-04T16:29:53.655Z" }, + { url = "https://files.pythonhosted.org/packages/29/55/b20b8c4c3dde8755bfcd5b08492a02d0cd2e26929aedf6199ca2a377d42a/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:78326f4cb4427a56ba4996c0762b63be45f06b85f086526420d2b3a66e40f84d", size = 578235, upload-time = "2026-10-04T16:29:55.157Z" }, + { url = "https://files.pythonhosted.org/packages/55/42/df3f7bbc3f7ab37a8a9db8d6c2ff2c985422899f1f7926afbf7ec3c0b8b4/rpds_py-2026.9.1-cp313-cp313-win32.whl", hash = "sha256:172e47169583f46ce118cbec68e6795d0da0f4606b488b6434f8276bca0a058c", size = 205293, upload-time = "2026-10-04T16:29:56.669Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/ba5a9569d719bfdd6ce863df4133ac6a1658cf1b07cc3534c31db729fbbc/rpds_py-2026.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:3e93b2cd69a9830be33e03945cd7cda940a0a8bfcfbff41d6144f0cb0d3d8bd9", size = 222409, upload-time = "2026-10-04T16:29:58.049Z" }, + { url = "https://files.pythonhosted.org/packages/35/72/f28ca566f6c23c35bbf7445f65eb0364577b25a026305995e24f78b83d94/rpds_py-2026.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f", size = 217681, upload-time = "2026-10-04T16:29:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b7/a662f367d4896dd0a10cef2fc91f10b7f08af1c10858e287e019563f338d/rpds_py-2026.9.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:1d77b649e6f7cdf12ca5c2a98dad0ad37f9ea9b6f960408a92f0cb12bb3d04d9", size = 347463, upload-time = "2026-10-04T16:30:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25", size = 342051, upload-time = "2026-10-04T16:30:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/7e/31/3dcd68c13d4bcc59c1f7eb33ac8e80698f06f3eb0d8a1e06419836071c20/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec450527cbf485e13c8d3602a54f428ab0432fdade0ede75efd74b735421c871", size = 374143, upload-time = "2026-10-04T16:30:51.508Z" }, + { url = "https://files.pythonhosted.org/packages/cf/0d/68c1f058a250fbd1380ebda9fc227cbf50117adf8ffe8161ef383ea79f68/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:306ee1850d8105b5baf977e78d45fcadd12c1a54678d614c9baf217708446e91", size = 378604, upload-time = "2026-10-04T16:30:53.206Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d6/2d4c59b85397cb4800594fadf692688ccf5ce556adc930e7a5bf21061a5e/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef6b65b03247c54692ad4fd9ee97cb772781927db72e3cb05e70b3db6d1ff14f", size = 490790, upload-time = "2026-10-04T16:30:54.925Z" }, + { url = "https://files.pythonhosted.org/packages/da/04/7e05dc3aebaf52f4e026766bd668fdd09a9d0e23f64a14686b36b3501892/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a575404ebc9cf2e91edd32eaf570ec1430eb900d4f56724ba7dd4bc1fc9c176d", size = 393146, upload-time = "2026-10-04T16:30:56.625Z" }, + { url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945", size = 372022, upload-time = "2026-10-04T16:30:58.295Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cb/8f8774df5134e23424372838bcc5c7ed4127d723e1ff52f7bebd4dcb2563/rpds_py-2026.9.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:7664419f27db41d4f1c43a78dccda7dd6e8ef2428df3ee01d0c2a07a6b071297", size = 377092, upload-time = "2026-10-04T16:30:59.984Z" }, + { url = "https://files.pythonhosted.org/packages/90/02/8d7095d73bf9114219be40230baa5df00611e0a82ed9517779ff9c19f82b/rpds_py-2026.9.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b26b03d9d2658ee2fa234f8f4f19f38a09773fe5261028025032e26d4d35af0", size = 400830, upload-time = "2026-10-04T16:31:01.721Z" }, + { url = "https://files.pythonhosted.org/packages/ec/02/8206856f8f363cd042a8315dc86b3912f5dcb6d3c66bbb24b69c2bfb0775/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:be3e47e2d91aa3942ff9bf4077a505226005abfc39b6f7554a91c1b9393986b9", size = 551478, upload-time = "2026-10-04T16:31:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/41/6b/36211f1bb1f0b0313f496d92f5905b74ea107f27cb16fa3355a82f04575e/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:6307a0da524939decb8ca4a3933b8ab62525794411d6984fca6726e732804af6", size = 615173, upload-time = "2026-10-04T16:31:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/35/77/cda0c4a6f055446b692f0ed5692f73707cafd3dff82672ede31b1a9b59de/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:159a7aab5c5e8b112c8830f54717ce56da1252ebdbb526f5be2df2309280b9e7", size = 579653, upload-time = "2026-10-04T16:31:07.065Z" }, + { url = "https://files.pythonhosted.org/packages/74/ec/d8385f446240aed643b9e92a5055cff3015cc04a13c61f73b2883c478ed5/rpds_py-2026.9.1-cp315-cp315-win32.whl", hash = "sha256:dbc2673f9223d420c91145599b3ba45a8a50c207d1976908e5fb5ddb0c9b9429", size = 206498, upload-time = "2026-10-04T16:31:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15", size = 223121, upload-time = "2026-10-04T16:31:10.84Z" }, + { url = "https://files.pythonhosted.org/packages/33/58/dba857c3bc8221b31b62eb170a3080f4f191de79f047200389b7ed1b06a7/rpds_py-2026.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:a431156bb41865fc14cd5d79bb9d7bbed83110b0159e34e62ae30951f96c0009", size = 218741, upload-time = "2026-10-04T16:31:12.592Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d0/320ab28ccc1415eeb509d68682b0014fb74690cd49f1c2d29a232475af50/rpds_py-2026.9.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ef0d8c843e2827d6c120ab4687e9423fb1d893db1df27b7c1506615bcb9734a0", size = 349719, upload-time = "2026-10-04T16:31:14.48Z" }, + { url = "https://files.pythonhosted.org/packages/56/88/f5b12f1358f443c08b7ce3cc8391d82d335f4872580e2e097847fd36087a/rpds_py-2026.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:45bc6bccf78b20fd834237d18db64965d7ee68ba7f60440a26c7ab71e7b8d51a", size = 343267, upload-time = "2026-10-04T16:31:16.827Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0c/c765b0059d532acb3b9c45d781ccc22f15a96dbe443d00903f643ba9df10/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d55198263bb51f557550c6ed2e6d1cb6a6fed6eb5c9120b741c5926bef8a45d", size = 374555, upload-time = "2026-10-04T16:31:18.931Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8a/cafddfda77564a10cd21184640c3bffda6a2b8d20972fb5dedd5e0166328/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8763f20692da7df39b0afdd1ba3042b004c50a45994f76c2d9a25641f7673db", size = 378346, upload-time = "2026-10-04T16:31:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/b9/01/5e626016eff72c183bf6c96539240cace15d402468647a453ec08415b2fc/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e43d4a1f673e8a1cbd8533e809e02b4bf9d4f2280269bb640436556312121250", size = 492373, upload-time = "2026-10-04T16:31:22.614Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9c/15a2469e9389242f46896b3f0a01d68caea8a5a35c011fcb05ef333aae73/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea394a937f17a54c51239348bdbe2e3518124c8d4a8951ba04a311d3095bd18f", size = 394371, upload-time = "2026-10-04T16:31:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/63/f5/c100ff77e1e6366e947c75969c258fdfc4b7bc5bbfe7351e62ffbf2a1228/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdeaa99ce822dca76cfb1b993e9120c5ea212f2eb66d48950ad63c349668a018", size = 375132, upload-time = "2026-10-04T16:31:26.588Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/fe0269c9de253e99c81cabc12b8971a5feaa083debdaff1221e06264d9e3/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:b4f062343e7ad3fa94f2c66e5ae667dee47ee74dd41a9057c4fbe163236a123d", size = 377642, upload-time = "2026-10-04T16:31:28.677Z" }, + { url = "https://files.pythonhosted.org/packages/05/65/b34a7b257baccff8f4a24a722933166d4941d5ebdc9c3f4bc4ffcd5ce4f4/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22ffd29a63d71fb1b81552c21f2c2b734949b7ac751a9be70675a939a900839b", size = 400855, upload-time = "2026-10-04T16:31:30.802Z" }, + { url = "https://files.pythonhosted.org/packages/98/32/844e54176b6071b90b38a564e6940bc6eb8f97b2890dc709c19db9dec0f4/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:08dae4a4095150a7c4545a1fb40b98e1ab1744fbc2770d92c977b9dadaa49ab6", size = 552573, upload-time = "2026-10-04T16:31:32.709Z" }, + { url = "https://files.pythonhosted.org/packages/17/73/6041d20729dffbfdf155c02d65be58bc225a1c1fb548fd87c23ef306138f/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9a0460d43603d1fd9ef59c30278531e15d78581721ddb538fa560aa7817ea4ad", size = 615840, upload-time = "2026-10-04T16:31:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/af/9e/418094adaee6b056ce199051b255448ed872829051e341b2294c80da0977/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1c2d1f6da5128eabf34e963d7163a818846075a52568250d006c4c953b40f903", size = 582764, upload-time = "2026-10-04T16:31:36.665Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9b/1698ebf6b840ddfe8b472198abbed6ace35c6e398faeecd6c47dae742a4e/rpds_py-2026.9.1-cp315-cp315t-win32.whl", hash = "sha256:5c6ee90dee3e85e055ddfd502d611643d9b0fd94c818220bda84ec3dacd9b27b", size = 203795, upload-time = "2026-10-04T16:31:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e2/91f70d804c61f8eac39a417e82aa1024f655ab9e8bdd7393b196242bc41b/rpds_py-2026.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:fe5ad0664ec772b02c45859041aa17655709cced7a31005817fbbbd988c25567", size = 223037, upload-time = "2026-10-04T16:31:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/f0eba548e297987964ef8088403db875b333963bf8189f3f563b6936b20e/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c0d2cb595a420b34d5086db0add011e26e2c09d6a024afbac4228bf8f863a30", size = 384112, upload-time = "2026-10-04T16:31:42.643Z" }, + { url = "https://files.pythonhosted.org/packages/33/89/891fa4d81f1eae7fe262ecd0aba2c51de94a116c7f3ccfb034fa75f4c537/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3d6ed6a98cfd19155996605474982cc470d7601746a6439078f1a5a3fa8b050", size = 389662, upload-time = "2026-10-04T16:31:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/4f/46/912cf623b2263f9c9c2d04e8508fc38dbe89983faeba5fb36ca4558ae550/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8171b44a054e5c67fd748ada04187f1250bf35b95f85e52ab64bcf3331a923bb", size = 504637, upload-time = "2026-10-04T16:31:46.667Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5c/7f3cfa7c5f01765c9f47c8398210ed0c1eb686c0e99c223cb1c6776d3fb9/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda1d96e542c37b6c804547dbf489c129fe7c97183a76a5ec275909ba1a063df", size = 402607, upload-time = "2026-10-04T16:31:49.068Z" }, + { url = "https://files.pythonhosted.org/packages/f0/76/2e915d4115ee453c33e6af17bc1df7e22ddc52974e178eeeb92a78c09983/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:270bdcdaac5d5b6f73c5e22e7e135c7f2a50e789f71d9e241d5be8d90026e19a", size = 382762, upload-time = "2026-10-04T16:31:51.24Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e5/844382925d7b84a4074ad79fc2b9d05eb6231ea243b34f247e68e519b606/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:84a6ecc0c940169190d2c23bd969debd48c94dbc855acd60188a68d71d421608", size = 389563, upload-time = "2026-10-04T16:31:53.549Z" }, + { url = "https://files.pythonhosted.org/packages/81/82/d6e3951602dc2e24790ef9d299aa24bf5e2493df5043f2c6da00480cc447/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab4b2fda7c2b542f7f9d886cc6a838c5079d2b76f72e6081411faba11adde2c9", size = 416842, upload-time = "2026-10-04T16:31:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1c/8d248e4ea671d525ab7daa0703ad81cc2175076fa0ccd7ef088facbd2b01/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:44b32a7c4f0da3d28af31c259e38ddcff096f855e205ed0671d02fcf44f1ea1c", size = 562434, upload-time = "2026-10-04T16:31:57.874Z" }, + { url = "https://files.pythonhosted.org/packages/ad/38/e7ac0db898120d15dea25e3661b1242a40cb3b23d1f4db86c15a01fe0ba1/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:a3dbc5ed9514908d5046107d7b1346bde71eea61de6e0e4919c19354f97e769f", size = 627909, upload-time = "2026-10-04T16:31:59.977Z" }, + { url = "https://files.pythonhosted.org/packages/03/c8/37e555514932cb1ee36c75349187aa882a98dee5d5654c761699ed8a0d27/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6eae33003518fd4cb4f83a218d5371469dd3001aa3b87128c005b07762f7fe5e", size = 593658, upload-time = "2026-10-04T16:32:02.14Z" }, + { url = "https://files.pythonhosted.org/packages/12/5c/5f69a91add94a8a33acea68a2b034a464946a654ff7296af9d957ef0d6a5/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_10_12_x86_64.whl", hash = "sha256:0483515261947e4e8b8e1375bf7463e7eb6ccfb3d86e7b554d90cd5285f20f32", size = 351769, upload-time = "2026-10-04T16:32:04.675Z" }, + { url = "https://files.pythonhosted.org/packages/41/22/35ea8bf3b2682b90ab687c74e4828418ff12298a5b0b0a0c033f61c86a30/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:4cfaf02209061880210819934de2f4f6aa83dc04dafe6770276acc240a56da31", size = 351719, upload-time = "2026-10-04T16:32:06.834Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/c0/cd761f018601bf954c9018420eb6707e3675292dfa514cfda44d06aff2ee/sqlglot-30.21.0.tar.gz", hash = "sha256:6787ac202a3adc0e25434f1512370a655e51cb3de8c792544de5f6ff9c618d25", size = 6064227, upload-time = "2026-09-30T17:35:09.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/93/acaa0862f841051e490326c1fc8c61d618d2df26f9e7aed26c331419ab15/sqlglot-30.21.0-py3-none-any.whl", hash = "sha256:816d1a4815b7562b3976efcc0b561c928a743f976efd0fb8ad6db7a5b96c069e", size = 763892, upload-time = "2026-09-30T17:35:07.976Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index e6c39ec0b..30cfac033 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -37,7 +37,7 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "OSSIE_SQL_2026"], + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL", "OSSIE_SQL_2026"], "description": "Supported SQL and expression language dialects" }, "Vendor": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 8f90773c6..99a39e329 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -61,6 +61,7 @@ Dialects in which metric and field expressions can be declared. | `SIGMA` | Sigma Computing's spreadsheet-style formula language | | `THOUGHTSPOT` | ThoughtSpot formula language | | `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | +| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | | `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in `expression_language.md` | `OSSIE_SQL_2026` is the only dialect whose semantics this specification defines. The other values diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index cba031c2e..887ee86de 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -41,6 +41,7 @@ dialects: - "SIGMA" # Sigma Computing spreadsheet-style formula language - "THOUGHTSPOT" # ThoughtSpot formula language (not SQL) - "DAX" # Data Analysis Expressions (Power BI / Analysis Services) + - "HOLISTICS_AQL" # Holistics AQL (Analytical Query Language) - "OSSIE_SQL_2026" # Ossie portable SQL expression language (see expression_language.md) # Supported logical data types for fields and metrics diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 232e786e1..261c31f23 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -35,6 +35,7 @@ class OssieDialect(str, Enum): SIGMA = "SIGMA" THOUGHTSPOT = "THOUGHTSPOT" DAX = "DAX" + HOLISTICS_AQL = "HOLISTICS_AQL" OSSIE_SQL_2026 = "OSSIE_SQL_2026" diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py index 474ffe951..925aab9df 100755 --- a/scripts/generate-spec-types.py +++ b/scripts/generate-spec-types.py @@ -66,6 +66,7 @@ "SIGMA": "Sigma Computing spreadsheet-style formula language", "THOUGHTSPOT": "ThoughtSpot formula language (not SQL)", "DAX": "Data Analysis Expressions (Power BI / Analysis Services)", + "HOLISTICS_AQL": "Holistics AQL (Analytical Query Language)", "OSSIE_SQL_2026": "Ossie portable SQL expression language (see expression_language.md)", }, }, diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 0e10c3c3d..ffd51da96 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -337,6 +337,26 @@ def test_ossie_sql_2026_maps_to_the_ansi_default_explicitly() -> None: assert "OSSIE_SQL_2026" not in _VALIDATE.SKIP_SQL_VALIDATION +def test_holistics_aql_is_skipped_rather_than_parsed_as_sql() -> None: + # AQL is relationship-aware and is not SQL, so sqlglot cannot parse it. + # Skipping it is what lets a Holistics field carry its body at all, the way + # MDX, MAQL, THOUGHTSPOT and DAX already do. + assert "HOLISTICS_AQL" in _VALIDATE.DIALECT_MAP + assert _VALIDATE.DIALECT_MAP["HOLISTICS_AQL"] is None + assert "HOLISTICS_AQL" in _VALIDATE.SKIP_SQL_VALIDATION + + +def test_a_holistics_aql_expression_is_not_reported_as_invalid_sql() -> None: + # `|` is AQL's pipe operator. Parsed as SQL it is a bitwise OR over two + # names, which sqlglot accepts, so the skip is asserted through a body that + # SQL cannot read at all. + body = "count(orders.id) | where(orders.status == 'delivered')" + expression = {"dialects": [{"dialect": "HOLISTICS_AQL", "expression": body}]} + dataset = {**_ORDERS, "fields": [{"name": "delivered", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") def test_validates_a_valid_ossie_sql_2026_expression() -> None: error = _VALIDATE.validate_sql_expression("SUM(amount)", "OSSIE_SQL_2026", "ctx") diff --git a/validation/validate.py b/validation/validate.py index 3799b0eb3..fe27f3ec3 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -85,10 +85,19 @@ "SIGMA": None, # Sigma's spreadsheet-style formula language, not SQL; skip validation "THOUGHTSPOT": None, # Not supported by sqlglot, skip validation "DAX": None, # Not supported by sqlglot, skip validation + "HOLISTICS_AQL": None, # Holistics AQL, not SQL; skip validation } # Dialects that sqlglot cannot parse -SKIP_SQL_VALIDATION = {"MDX", "TABLEAU", "MAQL", "SIGMA", "THOUGHTSPOT", "DAX"} +SKIP_SQL_VALIDATION = { + "MDX", + "TABLEAU", + "MAQL", + "SIGMA", + "THOUGHTSPOT", + "DAX", + "HOLISTICS_AQL", +} # Concepts every ontology includes implicitly (ontology.md, "Built-in concepts") BUILT_IN_CONCEPTS = {"Any", "Boolean", "Date", "DateTime", "Decimal", "Float", "Integer", "String"}