From f31238211d8294da1157ae52c1561474d768d5de Mon Sep 17 00:00:00 2001 From: Huynh Tan Date: Wed, 7 Oct 2026 12:26:35 +0700 Subject: [PATCH 01/12] feat(core-spec): register HOLISTICS_AQL as a dialect AQL is the Holistics analytical query language. It is not SQL: it has relationship-aware semantics, so `count(orders.id) | where(...) | of_all()` has no SQL rendering that is correct without the whole dataset's relationship graph. The specification defines `dialects[]` as a list of `{dialect, expression}` pairs so that a value a converter cannot translate still travels, tagged with the language it is valid in. Without a member for AQL, a Holistics field defined in AQL has nowhere to put its body, and `expression` is required on every field. Registering it dissolves that. Same pattern as THOUGHTSPOT in [#351](https://github.com/apache/ossie/pull/351): the schema enumeration, the specification table, the generated types and the Python enumeration. AQL goes into validate.py's SKIP_SQL_VALIDATION alongside MDX, TABLEAU, MAQL, SIGMA, THOUGHTSPOT and DAX, because sqlglot cannot parse it. Two tests cover it: that the dialect maps to None and sits in SKIP_SQL_VALIDATION, and that an AQL body with a pipe operator passes validate_sql rather than being reported as invalid SQL. Co-Authored-By: Claude Opus 5 --- core-spec/ossie-schema.json | 2 +- core-spec/spec.md | 1 + core-spec/spec.yaml | 1 + python/src/ossie/models.py | 1 + scripts/generate-spec-types.py | 1 + validation/tests/test_validate.py | 20 ++++++++++++++++++++ validation/validate.py | 11 ++++++++++- 7 files changed, 35 insertions(+), 2 deletions(-) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index e6c39ec0b..30cfac033 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -37,7 +37,7 @@ "$defs": { "Dialect": { "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "OSSIE_SQL_2026"], + "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL", "BIGQUERY", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL", "OSSIE_SQL_2026"], "description": "Supported SQL and expression language dialects" }, "Vendor": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 8f90773c6..99a39e329 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -61,6 +61,7 @@ Dialects in which metric and field expressions can be declared. | `SIGMA` | Sigma Computing's spreadsheet-style formula language | | `THOUGHTSPOT` | ThoughtSpot formula language | | `DAX` | Data Analysis Expressions (Power BI / Analysis Services) | +| `HOLISTICS_AQL` | Holistics AQL (Analytical Query Language) | | `OSSIE_SQL_2026` | Ossie's portable SQL expression language, defined in `expression_language.md` | `OSSIE_SQL_2026` is the only dialect whose semantics this specification defines. The other values diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index cba031c2e..887ee86de 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -41,6 +41,7 @@ dialects: - "SIGMA" # Sigma Computing spreadsheet-style formula language - "THOUGHTSPOT" # ThoughtSpot formula language (not SQL) - "DAX" # Data Analysis Expressions (Power BI / Analysis Services) + - "HOLISTICS_AQL" # Holistics AQL (Analytical Query Language) - "OSSIE_SQL_2026" # Ossie portable SQL expression language (see expression_language.md) # Supported logical data types for fields and metrics diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 232e786e1..261c31f23 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -35,6 +35,7 @@ class OssieDialect(str, Enum): SIGMA = "SIGMA" THOUGHTSPOT = "THOUGHTSPOT" DAX = "DAX" + HOLISTICS_AQL = "HOLISTICS_AQL" OSSIE_SQL_2026 = "OSSIE_SQL_2026" diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py index 474ffe951..925aab9df 100755 --- a/scripts/generate-spec-types.py +++ b/scripts/generate-spec-types.py @@ -66,6 +66,7 @@ "SIGMA": "Sigma Computing spreadsheet-style formula language", "THOUGHTSPOT": "ThoughtSpot formula language (not SQL)", "DAX": "Data Analysis Expressions (Power BI / Analysis Services)", + "HOLISTICS_AQL": "Holistics AQL (Analytical Query Language)", "OSSIE_SQL_2026": "Ossie portable SQL expression language (see expression_language.md)", }, }, diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index 0e10c3c3d..ffd51da96 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -337,6 +337,26 @@ def test_ossie_sql_2026_maps_to_the_ansi_default_explicitly() -> None: assert "OSSIE_SQL_2026" not in _VALIDATE.SKIP_SQL_VALIDATION +def test_holistics_aql_is_skipped_rather_than_parsed_as_sql() -> None: + # AQL is relationship-aware and is not SQL, so sqlglot cannot parse it. + # Skipping it is what lets a Holistics field carry its body at all, the way + # MDX, MAQL, THOUGHTSPOT and DAX already do. + assert "HOLISTICS_AQL" in _VALIDATE.DIALECT_MAP + assert _VALIDATE.DIALECT_MAP["HOLISTICS_AQL"] is None + assert "HOLISTICS_AQL" in _VALIDATE.SKIP_SQL_VALIDATION + + +def test_a_holistics_aql_expression_is_not_reported_as_invalid_sql() -> None: + # `|` is AQL's pipe operator. Parsed as SQL it is a bitwise OR over two + # names, which sqlglot accepts, so the skip is asserted through a body that + # SQL cannot read at all. + body = "count(orders.id) | where(orders.status == 'delivered')" + expression = {"dialects": [{"dialect": "HOLISTICS_AQL", "expression": body}]} + dataset = {**_ORDERS, "fields": [{"name": "delivered", "expression": expression}]} + + assert _VALIDATE.validate_sql(_document([dataset], [])) == [] + + @pytest.mark.skipif(not _VALIDATE.SQLGLOT_AVAILABLE, reason="sqlglot is not installed") def test_validates_a_valid_ossie_sql_2026_expression() -> None: error = _VALIDATE.validate_sql_expression("SUM(amount)", "OSSIE_SQL_2026", "ctx") diff --git a/validation/validate.py b/validation/validate.py index 3799b0eb3..fe27f3ec3 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -85,10 +85,19 @@ "SIGMA": None, # Sigma's spreadsheet-style formula language, not SQL; skip validation "THOUGHTSPOT": None, # Not supported by sqlglot, skip validation "DAX": None, # Not supported by sqlglot, skip validation + "HOLISTICS_AQL": None, # Holistics AQL, not SQL; skip validation } # Dialects that sqlglot cannot parse -SKIP_SQL_VALIDATION = {"MDX", "TABLEAU", "MAQL", "SIGMA", "THOUGHTSPOT", "DAX"} +SKIP_SQL_VALIDATION = { + "MDX", + "TABLEAU", + "MAQL", + "SIGMA", + "THOUGHTSPOT", + "DAX", + "HOLISTICS_AQL", +} # Concepts every ontology includes implicitly (ontology.md, "Built-in concepts") BUILT_IN_CONCEPTS = {"Any", "Boolean", "Date", "DateTime", "Decimal", "Float", "Integer", "String"} From 306e88d3134e788d562a1ed39f95a8a3413e5a2a Mon Sep 17 00:00:00 2001 From: Huynh Tan Date: Wed, 7 Oct 2026 12:27:00 +0700 Subject: [PATCH 02/12] feat(holistics): add the Holistics AML converter Offline conversion between a Holistics AML dataset and an Apache Ossie semantic model, in both directions. to-ossie reads the JSON `holistics aml compile` produces, and writes one Ossie semantic model. to-aml reads one Ossie semantic model, and writes AML source files. Reading goes through the compiled JSON because that is what the CLI emits. Writing produces AML source text directly, since nothing turns that JSON back into AML and AML source is what a person edits and what the CLI accepts. One AML `Dataset` becomes one Ossie semantic model. Its models become Ossie datasets, its relationships become Ossie relationships, and its measures and metrics become Ossie metrics. The two SQLs ------------ Holistics SQL is an AML `@sql` body: it interpolates through `{{ ... }}` and resolves inside the one model it sits in. Ossie SQL is the `expression` of a `dialects[]` entry: it writes a reference as `dataset.field`, and its dialect tag names the warehouse. Warehouse SQL is what Holistics generates and sends to the database. Every message and document here names which one it means. What the converter will not do ------------------------------ It translates no AQL into SQL. An AQL body travels verbatim under the HOLISTICS_AQL dialect. It rewrites no warehouse SQL between dialects. A computed `@sql` body is emitted under the dialect the operator names with --sql-dialect. AML records only `data_source_name`, an instance-local connection name, so reading a dialect out of it would be a guess. It resolves no query template. A `QueryModel` whose body uses `{{ #model }}`, `{{ #model.* }}` or `{% filter(param) %}` is carried unresolved with an ERROR naming the dataset. A filter block cannot be resolved at conversion time at all, because the param value exists only at query time, so resolving the other two would turn a body that is visibly not SQL into one that looks like SQL and still is not. Restoring AML interpolation --------------------------- The reverse path has to turn `CAST(created_at AS DATE)` back into `CAST({{ #SOURCE.created_at }} AS DATE)`. A bare column name does compile, and it is not equivalent: AQL reads the interpolation to learn which field a body depends on, and a body without it loses the query rewriting and the optimisation that link drives. That means telling a column apart from a keyword, a function name and a date part, which is why sqlglot is a runtime dependency here rather than dev-only. The converter parses at the dialect the expression declares, takes the character offsets sqlglot reports for each column, and splices the replacement into the original string. In BigQuery's `DATE_TRUNC(created_at, month)` sqlglot reads `created_at` as a column and `month` as a date part. The Ossie SQL is never written back out through sqlglot. That same string rendered through the default dialect returns `DATE_TRUNC(created_at, MONTH)`, which the default dialect then re-reads with the unit and the expression exchanged. A model-qualified reference is a column --------------------------------------- `{{ field }}` substitutes the sibling field's definition. `{{ #model.field }}` reads the column of that model's relation. Holistics renders `FLOOR({{ #ecommerce_users.age }} / 10)` as `FLOOR(`ecommerce_users`.`age` / 10)`, where `{{ age }}` would have rendered the whole `extract(year from age(...))` body. So a model-qualified reference naming the model it sits in becomes a bare column, and only one naming a different model becomes a cross-dataset reference. A measure is the exception, because a measure is never a relation column. Ossie SQL to AQL ---------------- An Ossie metric may name two datasets. A `@sql` body resolves inside one model, and a dataset-level AML `metric` accepts only `@aql`, so such a metric has an AML home only as AQL. The converter renders it one construct at a time: 51 function names, the operators, and `sql_number('NAME', ...)` for a function with no AQL name, recovered from the source offsets so `LPAD` does not come back as sqlglot's `PAD`. Rendering can also add a hidden dimension for a column no dataset declares, and a `number` type for a field the document never typed, because AQL resolves against declared fields where SQL resolves against the table. Window functions are refused with an ERROR. AQL has them, and the frame comes from the query grain rather than from an `OVER` clause, so translating one means choosing a grain. The Ossie specification defines no semantics for how a window function behaves when queried, so nothing says which grain that is. A metric that names another metric ---------------------------------- Which AML home the named metric landed in decides where the naming one can go, and placement follows the reference to get there. A measure on the same model is `{{ items_quantity }}`. A measure on another model, or a dataset-level metric, moves the naming metric onto the dataset where AQL resolves it. A `@sql` body pinned to a model by a stash or a qualifier, and naming either, is an ERROR: Holistics accepts the file and fails at SQL generation with "Cannot refer to external field in TableModel/QueryModel". A reference cycle is an ERROR as well, which `holistics aml validate` does not catch. Verification ------------ Snapshot tests over four fixtures, plus three checks against the real artifact. The converted document is validated against core-spec/ossie-schema.json. The generated AML is run through `holistics aml validate`. And the round trip compiles that AML back to JSON, converts it to Ossie, and compares it with the document the reverse path started from, which for the ecommerce fixture is byte-identical. The checks that drive the Holistics CLI skip when it is absent. CI installs it, so they do not skip there. See converters/holistics/README.md. tests/fixtures/tpcds/ is examples/tpcds_semantic_model.yaml converted to AML. That document carries no HOLISTICS stash, so it exercises every path that has to derive an AML property rather than read one. 5 of its 8 metrics span two datasets: 2 become AQL, and the 3 that are window functions are dropped with an ERROR each. tests/fixtures/aql/ covers each path through the translation, and tests/fixtures/metric_refs/ each way a metric-to-metric reference lands. Co-Authored-By: Claude Opus 5 --- .github/workflows/converter-holistics-ci.yml | 94 ++ converters/README.md | 1 + converters/holistics/.gitignore | 28 + converters/holistics/README.md | 179 ++ converters/holistics/docs/aql.md | 179 ++ converters/holistics/docs/expressions.md | 114 ++ converters/holistics/docs/fixture.md | 43 + converters/holistics/docs/limitations.md | 128 ++ converters/holistics/docs/mapping.md | 134 ++ converters/holistics/docs/measures.md | 63 + converters/holistics/docs/reverse.md | 214 +++ converters/holistics/docs/vendor-payload.md | 58 + converters/holistics/pyproject.toml | 73 + .../holistics/src/ossie_holistics/__init__.py | 22 + .../holistics/src/ossie_holistics/_yaml.py | 140 ++ .../holistics/src/ossie_holistics/aml.py | 312 ++++ .../src/ossie_holistics/aml_to_ossie.py | 603 +++++++ .../holistics/src/ossie_holistics/amlgen.py | 176 ++ .../holistics/src/ossie_holistics/aql.py | 360 +++++ .../holistics/src/ossie_holistics/cli.py | 264 +++ .../src/ossie_holistics/datatypes.py | 129 ++ .../holistics/src/ossie_holistics/errors.py | 22 + .../src/ossie_holistics/interpolation.py | 179 ++ .../holistics/src/ossie_holistics/issues.py | 96 ++ .../src/ossie_holistics/ossie_to_aml.py | 1041 ++++++++++++ .../holistics/src/ossie_holistics/sqlrefs.py | 180 +++ .../holistics/src/ossie_holistics/stash.py | 94 ++ converters/holistics/tests/conftest.py | 23 + .../fixtures/aql/expected_aml.issues.yaml | 86 + .../expected_aml/aql_translation.dataset.aml | 68 + .../fixtures/aql/expected_aml/items.model.aml | 48 + .../aql/expected_aml/orders.model.aml | 42 + .../fixtures/aql/expected_aml/users.model.aml | 36 + .../tests/fixtures/aql/translation.ossie.yaml | 137 ++ .../fixtures/ecommerce/aml/cities.model.aml | 66 + .../ecommerce/aml/delivered_orders.model.aml | 63 + .../ecommerce/aml/ecommerce.dataset.aml | 99 ++ .../ecommerce/aml/merchants.model.aml | 60 + .../aml/modules/reporting/calendar.model.aml | 45 + .../ecommerce/aml/order_items.model.aml | 60 + .../fixtures/ecommerce/aml/orders.model.aml | 95 ++ .../fixtures/ecommerce/aml/products.model.aml | 52 + .../fixtures/ecommerce/aml/users.model.aml | 98 ++ .../ecommerce/ecommerce.dataset.aml.json | 1437 +++++++++++++++++ .../fixtures/ecommerce/expected.issues.yaml | 108 ++ .../fixtures/ecommerce/expected.ossie.yaml | 651 ++++++++ .../ecommerce/expected_aml.issues.yaml | 18 + .../ecommerce/expected_aml/cities.model.aml | 65 + .../expected_aml/delivered_orders.model.aml | 56 + .../expected_aml/ecommerce.dataset.aml | 86 + .../expected_aml/ecommerce_orders.model.aml | 92 ++ .../expected_aml/ecommerce_users.model.aml | 93 ++ .../expected_aml/merchants.model.aml | 57 + .../modules/reporting/calendar.model.aml | 45 + .../expected_aml/order_items.model.aml | 58 + .../ecommerce/expected_aml/products.model.aml | 52 + .../metric_refs/expected_aml.issues.yaml | 45 + .../metric_refs/expected_aml/items.model.aml | 58 + .../metric_references.dataset.aml | 61 + .../metric_refs/expected_aml/orders.model.aml | 36 + .../metric_refs/expected_aml/users.model.aml | 30 + .../metric_refs/references.ossie.yaml | 128 ++ .../fixtures/tpcds/expected_aml.issues.yaml | 85 + .../tpcds/expected_aml/customer.model.aml | 67 + .../tpcds/expected_aml/date_dim.model.aml | 60 + .../tpcds/expected_aml/item.model.aml | 67 + .../tpcds/expected_aml/store.model.aml | 67 + .../tpcds/expected_aml/store_sales.model.aml | 105 ++ .../tpcds_retail_model.dataset.aml | 51 + converters/holistics/tests/snapshots.py | 152 ++ .../holistics/tests/test_cross_vendor.py | 137 ++ .../holistics/tests/test_issue_codes.py | 70 + .../holistics/tests/test_malformed_input.py | 68 + converters/holistics/tests/test_snapshots.py | 538 ++++++ converters/holistics/tools/check_links.py | 92 ++ .../holistics/tools/update_snapshots.py | 50 + converters/holistics/uv.lock | 560 +++++++ 77 files changed, 11349 insertions(+) create mode 100644 .github/workflows/converter-holistics-ci.yml create mode 100644 converters/holistics/.gitignore create mode 100644 converters/holistics/README.md create mode 100644 converters/holistics/docs/aql.md create mode 100644 converters/holistics/docs/expressions.md create mode 100644 converters/holistics/docs/fixture.md create mode 100644 converters/holistics/docs/limitations.md create mode 100644 converters/holistics/docs/mapping.md create mode 100644 converters/holistics/docs/measures.md create mode 100644 converters/holistics/docs/reverse.md create mode 100644 converters/holistics/docs/vendor-payload.md create mode 100644 converters/holistics/pyproject.toml create mode 100644 converters/holistics/src/ossie_holistics/__init__.py create mode 100644 converters/holistics/src/ossie_holistics/_yaml.py create mode 100644 converters/holistics/src/ossie_holistics/aml.py create mode 100644 converters/holistics/src/ossie_holistics/aml_to_ossie.py create mode 100644 converters/holistics/src/ossie_holistics/amlgen.py create mode 100644 converters/holistics/src/ossie_holistics/aql.py create mode 100644 converters/holistics/src/ossie_holistics/cli.py create mode 100644 converters/holistics/src/ossie_holistics/datatypes.py create mode 100644 converters/holistics/src/ossie_holistics/errors.py create mode 100644 converters/holistics/src/ossie_holistics/interpolation.py create mode 100644 converters/holistics/src/ossie_holistics/issues.py create mode 100644 converters/holistics/src/ossie_holistics/ossie_to_aml.py create mode 100644 converters/holistics/src/ossie_holistics/sqlrefs.py create mode 100644 converters/holistics/src/ossie_holistics/stash.py create mode 100644 converters/holistics/tests/conftest.py create mode 100644 converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml create mode 100644 converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml create mode 100644 converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml create mode 100644 converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml create mode 100644 converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml create mode 100644 converters/holistics/tests/fixtures/aql/translation.ossie.yaml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml create mode 100644 converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml create mode 100644 converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml create mode 100644 converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml create mode 100644 converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml create mode 100644 converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml create mode 100644 converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml create mode 100644 converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml create mode 100644 converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml create mode 100644 converters/holistics/tests/snapshots.py create mode 100644 converters/holistics/tests/test_cross_vendor.py create mode 100644 converters/holistics/tests/test_issue_codes.py create mode 100644 converters/holistics/tests/test_malformed_input.py create mode 100644 converters/holistics/tests/test_snapshots.py create mode 100644 converters/holistics/tools/check_links.py create mode 100644 converters/holistics/tools/update_snapshots.py create mode 100644 converters/holistics/uv.lock diff --git a/.github/workflows/converter-holistics-ci.yml b/.github/workflows/converter-holistics-ci.yml new file mode 100644 index 000000000..4831ab8c6 --- /dev/null +++ b/.github/workflows/converter-holistics-ci.yml @@ -0,0 +1,94 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Holistics CI +on: + push: + branches: [ "main" ] + paths: + - 'converters/holistics/**' + # core-spec/** is the convention every sibling workflow adopted in #396. + # examples/** is load-bearing here rather than merely conventional: the + # reverse-path snapshot converts examples/tpcds_semantic_model.yaml, so an + # edit there fails this converter's tests without touching a converter file. + - 'core-spec/**' + - 'examples/**' + - '.github/workflows/converter-holistics-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/holistics/**' + - 'core-spec/**' + - 'examples/**' + - '.github/workflows/converter-holistics-ci.yml' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.19" + + - name: Set up Node + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0 + with: + node-version: "22" + + - name: Install the Holistics CLI + # The tests that run `holistics aml compile` and `holistics aml validate` + # skip without the CLI. @holistics/cli-core is the CLI's logic as one + # bundled JavaScript file, with no native binary and no platform + # restriction, so it runs on the runner as it does on a developer's + # machine. @commander-js/extra-typings is imported by its `bin` script + # and missing from its dependencies, so it is installed alongside. + # + # Pinned: 2.3.32 is the first version whose compiler accepts the + # composite relationship in the ecommerce fixture. + run: | + npm install --no-package-lock --prefix "$RUNNER_TEMP/holistics" \ + @holistics/cli-core@2.3.32 @commander-js/extra-typings@14.0.0 + echo "$RUNNER_TEMP/holistics/node_modules/.bin" >> "$GITHUB_PATH" + + - name: Sync dependencies + working-directory: converters/holistics + run: | + uv sync + + - name: Unit Tests + working-directory: converters/holistics + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 62cdf72cc..4456ce416 100644 --- a/converters/README.md +++ b/converters/README.md @@ -90,6 +90,7 @@ The Ossie specification currently defines extensions for the following vendors: | `THOUGHTSPOT` | ThoughtSpot TML (Model + Table/SQL View) | | `CUBE` | Cube data model | | `HEX` | Hex semantic project | +| `HOLISTICS` | Holistics AML dataset | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/holistics/.gitignore b/converters/holistics/.gitignore new file mode 100644 index 000000000..80f74e214 --- /dev/null +++ b/converters/holistics/.gitignore @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +__pycache__/ +*.py[cod] +.pytest_cache/ +*.egg-info/ +dist/ +build/ +.venv/ +venv/ + +# Rendered previews +*.html diff --git a/converters/holistics/README.md b/converters/holistics/README.md new file mode 100644 index 000000000..3d25c8d35 --- /dev/null +++ b/converters/holistics/README.md @@ -0,0 +1,179 @@ + + +# Apache Ossie Holistics converter + +Converts between a [Holistics](https://docs.holistics.io) AML dataset and an +[Apache Ossie](https://github.com/apache/ossie) semantic model. + +- `to-ossie` reads the JSON `holistics aml compile` produces, and writes one Ossie semantic model. +- `to-aml` reads one Ossie semantic model, and writes AML source files. + +One AML `Dataset` becomes one Ossie semantic model. + +| AML | Ossie | +|---|---| +| the `Dataset` | the document root | +| each model | a `datasets[]` entry | +| each model `dimension` | a `fields[]` entry on that dataset | +| each model `measure` | a `metrics[]` entry named `__` | +| each dataset-level `dimension` | a `fields[]` entry on the model it names | +| each dataset-level `metric` | a `metrics[]` entry | +| each `relationship` | a `relationships[]` entry | + +[Mapping](docs/mapping.md) covers data types, relationship kinds and the rest. + +AQL travels under the `HOLISTICS_AQL` dialect, registered upstream the way `THOUGHTSPOT` was in +[#351](https://github.com/apache/ossie/pull/351). The converter does not translate AQL into SQL, and does not +rewrite warehouse SQL from one dialect into another. [Limitations](docs/limitations.md) lists what that costs. + +## The two SQLs + +Both sides write SQL, and the two are not the same text. + +- **Holistics SQL** is an AML `@sql` body. It interpolates through `{{ ... }}`, and it resolves inside the one + model it sits in. +- **Ossie SQL** is the `expression` of a `dialects[]` entry. It writes a reference as `dataset.field`, and its + dialect tag names the warehouse it was written for. + +**Warehouse SQL** is what Holistics generates and sends to the database. This converter reads and writes the +first two, and Holistics generates the third. + +## Installation + +```bash +pip install -e . # from a checkout of this directory +``` + +Python 3.10+. The runtime dependencies are `PyYAML` and `sqlglot`. `sqlglot` is here for `to-aml`, which uses +it to find the column references in an Ossie SQL expression. [Ossie to AML](docs/reverse.md#expressions) says +why that is needed. + +## Usage + +AML to Ossie. `to-ossie` takes compiled JSON, so run the Holistics CLI first: + +```bash +holistics aml compile ecommerce.dataset.aml -r . -o build/ +ossie-holistics to-ossie build/ecommerce.dataset.aml.json -o ecommerce.yaml --sql-dialect BIGQUERY +``` + +Ossie to AML. `holistics aml validate` checks the result and needs no account: + +```bash +ossie-holistics to-aml ecommerce.yaml -o aml/ --sql-dialect BIGQUERY --data-source-name my_warehouse +holistics aml validate aml/ecommerce.dataset.aml -r aml/ +``` + +Options: + +- `-o`, required in both directions. A file for `to-ossie`, a directory for `to-aml`. +- `--sql-dialect DIALECT`, required in both directions. The choices are `ANSI_SQL`, `OSSIE_SQL_2026`, + `BIGQUERY`, `DATABRICKS` and `SNOWFLAKE`, the Ossie dialects sqlglot can parse. + - On `to-ossie` it tags every `@sql` body. AML does not record which warehouse a body was written for. + - On `to-aml` it names the warehouse the Holistics connection reads. An expression already in that dialect + is written as it stands. One in `ANSI_SQL` or `OSSIE_SQL_2026` is rendered into it, with a WARNING + showing both texts. An expression offering neither gets an ERROR, and its body is written unchanged. +- `--data-source-name NAME`, on `to-aml`. The Holistics connection the models read from. AML requires it on a + dataset and Ossie has no field for it. Required when the Ossie document carries no `HOLISTICS` entry in + `custom_extensions`. +- `--issues FILE`. Where to write the issue log, a YAML sequence. Defaults to stderr. Never mixed into the + document output. A message runs to a paragraph, so YAML folds it rather than putting it on one line. +- `--force`. Overwrite existing output files. Without it, an existing target stops the run before anything is + written. + +The process exits `1` when the issue log holds an ERROR-severity issue, and `0` otherwise. + +`to-aml` writes one dataset file plus one model file per Ossie dataset. An Ossie dataset named +`reporting__calendar` goes to `modules/reporting/calendar.model.aml`: + +``` +aml/ + ecommerce.dataset.aml + ecommerce_orders.model.aml + modules/reporting/calendar.model.aml +``` + +## Documents + +| Document | What it covers | +|---|---| +| [The fixture](docs/fixture.md) | The AML source, how to recompile it, and the shape of the compiled JSON | +| [Mapping](docs/mapping.md) | Which AML construct becomes which Ossie construct, including relationships and data types | +| [Expression translation](docs/expressions.md) | What the converter emits for each definition body, and what it leaves untranslated | +| [Ossie to AML](docs/reverse.md) | The generated file layout, and what AML cannot express | +| [Ossie SQL to AQL](docs/aql.md) | How a cross-dataset metric's Ossie SQL becomes AQL, and what is not translated | +| [Measures](docs/measures.md) | How `aggregation_type` and a definition body compose into one Ossie metric expression | +| [The HOLISTICS payload](docs/vendor-payload.md) | Everything carried in `custom_extensions` | +| [Limitations](docs/limitations.md) | The coverage matrix, and what is known not to work | + +## Fixtures + +`tests/fixtures/ecommerce/` holds the AML source of the `ecommerce` dataset, written for this converter, and +that source compiled by the Holistics CLI. The `expected*` files are what the converter produces from it in +both directions. + +`tests/fixtures/tpcds/expected_aml/` is `examples/tpcds_semantic_model.yaml` converted to AML. That document +carries no `HOLISTICS` stash, so every AML property with no Ossie field is derived or supplied rather than +read back. + +`tests/fixtures/aql/` is written for the Ossie SQL to AQL translation. Every metric in it spans two datasets, +which is what makes AML take an `@aql` body, and each covers a different path: a function AQL has, a function +it does not, a column no dataset declares, a field with no `datatype`, and two that are dropped with an +ERROR. + +`tests/fixtures/metric_refs/` is written for a metric that names another metric. AML gives a metric two homes +that take different bodies, so the home the named metric landed in decides where the naming one can go. Each +metric covers one way that lands. + +## Development + +```bash +uv run pytest +python3 tools/check_links.py # every cross-document link and anchor resolves +python3 tools/update_snapshots.py # rewrite the snapshots after a deliberate change +``` + +Most tests compare a committed snapshot against what the converter produces now. The rest check the output +against something other than itself. + +These run the Holistics CLI: + +- the generated AML through `holistics aml validate`, for all three fixtures +- the committed compiled JSON against what `holistics aml compile` produces from the committed AML source +- a round trip: compile the generated AML, convert that back to Ossie, and compare it with the document + `to-aml` started from. For `ecommerce` the two are byte-identical. +- `to-aml` over every other Ossie document in this repository, each one compiled. See + `tests/test_cross_vendor.py`, which found two crashes the two fixtures missed. + +These need nothing beyond the package: + +- the converted document against `core-spec/ossie-schema.json` +- every qualified `.` reference against the fields the document declares +- the issue log read back from its folded YAML, which a fold inside a URL would corrupt +- each path through the Ossie SQL to AQL translation, pinned by metric name so a snapshot update cannot quietly + stop covering one +- the issue-code table in `docs/limitations.md` against the codes the source raises, in both directions. See + `tests/test_issue_codes.py` +- a malformed document reaching `to-aml`, which must report rather than raise. See + `tests/test_malformed_input.py`. + +CI installs the CLI, so the first group runs there too: + +```bash +npm install --no-package-lock --prefix /tmp/holistics \ + @holistics/cli-core@2.3.32 @commander-js/extra-typings@14.0.0 +export PATH="/tmp/holistics/node_modules/.bin:$PATH" +``` + +`@holistics/cli-core` is the CLI's logic as one bundled JavaScript file, so it needs only Node. The +[native CLI](https://docs.holistics.io/docs/cli) works the same way for these tests. With neither installed, +that first group skips and the rest of the suite still runs. diff --git a/converters/holistics/docs/aql.md b/converters/holistics/docs/aql.md new file mode 100644 index 000000000..339d1541f --- /dev/null +++ b/converters/holistics/docs/aql.md @@ -0,0 +1,179 @@ + + +# Ossie SQL to AQL + +AML gives a metric that spans datasets one body type, `@aql`. So `to-aml` renders its Ossie SQL as AQL, one +construct at a time. Along with rendering a portable body into `--sql-dialect`, this is the only place the +converter rewrites an expression rather than carrying it. + +## Why a metric is translated + +An AQL expression can name fields from several models. A `@sql` expression can name only fields of the model +it sits in. Naming another model's field from a `@sql` body fails with +`Cannot refer to external field in TableModel/QueryModel`. + +A metric whose expression names two datasets therefore has to be AQL, in either place AML allows one. + +| Where it is written | Body it accepts | +|---|---| +| a dataset-level `metric` | `@aql` only. The compiler rejects `@sql` there with `Incompatible property 'definition' of 'Metric'` | +| a model-level `measure` | `@sql` or `@aql`, and a `@sql` one names only this model's fields | + +`customer_lifetime_value` in `examples/tpcds_semantic_model.yaml` is +`SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)` and becomes +`sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk)`. + +## What translation changes in the models + +AQL resolves a reference differently from Ossie SQL, so rendering a metric can need two changes to the models it +names. The converter reports each one. + +| What the Ossie SQL does | What the converter writes | Why | +|---|---|---| +| reads a column no dataset declares as a field | a hidden `dimension` on that model over the column of that name | AQL resolves against declared fields only, answering ``Field `subtotal` not found in model `orders` `` where Ossie SQL resolves against the table | +| sums or averages a field with no Ossie `datatype` | `type: 'number'` on that dimension | the fallback is `text`, and AQL then answers ``The `sum` function expects `Dimension(Number)`, but got `Text` `` | + +In the second row the metric is the only evidence the field is a number, because the Ossie document +never said so. + +## A name that is another metric + +An Ossie SQL body may name a metric where a column would go. AQL resolves both, and spells them differently. + +| The metric named | The AQL written | +|---|---| +| a dataset-level `metric` | `quantity_per_customer`, the bare name | +| a `measure` on a model | `items.items_quantity`, qualified by the model holding it | + +[Ossie to AML](reverse.md#a-metric-that-names-another-metric) covers which home a metric lands in, which is +what picks the row. + +## A window function depends on how the metric is queried + +A window function computes over a set of rows, and the query decides which rows those are. The two AML +homes handle that differently. + +**On a model, as a `@sql` measure.** AML carries the body as written, and it compiles. The warehouse SQL it +generates is valid only for a query that groups by the columns in the frame. +`SUM(SUM(q)) OVER (ORDER BY oid ...)` queried alongside `oid` gives: + +```sql +SELECT oid, SUM(SUM(quantity)) OVER (ORDER BY oid ...) FROM t GROUP BY 1 +``` + +Queried on its own, as a grand total, it gives the same expression with no `GROUP BY`, which the warehouse +rejects. Holistics checks none of this at conversion time, so the converter raises a WARNING naming the +columns the frame reads. + +**On the dataset, as a `metric`.** The body has to be AQL, and the next section is why that cannot be +written mechanically. + +## A window function cannot be translated + +Even though AQL has window functions, this converter cannot output one. + +The two languages take the frame from different places. Ossie SQL carries the partition and the ordering in the +expression, inside the `OVER` clause. AQL takes both from the grain of the query being run. + +Translating one into the other means deciding which query grain a given `OVER` clause corresponds to. The +Ossie specification defines no semantics for how a window function behaves when queried, so nothing says +which grain that is. Picking one changes what the metric computes. + +The ERROR therefore records a limit of this converter. AQL can express the metric, so write the AQL by hand +to carry one of these over. + +`brand_rank_in_store` in `examples/tpcds_semantic_model.yaml` is one of these: +`RANK() OVER (PARTITION BY store.s_store_sk ORDER BY SUM(store_sales.ss_ext_sales_price) DESC)`. + +AQL's window functions are listed under Window Function in the +[AQL function reference](https://docs.holistics.io/reference/aql/functions), and +[`rank`](https://docs.holistics.io/reference/aql/rank) shows one in full. + +## The function table + +Most Ossie SQL functions keep their name in AQL, lowercased. `SUM(x)` becomes `sum(x)`, `ROUND(x, 2)` becomes +`round(x, 2)`, `SAFE_DIVIDE(a, b)` becomes `safe_divide(a, b)`. 45 of the table's 51 entries work that way, +so the table is worth reading only for the ones that do not. + +| Ossie SQL | AQL | +|---|---| +| `LENGTH` | `len` | +| `POWER` | `pow` | +| `STDDEV` | `stdev` | +| `STDDEV_POP` | `stdevp` | +| `VARIANCE` | `var` | +| `VARIANCE_POP` | `varp` | +| `COUNT(DISTINCT x)` | `count_distinct(x)` | + +The last is not a rename but a reshape. Ossie SQL puts `DISTINCT` inside the argument list and AQL puts it in the +function name, so `COUNT` reads as either `count` or `count_distinct` depending on what it wraps. + +The table itself comes from parsing every name AQL registers and reading which node sqlglot produced, rather +than from matching names by eye. + +The table leaves out some names. What happens to one depends on whether the original call can be read back +out of the source text, which [the passthrough](#the-passthrough) explains. + +**A name that is one sqlglot node with several spellings.** `LPAD` and `RPAD` are both `Pad`, `LTRIM` and +`RTRIM` are both `Trim`, `LOG` and `LOG10` are both `Log`. The node says which shape was parsed, not which +name was written, so rendering from it would turn `LPAD(s, 5)` into `PAD(s, 5)`. The node's offset points at +the name in the source, so the passthrough recovers it. + +**A name whose arguments sqlglot rewrote.** `LOG10(x)` gains a base argument the source never had, and +`DATE_TRUNC('month', d)` has its unit normalized to `'MONTH'`. Either way the argument carries no offset. +Dropping the invented base is right and dropping the normalized unit is not, and an absent offset does not +say which case it is, so both raise an ERROR. + +**A name whose arguments mean something else in AQL.** Ossie SQL `SUBSTRING(s, start, len)` and `STRPOS(s, sub)` +do not line up with AQL's `mid` and `find`, so neither is in the table. The passthrough carries `STRPOS` +under its own name with the warehouse's own meaning. `SUBSTRING`'s node carries no offset, so the converter +raises an ERROR for it. + +## The passthrough + +A function with no AQL name becomes `sql_number('NAME', arg, ...)`, AQL's raw-SQL call, which hands the +warehouse the original call with AQL-resolved arguments. + +``` +MY_WAREHOUSE_UDF(COUNT(users.id), 3) -> sql_number('MY_WAREHOUSE_UDF', count(users.id), 3) +LPAD(users.full_name, 5) -> sql_number('LPAD', users.full_name, 5) +``` + +The name and the argument order come from the source text rather than from sqlglot, which names a node +after the shape it parsed. A function sqlglot does not model keeps its name and arguments in an `Anonymous` +node already. For one it does model, the node carries the offset of the name token and each argument carries +its own, so the converter reads both back out of the source. + +Every argument has to carry an offset for that to work. An argument sqlglot synthesized or normalized has +none, and the converter raises an ERROR rather than rebuild the call without it. + +The family is typed by return value: `sql_number`, `sql_text`, `sql_date`, `sql_datetime` and +`sql_truefalse`. A metric body is numeric, so the converter uses `sql_number`. + +## Reading it back + +`tests/fixtures/aql/translation.ossie.yaml` holds one metric for each path through this document. + +| Metric | Path | +|---|---| +| `quantity_per_customer` | aggregates with AQL names | +| `rounded_ratio` | scalar functions with AQL names | +| `untyped_total` | a field with no Ossie `datatype` | +| `borrowed_column_total` | a column no dataset declares as a field | +| `passthrough_call` | a function with no AQL name | +| `windowed_rank` | a window function, dropped with an ERROR | +| `unmapped_function` | a call sqlglot rewrote, dropped with an ERROR | + +`tests/test_snapshots.py` pins each result by metric name, and runs the generated AML through +`holistics aml validate`, which type-checks the AQL rather than only parsing the file. + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/expressions.md b/converters/holistics/docs/expressions.md new file mode 100644 index 000000000..8ea49d377 --- /dev/null +++ b/converters/holistics/docs/expressions.md @@ -0,0 +1,114 @@ + + +# Expression translation + +AML writes a field body in one of two languages, named by the heredoc. + +| Heredoc | What it holds | Ossie dialect | +|---|---|---| +| `@sql` | Holistics SQL: warehouse SQL with `{{ ... }}` interpolation | the `--sql-dialect` argument | +| `@aql` | AQL, the Holistics analytical query language | `HOLISTICS_AQL` | + +The converter substitutes the interpolation and passes the rest of each body through as written. It does not +translate AQL into SQL, and it does not re-render warehouse SQL from one dialect into another. + +## Why AQL travels untranslated + +AQL is not SQL. `count(ecommerce_orders.id) | where(...) | of_all()` is relationship-aware, so no SQL +rendering of it is correct without the dataset's whole relationship graph. + +The specification defines `dialects[]` as `{dialect, expression}` pairs, so a value a converter cannot +translate still travels, tagged with the language it is valid in. `HOLISTICS_AQL` is a member of the +`Dialect` enumeration, registered the way `THOUGHTSPOT` was in +[#351](https://github.com/apache/ossie/pull/351). An AQL body is therefore carried rather than stashed, and +every field keeps the `expression` that Ossie requires. + +## Interpolation + +A `@sql` body interpolates through `{{ ... }}`. + +| Form | Names | Becomes | +|---|---|---| +| `{{ #SOURCE.column }}` | a column of this model's relation | the column name | +| `{{ field }}` | another dimension of the same model | `.` | +| `{{ field }}` where `field` is a measure | a measure of the same model | the metric's own name | + +A measure is the one reference that is not dataset-qualified, because a measure becomes a root-level Ossie +metric rather than a field inside a dataset. See [measures](measures.md#naming). + +References chain. `cities.downstream_measure` is `{{ upstream_measure }} + 1`, and `upstream_measure` is +`COUNT({{ id }})`. + +## What the converter emits + +| AML definition | Ossie expression | Example | +|---|---|---| +| absent | the column sharing the field's name | `ecommerce_orders.id` becomes `id` | +| exactly one interpolation | what that interpolation resolves to | `{{ #SOURCE.quantity }}` becomes `quantity` | +| anything else | the body, with every interpolation resolved | `CAST({{ #SOURCE.created_at }} AS DATE)` becomes `CAST(created_at AS DATE)` | +| `@aql` | the body, unchanged | | + +Every `@sql` row takes the `--sql-dialect` tag, and the `@aql` row takes `HOLISTICS_AQL`. Holistics SQL is +warehouse SQL throughout, so one dialect covers all of them and a reader needs no rule for which field +carries which tag. + +A bare reference runs on any warehouse. A body with SQL around it runs on the one it was written for, and +`ecommerce_users.signup_month` shows why: it is `DATE_TRUNC({{ #SOURCE.created_at }}, month)`, which is +BigQuery argument order. + +### An omitted definition + +An omitted `definition` is a default rather than a missing value. AML fills it with +`{{ #SOURCE. }}`, the source column sharing the field's name. + +- Only a dimension may omit it. A measure must carry one, and the compiler says so rather than defaulting: + `Property 'definition' is missing on element but required on type 'Measure'`. +- The default is the same in a `TableModel` and a `QueryModel`. A `QueryModel` resolves it against the + columns its query returns rather than against a table. +- Omitting it is common. Most dimensions in the Holistics AMQL test corpus do. + +### Why a sibling reference is qualified + +A `{{ field }}` names another AML field in the same model, and that field becomes an Ossie field in the same +dataset, so the reference travels as `dataset.field` in a dimension body and a measure body alike. + +- `ecommerce_users.age` is `extract(year from age({{ birth_date }}))` and becomes + `extract(year from age(ecommerce_users.birth_date))`. +- `ecommerce_orders.last_created_at` is `{{ created_at }}` under `aggregation_type: max` and becomes + `MAX(ecommerce_orders.created_at)`. + +A bare name would be right only by accident. An AML field usually shares its name with the column it reads, +which makes `birth_date` look like it would resolve on its own, and AML does not require the two to match. +`dimension created_at { definition: @sql {{ #SOURCE.sign_up_at }} }` is a field named `created_at` over a +column named `sign_up_at`, so a bare `created_at` would name a column that does not exist. + +The shipped examples read the same way. `customer.customer_full_name` in +`examples/tpcds_semantic_model.yaml` is `c_first_name || ' ' || c_last_name`, where the field `c_first_name` +has the expression `c_first_name`, so the bare name could be either. The metric expressions settle it, and +they qualify: `SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)`. + +Referencing also beats inlining. Inlining would copy the referenced field's body into every dependent +expression, so one warehouse-specific body spreads across many and an edit to the referenced field stops +reaching them. The specification points the same way: its "Not Supported in Expressions" table answers both +subqueries and CTEs with "use field references instead". + +## Where the dialect comes from + +A `@sql` body is the warehouse's SQL, and AML records nothing that says which warehouse. `data_source_name` +is the closest thing to it, and it is an instance-local connection name chosen by whoever created the +connection. + +`to-ossie` therefore requires `--sql-dialect`. Its value is one of `ANSI_SQL`, `OSSIE_SQL_2026`, `BIGQUERY`, +`DATABRICKS` or `SNOWFLAKE`, the Ossie dialects sqlglot can parse. The reverse path parses the expression to +restore its field references, so a dialect outside that set would convert in one direction only. + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/fixture.md b/converters/holistics/docs/fixture.md new file mode 100644 index 000000000..4878e1abf --- /dev/null +++ b/converters/holistics/docs/fixture.md @@ -0,0 +1,43 @@ + + +# Produce the fixture + +The forward path reads compiled JSON, not AML source. The [Holistics CLI](https://docs.holistics.io/docs/cli) does the compilation. The reverse path writes AML source directly, and never reads this fixture. See [Ossie to AML](reverse.md). + +```bash +holistics aml compile ecommerce.dataset.aml -r . -o +``` + +`-r` names the root of the AML repository. The compiler needs it to resolve module paths such as `reporting::calendar`. Run the +command from the directory that holds the dataset file, or pass the file path and set `-r` to the repository root. + +The AML source sits in `tests/fixtures/ecommerce/aml/`. Regenerate the compiled JSON from it with Holistics CLI 2.3.32 or later. The composite relationship needs that version: + +```bash +cd tests/fixtures/ecommerce/aml +holistics aml compile ecommerce.dataset.aml -r . -o .. +``` + +The source is small. It models a marketplace in eight models, and each one covers a row of the mapping, so trimming it further drops a construct the mapping describes. + +## What the compiled JSON looks like + +The CLI emits one self-contained JSON document per dataset. The document inlines every model it references, so +it needs no sibling files. It also inlines a full copy of a model under each dataset-level dimension that names +that model, and under each `QueryModel` for every model its query references, so most of the file is +duplication. Read `models` and `relationships` as the single source of truth, and treat the copies under +`dimension[].model` and `models[].models` as redundant. + +Each node carries a `__type__` tag. The converter dispatches on that tag rather than on the shape of the object. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/limitations.md b/converters/holistics/docs/limitations.md new file mode 100644 index 000000000..c0cd4fde9 --- /dev/null +++ b/converters/holistics/docs/limitations.md @@ -0,0 +1,128 @@ + + +# Limitations + +## Coverage matrix + +Every construct the converter does not carry, with its consequence. A construct it drops or changes raises a +structured issue at conversion time, listed under [Issue codes](#issue-codes). + +| Construct | Limitation | Consequence | +|---|---|---| +| AQL field and metric bodies | Carried under `HOLISTICS_AQL`, never translated to SQL | A consumer that does not speak AQL cannot evaluate these, though it can read and round-trip them | +| Inactive relationships | `active: false` has no Ossie marker. **WARNING severity**, one issue per relationship | A consumer joins across a relationship that Holistics leaves out by default | +| Params | Runtime inputs with no Ossie equivalent | A `@sql` body that reads a param cannot be resolved statically | +| `number` precision | AML does not distinguish exact from approximate | A consumer reading `Decimal` may infer exactness AML never stated. The stash holds the original AML type | +| Warehouse SQL portability | Emitted under `--sql-dialect`, never rewritten | A document converted for one warehouse does not run on another | +| Dashboards, reports and canvases | Out of scope. Ossie models semantics, not presentation | No loss to the semantic model | +| A query body using Holistics template syntax | Detected and carried unresolved, never resolved. **ERROR severity** | The dataset's `source` is not runnable SQL. A consumer must resolve it through Holistics or drop the dataset | +| `RangeRelationship` and `ManyToManyRelationship` | Neither has an Ossie form. **ERROR severity**, one issue per relationship | A range or many-to-many join is dropped, so a consumer sees the two datasets as unrelated. Both carry an AQL `match` rather than column pairs, which is why no column mapping exists | +| A relationship `where` filter | A `RelationshipFilter` restricts the join condition. No Ossie equivalent | The join widens to every matching row, so a consumer counts rows Holistics excludes. **WARNING severity** | +| A `QueryModel` becoming a query in `source` | `source` is one string, so a consumer reads the text to tell a query from a table reference. **WARNING severity** | Whether a given target reads it as a query depends on that target. See below | + +These are the reverse path's, writing AML from Ossie. + +| Construct | Limitation | Consequence | +|---|---|---| +| A metric spanning datasets, carrying Ossie SQL | Rendered as AQL, which is the only body a dataset-level `metric` accepts. **WARNING severity** | The AQL is a construct-for-construct map of the Ossie SQL, so it is worth reading once. See [Ossie SQL to AQL](aql.md) | +| A metric whose Ossie SQL this converter does not write as AQL | A window function, or a construct outside the map. **ERROR severity** | The metric is dropped. AQL may well express it; the converter does not translate it. See [Ossie SQL to AQL](aql.md) | +| A `@sql` body with a window function | The warehouse SQL it generates is valid only for a query that groups by the frame's columns. **WARNING severity** | A grand total over it produces warehouse SQL the database rejects. See [Ossie SQL to AQL](aql.md#a-window-function-depends-on-how-the-metric-is-queried) | +| An expression sqlglot cannot parse, or tagged with a dialect that is not SQL | Its column references cannot be located, so they cannot be written as interpolation. **ERROR severity** | The body is written out unchanged. It compiles, and AQL cannot see which fields it reads, so query rewriting and optimisation lose that link | +| An expression in a warehouse dialect that is not `--sql-dialect` | Translating Ossie SQL between warehouses is out of scope. **ERROR severity** | The body is written unchanged and will not run on the connection. An `ANSI_SQL` or `OSSIE_SQL_2026` body is rendered into the target instead, with a WARNING | +| A qualifier naming no dataset in the document | A table alias from a subquery has no AML form. **ERROR severity** | The reference is left as written, and the body compiles without the field linkage | +| A `@sql` body naming a field of another dataset | A `@sql` expression names only fields of the model it sits in. **ERROR severity** | Holistics rejects it at SQL generation with "Cannot refer to external field in TableModel/QueryModel". An expression that names two models belongs in AQL | +| A field with no `datatype` | AML requires `type` on every dimension. **WARNING severity** | `text` is written. `date_dim.d_quarter_name` in `examples/tpcds_semantic_model.yaml` is one | +| A name AML cannot spell | An Ossie name is any non-empty string. An AML identifier is letters, digits and underscore, with no leading digit. **WARNING severity** | Each illegal character becomes an underscore, so `Flights semantic model` in `examples/flights.semantic_model.yaml` is written `Flights_semantic_model`. Two names that collapse onto one spelling stop the run | +| A semantic model sharing a name with one of its datasets | Ossie keeps the two in separate namespaces and AML has one namespace for `Dataset` and `Model`. **WARNING severity** | The AML dataset takes a `_dataset` suffix. Models and relationships are unaffected, since nothing references it. `converters/cube/tests/fixtures/databricks_ossie.yaml` names both `orders` | +| File layout | The compiled JSON records no file provenance | `AML to Ossie to AML` returns the same models, not the same files. See [Ossie to AML](reverse.md) | +| Comments and property order | Neither survives the forward path | A diff against the original AML shows both, even where nothing semantic changed | + + +## Issue codes + +Every code the converter writes to its issue log, with the severity it carries. The sections above give the +reasoning behind the ones that need it. + +`to-ossie`: + +| Code | Severity | Raised when | +|---|---|---| +| `HOLISTICS_AQL_NOT_TRANSLATED` | INFO | A body is AQL, carried under `HOLISTICS_AQL` | +| `HOLISTICS_INACTIVE_RELATIONSHIP` | WARNING | `active: false`, which Ossie cannot express | +| `HOLISTICS_PARAM_DROPPED` | WARNING | A model declares params, stashed rather than carried | +| `HOLISTICS_QUERY_SOURCE` | WARNING | A `QueryModel` becomes a query in `source` | +| `HOLISTICS_RELATIONSHIP_FILTER` | WARNING | A relationship carries a `where` filter | +| `HOLISTICS_UNKNOWN_DATATYPE` | WARNING | An AML type outside the data type table | +| `HOLISTICS_UNRESOLVED_QUERY_TEMPLATE` | ERROR | A query body uses Holistics template syntax | +| `HOLISTICS_UNSUPPORTED_RELATIONSHIP` | ERROR | A range or many-to-many relationship | +| `HOLISTICS_UNKNOWN_AGGREGATION` | ERROR | An `aggregation_type` outside the aggregation table | +| `HOLISTICS_UNRESOLVED_REFERENCE` | ERROR | An interpolation naming nothing in the dataset | + +`to-aml`: + +| Code | Severity | Raised when | +|---|---|---| +| `HOLISTICS_ASSUMED_TYPE` | WARNING | A field has no `datatype`, so `text` is written | +| `HOLISTICS_NAME_NOT_SPELLABLE_IN_AML` | WARNING | A name holds characters AML identifiers do not | +| `HOLISTICS_DATASET_NAME_COLLIDES_WITH_MODEL` | WARNING | The model name is also a dataset name | +| `HOLISTICS_EXPRESSION_RENDERED_INTO_TARGET_DIALECT` | WARNING | A portable body was rendered into `--sql-dialect` | +| `HOLISTICS_AGGREGATION_NOT_STRIPPED` | WARNING | A stashed aggregation does not wrap the whole expression | +| `HOLISTICS_METRIC_RENDERED_AS_AQL` | WARNING | A cross-dataset metric's Ossie SQL became AQL | +| `HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION` | WARNING | An AQL metric reads a column no dataset declares | +| `HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING` | WARNING | A `@sql` body has a window function | +| `HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL` | ERROR | A construct the converter does not write as AQL | +| `HOLISTICS_CROSS_MODEL_SQL_BODY` | ERROR | A `@sql` body names a field of another dataset, or a metric it cannot spell | +| `HOLISTICS_METRIC_REFERENCE_CYCLE` | ERROR | A metric references itself through another metric | +| `HOLISTICS_UNKNOWN_QUALIFIER` | ERROR | A qualifier naming no dataset in the document | +| `HOLISTICS_UNPARSEABLE_SQL` | ERROR | sqlglot cannot read the body at its declared dialect | +| `HOLISTICS_NO_USABLE_DIALECT` | ERROR | No body in the target dialect and none portable | + +## Known limitations + +### Name flattening can collide + +The converter replaces `::` with `__` to turn an AML module path into a flat Ossie dataset name, so +`reporting::calendar` becomes `reporting__calendar`. A doubled separator is used rather than a single `_` because a +single one collides with any model name that already contains an underscore, which most do. A root model named +`reporting_calendar` would take the name `reporting::calendar` wants. + +A collision is still reachable, just rarer. A root model named `reporting__calendar` still takes it, and the +converter fails naming both rather than letting one dataset overwrite the other. The unflattened `__fqn__` +goes to the stash either way, so the reverse path reads the module boundaries from there. + +### A query model becomes a query in `source` + +A `QueryModel` has no table name, only a Holistics SQL body, so its Ossie `source` is that query. The specification +provides for this. `core-spec/spec.md` describes `source` as a "Reference to underlying physical table/view +(e.g., `database.schema.table`) or query". + +`source` is a single string, `{"type": "string", "minLength": 1}`, so a consumer tells a query from a table +reference by reading the text. Several converters do, each with its own test, and they do not all draw the +line in the same place. `(SELECT * FROM public.users)` and `EXEC sp_get_orders` are two spellings where the +answers differ. + +So the converter emits a `QueryModel` as a query in `source` and raises a WARNING naming the model. Whether a +given target reads it as a query depends on that target, and a model worth sharing widely is better +materialised as a table in Holistics first. + +### Warehouse SQL has no dialect of its own + +`Dialect` in `core-spec/ossie-schema.json` is a closed enumeration. Its members +today are `ANSI_SQL`, `SNOWFLAKE`, `MDX`, `TABLEAU`, `DATABRICKS`, `MAQL`, `BIGQUERY`, `SIGMA`, `THOUGHTSPOT`, `DAX`, `HOLISTICS_AQL`, and +`OSSIE_SQL_2026`. + +`HOLISTICS_AQL` covers AQL. It does not cover a `@sql` body, because that body is the warehouse's SQL rather than a +Holistics language, and AML does not record which warehouse. A computed `@sql` body takes its dialect from +`--sql-dialect`, which must name an existing member. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md) and [Ossie SQL to AQL](aql.md). diff --git a/converters/holistics/docs/mapping.md b/converters/holistics/docs/mapping.md new file mode 100644 index 000000000..f86b0990b --- /dev/null +++ b/converters/holistics/docs/mapping.md @@ -0,0 +1,134 @@ + + +# Mapping + +| AML | Ossie | Notes | +|---|---|---| +| `Dataset` | the semantic model (the document root) | One dataset per document, its fields at the root beside `version` | +| `Dataset.description` | `description` | Falling back to `label` when the dataset declares no description. The unused one goes to the stash | +| `TableModel` | `dataset` with `source` from `table_name` | `table_name` arrives already quoted for the warehouse, for example `` `ecommerce`.`order_items` `` | +| `QueryModel` | `dataset` with `source` derived from the `query` body | The body is not plain SQL and cannot be copied verbatim. See [Query model bodies](#query-model-bodies) | +| model `__fqn__` | `dataset.name` | AML joins module paths with `::`. Ossie names are flat, so each `::` becomes `__` | +| `dimension` | `dataset.fields[]` entry | See [Expression translation](expressions.md) | +| `measure` | `metrics[]` entry named `__` | See [Measures](measures.md#naming) | +| dimension `primary_key: true` | `dataset.primary_key` | AML marks the key on the dimension. Ossie carries it on the dataset | +| `dimension.type` | `field.datatype` | See [Data types](#data-types) | +| `relationship(a.x > b.y, true)` | `relationships[]` entry | `>` is many-to-one. `a` becomes `from`, `b` becomes `to` | +| `relationship(a.x - b.y, true)` | `relationships[]` entry | `-` is one-to-one. Ossie has no one-to-one marker, so the kind goes to the stash | +| dataset-level `dimension` | `dataset.fields[]` entry on the model it names | The AML dimension names its owning model in `model` | +| dataset-level `metric` | `metrics[]` entry | Always AQL in practice, so always tagged `HOLISTICS_AQL` | +| everything else | `custom_extensions[HOLISTICS]` | See [The HOLISTICS payload](vendor-payload.md) | + +## Relationships + +AML writes a relationship as a pair of field references plus an active flag. Ossie needs a name, and AML gives +none, so the converter synthesizes `_to_`, the spelling Cube, GoodData, NVIDIA and Orion Belt all use +in their Ossie fixtures. A second relationship between the same pair takes `_2`, then `_3`. The reverse path +reads the endpoints from `from`, `to`, `from_columns` and `to_columns`, so the name it was given does not +change the AML it writes. + +AML has three relationship types, and only one of them has an Ossie form. + +| AML type | Ossie | +|---|---| +| `EqualityRelationship` (`many_to_one`, `one_to_one`) | a `relationships[]` entry | +| `RangeRelationship` (`range`) | none. Its `match` is an AQL condition, not a column pair | +| `ManyToManyRelationship` (`many_to_many`) | none. Ossie relationships run many to one | + +An `EqualityRelationship` carries its first column pair in `from` and `to`, and each further pair in a +repeated `on` block. Blocks combine with AND, which is exactly Ossie's positional `from_columns` and +`to_columns`: + +``` +from: FieldRef { model: 'merchants', field: 'city_id' } from_columns: [city_id, country_code] +to: FieldRef { model: 'cities', field: 'id' } to_columns: [id, country_code] +on { from: merchants.country_code to: cities.country_code } +``` + +The `relationship(a.x > b.y, true)` shorthand carries one pair only. A composite needs the long +`RelationshipConfig` form, which is what the fixture uses for `merchants` to `cities`. + +A relationship endpoint and a model spell the same name two different ways. The endpoint uses dots, as in +`reporting.calendar`, while the model's own `__fqn__` uses colons, as in `reporting::calendar`. The compiler +fills a `__refFqn__` shortcut on some endpoints and leaves it `null` on others, and in the fixture the one it +leaves null names a model inside a module. The converter therefore ignores `__refFqn__` and resolves an +endpoint by replacing each `.` with `::` and looking the result up in `models`. + +A relationship may set `active: false`. Holistics does not join across an inactive relationship unless a query asks +for it. Ossie has no inactive marker. Emitting an inactive relationship as an ordinary Ossie relationship would change +which joins a consumer makes, so the converter records the flag in the stash and raises a WARNING for each one. + +## Query model bodies + +A `TableModel` names a table, so its `source` is that name. A `QueryModel` carries Holistics SQL in `query` instead, +and that body may use Holistics template syntax. The compiler does not resolve it. The body is passed +through, and each referenced model is inlined under `models[]`. + +`delivered_orders` in the fixture compiles to this: + +``` +query: select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} +models[]: [ecommerce_orders, inlined whole, table_name `ecommerce`.`orders`] +``` + +| Form | Means | +|---|---| +| `{{ #model }}` | the referenced model's relation | +| `{{ #model.* }}` | every column of the referenced model | +| `{% filter(param) %} ... {% end %}` | a predicate, once a param value arrives | + +### What the converter does + +A query body that matches none of those forms is plain SQL. It becomes the dataset's `source` unchanged. + +A query body that matches any of them is carried unresolved. The converter puts the verbatim body in +`source`, records the forms it found in the stash, and raises an ERROR naming the dataset. + +### Why it resolves nothing, including what it could + +`models[]` inlines the referenced model with its `table_name`, so `{{ #ecommerce_orders }}` could be resolved +to `` `ecommerce`.`orders` ``. The converter does not do it. + +A filter block cannot be resolved at conversion time, because the param value arrives at query time. So a +body can carry both a form the converter could resolve and a form it cannot. Resolving the first leaves a +`source` that reads as SQL and is not, and a reader would have to know which forms this converter handles to +tell. Leaving every form in place keeps the body uniformly unresolved. + +Resolving them would also mean a second implementation of Holistics' own resolution rules, kept in step by +hand. That is the same reason the converter does not translate AQL and does not rewrite warehouse SQL. See +[expression translation](expressions.md). + +A dbt semantic manifest carries `node_relation.relation_name`, which dbt resolved during its own +compilation, so `ossie_to_msi.py` reads a finished string and the question does not arise there. The +Holistics CLI passes the template through instead. + +## Data types + +| AML `type` | Ossie `datatype` | `dimension.is_time` | +|---|---|---| +| `text` | `String` | defaults to false | +| `truefalse` | `Boolean` | defaults to false | +| `date` | `Date` | defaults to true | +| `datetime` | `DateTime` | defaults to true | +| `number` | `Decimal` | defaults to false | + +`number` is the one lossy row. AML does not say whether a `number` is exact or approximate, so no Ossie member matches it +exactly. `Decimal` is the closest, because the Ossie specification leaves its precision and scale unspecified. The +converter writes the original AML `type` into the stash, so an Ossie document converted back to AML recovers the AML type +regardless of this choice. + +`is_time` follows the Ossie default rule for each mapped `datatype`, so the converter writes no explicit `is_time`. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/measures.md b/converters/holistics/docs/measures.md new file mode 100644 index 000000000..4ab4c4b6d --- /dev/null +++ b/converters/holistics/docs/measures.md @@ -0,0 +1,63 @@ + + +# Measures + +A measure is the one AML construct where two properties compose into one Ossie expression. `aggregation_type` names the +aggregate, and `definition` gives the body. + +## Naming + +A measure belongs to a model. An Ossie metric sits at the document root, where one namespace covers the whole +document, so two models with a `total_quantity` each would collide. The converter therefore names a metric +`__`: `order_items.total_quantity` becomes the metric `order_items__total_quantity`. A +dataset-level AML metric keeps its own name, because it was already document-wide. + +The fixture needs this. `ecommerce_orders` has a measure `count_orders` and the dataset has a metric +`count_orders`, and the two become `ecommerce_orders__count_orders` and `count_orders`. + +`__` rather than `.` because a metric reference inside an expression is the metric's own name, and the +sibling converters that face the same collision spell it the same way: Cube writes `orders__count`, and Omni +writes `__`. A field reference stays dotted, as in `cities.id`, because fields are namespaced +per dataset and the shipped examples qualify them that way. + +| `aggregation_type` | Meaning | Ossie expression | +|---|---|---| +| `sum`, `count`, `count distinct`, `min`, `max`, `average`, `median` | The aggregate wraps the body | `SUM()` and so on | +| `custom` | The body already aggregates | the resolved body, unchanged | + +Worked examples from the fixture. + +`order_items.total_quantity` has `aggregation_type: sum` and the body `{{ #SOURCE.quantity }}`. The body is a bare source +column, so it resolves to `quantity`, and the metric expression is `SUM(quantity)` under `--sql-dialect`. + +`order_items.custom_total_quantity` has `aggregation_type: custom` and the body `sum({{ #SOURCE.quantity }})`. The aggregate +is already in the body, so the converter wraps nothing and emits `sum(quantity)`. + +`ecommerce_orders.last_created_at` has `aggregation_type: max` and the body `{{ created_at }}`. That is a sibling reference, +not a source column. `created_at` is a dimension in the same model, so it becomes a field in the same dataset and the metric expression is `MAX(ecommerce_orders.created_at)`. + +`cities.downstream_measure` has `aggregation_type: custom` and the body `{{ upstream_measure }} + 1`. `upstream_measure` is a measure in the same model, so it becomes a metric and the expression is `cities__upstream_measure + 1`. `upstream_measure` itself is `COUNT({{ id }})`, whose sibling `id` is `{{ #SOURCE.id }}`, so it becomes `COUNT(cities.id)`. +Referencing rather than inlining means the converter never walks that chain, so a reference cycle in AML cannot make it recurse. + +Referencing handles a case inlining could not. A measure whose body names another measure, +under a named `aggregation_type`, would inline to `SUM(COUNT(x))`, which no warehouse accepts. As a reference it is +`SUM(cities__upstream_measure)`, a metric naming a metric, which the semantic layer resolves rather than the warehouse. + +`core-spec/expression_language.md` lists "Column and Metric references" among the supported constructs. The +converter emits the dataset-qualified form the shipped examples use for a field, and the metric's own name for +a metric, and raises a WARNING naming the metric so a consumer can check the spelling against its own +resolver. + +A measure written in `@aql` travels under `HOLISTICS_AQL` whatever its `aggregation_type` says, because the body already contains its own aggregation. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [the HOLISTICS payload](vendor-payload.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/reverse.md b/converters/holistics/docs/reverse.md new file mode 100644 index 000000000..7331aa3dd --- /dev/null +++ b/converters/holistics/docs/reverse.md @@ -0,0 +1,214 @@ + + +# Ossie to AML + +The reverse path writes AML source text. + +``` +AML source --(holistics aml compile)--> JSON --(this converter)--> Ossie YAML +AML source <---------------------(this converter)------------------ Ossie YAML +``` + +`holistics aml compile` reads AML source and writes JSON, and it is the only tool that moves between the +two. So AML source is the format a person edits, a repository holds, and the CLI reads back, and generated +JSON would be a file with no reader. + +## Output layout + +One Ossie document becomes one dataset file plus one model file per dataset: + +``` +/ + .dataset.aml + orders.model.aml a dataset named orders + modules/reporting/calendar.model.aml a dataset named reporting__calendar +``` + +A dataset name splits on `__` back into a module path, so `reporting__calendar` becomes +`modules/reporting/calendar.model.aml` holding `Model calendar`. When the stash carries the original `__fqn__`, +that wins over splitting, because it records where the module boundaries actually were. + +The input file layout is lost on the way in. The compiled JSON records which models a dataset holds and not +which file each came from, so by the time a document reaches Ossie the layout is already gone. AML accepts any +arrangement, so the generator picks one. `AML to Ossie to AML` returns the same models in the generator's +layout. + +## Mapping + +| Ossie | AML | +|---|---| +| the document root | `Dataset { ... }` | +| `description` | the dataset's `description` | +| `dataset` whose `source` is a dotted reference | `Model { type: 'table' table_name: '' }` | +| `dataset` whose `source` is a query | `Model { type: 'query' query: @sql ;; }` | +| `dataset.name` | the model name, `__` split back into a module path | +| `dataset.fields[]` | `dimension { ... }` | +| `dataset.primary_key` | `primary_key: true` on each named dimension | +| `metrics[]` | a `measure` on one model, or a dataset-level `metric` | +| `relationships[]`, one column pair | `relationship(from.col > to.col, true)` in the dataset block | +| `relationships[]`, composite | the long `RelationshipConfig` form, extra pairs in repeated `on` blocks | +| `custom_extensions[HOLISTICS]` | restores `hidden`, `format`, `type`, `aggregation_type`, `param`, `persistence`, `owner`, `table_name`, the relationship kind and `active` | + +## Properties AML requires and Ossie has no field for + +Four properties are mandatory in AML. `label` and `type` on every dimension and measure, and +`data_source_name` and `relationships` on the dataset. The compiler says so rather than defaulting: +`Property 'label' is missing on element but required on type 'Dimension'`. + +| Property | Where it comes from | +|---|---| +| `label` | the stash, else the name titleized the way AML itself does it, so `created_at` becomes `Created At` | +| `type` | the stash, else `datatype` through the reverse of the type table, else `text` with a WARNING | +| `data_source_name` | the stash, else the `--data-source-name` argument. Without either, the run fails | +| a legal identifier | each illegal character in an Ossie name becomes an underscore, and the rename is reported | +| a dataset name no model holds | AML has one namespace for `Dataset` and `Model`, so a collision moves the dataset to `_dataset` | +| `relationships` | the Ossie `relationships[]`, written as `[]` when there are none | + +## Expressions + +An Ossie `expression` may carry several dialects. `--sql-dialect` names the warehouse the Holistics +connection reads, and the converter picks against it: + +| The expression carries | Written as | Issue | +|---|---|---| +| `HOLISTICS_AQL` | `definition: @aql ;;`, unchanged | | +| the `--sql-dialect` dialect | `definition: @sql ;;`, unchanged | | +| `ANSI_SQL` or `OSSIE_SQL_2026` | `@sql`, rendered into `--sql-dialect` by sqlglot | WARNING, showing both texts | +| none of those | `@sql`, written unchanged | ERROR. The body will not run on this connection | + +AQL comes first and is exact, because the body came out of AML unchanged and goes back unchanged. + +A warehouse dialect that is not the target gets an ERROR rather than a rendering. Turning a `SNOWFLAKE` body +into a BigQuery one is a translation between warehouses, which this converter does not do. `ANSI_SQL` and +`OSSIE_SQL_2026` are the exception, because they are the portable forms the specification provides, and +every sibling converter reads `OSSIE_SQL_2026` as an `ANSI_SQL` equivalent. + +A `@sql` body needs every reference written back as Holistics interpolation. A bare column name does compile, +and it is not equivalent: AQL reads the interpolation to learn which field a body depends on, and a body +without it loses the query rewriting and the optimisation that link drives. + +| Ossie expression | AML | +|---|---| +| a bare column name | `{{ #SOURCE.column }}` | +| `.` | `{{ field }}` | +| a metric name owned by this model | `{{ }}` | +| a reference to a field of another dataset | no `@sql` form, so it is written model-qualified and an ERROR names it. See [Ossie SQL to AQL](aql.md) | + +Restoring those references means telling a column apart from a keyword, a function name and a date part. The +converter parses the expression with sqlglot at the dialect the expression declares, takes the character +offsets sqlglot reports for each column it found, and splices the replacement text into the original string. +In BigQuery's `DATE_TRUNC(created_at, month)` sqlglot reads `created_at` as a column and `month` as a date +part, so the result is `DATE_TRUNC({{ #SOURCE.created_at }}, month)`. + +The Ossie SQL is never written back out through sqlglot. `DATE_TRUNC(created_at, month)` rendered through the +default dialect returns `DATE_TRUNC(created_at, MONTH)`, which that same default dialect then re-reads with +the unit and the expression exchanged. Splicing on offsets keeps every character outside a replaced span +exactly as it was, so no round trip through the writer can happen. + +An expression the parser cannot read, or one tagged with a dialect that is not SQL, raises an ERROR naming +the field. The body is written out unchanged, which compiles and loses the field linkage, and the error says +so. + +## Metrics have two possible homes + +An Ossie metric sits at the document root and may span datasets. AML offers two places to put one. A stashed +`model` decides it, because it records which model the measure came from. Without one the expression +decides: + +- An expression naming exactly one dataset becomes a `measure` on that model. +- An expression naming several, or none, becomes a dataset-level `metric`. +- An expression naming another metric counts the datasets that metric resolves to as its own. + +### A metric that names another metric + +An Ossie body may name a metric where a column would go. `cities__upstream_measure + 1` in +`tests/fixtures/ecommerce` is one, and the forward path writes that shape every time one AML measure +references another. + +Which home the named metric landed in decides where the naming one can go. + +| The named metric is | The naming metric becomes | +|---|---| +| a `measure` on the same model | a `measure` on that model, written `{{ items_quantity }}` | +| a `measure` on another model | a dataset-level `metric`, written `items.items_quantity` in AQL | +| a dataset-level `metric` | a dataset-level `metric`, written as the bare name in AQL | + +AQL names a measure in any model, and a `@sql` body has no spelling for a dataset-level metric. So the last +two rows move the naming metric onto the dataset, where AQL resolves the reference. + +The third bullet above is what puts a metric on the right model. `items_quantity + 1` names no dataset of +its own. Counting only its columns sends it to the dataset, and `items_quantity` is an Ossie SQL name that AQL does +not resolve there, so the metric is dropped. + +Two shapes are still an ERROR, and a stash or a qualifier that disagrees with the body produces both: + +- A `@sql` body naming a dataset-level metric. A qualifier gives this: `orders.spanning` reads as a field of + `orders`, which has no such field. +- A measure stashed onto one model, naming a measure of another. Holistics rejects it at SQL generation with + `Cannot refer to external field in TableModel/QueryModel`. + +A metric that names itself through another metric is an ERROR as well. `holistics aml validate` accepts a +cycle, and Holistics inlines one metric into the body of the next, so the pair has no warehouse SQL. + +`tests/fixtures/metric_refs` covers each row of the table, and `holistics aml validate` runs over the AML it +generates. + +### A cross-dataset metric becomes AQL + +A `@sql` expression names only fields of the model it sits in, so a metric naming two datasets is rendered +into AQL whichever of the two places AML allows it. The converter raises a WARNING showing the Ossie SQL and +the AQL side by side. A metric that came from Holistics arrives as `HOLISTICS_AQL` already and is untouched. + +A construct the converter does not write as AQL drops the metric with an ERROR. A window function is the one +to expect. [Ossie SQL to AQL](aql.md) covers the function table, the raw-SQL call for a function AQL has no name +for, and the two changes rendering can make to the models a metric names. + +`aggregation_type` and the body compose into one Ossie expression on the way out, so the reverse path splits +them apart again. + +| Stash | AML written | +|---|---| +| a named `aggregation_type` wrapping the whole expression | that aggregation, and the body from inside the wrapper. `sum` over `SUM(quantity)` gives `aggregation_type: 'sum'` and `{{ #SOURCE.quantity }}` | +| no `aggregation_type` | `custom`, with the expression kept whole. The aggregate already sits inside it | +| a named `aggregation_type` that does not wrap the whole expression | `custom`, with the expression kept whole, and a WARNING. The expression was edited after the forward path wrote it, and aggregating it again would double the aggregate | + +TPC-DS takes the second row throughout, because it carries no stash. Its `total_sales` is +`SUM(store_sales.ss_ext_sales_price)` and becomes `definition: @sql SUM({{ ss_ext_sales_price }});;` under +`custom`, where `store_sales` is the model being written so the reference is a sibling. An unqualified +`SUM(ss_ext_sales_price)` would have become `SUM({{ #SOURCE.ss_ext_sales_price }})` instead. + +## Verifying the output + +The generated AML is checkable without a Holistics account: + +```bash +holistics aml validate /.dataset.aml -r +``` + +That parses every generated file and type-checks the AQL. `tests/test_snapshots.py` runs it over both +fixtures and asserts a clean exit, and `tests/test_cross_vendor.py` runs it over every other Ossie document in +this repository. "The output is valid AML" is measured rather than asserted. CI installs the CLI, so those +tests run there too. + +Compiling goes further than validating. `test_the_round_trip_returns_the_same_ossie_document` runs +`holistics aml compile` over the generated AML and converts the result back to Ossie, then compares it with +the document the reverse path started from. For the `ecommerce` fixture the two are byte-identical. + +## What does not survive + +Beyond the file layout above: + +- Comments. The compiled JSON keeps `__doc__` fields but the forward path does not carry them into Ossie. +- Property order inside a block, which the generator chooses. +- Anything the forward path already declared lost. See [limitations](limitations.md). + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [the HOLISTICS payload](vendor-payload.md) and [limitations](limitations.md). diff --git a/converters/holistics/docs/vendor-payload.md b/converters/holistics/docs/vendor-payload.md new file mode 100644 index 000000000..90e10b8a1 --- /dev/null +++ b/converters/holistics/docs/vendor-payload.md @@ -0,0 +1,58 @@ + + +# The HOLISTICS payload + +AML carries properties that the Ossie core specification has no field for. The converter stashes each one under a single +`custom_extensions` entry with `vendor_name: HOLISTICS`, attached to the Ossie object it came from. + +```yaml +custom_extensions: +- vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo"}' +``` + +`data` is always a single JSON-encoded string, never a nested object, because the core specification requires that. It +carries `_v`, a shape version, currently 1. The converter fails on an unrecognised version rather than misreading a +shape it has never seen. Keys are sorted, so a change to the stash shows up as a diff on the keys that changed. + +What goes into the stash: + +| AML property | Scope | Why Ossie has no field for it | +|---|---|---| +| `aggregation_type` | metric | Ossie carries the whole aggregate in one expression | +| `hidden` | field, metric | Ossie models semantics, not visibility | +| `format` | field, metric | A display format, not semantics | +| `type` | field, metric | The AML type, kept verbatim so the return trip recovers it | +| query template forms | dataset | Which of the three forms the body uses, so a consumer knows why `source` is not runnable | +| `param` | dataset | A runtime input with no Ossie equivalent | +| `persistence` | dataset | A materialization directive, not semantics | +| `data_source_name` | model, dataset | An instance-local connection name | +| `owner` | model, dataset | An instance-local email address | +| `table_name` | dataset | Kept verbatim, because `source` normalizes the quoting | +| relationship kind | relationship | Ossie has no one-to-one marker | +| `active` | relationship | See [Relationships](mapping.md#relationships) | +| `where` | relationship | A `RelationshipFilter` narrowing the join condition. No Ossie equivalent | +| `direction`, `nullable`, `rlp_propagation` | relationship | No Ossie equivalent | +| `__fqn__` | dataset | The unflattened module path, kept so the return trip restores it | +| `aml_type` | model, dataset | `TableModel`, `QueryModel` or `Dataset`, so the return trip does not have to guess a query from the `source` text | +| `label` | field, metric, dataset | AML requires a label on every dimension and measure, and Ossie's `label` means categorization rather than a display name | +| `model` | metric | The model a measure came from. The reverse path reads this rather than splitting the metric name, which `__` makes ambiguous | +| `aml_name` | metric | The measure's own name, since the metric carries `__` | +| `scope` | field | `dataset` when the field came from a dataset-level AML `dimension`. Without the marker the reverse path would write it back inside a model, where a body spanning two models does not resolve | + +A foreign vendor's `custom_extensions` entry on the same object passes through untouched. The `HOLISTICS` +entry appears only on an object that has something to carry. + +`distinct_rows` is absent from the table above. It is deprecated in AML, so the converter neither reads it nor carries it. + + +Part of the [Apache Ossie Holistics converter](../README.md). See also [the fixture](fixture.md), [the mapping](mapping.md), [expression translation](expressions.md), [measures](measures.md), [Ossie to AML](reverse.md), [Ossie SQL to AQL](aql.md) and [limitations](limitations.md). diff --git a/converters/holistics/pyproject.toml b/converters/holistics/pyproject.toml new file mode 100644 index 000000000..13f6fdbff --- /dev/null +++ b/converters/holistics/pyproject.toml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + # Dev/test-only: tests/test_forward.py validates the converted document + # against core-spec/ossie-schema.json. Guarded by pytest.importorskip, so + # the package's only runtime dependency stays PyYAML. Same dev-group + # placement as converters/thoughtspot's pyproject.toml. + "jsonschema>=4.26.0", +] + +[project] +name = "apache-ossie-holistics" +version = "0.1.0" +description = "Convert between Holistics AML and the Apache Ossie semantic model" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.10" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Holistics", + "AML", +] +dependencies = [ + "PyYAML>=6.0", + # Runtime, not dev-only, unlike the sibling converters that use sqlglot for + # validation. ossie_to_aml needs it to tell a column apart from a keyword or a + # date part, which is what restores the AML interpolation AQL reads to learn + # a body's field dependencies. Without it the reverse path cannot produce + # usable AML at all. See src/ossie_holistics/sqlrefs.py. + "sqlglot>=30.12.0", +] + +[project.scripts] +ossie-holistics = "ossie_holistics.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_holistics"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] diff --git a/converters/holistics/src/ossie_holistics/__init__.py b/converters/holistics/src/ossie_holistics/__init__.py new file mode 100644 index 000000000..f00db3d2a --- /dev/null +++ b/converters/holistics/src/ossie_holistics/__init__.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert between Holistics AML and the Apache Ossie semantic model.""" + +from . import aml, aml_to_ossie, ossie_to_aml + +__all__ = ["aml", "aml_to_ossie", "ossie_to_aml"] diff --git a/converters/holistics/src/ossie_holistics/_yaml.py b/converters/holistics/src/ossie_holistics/_yaml.py new file mode 100644 index 000000000..60910f148 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/_yaml.py @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""YAML codec with the boolean resolver narrowed to YAML 1.2. + +PyYAML implements YAML 1.1, in which `on`, `off`, `yes`, `no`, `y` and `n` +resolve to booleans. An AML identifier may be any of them, so a bare +`yaml.safe_load` corrupts them silently. Only the boolean resolver is +narrowed here. No other YAML 1.1/1.2 divergence (e.g. octal/sexagesimal +number parsing) is addressed. + +Both directions matter. The loader stops 1.1 bool tokens becoming booleans. The +dumper quotes them on the way out, so the next reader cannot re-resolve them +even if it implements 1.1. +""" +import re + +import yaml + +from .errors import ConversionError + +#: YAML 1.2 core schema: only these spellings are booleans. +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") + +#: Bare scalars YAML 1.1 resolves as booleans and YAML 1.2 does not. +_YAML11_ONLY_BOOLS = frozenset( + {"y", "Y", "yes", "Yes", "YES", "n", "N", "no", "No", "NO", + "on", "On", "ON", "off", "Off", "OFF"} +) + + +class Yaml12Loader(yaml.SafeLoader): + """SafeLoader with the YAML 1.1 boolean resolver narrowed to the 1.2 set.""" + + +# Drop the inherited bool resolver outright, then reinstate the 1.2-only one. +# Mutating in place would affect SafeLoader itself, so rebuild the mapping. +Yaml12Loader.yaml_implicit_resolvers = { + key: [(tag, regexp) for tag, regexp in resolvers if tag != "tag:yaml.org,2002:bool"] + for key, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} +Yaml12Loader.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +class Yaml12Dumper(yaml.SafeDumper): + """SafeDumper that quotes strings a YAML 1.1 reader would take for booleans.""" + + +def _represent_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + style = "'" if data in _YAML11_ONLY_BOOLS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +Yaml12Dumper.add_representer(str, _represent_str) + + +def load(text: str) -> object: + """Parse YAML text under YAML 1.2 boolean rules. + + A malformed document raises `ConversionError` naming the failure, never a + bare `yaml.YAMLError` traceback. `stash.py` holds the same contract for + malformed `custom_extensions` JSON. + """ + try: + return yaml.load(text, Loader=Yaml12Loader) + except yaml.YAMLError as exc: + raise ConversionError(f"malformed YAML: {exc}") from exc + + +def dump(data: object) -> str: + """Serialise to YAML, preserving insertion order and quoting 1.1 bool tokens. + + `allow_unicode=True` so a non-ASCII value (e.g. a display label) emits as + a literal character rather than a `\\xXX`/`\\uXXXX` escape. Ossie + documents are read by people. + """ + return yaml.dump( + data, + Dumper=Yaml12Dumper, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + ) + + +#: Width to fold an issue message at. Wide enough to keep a line meaningful, +#: narrow enough to read in a side-by-side diff. +ISSUE_WIDTH = 96 + + +class IssueDumper(Yaml12Dumper): + """Dumper that folds a long message instead of running it off the screen.""" + + +def _represent_issue_str(dumper: yaml.SafeDumper, data: str) -> yaml.ScalarNode: + """Fold a long single-line string, quote a YAML 1.1 bool token, else plain. + + Folded style (`>-`) breaks only at spaces, so a URL stays on one line, and a + reader joins the lines back with spaces. `load` returns the original string, + which `tests/test_snapshots.py` asserts. + """ + if data in _YAML11_ONLY_BOOLS: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="'") + style = ">" if len(data) > ISSUE_WIDTH and "\n" not in data else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +IssueDumper.add_representer(str, _represent_issue_str) + + +def dump_issues(issues: list[dict[str, object]]) -> str: + """The issue log as YAML, one block per issue, long messages folded. + + JSON put each message on one unbroken line, which a reviewer had to scroll + sideways to read. The same log as folded YAML fits a diff. + """ + if not issues: + return "[]\n" + return yaml.dump( + issues, + Dumper=IssueDumper, + sort_keys=False, + default_flow_style=False, + allow_unicode=True, + width=ISSUE_WIDTH, + ) diff --git a/converters/holistics/src/ossie_holistics/aml.py b/converters/holistics/src/ossie_holistics/aml.py new file mode 100644 index 000000000..94b5e3d1b --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aml.py @@ -0,0 +1,312 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A typed reading of `holistics aml compile` output. + +The compiled JSON is the boundary. Everything past this module works on the +dataclasses below rather than on raw dictionaries, so a shape the compiler +changes breaks here with a named error instead of silently reaching the +converted document. + +Three things in the payload are deliberately not read: + +`distinct_rows` is deprecated in AML. + +`models[]` on a `QueryModel` is a full inlined copy of every model the query +references. The same models are already in the dataset's own `models` list, so +reading the copy would duplicate every dataset. + +`__doc__` holds the AML comment text. The forward path has nowhere to put it, +and `reverse.md` records comments as not surviving the round trip. +""" +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + +from .errors import ConversionError + + +class ModelKind(str, Enum): + TABLE = "table" + QUERY = "query" + + +class RelationshipKind(str, Enum): + MANY_TO_ONE = "many_to_one" + ONE_TO_ONE = "one_to_one" + MANY_TO_MANY = "many_to_many" + RANGE = "range" + + +#: Only an equality relationship has an Ossie form. The other two carry an AQL +#: `match` condition rather than column pairs, so no column mapping exists. +EQUALITY_KINDS = frozenset({RelationshipKind.MANY_TO_ONE, RelationshipKind.ONE_TO_ONE}) + + +@dataclass(frozen=True) +class Heredoc: + """An AML `@sql` or `@aql` body. `language` is the heredoc's own name.""" + + language: str + content: str + + @property + def is_aql(self) -> bool: + return self.language == "aql" + + +@dataclass(frozen=True) +class Field: + """One `dimension` or `measure`. + + `aggregation_type` is None on a dimension and set on a measure, which is the + only structural difference between the two in the compiled payload. + """ + + name: str + label: str | None + description: str | None + hidden: bool + aml_type: str | None + format: str | None + definition: Heredoc | None + primary_key: bool + aggregation_type: str | None + #: Set on a dataset-level dimension, naming the model it attaches to. + owner_model_fqn: str | None = None + + +@dataclass(frozen=True) +class Param: + name: str + label: str | None + aml_type: str | None + + +@dataclass(frozen=True) +class Model: + name: str + fqn: str + label: str | None + description: str | None + owner: str | None + data_source_name: str | None + kind: ModelKind + table_name: str | None + query: Heredoc | None + dimensions: list[Field] + measures: list[Field] + params: list[Param] + persistence: dict[str, Any] | None + aml_type: str + + +@dataclass(frozen=True) +class Endpoint: + """One side of a relationship. `model` is spelled with dots, not `::`.""" + + model: str + field: str + + +@dataclass(frozen=True) +class Relationship: + kind: RelationshipKind + #: Column pairs, `from` first then each `on` block. They combine with AND. + pairs: list[tuple[Endpoint, Endpoint]] + active: bool + direction: str | None + nullable: bool | None + rlp_propagation: str | None + where: Any | None + aml_type: str + + +@dataclass(frozen=True) +class Dataset: + name: str + fqn: str + label: str | None + description: str | None + owner: str | None + data_source_name: str | None + models: list[Model] + relationships: list[Relationship] + #: Dataset-level `dimension` entries, each naming its owning model. + dimensions: list[Field] + #: Dataset-level `metric` entries, always AQL in practice. + metrics: list[Field] + + def model_by_fqn(self) -> dict[str, Model]: + return {model.fqn: model for model in self.models} + + +def _require(payload: Any, key: str, where: str) -> Any: + if not isinstance(payload, dict) or key not in payload: + raise ConversionError(f"{where}: compiled AML is missing {key!r}") + return payload[key] + + +def _optional_text(payload: dict[str, Any], key: str) -> str | None: + """A string property, with the compiler's empty string read as absent. + + `holistics aml compile` writes `""` for a property the source omits, and an + empty description is not a description. + """ + value = payload.get(key) + return value if isinstance(value, str) and value else None + + +def _heredoc(payload: Any, where: str) -> Heredoc | None: + if payload is None: + return None + if not isinstance(payload, dict): + raise ConversionError(f"{where}: expected a heredoc object, got {type(payload).__name__}") + return Heredoc( + language=_require(payload, "name", where), + content=_require(payload, "content", where), + ) + + +def _ordered(container: Any, where: str) -> list[dict[str, Any]]: + """Members of a `dimension`/`measure`/`param` block in AML source order. + + The compiler writes these as an object keyed by name, and a dataset-level + `dimension` as an array. `__childIdx__` is the source position, used as the + sort key so the converted document lists fields in the order the AML author + wrote them rather than in whatever order the payload happens to hold. + """ + if container is None: + return [] + if isinstance(container, dict): + members = list(container.values()) + elif isinstance(container, list): + members = list(container) + else: + raise ConversionError(f"{where}: expected an object or array") + return sorted(members, key=lambda m: m.get("__childIdx__", 0)) + + +def _field(payload: dict[str, Any], where: str) -> Field: + name = _require(payload, "name", where) + return Field( + name=name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + hidden=bool(payload.get("hidden", False)), + aml_type=payload.get("type"), + format=_optional_text(payload, "format"), + definition=_heredoc(payload.get("definition"), f"{where}.{name}"), + primary_key=bool(payload.get("primary_key", False)), + aggregation_type=payload.get("aggregation_type"), + owner_model_fqn=(payload.get("model") or {}).get("__fqn__"), + ) + + +def _param(payload: dict[str, Any], where: str) -> Param: + return Param( + name=_require(payload, "name", where), + label=_optional_text(payload, "label"), + aml_type=payload.get("type"), + ) + + +def _model(payload: dict[str, Any]) -> Model: + name = _require(payload, "name", "model") + where = f"model {name!r}" + raw_kind = _require(payload, "type", where) + try: + kind = ModelKind(raw_kind) + except ValueError as exc: + raise ConversionError(f"{where}: unknown model type {raw_kind!r}") from exc + return Model( + name=name, + fqn=payload.get("__fqn__") or name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + owner=_optional_text(payload, "owner"), + data_source_name=_optional_text(payload, "data_source_name"), + kind=kind, + table_name=_optional_text(payload, "table_name"), + query=_heredoc(payload.get("query"), where), + dimensions=[_field(d, where) for d in _ordered(payload.get("dimension"), where)], + measures=[_field(m, where) for m in _ordered(payload.get("measure"), where)], + params=[_param(p, where) for p in _ordered(payload.get("param"), where)], + persistence=payload.get("persistence"), + aml_type=payload.get("__type__", ""), + ) + + +def _endpoint(payload: dict[str, Any], where: str) -> Endpoint: + return Endpoint( + model=_require(payload, "model", where), + field=_require(payload, "field", where), + ) + + +def _relationship(payload: dict[str, Any], index: int) -> Relationship: + where = f"relationship {index}" + rel = _require(payload, "rel", where) + raw_kind = _require(rel, "type", where) + try: + kind = RelationshipKind(raw_kind) + except ValueError as exc: + raise ConversionError(f"{where}: unknown relationship type {raw_kind!r}") from exc + + pairs: list[tuple[Endpoint, Endpoint]] = [] + if "from" in rel and "to" in rel: + pairs.append((_endpoint(rel["from"], where), _endpoint(rel["to"], where))) + for extra in _ordered(rel.get("on"), where): + pairs.append((_endpoint(extra["from"], where), _endpoint(extra["to"], where))) + + return Relationship( + kind=kind, + pairs=pairs, + active=bool(payload.get("active", True)), + direction=payload.get("direction"), + nullable=payload.get("nullable"), + rlp_propagation=payload.get("rlp_propagation"), + where=rel.get("where"), + aml_type=rel.get("__type__", ""), + ) + + +def parse(payload: Any) -> Dataset: + """Read one compiled `*.dataset.aml.json` document.""" + if not isinstance(payload, dict): + raise ConversionError("compiled AML is not an object") + if payload.get("__type__") != "Dataset": + raise ConversionError( + f"expected a compiled Dataset, got __type__={payload.get('__type__')!r}" + ) + name = _require(payload, "name", "dataset") + return Dataset( + name=name, + fqn=payload.get("__fqn__") or name, + label=_optional_text(payload, "label"), + description=_optional_text(payload, "description"), + owner=_optional_text(payload, "owner"), + data_source_name=_optional_text(payload, "data_source_name"), + models=[_model(m) for m in payload.get("models") or []], + relationships=[ + _relationship(r, i) for i, r in enumerate(payload.get("relationships") or []) + ], + dimensions=[_field(d, "dataset") for d in _ordered(payload.get("dimension"), "dataset")], + metrics=[_field(m, "dataset") for m in _ordered(payload.get("metric"), "dataset")], + ) diff --git a/converters/holistics/src/ossie_holistics/aml_to_ossie.py b/converters/holistics/src/ossie_holistics/aml_to_ossie.py new file mode 100644 index 000000000..1609a2e6c --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aml_to_ossie.py @@ -0,0 +1,603 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Compiled AML to one Apache Ossie semantic model. + +`convert` takes the parsed `holistics aml compile` output and returns the Ossie +document plus the issues the conversion raised. A construct it drops or changes +raises an `ISSUE_*` code, and `docs/limitations.md` lists every one. + +Naming is the decision that shapes the rest. An AML module path joins with +`::`, so `reporting::calendar` flattens to the Ossie dataset name +`reporting__calendar`. A model measure becomes a root-level Ossie metric, and +root level has one namespace for the whole document, so the metric takes the +name `__`. A reference to a field inside an expression is +dataset-qualified (`ecommerce_orders.created_at`), and a reference to a metric +is the metric's own name, because that is what each one resolves against. +""" +from __future__ import annotations + +from collections import Counter +from typing import Any + +from . import aml, interpolation, stash +from .datatypes import AGGREGATIONS, AML_TO_OSSIE_DATATYPE, CUSTOM_AGGREGATION, apply_aggregation +from .errors import ConversionError +from .issues import IssueLog, Severity + +SPEC_VERSION = "0.2.0.dev0" + +AQL_DIALECT = "HOLISTICS_AQL" + +ISSUE_AQL_NOT_TRANSLATED = "HOLISTICS_AQL_NOT_TRANSLATED" +ISSUE_INACTIVE_RELATIONSHIP = "HOLISTICS_INACTIVE_RELATIONSHIP" +ISSUE_UNSUPPORTED_RELATIONSHIP = "HOLISTICS_UNSUPPORTED_RELATIONSHIP" +ISSUE_RELATIONSHIP_FILTER = "HOLISTICS_RELATIONSHIP_FILTER" +ISSUE_QUERY_SOURCE = "HOLISTICS_QUERY_SOURCE" +ISSUE_UNRESOLVED_QUERY = "HOLISTICS_UNRESOLVED_QUERY_TEMPLATE" +ISSUE_UNKNOWN_DATATYPE = "HOLISTICS_UNKNOWN_DATATYPE" +ISSUE_UNKNOWN_AGGREGATION = "HOLISTICS_UNKNOWN_AGGREGATION" +ISSUE_UNRESOLVED_REFERENCE = "HOLISTICS_UNRESOLVED_REFERENCE" +ISSUE_PARAM_DROPPED = "HOLISTICS_PARAM_DROPPED" + + +class Result: + def __init__(self, model: dict[str, Any], issues: IssueLog): + self.model = model + self.issues = issues + + +def flatten(fqn: str) -> str: + """An AML module path as a flat Ossie name. + + `__` rather than `_` because a single underscore collides with any model + name that already contains one, which most do. `reporting::calendar` and a + root model named `reporting_calendar` would otherwise take the same name. + """ + return fqn.replace("::", "__") + + +def _expression(expression: str, dialect: str) -> dict[str, Any]: + return {"dialects": [{"dialect": dialect, "expression": expression}]} + + +class _Converter: + """One conversion. Holds the name tables every expression resolves against.""" + + def __init__(self, dataset: aml.Dataset, sql_dialect: str): + self.dataset = dataset + self.sql_dialect = sql_dialect + self.issues = IssueLog() + + self.models_by_fqn = dataset.model_by_fqn() + self.dataset_name: dict[str, str] = {} + claimed_by: dict[str, str] = {} + for model in dataset.models: + flat = flatten(model.fqn) + clash = claimed_by.get(flat) + if clash is not None: + raise ConversionError( + f"models {clash!r} and {model.fqn!r} both flatten to the Ossie " + f"dataset name {flat!r}; rename one of them in AML" + ) + claimed_by[flat] = model.fqn + self.dataset_name[model.fqn] = flat + + #: Dataset-level dimensions, grouped by the model they attach to. + self.extra_dimensions: dict[str, list[aml.Field]] = {} + for field in dataset.dimensions: + owner = field.owner_model_fqn + if owner is None or owner not in self.models_by_fqn: + raise ConversionError( + f"dataset dimension {field.name!r} names model {owner!r}, " + f"which is not in this dataset" + ) + self.extra_dimensions.setdefault(owner, []).append(field) + + #: Every field name reachable on a model, for resolving `{{ field }}`. + self.model_fields: dict[str, set[str]] = {} + self.model_measures: dict[str, set[str]] = {} + for model in dataset.models: + extra = self.extra_dimensions.get(model.fqn, []) + self.model_fields[model.fqn] = {d.name for d in model.dimensions} | { + d.name for d in extra + } + self.model_measures[model.fqn] = {m.name for m in model.measures} + + # -- names ------------------------------------------------------------ + + def metric_name(self, model_fqn: str, measure_name: str) -> str: + return f"{self.dataset_name[model_fqn]}__{measure_name}" + + def resolve_model(self, reference: str) -> str | None: + """An AML model reference to its `__fqn__`. + + A relationship endpoint and a model spell the same name differently. The + endpoint uses dots (`reporting.calendar`) and the model's own `__fqn__` + uses colons (`reporting::calendar`). The compiler fills a `__refFqn__` + shortcut on some endpoints and leaves it null on others, so this + rewrites the separator and looks the result up instead. + """ + for candidate in (reference, reference.replace(".", "::")): + if candidate in self.models_by_fqn: + return candidate + return None + + # -- expressions ------------------------------------------------------ + + def resolve_reference(self, model_fqn: str, name: str, scope: str) -> str: + """`{{ field }}` as the Ossie name it resolves to. + + A dimension becomes a dataset-qualified field reference. A measure + becomes a metric reference, which is the metric's own name because + metrics live in one root namespace. + """ + if name in self.model_fields.get(model_fqn, ()): + return f"{self.dataset_name[model_fqn]}.{name}" + if name in self.model_measures.get(model_fqn, ()): + return self.metric_name(model_fqn, name) + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"expression references {name!r}, which is not a field or measure of this model", + object_ref=scope, + remedy="fix the reference in AML, or drop the field", + ) + return name + + def resolve_model_field( + self, model_fqn: str, reference: str, field_name: str, scope: str + ) -> str: + """`{{ #model.field }}` as an Ossie reference. + + Naming this model is not the same as `{{ field }}`. `{{ field }}` inlines + the sibling field's own definition, while `{{ #model.field }}` reads the + column of that model's relation, exactly as `{{ #SOURCE.field }}` does. + `ecommerce_users.age_bucket` is `FLOOR({{ #ecommerce_users.age }} / 10)` + and Holistics renders it `FLOOR(`ecommerce_users`.`age` / 10)`, where + `{{ age }}` would have rendered the whole `extract(...)` body instead. + Collapsing the two changes the warehouse SQL, so a self-naming reference becomes a + column and only a measure, which is never a relation column, stays a + metric reference. + """ + target = self.resolve_model(reference) + if target is None: + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"expression references model {reference!r}, which is not in this dataset", + object_ref=scope, + remedy="add the model to the dataset, or drop the field", + ) + return f"{flatten(reference)}.{field_name}" + if target == model_fqn and field_name not in self.model_measures.get(target, ()): + return field_name + return self.resolve_reference(target, field_name, scope) + + def resolve_interpolation( + self, model_fqn: str, part: interpolation.Interpolation, scope: str + ) -> str: + """One interpolation as the Ossie text that replaces it.""" + if isinstance(part, interpolation.Source): + return part.column + if isinstance(part, interpolation.Sibling): + return self.resolve_reference(model_fqn, part.field, scope) + if isinstance(part, interpolation.ModelField): + return self.resolve_model_field(model_fqn, part.model, part.field, scope) + self.issues.add( + code=ISSUE_UNRESOLVED_REFERENCE, + severity=Severity.ERROR, + message=f"unrecognised AML interpolation {part.to_aml()!r}, carried verbatim", + object_ref=scope, + remedy="rewrite the body in AML, or drop the field", + ) + return part.to_aml() + + def translate_sql(self, model_fqn: str, body: str, scope: str) -> str: + """A `@sql` body with its interpolations replaced by Ossie references.""" + return "".join( + part if isinstance(part, str) else self.resolve_interpolation(model_fqn, part, scope) + for part in interpolation.parse_sql_body(body) + ) + + def sql_body(self, model_fqn: str, body: str, scope: str) -> str: + """A `@sql` body as an Ossie expression, with its references resolved. + + Every `@sql` body is tagged `--sql-dialect`, including one that is a + bare reference. Holistics SQL is warehouse SQL throughout, so + one dialect describes the whole document and a reader needs no rule for + which fields carry which tag. + """ + sole = interpolation.sole_reference(body) + if sole is not None: + return self.resolve_interpolation(model_fqn, sole, scope) + return self.translate_sql(model_fqn, body, scope) + + def field_expression(self, model_fqn: str, field: aml.Field, scope: str) -> dict[str, Any]: + """The `expression` for one dimension. + + An omitted `definition` is a default rather than a missing value: AML + fills it with `{{ #SOURCE. }}`, so it resolves exactly like + a bare source column written out in full. + """ + definition = field.definition + if definition is None: + return _expression(field.name, self.sql_dialect) + if definition.is_aql: + self._note_aql(scope) + return _expression(definition.content, AQL_DIALECT) + return _expression(self.sql_body(model_fqn, definition.content, scope), self.sql_dialect) + + def measure_expression(self, model_fqn: str, measure: aml.Field, scope: str) -> dict[str, Any]: + """The `expression` for one measure, aggregation and body composed. + + An AQL body carries its own aggregation, so `aggregation_type` is not + applied on top of it whatever it says. + """ + definition = measure.definition + if definition is None: + raise ConversionError( + f"{scope}: a measure must carry a definition, and this one has none" + ) + if definition.is_aql: + self._note_aql(scope) + return _expression(definition.content, AQL_DIALECT) + + body = self.sql_body(model_fqn, definition.content, scope) + + aggregation = measure.aggregation_type + if aggregation in (None, CUSTOM_AGGREGATION): + return _expression(body, self.sql_dialect) + if aggregation not in AGGREGATIONS: + self.issues.add( + code=ISSUE_UNKNOWN_AGGREGATION, + severity=Severity.ERROR, + message=f"unknown aggregation_type {aggregation!r}, body emitted unaggregated", + object_ref=scope, + remedy="add the aggregation to datatypes.AGGREGATIONS", + ) + return _expression(body, self.sql_dialect) + return _expression(apply_aggregation(aggregation, body), self.sql_dialect) + + def _note_aql(self, scope: str) -> None: + self.issues.add( + code=ISSUE_AQL_NOT_TRANSLATED, + severity=Severity.INFO, + message=( + "body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does " + "not speak AQL can read and round-trip it but cannot evaluate it" + ), + object_ref=scope, + remedy="evaluate through Holistics, or rewrite the body as Holistics SQL", + ) + + # -- objects ---------------------------------------------------------- + + def datatype(self, field: aml.Field, scope: str) -> str | None: + if field.aml_type is None: + return None + mapped = AML_TO_OSSIE_DATATYPE.get(field.aml_type) + if mapped is None: + self.issues.add( + code=ISSUE_UNKNOWN_DATATYPE, + severity=Severity.WARNING, + message=f"unknown AML type {field.aml_type!r}, datatype omitted", + object_ref=scope, + remedy="add the type to datatypes.AML_TO_OSSIE_DATATYPE", + ) + return mapped + + def field_stash(self, field: aml.Field) -> dict[str, Any]: + data: dict[str, Any] = {} + if field.hidden: + data["hidden"] = True + if field.format: + data["format"] = field.format + if field.aml_type: + data["type"] = field.aml_type + if field.label: + data["label"] = field.label + return data + + def convert_field( + self, model_fqn: str, field: aml.Field, dataset_level: bool = False + ) -> dict[str, Any]: + """One dimension as an Ossie field. + + A dataset-level AML dimension becomes a field on the model it names, and + the stash records where it was declared. Without that marker the reverse + path would write it back as a model dimension, and an AQL body spanning + two models does not resolve from inside one model. + """ + scope = f"{model_fqn}.{field.name}" + out: dict[str, Any] = { + "name": field.name, + "expression": self.field_expression(model_fqn, field, scope), + } + if field.description: + out["description"] = field.description + datatype = self.datatype(field, scope) + if datatype: + out["datatype"] = datatype + data = self.field_stash(field) + if dataset_level: + data["scope"] = "dataset" + stash.attach(out, data) + return out + + def convert_measure(self, model_fqn: str, measure: aml.Field) -> dict[str, Any]: + scope = f"{model_fqn}.{measure.name}" + out: dict[str, Any] = { + "name": self.metric_name(model_fqn, measure.name), + "expression": self.measure_expression(model_fqn, measure, scope), + } + if measure.description: + out["description"] = measure.description + datatype = self.datatype(measure, scope) + if datatype: + out["datatype"] = datatype + data = self.field_stash(measure) + data["model"] = model_fqn + data["aml_name"] = measure.name + if measure.aggregation_type: + data["aggregation_type"] = measure.aggregation_type + stash.attach(out, data) + return out + + def convert_dataset_metric(self, metric: aml.Field) -> dict[str, Any]: + scope = f"{self.dataset.name}.{metric.name}" + definition = metric.definition + if definition is None: + raise ConversionError(f"{scope}: a dataset metric must carry a definition") + if not definition.is_aql: + raise ConversionError( + f"{scope}: a dataset-level metric spans models, so its body has no single " + f"model to resolve `@sql` against" + ) + self._note_aql(scope) + out: dict[str, Any] = { + "name": metric.name, + "expression": _expression(definition.content, AQL_DIALECT), + } + if metric.description: + out["description"] = metric.description + datatype = self.datatype(metric, scope) + if datatype: + out["datatype"] = datatype + stash.attach(out, self.field_stash(metric)) + return out + + def model_source(self, model: aml.Model) -> str: + scope = model.fqn + if model.kind is aml.ModelKind.TABLE: + if not model.table_name: + raise ConversionError(f"{scope}: a table model has no table_name") + return model.table_name + if model.query is None: + raise ConversionError(f"{scope}: a query model has no query") + body = model.query.content + self.issues.add( + code=ISSUE_QUERY_SOURCE, + severity=Severity.WARNING, + message=( + "a query model becomes a query in `source`, which is one string, so a " + "consumer reads the text to tell it from a table reference" + ), + object_ref=scope, + remedy="materialise the query as a table in Holistics before converting", + ) + forms = interpolation.query_forms_used(body) + if forms: + self.issues.add( + code=ISSUE_UNRESOLVED_QUERY, + severity=Severity.ERROR, + message=( + f"the query body uses Holistics template syntax ({', '.join(forms)}) " + f"and is carried unresolved. `source` is not runnable SQL" + ), + object_ref=scope, + remedy="resolve the query through Holistics, or drop the model", + ) + return body + + def model_stash(self, model: aml.Model) -> dict[str, Any]: + data: dict[str, Any] = {"aml_type": model.aml_type} + if model.fqn != model.name: + data["fqn"] = model.fqn + if model.label: + data["label"] = model.label + if model.owner: + data["owner"] = model.owner + if model.data_source_name: + data["data_source_name"] = model.data_source_name + if model.table_name: + data["table_name"] = model.table_name + if model.persistence: + data["persistence"] = model.persistence + if model.params: + data["param"] = [ + {"name": p.name, "label": p.label, "type": p.aml_type} for p in model.params + ] + self.issues.add( + code=ISSUE_PARAM_DROPPED, + severity=Severity.WARNING, + message=( + f"{len(model.params)} param(s) have no Ossie equivalent and are stashed. " + f"A body that reads one cannot be resolved statically" + ), + object_ref=model.fqn, + remedy="supply the param value in Holistics before converting", + ) + if model.kind is aml.ModelKind.QUERY and model.query is not None: + forms = interpolation.query_forms_used(model.query.content) + if forms: + data["query_forms"] = forms + return data + + def convert_model(self, model: aml.Model) -> dict[str, Any]: + out: dict[str, Any] = { + "name": self.dataset_name[model.fqn], + "source": self.model_source(model), + } + primary_key = [d.name for d in model.dimensions if d.primary_key] + if primary_key: + out["primary_key"] = primary_key + description = model.description or model.label + if description: + out["description"] = description + out["fields"] = [self.convert_field(model.fqn, f) for f in model.dimensions] + [ + self.convert_field(model.fqn, f, dataset_level=True) + for f in self.extra_dimensions.get(model.fqn, []) + ] + stash.attach(out, self.model_stash(model)) + return out + + def convert_relationship( + self, relationship: aml.Relationship, index: int, taken: set[str] + ) -> dict[str, Any] | None: + scope = f"relationship {index}" + if relationship.kind not in aml.EQUALITY_KINDS: + self.issues.add( + code=ISSUE_UNSUPPORTED_RELATIONSHIP, + severity=Severity.ERROR, + message=( + f"a {relationship.kind.value} relationship carries an AQL match condition " + f"rather than column pairs, so it has no Ossie form and is dropped" + ), + object_ref=scope, + remedy="model the join as an equality relationship in AML", + ) + return None + + from_fqn = self.resolve_model(relationship.pairs[0][0].model) + to_fqn = self.resolve_model(relationship.pairs[0][1].model) + if from_fqn is None or to_fqn is None: + missing = relationship.pairs[0][0 if from_fqn is None else 1].model + raise ConversionError(f"{scope}: endpoint names model {missing!r}, not in this dataset") + + from_name = self.dataset_name[from_fqn] + to_name = self.dataset_name[to_fqn] + name = f"{from_name}_to_{to_name}" + base, suffix = name, 2 + while name in taken: + name, suffix = f"{base}_{suffix}", suffix + 1 + taken.add(name) + scope = name + + out: dict[str, Any] = { + "name": name, + "from": from_name, + "to": to_name, + "from_columns": [pair[0].field for pair in relationship.pairs], + "to_columns": [pair[1].field for pair in relationship.pairs], + } + + if not relationship.active: + self.issues.add( + code=ISSUE_INACTIVE_RELATIONSHIP, + severity=Severity.WARNING, + message=( + "the relationship is inactive in AML, and Ossie has no inactive marker. " + "A consumer joins across it where Holistics leaves it out by default" + ), + object_ref=scope, + remedy="drop the relationship, or activate it in AML", + ) + if relationship.where is not None: + self.issues.add( + code=ISSUE_RELATIONSHIP_FILTER, + severity=Severity.WARNING, + message=( + "the relationship carries a filter that narrows the join, and Ossie has " + "no equivalent. The join widens to every matching row" + ), + object_ref=scope, + remedy="move the condition into a dimension in AML", + ) + + data: dict[str, Any] = {"type": relationship.kind.value, "active": relationship.active} + for key, value in ( + ("direction", relationship.direction), + ("nullable", relationship.nullable), + ("rlp_propagation", relationship.rlp_propagation), + ("where", relationship.where), + ): + if value is not None: + data[key] = value + stash.attach(out, data) + return out + + # -- entry point ------------------------------------------------------ + + def run(self) -> Result: + datasets = [self.convert_model(m) for m in self.dataset.models] + + metrics: list[dict[str, Any]] = [] + for model in self.dataset.models: + metrics.extend(self.convert_measure(model.fqn, m) for m in model.measures) + metrics.extend(self.convert_dataset_metric(m) for m in self.dataset.metrics) + counts = Counter(m["name"] for m in metrics) + duplicates = sorted(name for name, count in counts.items() if count > 1) + if duplicates: + raise ConversionError( + f"metric name(s) {', '.join(duplicates)} derived twice; " + f"rename the colliding measures in AML" + ) + + taken: set[str] = set() + relationships = [ + converted + for index, relationship in enumerate(self.dataset.relationships) + if (converted := self.convert_relationship(relationship, index, taken)) is not None + ] + + model: dict[str, Any] = { + "version": SPEC_VERSION, + "name": flatten(self.dataset.fqn), + } + description = self.dataset.description or self.dataset.label + if description: + model["description"] = description + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + data: dict[str, Any] = {"aml_type": "Dataset"} + if self.dataset.fqn != self.dataset.name: + data["fqn"] = self.dataset.fqn + if self.dataset.label: + data["label"] = self.dataset.label + if self.dataset.owner: + data["owner"] = self.dataset.owner + if self.dataset.data_source_name: + data["data_source_name"] = self.dataset.data_source_name + stash.attach(model, data) + return Result(model, self.issues) + + +def convert(payload: Any, sql_dialect: str) -> Result: + """One compiled AML dataset to one Ossie semantic model. + + `sql_dialect` names the warehouse a computed `@sql` body is written for. AML + does not record it, because `data_source_name` is an instance-local + connection name rather than a dialect declaration. It is required even for a + dataset whose bodies are all bare references, so that adding one computed + body later does not change the command a caller has to run. + """ + return _Converter(aml.parse(payload), sql_dialect).run() diff --git a/converters/holistics/src/ossie_holistics/amlgen.py b/converters/holistics/src/ossie_holistics/amlgen.py new file mode 100644 index 000000000..d29a0eb86 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/amlgen.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Writing AML source text. + +The reverse path produces AML source rather than compiled JSON, because +`holistics aml compile` only runs one way and nothing reads its output back. +AML source is what a person edits, a repository holds, and the CLI accepts. + +Everything here is about the text: quoting, indentation, identifiers and +heredocs. What to write is `ossie_to_aml`'s decision. +""" +from __future__ import annotations + +import re + +from .errors import ConversionError + +def sanitize(name: str) -> str: + """An Ossie name as a legal AML identifier. + + An Ossie `name` is any non-empty string, so a valid document can carry one + AML cannot spell. `examples/flights.semantic_model.yaml` is named + `Flights semantic model`. Each illegal character becomes an underscore, and + a name starting with a digit gains a leading one. + + The caller reports the rename and checks for a collision. This function only + does the spelling. + """ + if not name: + raise ConversionError("an Ossie name is empty, and AML has no spelling for that") + cleaned = re.sub(r"[^A-Za-z0-9_]", "_", name) + return cleaned if cleaned[0].isalpha() or cleaned[0] == "_" else f"_{cleaned}" + + +def quote(value: str) -> str: + """An AML single-quoted string. A backslash escapes a quote or a backslash.""" + escaped = value.replace("\\", "\\\\").replace("'", "\\'") + return f"'{escaped}'" + + +def titleize(name: str) -> str: + """The label Holistics generates for a field that declares none. + + `label` is required on a dimension and on a measure, so a document with no + stashed label still needs one, and this is the spelling AML itself uses: + `created_at` becomes `Created At`. + """ + return " ".join(word.capitalize() for word in name.split("_") if word) or name + + +class Block: + """One `Name header { ... }` block, rendered with its children.""" + + def __init__(self, header: str): + self.header = header + self.lines: list[str | Block] = [] + + def property(self, key: str, value: str) -> "Block": + """A `key: value` line. `value` is already rendered AML.""" + self.lines.append(f"{key}: {value}") + return self + + def text(self, key: str, value: str | None) -> "Block": + """A `key: 'value'` line, skipped when `value` is None or empty.""" + if value: + self.property(key, quote(value)) + return self + + def flag(self, key: str, value: bool) -> "Block": + """A `key: true` line, written only when true. + + AML defaults every one of these to false, so writing `false` would add + noise without adding meaning. + """ + if value: + self.property(key, "true") + return self + + def heredoc(self, key: str, language: str, body: str) -> "Block": + """A `key: @sql ... ;;` property. + + A single-line body stays on one line, which is how AML is normally + written. A multi-line body is indented under the opener with the + terminator on its own line, because `;;` on the last content line would + read as part of that line. + """ + if "\n" not in body: + return self.property(key, f"@{language} {body};;") + self.lines.append(_Heredoc(key, language, body)) + return self + + def block(self, header: str) -> "Block": + child = Block(header) + self.lines.append(child) + return child + + def blank(self) -> "Block": + self.lines.append("") + return self + + def render(self, indent: int = 0) -> str: + pad = " " * indent + out = [f"{pad}{self.header} {{"] + for line in self.lines: + if isinstance(line, Block): + out.append(line.render(indent + 1)) + elif isinstance(line, _Heredoc): + out.append(line.render(indent + 1)) + elif line == "": + out.append("") + else: + out.append(f"{pad} {line}") + out.append(f"{pad}}}") + return "\n".join(out) + + +class _Heredoc: + def __init__(self, key: str, language: str, body: str): + self.key = key + self.language = language + self.body = body + + def render(self, indent: int) -> str: + pad = " " * indent + body = "\n".join(f"{pad} {line}" if line.strip() else "" for line in self.body.split("\n")) + return f"{pad}{self.key}: @{self.language}\n{body}\n{pad};;" + + +def array(items: list[str], indent: int = 1) -> str: + """A multi-line AML array with a trailing comma, as AML files are written.""" + if not items: + return "[]" + pad = " " * (indent + 1) + closing = " " * indent + body = "\n".join(f"{pad}{item}," for item in items) + return f"[\n{body}\n{closing}]" + + +LICENSE_HEADER = """\ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +""" + + +def document(block: Block, license_header: bool) -> str: + head = f"{LICENSE_HEADER}\n" if license_header else "" + return f"{head}{block.render()}\n" diff --git a/converters/holistics/src/ossie_holistics/aql.py b/converters/holistics/src/ossie_holistics/aql.py new file mode 100644 index 000000000..c57fb9572 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/aql.py @@ -0,0 +1,360 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Rendering an Ossie SQL expression as AQL. + +An AQL expression can name fields from several models. A `@sql` expression can +name only fields of the model it sits in, and a dataset-level AML `metric` +accepts only `@aql`. So an Ossie metric naming two datasets has no AML home +unless its Ossie SQL becomes AQL. + +What this translates is a name map and nothing else. `FUNCTIONS` pairs a sqlglot +node with the AQL function of the same meaning, references are already +`dataset.field` on both sides, and the operators match. Anything outside the +table raises `Untranslatable`, so the caller reports it rather than guessing. + +Window functions are the deliberate omission. Even though AQL has `rank`, +`previous`, `window_sum` and the rest, this converter cannot output one. + +Ossie SQL carries the partition and the ordering in the `OVER` clause, while AQL takes +both from the grain of the query being run. Translating one into the other means +deciding which query grain a given `OVER` clause corresponds to. The Ossie +specification says nothing about how a window function should behave when +queried, so nothing says which grain that is, and picking one would change what +the metric computes. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from . import sqlrefs + +if sqlrefs.AVAILABLE: # pragma: no branch - exercised by the packaging test + import sqlglot + from sqlglot import exp + + +#: The AQL cheatsheet, whose Window Function section lists `rank`, `previous`, +#: `next`, `window_sum` and the rest. +WINDOW_FUNCTION_DOCS = "https://docs.holistics.io/reference/aql/functions" + +#: One window function in full, showing that the grain comes from the query. +RANK_DOCS = "https://docs.holistics.io/reference/aql/rank" + +#: AQL's raw-SQL escape hatch, `sql_number(func_name, arg1, arg2, ...)`. The +#: family is typed by return value, and a metric body is numeric. +SQL_PASSTHROUGH = "sql_number" + + +@dataclass(frozen=True) +class _Context: + """What every `_render` call needs: the source text and the caller's hooks.""" + + source: str + functions: dict + reference: object + numeric: set + + +class Untranslatable(Exception): + """Raised when this converter will not write an expression as AQL. + + The expression may well have an AQL form. A window function does. This says + the converter did not write one, not that AQL cannot express it. + """ + + +def _functions(): + """sqlglot node to the AQL function of the same meaning and argument order. + + Built lazily because the keys are sqlglot classes, and sqlglot is imported + at module scope only when it is installed. + + Membership was derived by parsing each name AQL registers in + `src/checker/row_level.ts` and reading which node sqlglot produced, rather + than by matching names by eye. A name outside it falls to `_from_source`, + which rebuilds the call from the source text when the offsets allow it: + + A name that is one sqlglot node with several spellings. `LPAD` and `RPAD` + are both `Pad`, `LTRIM` and `RTRIM` are both `Trim`. The node's offset + points at the name in the source, so the passthrough recovers it. + + A name whose arguments sqlglot rewrote. `LOG10(x)` gains a base the source + never had, and `DATE_TRUNC('month', d)` has its unit normalized. Neither + argument carries an offset, so neither call can be rebuilt faithfully. + + A name whose arguments mean something else in AQL. Ossie SQL `SUBSTRING(s, start, + len)` and `STRPOS(s, sub)` do not line up with AQL's `mid` and `find`, so + the passthrough carries them under their own names instead. + """ + return { + # Aggregates. + exp.Sum: "sum", + exp.Avg: "avg", + exp.Min: "min", + exp.Max: "max", + exp.Stddev: "stdev", + exp.StddevPop: "stdevp", + exp.Variance: "var", + exp.VariancePop: "varp", + # Numeric. + exp.Abs: "abs", + exp.Ceil: "ceil", + exp.Floor: "floor", + exp.Round: "round", + exp.Trunc: "trunc", + exp.Sqrt: "sqrt", + exp.Pow: "pow", + exp.Exp: "exp", + exp.Ln: "ln", + exp.Mod: "mod", + exp.Sign: "sign", + exp.SafeDivide: "safe_divide", + # Trigonometric. + exp.Acos: "acos", + exp.Asin: "asin", + exp.Atan: "atan", + exp.Atan2: "atan2", + exp.Cos: "cos", + exp.Cot: "cot", + exp.Sin: "sin", + exp.Tan: "tan", + exp.Degrees: "degrees", + exp.Radians: "radians", + # Text. + exp.Concat: "concat", + exp.Lower: "lower", + exp.Upper: "upper", + exp.Length: "len", + exp.Left: "left", + exp.Right: "right", + exp.Replace: "replace", + exp.SplitPart: "split_part", + exp.RegexpExtract: "regexp_extract", + exp.RegexpLike: "regexp_like", + exp.RegexpReplace: "regexp_replace", + # Date parts, which take one date argument and so carry no order risk. + exp.Year: "year", + exp.Quarter: "quarter", + exp.Month: "month", + exp.Week: "week", + exp.Day: "day", + exp.Hour: "hour", + exp.Minute: "minute", + exp.LastDay: "last_day", + # Conditional. + exp.Nullif: "nullif", + exp.Coalesce: "coalesce", + } + + +#: Aggregates whose argument must be numeric. A reference inside one of these +#: tells the caller the field's type, which matters when the Ossie document +#: declares no `datatype`: AQL answers `sum` over a text dimension with +#: "The `sum` function expects `Dimension(Number)`, but got `Text`". +def _numeric_aggregates(): + return (exp.Sum, exp.Avg, exp.Stddev, exp.StddevPop, exp.Variance, exp.VariancePop) + + +#: Ossie SQL binary operators that are spelled the same in AQL. +OPERATORS = { + "Add": "+", + "Sub": "-", + "Mul": "*", + "Div": "/", + "EQ": "==", + "NEQ": "!=", + "GT": ">", + "GTE": ">=", + "LT": "<", + "LTE": "<=", + "And": "and", + "Or": "or", +} + + +def translate(expression: str, dialect: str, reference) -> tuple[str, set[tuple[str, str]]]: + """`expression` as AQL, with the columns it aggregates numerically. + + `reference(table, column)` returns the AQL spelling of one column, so the + caller owns the mapping from Ossie dataset names to AML model names. Raises + `Untranslatable` naming what stopped it. + """ + if not sqlrefs.AVAILABLE: + raise Untranslatable("sqlglot is not installed") + try: + tree = sqlglot.parse_one(expression, read=sqlrefs.SQLGLOT_DIALECT.get(dialect)) + except Exception as exc: # noqa: BLE001 - sqlglot raises several error types + raise Untranslatable(f"sqlglot cannot read it as {dialect}. {exc}") from exc + numeric: set[tuple[str, str]] = set() + context = _Context(source=expression, functions=_functions(), reference=reference, numeric=numeric) + return _render(tree, context), numeric + + +def _render(node, ctx) -> str: + if isinstance(node, exp.Window): + raise Untranslatable( + "It is a window function. Even though AQL has window functions, this converter " + "cannot output one. Ossie SQL carries the partition and the ordering in the OVER " + "clause, " + "while AQL takes both from the grain of the query being run. Translating one " + "into the other means deciding which query grain this OVER clause corresponds " + "to, and the Ossie specification defines no semantics for how a window function " + "behaves when queried, so nothing says which grain that is. See " + f"{WINDOW_FUNCTION_DOCS} for AQL's window functions and {RANK_DOCS} for one in " + "full, then write the AQL by hand." + ) + if isinstance(node, exp.Column): + return ctx.reference(node.table, node.name) + if isinstance(node, exp.Literal): + return node.sql() + if isinstance(node, exp.Boolean): + return "true" if node.this else "false" + if isinstance(node, exp.Null): + return "null" + if isinstance(node, exp.Paren): + return f"({_render(node.this, ctx)})" + if isinstance(node, exp.Neg): + return f"-{_render(node.this, ctx)}" + if isinstance(node, exp.Count): + # sqlglot holds the DISTINCT as a child node rather than a flag, so + # `COUNT(DISTINCT x)` parses to Count(this=Distinct(expressions=[x])). + name = "count_distinct" if isinstance(node.this, exp.Distinct) else "count" + return f"{name}({', '.join(_arguments(node, ctx))})" + if isinstance(node, exp.Distinct): + inner = node.expressions + if len(inner) != 1: + raise Untranslatable( + "It applies DISTINCT to more than one expression, which this converter does " + "not write as AQL." + ) + return _render(inner[0], ctx) + + for kind, aql_name in ctx.functions.items(): + if isinstance(node, kind): + if isinstance(node, _numeric_aggregates()): + ctx.numeric.update( + (c.table, c.name) for c in node.find_all(exp.Column) + ) + return f"{aql_name}({', '.join(_arguments(node, ctx))})" + + operator = OPERATORS.get(type(node).__name__) + if operator and node.this is not None and node.args.get("expression") is not None: + left = _render(node.this, ctx) + right = _render(node.args["expression"], ctx) + return f"{left} {operator} {right}" + + if isinstance(node, exp.Anonymous): + # AQL's escape hatch for a warehouse function it has no name of its own + # for: `sql_number('APPROX_TOP_COUNT', x, 3)` calls it with AQL-resolved + # arguments. The family is typed by return value, and a metric body is + # numeric, so `sql_number` is the member to use. + # + # Only `Anonymous` qualifies, because only there do the name and the + # argument order come from the source text. A function sqlglot models + # carries sqlglot's own name and slot order instead, which produced + # `sql_number('PAD', s, 5)` for `LPAD(s, 5)` and + # `sql_number('LOG', 10, x)` for `LOG10(x)`. Neither is a call any + # warehouse answers. + arguments = [_render(a, ctx) for a in node.expressions] + return f"{SQL_PASSTHROUGH}('{node.name}'{''.join(', ' + a for a in arguments)})" + + if isinstance(node, exp.Func): + rebuilt = _from_source(node, ctx) + if rebuilt is not None: + return rebuilt + raise Untranslatable( + f"{node.sql_name()} is not in this converter's AQL function table, and its call " + f"cannot be read back from the source text, so there is no faithful way to pass " + f"it through." + ) + + raise Untranslatable( + f"It contains a {type(node).__name__}, which this converter does not write as AQL." + ) + + +def _span(node) -> int | None: + """Where `node` starts in the source, or None when sqlglot made it up. + + sqlglot records an offset for a token it read. An argument it synthesized, + such as the `10` in `LOG10(x)`, or normalized, such as `'month'` becoming + `'MONTH'` in `DATE_TRUNC`, carries none. + """ + meta = node.meta + if not meta and isinstance(getattr(node, "this", None), exp.Expression): + meta = node.this.meta + return meta.get("start") + + +def _from_source(node, ctx) -> str | None: + """A function sqlglot models, passed through under the name the source used. + + sqlglot names a node after the shape it parsed, not after the text. `LPAD` + and `RPAD` are both `Pad`, `LTRIM` and `RTRIM` are both `Trim`, so rendering + from the node loses which one was written. The node's own offset points at + the name token in the source, so the original name is still there to read. + + Returns None when the call cannot be rebuilt faithfully, which happens when + sqlglot synthesized or normalized an argument and so left it with no offset. + `LOG10(x)` gains a base argument the source never had, and + `DATE_TRUNC('month', d)` has its unit rewritten to `'MONTH'`. Dropping the + first is right and dropping the second is not, and an absent offset does not + say which case it is, so neither is passed through. + """ + start = node.meta.get("start") + if start is None: + return None + opening = ctx.source.find("(", start) + if opening < 0: + return None + name = ctx.source[start:opening].strip() + if not name.replace("_", "").isalnum(): + return None + + arguments = [ + value + for slot in node.args.values() + for value in (slot if isinstance(slot, list) else [slot]) + if isinstance(value, exp.Expression) + ] + offsets = [_span(argument) for argument in arguments] + if any(offset is None for offset in offsets): + return None + + in_source_order = [a for _, a in sorted(zip(offsets, arguments), key=lambda pair: pair[0])] + rendered = [_render(argument, ctx) for argument in in_source_order] + return f"{SQL_PASSTHROUGH}('{name}'{''.join(', ' + r for r in rendered)})" + + +def _arguments(node, ctx) -> list[str]: + """Every argument of a function node, in the order sqlglot declares them. + + Each node names its own slots, and they are not the same from one function + to the next: `NULLIF` uses `this` and `expression`, `COALESCE` uses `this` + and `expressions`, `ROUND` uses `this` and `decimals`. Reading a fixed set + of slot names drops the rest without saying so, which is how + `ROUND(SUM(x), 2)` first rendered as `sql_number('ROUND', sum(x))`. + `arg_types` gives the declared order, so no slot is missed. + """ + rendered = [] + for slot in getattr(node, "arg_types", {"this": True}): + value = node.args.get(slot) + for item in value if isinstance(value, list) else [value]: + if isinstance(item, exp.Expression): + rendered.append(_render(item, ctx)) + return rendered diff --git a/converters/holistics/src/ossie_holistics/cli.py b/converters/holistics/src/ossie_holistics/cli.py new file mode 100644 index 000000000..c7e8ef890 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/cli.py @@ -0,0 +1,264 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Holistics AML converter. + + ossie-holistics to-ossie -o --sql-dialect BIGQUERY + ossie-holistics to-aml -o --sql-dialect BIGQUERY + +`to-ossie` reads the JSON `holistics aml compile` produces. `to-aml` writes AML +source text, which is the format that CLI reads back: + + holistics aml compile .dataset.aml -r -o + holistics aml validate /.dataset.aml -r + +Every declared loss is written as a YAML sequence of issues, to `--issues` when +given and to stderr otherwise, never mixed into the document output. The process +exits 1 when that log holds an ERROR-severity issue and 0 otherwise. A +conversion that only warned is still a successful conversion, and failing on a +warning would teach scripts to ignore it. + +Neither subcommand overwrites an existing output file unless `--force` is given. +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import Counter +from pathlib import Path + +from . import _yaml, aml_to_ossie, ossie_to_aml, sqlrefs +from .errors import ConversionError +from .issues import IssueLog + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="ossie-holistics", + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + sub = parser.add_subparsers(dest="command") + sub.required = True + + to_ossie = sub.add_parser( + "to-ossie", help="compiled Holistics AML JSON -> one Apache Ossie semantic model" + ) + to_ossie.add_argument( + "input", metavar="COMPILED_JSON", help="output of `holistics aml compile`" + ) + to_ossie.add_argument( + "-o", "--output", required=True, metavar="FILE", help="output Apache Ossie YAML file" + ) + to_ossie.add_argument( + "--sql-dialect", + required=True, + metavar="DIALECT", + choices=sorted(sqlrefs.SQLGLOT_DIALECT), + help="the Ossie dialect a computed `@sql` body is written for. AML does not " + "record the warehouse. The choices are the dialects sqlglot can parse, because " + "`to-aml` has to read the expression back to restore its field references", + ) + to_ossie.add_argument( + "--issues", metavar="FILE", help="write the issue log as YAML here (default: stderr)" + ) + to_ossie.add_argument( + "--force", action="store_true", help="overwrite an existing output file" + ) + + to_aml = sub.add_parser("to-aml", help="one Apache Ossie semantic model -> AML source files") + to_aml.add_argument("input", metavar="OSSIE_YAML", help="Apache Ossie YAML document") + to_aml.add_argument( + "-o", + "--output", + required=True, + metavar="DIR", + help="output directory. One dataset file plus one model file per dataset is " + "written here, created if it does not already exist", + ) + to_aml.add_argument( + "--sql-dialect", + required=True, + metavar="DIALECT", + choices=sorted(sqlrefs.SQLGLOT_DIALECT), + help="the warehouse the Holistics connection reads. Every `@sql` body is " + "written in this dialect. An expression already in it is written as it stands, " + "and an ANSI_SQL or OSSIE_SQL_2026 one is rendered into it", + ) + to_aml.add_argument( + "--data-source-name", + metavar="NAME", + help="the Holistics connection the models read from. AML requires it and Ossie " + "has no field for it, so a document this converter did not write needs it", + ) + to_aml.add_argument( + "--issues", metavar="FILE", help="write the issue log as YAML here (default: stderr)" + ) + to_aml.add_argument( + "--force", action="store_true", help="overwrite existing output files" + ) + return parser + + +def _safe_target(directory: Path, relative: str) -> Path: + """`directory / relative`, which has to resolve inside `directory`. + + A model name comes from user-controlled document content, and the reverse + path turns it into a file path. Both sides are resolved before comparing, + because a string-prefix check is wrong on a filesystem with symlinks, as on + macOS where `/tmp` links to `/private/tmp`. + """ + root = directory.resolve() + target = (root / relative).resolve() + if target == root or root not in target.parents: + raise ConversionError( + f"{relative!r} resolves outside the output directory {root}, so nothing is written" + ) + return target + + +def _colliding(paths: list[Path]) -> list[Path]: + """Any path this run would write more than once. + + Compared after `resolve()` and case-folded, because two spellings of one + file are the same file on disk, and the default filesystem on macOS and + Windows folds case. Checked before any write, so `--issues` pointing at `-o` + cannot replace the converted document with the issue log. + """ + seen: dict[str, Path] = {} + counts: Counter[str] = Counter() + for path in paths: + resolved = path.resolve() + key = str(resolved).casefold() + seen.setdefault(key, resolved) + counts[key] += 1 + return sorted(seen[key] for key, count in counts.items() if count > 1) + + +def _blocked(paths: list[Path], reason: str) -> str: + return f"{reason}: {', '.join(str(p) for p in paths)}" + + +def _write_issues(issues: IssueLog, path: Path | None) -> None: + """Issues as a YAML sequence, always written even when empty. + + A caller scripting against this output relies on the shape, not on whether + anything was said. YAML rather than JSON because a message runs to a + paragraph, and folded YAML wraps it instead of putting it on one long line. + """ + text = _yaml.dump_issues(issues.as_dicts()) + if path is None: + print(text, end="", file=sys.stderr) + return + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _guard(targets: list[Path], issues: Path | None, force: bool) -> str | None: + collisions = _colliding(targets) + if collisions: + issue_path = str(issues.resolve()).casefold() if issues else None + if issue_path and any(str(p).casefold() == issue_path for p in collisions): + reason = ( + "this run would write the same path twice. --issues must name a " + "different file from the converted output" + ) + else: + reason = ( + "this run would write the same path twice. Two datasets produce " + "this filename, and the filesystem may not tell them apart by case. " + "Rename one of them in Ossie" + ) + return _blocked(collisions, reason) + if not force: + existing = [p for p in targets if p.exists()] + if existing: + return _blocked(existing, "file(s) already exist; pass --force to overwrite") + return None + + +def _cmd_to_ossie(args: argparse.Namespace) -> int: + output = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + try: + payload = json.loads(Path(args.input).read_text(encoding="utf-8")) + result = aml_to_ossie.convert(payload, sql_dialect=args.sql_dialect) + except (ConversionError, OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + blocked = _guard( + [output] + ([issues_path] if issues_path else []), issues_path, args.force + ) + if blocked: + print(f"Error: {blocked}", file=sys.stderr) + return 1 + + try: + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(_yaml.dump(result.model), encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 1 if result.issues.has_errors() else 0 + + +def _cmd_to_aml(args: argparse.Namespace) -> int: + output_dir = Path(args.output) + issues_path = Path(args.issues) if args.issues else None + if output_dir.exists() and not output_dir.is_dir(): + print(f"Error: output path {output_dir} exists and is not a directory", file=sys.stderr) + return 1 + + try: + document = _yaml.load(Path(args.input).read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, args.sql_dialect, data_source_name=args.data_source_name + ) + targets = [(_safe_target(output_dir, name), text) for name, text in result.files] + except (ConversionError, OSError, UnicodeDecodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + + paths = [path for path, _ in targets] + ([issues_path] if issues_path else []) + blocked = _guard(paths, issues_path, args.force) + if blocked: + print(f"Error: {blocked}", file=sys.stderr) + return 1 + + try: + for path, text in targets: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + _write_issues(result.issues, issues_path) + except OSError as exc: + print(f"Error: {exc}", file=sys.stderr) + return 1 + return 1 if result.issues.has_errors() else 0 + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + if args.command == "to-ossie": + return _cmd_to_ossie(args) + return _cmd_to_aml(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/holistics/src/ossie_holistics/datatypes.py b/converters/holistics/src/ossie_holistics/datatypes.py new file mode 100644 index 000000000..5dee0d0b5 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/datatypes.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tables mapping AML vocabularies onto Ossie ones. + +Every mapping here is a dict rather than a branch, so adding an AML type or an +aggregation is one row and the reverse direction reads the same table backwards. +""" +from __future__ import annotations + +#: AML `type` to Ossie `datatype`. `number` is the lossy row: AML does not say +#: whether a number is exact or approximate, and `Decimal` is the closest Ossie +#: member because the specification leaves its precision and scale unspecified. +#: The original AML type goes to the stash either way, so the reverse path reads +#: it from there rather than from this table. +AML_TO_OSSIE_DATATYPE: dict[str, str] = { + "text": "String", + "truefalse": "Boolean", + "date": "Date", + "datetime": "DateTime", + "number": "Decimal", +} + +#: Ossie `datatype` to AML `type`, for a document with no stashed AML type. +#: `Decimal`, `Float` and `Integer` all collapse onto `number`, which is why the +#: forward path stashes the original rather than relying on this table. +OSSIE_TO_AML_TYPE: dict[str, str] = { + "String": "text", + "Boolean": "truefalse", + "Date": "date", + "Time": "datetime", + "DateTime": "datetime", + "DateTimeTz": "datetime", + "Decimal": "number", + "Float": "number", + "Integer": "number", + "Opaque": "text", +} + +#: AML `aggregation_type` to the `(prefix, suffix)` that wraps the measure body. +#: Stored as a pair rather than a function name so the reverse path strips what +#: the forward path added, instead of re-deriving it from the Ossie SQL. +#: `custom` is absent on purpose: its body already aggregates, so the converter +#: wraps nothing. A key missing from this table is an aggregation this converter +#: has never seen, and `aml_to_ossie` reports it rather than guessing. +AGGREGATIONS: dict[str, tuple[str, str]] = { + "sum": ("SUM(", ")"), + "count": ("COUNT(", ")"), + "count distinct": ("COUNT(DISTINCT ", ")"), + "min": ("MIN(", ")"), + "max": ("MAX(", ")"), + "average": ("AVG(", ")"), + "median": ("MEDIAN(", ")"), +} + +CUSTOM_AGGREGATION = "custom" + +#: Characters that open a quoted region in a SQL expression. A string literal +#: uses `'`, and a delimited identifier uses `"` or a backtick. +QUOTES = frozenset("'\"`") + + +def apply_aggregation(aggregation_type: str, body: str) -> str: + prefix, suffix = AGGREGATIONS[aggregation_type] + return f"{prefix}{body}{suffix}" + + +def strip_aggregation(aggregation_type: str, expression: str) -> str | None: + """The body inside the aggregate, or None when `expression` is not wrapped. + + The reverse path calls this to recover a measure `definition` from an Ossie + metric expression. A None result means the expression was edited after the + forward path wrote it, so the reverse path writes `aggregation_type: + 'custom'` and keeps the expression whole rather than producing a measure + that aggregates twice. + """ + pair = AGGREGATIONS.get(aggregation_type) + if pair is None: + return None + prefix, suffix = pair + text = expression.strip() + if not (text.startswith(prefix) and text.endswith(suffix)): + return None + if not _closes_at_end(text, prefix.index("(")): + return None + return text[len(prefix) : len(text) - len(suffix)] + + +def _closes_at_end(text: str, open_index: int) -> bool: + """Whether the parenthesis at `open_index` is closed by the last character. + + Without this, `SUM(a) + SUM(b)` matches the `sum` prefix and suffix and + strips to `a) + SUM(b`, which is not an expression. + + Quoted regions are skipped, so a parenthesis inside one does not move the + depth. All three quoting characters count: a string literal uses `'`, and a + delimited identifier uses `"` or, on BigQuery and Databricks, a backtick. + """ + depth = 0 + index = open_index + while index < len(text): + character = text[index] + if character in QUOTES: + closer = character + index += 1 + while index < len(text) and text[index] != closer: + index += 2 if text[index] == "\\" else 1 + elif character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth == 0: + return index == len(text) - 1 + index += 1 + return False diff --git a/converters/holistics/src/ossie_holistics/errors.py b/converters/holistics/src/ossie_holistics/errors.py new file mode 100644 index 000000000..d81421963 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/errors.py @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Hard failures. Distinct from ConverterIssue, which records a survivable loss.""" + + +class ConversionError(Exception): + """Raised when the converter cannot proceed, e.g. a malformed stash.""" diff --git a/converters/holistics/src/ossie_holistics/interpolation.py b/converters/holistics/src/ossie_holistics/interpolation.py new file mode 100644 index 000000000..ab889ddb6 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/interpolation.py @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `{{ ... }}` and `{% ... %}` syntax AML embeds in a `@sql` body. + +One `{{ ... }}` spelling covers three unrelated things, and telling them apart +decides what the forward path may emit: + + {{ #SOURCE.created_at }} a physical column of this model's own source + {{ created_at }} another AML field of the same model + {{ #orders.created_at }} a field reached through a named model + +`parse_sql_body` returns the body as an alternating list of literal text and +`Interpolation` values, so a caller rewrites the references and leaves every +other character untouched. Both directions use it: the forward path turns an +interpolation into an Ossie reference, and the reverse path turns an Ossie +reference back into an interpolation. + +A query body uses a second, disjoint set of forms, listed in `QUERY_FORMS`. +Those are never resolved. `query_forms_used` only reports which ones appear, so +the forward path can say why the dataset's `source` is not runnable SQL. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass + +#: The sentinel model name meaning "this model's own source relation". +SOURCE = "SOURCE" + +#: An AML identifier. `::` is allowed because a model reference inside a body +#: may name a model that lives in a module, as in `{{ #reporting::calendar.date }}`. +_IDENT = r"[A-Za-z_][A-Za-z0-9_]*(?:::[A-Za-z_][A-Za-z0-9_]*)*" + +_MOUSTACHE = re.compile(r"\{\{(.*?)\}\}", re.DOTALL) +_SOURCE_REF = re.compile(rf"^#{SOURCE}\.({_IDENT})$") +_MODEL_REF = re.compile(rf"^#({_IDENT})\.({_IDENT})$") +_SIBLING_REF = re.compile(rf"^({_IDENT})$") + + +@dataclass(frozen=True) +class Source: + """`{{ #SOURCE.column }}`. A physical column, so it carries no semantics.""" + + column: str + + def to_aml(self) -> str: + return f"{{{{ #{SOURCE}.{self.column} }}}}" + + +@dataclass(frozen=True) +class Sibling: + """`{{ field }}`. Another AML field of the same model.""" + + field: str + + def to_aml(self) -> str: + return f"{{{{ {self.field} }}}}" + + +@dataclass(frozen=True) +class ModelField: + """`{{ #model.field }}`. A field reached through a named model.""" + + model: str + field: str + + def to_aml(self) -> str: + return f"{{{{ #{self.model}.{self.field} }}}}" + + +@dataclass(frozen=True) +class Unrecognised: + """A `{{ ... }}` matching none of the three forms, kept verbatim.""" + + text: str + + def to_aml(self) -> str: + return f"{{{{{self.text}}}}}" + + +Interpolation = Source | Sibling | ModelField | Unrecognised + + +def classify(inner: str) -> Interpolation: + """Classify the text between one pair of braces.""" + stripped = inner.strip() + match = _SOURCE_REF.match(stripped) + if match: + return Source(match.group(1)) + match = _MODEL_REF.match(stripped) + if match: + return ModelField(match.group(1), match.group(2)) + match = _SIBLING_REF.match(stripped) + if match: + return Sibling(match.group(1)) + return Unrecognised(inner) + + +def parse_sql_body(body: str) -> list[str | Interpolation]: + """Split a `@sql` body into literal text and interpolations, in order. + + Literal segments are kept even when empty, so a body that is exactly one + interpolation returns `["", Source(...), ""]`. `render` rebuilds the input + from the output, so a caller that changes nothing changes nothing. + """ + parts: list[str | Interpolation] = [] + position = 0 + for match in _MOUSTACHE.finditer(body): + parts.append(body[position : match.start()]) + parts.append(classify(match.group(1))) + position = match.end() + parts.append(body[position:]) + return parts + + +def render(parts: list[str | Interpolation]) -> str: + """Rebuild an AML body from `parse_sql_body` output.""" + return "".join(p if isinstance(p, str) else p.to_aml() for p in parts) + + +def interpolations(body: str) -> list[Interpolation]: + return [p for p in parse_sql_body(body) if not isinstance(p, str)] + + +def sole_reference(body: str) -> Interpolation | None: + """The interpolation when `body` is exactly one, surrounded by whitespace. + + This is the test for "carries no Holistics SQL of its own". `{{ #SOURCE.quantity }}` + and `{{ created_at }}` both pass, and each becomes a plain Ossie reference + under `ANSI_SQL`. `CAST({{ #SOURCE.created_at }} AS DATE)` fails, because + the literal text around the interpolation is warehouse SQL that this + converter will not rewrite. + + An `Unrecognised` interpolation returns None. It is not a reference this + converter can resolve, so it must not be treated as one. + """ + parts = parse_sql_body(body) + if len(parts) != 3: + return None + before, middle, after = parts + if before.strip() or after.strip(): + return None + return None if isinstance(middle, Unrecognised) else middle + + +#: Every form that makes a query body something other than plain SQL. The names +#: are stashed verbatim, so a consumer reading the stash learns which Holistics +#: feature it must resolve before the `source` will run. +QUERY_FORMS: dict[str, re.Pattern[str]] = { + "model_columns": re.compile(rf"\{{\{{\s*#{_IDENT}\.\*\s*\}}\}}"), + "model_relation": re.compile(rf"\{{\{{\s*#{_IDENT}\s*\}}\}}"), + "filter_block": re.compile(r"\{%\s*filter\s*\(.*?\)\s*%\}", re.DOTALL), +} + + +def query_forms_used(body: str) -> list[str]: + """Which `QUERY_FORMS` appear in `body`, in declaration order. + + `model_columns` is tested before `model_relation` and the two never both + match one site, because `{{ #orders.* }}` ends in `.*` and the + `model_relation` pattern requires the closing braces straight after the + identifier. + """ + return [name for name, pattern in QUERY_FORMS.items() if pattern.search(body)] diff --git a/converters/holistics/src/ossie_holistics/issues.py b/converters/holistics/src/ossie_holistics/issues.py new file mode 100644 index 000000000..adb08ab0d --- /dev/null +++ b/converters/holistics/src/ossie_holistics/issues.py @@ -0,0 +1,96 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured, never-silent loss reporting. + +Discussion apache/ossie#325 treats a silently dropped field as a contract +violation rather than a documentation gap, so every declared loss produces an +issue here at conversion time. The same discussion treats a warning storm as a +defect, which is why severity is first-class and why callers can summarise via +count_by_severity() instead of printing every line. +""" +from collections import Counter +from dataclasses import dataclass, field +from enum import Enum + + +class Severity(str, Enum): + INFO = "INFO" + WARNING = "WARNING" + ERROR = "ERROR" + + +@dataclass(frozen=True) +class ConverterIssue: + """One declared loss or degradation, traceable to a specific object. + + object_ref is mandatory: an issue a reader cannot trace to an object cannot + be acted on, which is the complaint #325 raises about warning noise. + """ + + code: str + severity: Severity + message: str + object_ref: str + remedy: str | None = None + + def as_dict(self) -> dict[str, str | None]: + return { + "code": self.code, + "severity": self.severity.value, + "message": self.message, + "object_ref": self.object_ref, + "remedy": self.remedy, + } + + +@dataclass +class IssueLog: + """Ordered collection of issues raised during one conversion.""" + + issues: list[ConverterIssue] = field(default_factory=list) + + def add( + self, + *, + code: str, + severity: Severity, + message: str, + object_ref: str, + remedy: str | None = None, + ) -> None: + self.issues.append( + ConverterIssue( + code=code, + severity=severity, + message=message, + object_ref=object_ref, + remedy=remedy, + ) + ) + + def extend(self, other: "IssueLog") -> None: + self.issues.extend(other.issues) + + def as_dicts(self) -> list[dict[str, str | None]]: + return [i.as_dict() for i in self.issues] + + def has_errors(self) -> bool: + return any(i.severity is Severity.ERROR for i in self.issues) + + def count_by_severity(self) -> dict[str, int]: + return dict(Counter(i.severity.value for i in self.issues)) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py new file mode 100644 index 000000000..91eea8397 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -0,0 +1,1041 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""One Apache Ossie semantic model to a set of AML source files. + +`convert` returns `(relative path, text)` pairs rather than writing them, so the +CLI owns every filesystem decision and a test can read the output without a +temporary directory. + +The layout is one dataset file plus one model file per Ossie dataset: + + .dataset.aml + orders.model.aml an Ossie dataset named orders + modules/reporting/calendar.model.aml an Ossie dataset named reporting__calendar + +The input file layout is lost on the way in. The compiled JSON the forward path +reads records which models a dataset holds and not which file each came from, so +the layout is already gone by the time a document reaches Ossie. AML accepts any +arrangement, so the generator picks the one above. + +Four AML properties are required and have no Ossie field: `label` and `type` on +every dimension and measure, and `data_source_name` and `relationships` on the +dataset. A stashed value wins. Otherwise `label` is derived from the name the +way AML itself derives it, `type` from `datatype`, and `data_source_name` from +the `--data-source-name` argument. +""" +from __future__ import annotations + +from dataclasses import dataclass, field as dataclass_field +from typing import Any + +from . import amlgen, aql, interpolation, sqlrefs, stash +from .amlgen import Block, quote, sanitize, titleize +from .datatypes import AGGREGATIONS, CUSTOM_AGGREGATION, OSSIE_TO_AML_TYPE, strip_aggregation +from .errors import ConversionError +from .issues import IssueLog, Severity + +AQL_DIALECT = "HOLISTICS_AQL" + +ISSUE_CROSS_MODEL_SQL = "HOLISTICS_CROSS_MODEL_SQL_BODY" +ISSUE_UNPARSEABLE_SQL = "HOLISTICS_UNPARSEABLE_SQL" +ISSUE_UNKNOWN_QUALIFIER = "HOLISTICS_UNKNOWN_QUALIFIER" +ISSUE_METRIC_NOT_TRANSLATED = "HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL" +ISSUE_ASSUMED_TYPE = "HOLISTICS_ASSUMED_TYPE" +ISSUE_AGGREGATION_NOT_STRIPPED = "HOLISTICS_AGGREGATION_NOT_STRIPPED" +ISSUE_RENAMED = "HOLISTICS_NAME_NOT_SPELLABLE_IN_AML" +ISSUE_DATASET_RENAMED = "HOLISTICS_DATASET_NAME_COLLIDES_WITH_MODEL" +ISSUE_TRANSPILED = "HOLISTICS_EXPRESSION_RENDERED_INTO_TARGET_DIALECT" +ISSUE_NO_USABLE_DIALECT = "HOLISTICS_NO_USABLE_DIALECT" +ISSUE_RENDERED_AS_AQL = "HOLISTICS_METRIC_RENDERED_AS_AQL" +ISSUE_BORROWED_COLUMN = "HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION" +ISSUE_WINDOW_NEEDS_GROUPING = "HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING" +ISSUE_METRIC_CYCLE = "HOLISTICS_METRIC_REFERENCE_CYCLE" + +#: Placement markers for `_datasets_named`. A metric that finds one still marked +#: `_RESOLVING` closed a reference cycle, which no single model can hold. +_UNVISITED = object() +_RESOLVING = object() + +#: The AML type written for a field whose Ossie `datatype` is absent. +FALLBACK_TYPE = "text" + +#: The AML type for a column borrowed into a model by an AQL metric. Ossie +#: declares nothing about it, and every borrowed column so far is an aggregate's +#: argument, so `number` is the reading that lets the metric compile. +FALLBACK_TYPE_FOR_BORROWED = "number" + + +class Result: + def __init__(self, files: list[tuple[str, str]], issues: IssueLog): + self.files = files + self.issues = issues + + +@dataclass +class Model: + """One Ossie dataset, as the AML model it will be written as.""" + + ossie_name: str + fqn: str + payload: dict[str, Any] + stash: dict[str, Any] + measures: list[dict[str, Any]] = dataclass_field(default_factory=list) + #: Columns an AQL metric needed that the dataset does not declare as a + #: field. Each becomes a hidden dimension so the AQL reference resolves. + borrowed: list[str] = dataclass_field(default_factory=list) + #: Fields an AQL metric aggregates numerically. The metric is evidence of + #: the type when the Ossie document declares no `datatype`. + numeric: set[str] = dataclass_field(default_factory=set) + + @property + def segments(self) -> list[str]: + return [sanitize(part) for part in self.fqn.split("::")] + + @property + def aml_fqn(self) -> str: + return "::".join(self.segments) + + @property + def aml_name(self) -> str: + return self.segments[-1] + + @property + def path(self) -> str: + parts = self.segments + if len(parts) == 1: + return f"{parts[0]}.model.aml" + return "/".join(["modules", *parts[:-1], f"{parts[-1]}.model.aml"]) + + @property + def reference(self) -> str: + """How the dataset block names this model. Dots, not `::`.""" + return ".".join(self.segments) + + +def unflatten(name: str) -> str: + """An Ossie dataset name as an AML module path. + + The forward path writes `reporting::calendar` as `reporting__calendar`, so + this reverses that. A stashed `fqn` wins over this rule, because it records + where the module boundaries actually were rather than inferring them. + """ + return name.replace("__", "::") + + +#: The dialects a portable body may be written in. Every sibling converter that +#: reads Ossie treats `OSSIE_SQL_2026` as an `ANSI_SQL` equivalent, including +#: Snowflake's `_extract_expression` and Cube's `pick_expression`. +PORTABLE_DIALECTS = ("ANSI_SQL", "OSSIE_SQL_2026") + + +def _named_objects(container: Any, where: str) -> list[dict[str, Any]]: + """`container` as a list of named mappings, or a `ConversionError` saying why. + + A document this converter did not write is parsed at the boundary rather + than indexed into. Without this, `datasets` given as a mapping raised + `AttributeError: 'str' object has no attribute 'get'` and a field with no + `name` raised `KeyError`, neither of which the CLI catches, so a malformed + document produced a traceback instead of a message. apache/ossie#297 and + apache/ossie#407 both asked a sibling converter for the same guard. + """ + if container is None: + return [] + if not isinstance(container, list): + raise ConversionError( + f"{where} is {type(container).__name__}, and the specification defines it as a list" + ) + for index, member in enumerate(container): + if not isinstance(member, dict): + raise ConversionError( + f"{where}[{index}] is {type(member).__name__}, and must be a mapping" + ) + name = member.get("name") + if not isinstance(name, str) or not name: + raise ConversionError(f"{where}[{index}] has no name") + return container + + +def dialects_of(expression: Any) -> dict[str, str]: + dialects = (expression or {}).get("dialects") or [] + if not dialects: + raise ConversionError("expression has no dialects") + return {entry.get("dialect", "ANSI_SQL"): entry.get("expression", "") for entry in dialects} + + +class _Reverse: + def __init__( + self, document: dict[str, Any], data_source_name: str | None, sql_dialect: str + ): + self.document = document + self.sql_dialect = sql_dialect + self.issues = IssueLog() + self.root_stash = stash.read(document) + self.data_source_name = self.root_stash.get("data_source_name") or data_source_name + if not self.data_source_name: + raise ConversionError( + "the document carries no HOLISTICS data_source_name, so one must be " + "supplied: pass --data-source-name naming the Holistics connection" + ) + + #: AML identifier to the Ossie name it was spelled from, so two names + #: that collapse onto one spelling fail rather than overwrite. + self._renamed: dict[str, str] = {} + + datasets = _named_objects(document.get("datasets"), "datasets") + if not datasets: + raise ConversionError( + "the document declares no datasets, and the specification requires at least one" + ) + for dataset in datasets: + _named_objects(dataset.get("fields"), f"datasets[{dataset['name']}].fields") + _named_objects(document.get("metrics"), "metrics") + + self.models: list[Model] = [] + for dataset in datasets: + name = dataset["name"] + data = stash.read(dataset) + self.models.append( + Model( + ossie_name=name, + fqn=data.get("fqn") or unflatten(name), + payload=dataset, + stash=data, + ) + ) + self.by_ossie_name = {m.ossie_name: m for m in self.models} + + taken: dict[str, str] = {} + for model in self.models: + clash = taken.setdefault(model.aml_fqn, model.ossie_name) + if clash != model.ossie_name: + raise ConversionError( + f"datasets {clash!r} and {model.ossie_name!r} both spell as the AML " + f"model {model.aml_fqn!r}; rename one of them in Ossie" + ) + + self.dataset_name = self._dataset_name() + + #: Every metric that belongs on a model, indexed by Ossie metric name, + #: alongside the AML measure name it is written as. + self.metric_owner: dict[str, Model] = {} + self.metric_aml_name: dict[str, str] = {} + self.dataset_metrics: list[dict[str, Any]] = [] + self._metric_payloads: dict[str, dict[str, Any]] = {} + self._metric_datasets: dict[str, Any] = {} + self._cyclic_metrics: set[str] = set() + self._place_metrics() + + # -- placement -------------------------------------------------------- + + def _place_metrics(self) -> None: + """Decide where each Ossie metric goes: on a model, or on the dataset. + + A stashed `model` settles it. Without one, the expression decides: a + body naming exactly one dataset becomes a measure on that model, and a + body naming several becomes a dataset-level metric, which is the only + AML construct that spans models. A body may also name another metric, + and `_scan_for_datasets` covers what that counts as. + """ + metrics = self.document.get("metrics") or [] + for metric in metrics: + name = metric.get("name") + self.metric_aml_name[name] = stash.read(metric).get("aml_name") or name + self._metric_payloads[name] = metric + for metric in metrics: + name = metric.get("name") + named = self._datasets_named(name) or frozenset() + owner = self.by_ossie_name[next(iter(named))] if len(named) == 1 else None + if owner is None: + self.dataset_metrics.append(metric) + else: + owner.measures.append(metric) + self.metric_owner[name] = owner + self._report_metric_cycles() + + def _datasets_named(self, name: str) -> frozenset[str] | None: + """The Ossie datasets metric `name` resolves to, or None when that is unknown. + + None covers every reading that cannot put the metric on a single model: + an AQL body, Ossie SQL this converter cannot read, and a reference cycle. + """ + resolved = self._metric_datasets.get(name, _UNVISITED) + if resolved is _RESOLVING: + self._cyclic_metrics.add(name) + return None + if resolved is not _UNVISITED: + return resolved + self._metric_datasets[name] = _RESOLVING + result = self._scan_for_datasets(name) + self._metric_datasets[name] = result + return result + + def _scan_for_datasets(self, name: str) -> frozenset[str] | None: + """Which datasets one metric body names, directly or through a metric. + + An Ossie metric body may name another metric instead of a column, as + `cities__upstream_measure + 1` in `tests/fixtures/ecommerce` does. That + body names no dataset of its own, and the datasets the metric it + references resolves to stand in for it. Without that substitution a + metric built only from other metrics lands on the dataset, where AML + takes an AQL body, and the Ossie SQL name it was written with does not + resolve. + """ + metric = self._metric_payloads[name] + data = stash.read(metric) + owner_fqn = data.get("model") + if owner_fqn: + owner = next((m for m in self.models if m.fqn == owner_fqn), None) + if owner is None: + raise ConversionError( + f"metric {name!r} is stashed against model {owner_fqn!r}, " + f"which is not a dataset in this document" + ) + return frozenset({owner.ossie_name}) + available = dialects_of(metric.get("expression")) + if AQL_DIALECT in available: + return None + dialect = next( + (d for d in (self.sql_dialect, *PORTABLE_DIALECTS) if d in available), None + ) + if dialect is None: + return None + try: + references = sqlrefs.columns(available[dialect], dialect) + except sqlrefs.UnparseableSQL: + return None + named = {c.table for c in references if c.table in self.by_ossie_name} + for reference in references: + if reference.table or reference.name not in self._metric_payloads: + continue + through = self._datasets_named(reference.name) + if through is None: + return None + named |= through + return frozenset(named) + + def _report_metric_cycles(self) -> None: + for name in sorted(self._cyclic_metrics): + self.issues.add( + code=ISSUE_METRIC_CYCLE, + severity=Severity.ERROR, + message=( + f"metric {name!r} references itself through another metric. Holistics " + f"inlines one metric into the body of the next, so a cycle has no warehouse SQL, " + f"and `holistics aml validate` accepts the file without catching it" + ), + object_ref=name, + remedy="break the cycle in Ossie", + ) + + # -- names ------------------------------------------------------------ + + def _dataset_name(self) -> str: + """The AML `Dataset` name, moved aside when a model already has it. + + Ossie keeps the semantic model's name and its dataset names in separate + namespaces, so a document may use one name for both. Three fixtures in + this repository do, including + `converters/cube/tests/fixtures/databricks_ossie.yaml`, whose root name + and first dataset are both `orders`. AML has one namespace for `Dataset` + and `Model`, and rejects the pair with "Duplicated name 'orders'". + + Nothing references the AML dataset name. Models reference each other and + relationships reference models, so the dataset is the safe side to move. + """ + name = self.root_stash.get("fqn") or self.document.get("name") + if not name: + raise ConversionError("the document has no name") + spelled = self.name_in_aml(name, str(name)) + taken = {model.aml_fqn for model in self.models} | { + model.aml_name for model in self.models + } + if spelled not in taken: + return spelled + candidate, suffix = f"{spelled}_dataset", 2 + while candidate in taken: + candidate, suffix = f"{spelled}_dataset_{suffix}", suffix + 1 + self.issues.add( + code=ISSUE_DATASET_RENAMED, + severity=Severity.WARNING, + message=( + f"a model is already named {spelled!r}, and AML holds Dataset and Model " + f"in one namespace, so the dataset is written as {candidate!r}" + ), + object_ref=str(name), + remedy="rename the semantic model or the dataset in Ossie", + ) + return candidate + + def name_in_aml(self, name: str, scope: str) -> str: + """An Ossie name as the AML identifier it is written as. + + Lookups elsewhere stay keyed on the Ossie name, because that is what an + expression references. Only the output spelling changes, and every site + that writes a name goes through here, so a declaration and a reference + to it cannot disagree. + """ + spelled = sanitize(name) + if spelled != name: + previous = self._renamed.setdefault(spelled, name) + if previous != name: + raise ConversionError( + f"{scope}: {name!r} and {previous!r} both spell as the AML " + f"identifier {spelled!r}; rename one of them in Ossie" + ) + self.issues.add( + code=ISSUE_RENAMED, + severity=Severity.WARNING, + message=( + f"{name!r} is not a legal AML identifier and is written as " + f"{spelled!r}. AML allows letters, digits and underscore, and no " + f"leading digit" + ), + object_ref=scope, + remedy=f"rename it to {spelled!r} in Ossie to keep the two in step", + ) + return spelled + + # -- expressions ------------------------------------------------------ + + def pick_body(self, expression: Any, scope: str) -> tuple[str, str]: + """The body to write and the dialect it is in, after any rendering. + + AQL wins outright. It came out of AML unchanged and goes back unchanged. + + For Ossie SQL the target is `--sql-dialect`, the warehouse the Holistics + connection reads. A body already in that dialect is written as it + stands. Otherwise a portable body is rendered into the target with + sqlglot, because the alternative is AML that will not run. An expression + offering neither has no body this connection can execute, and the + converter says so rather than writing Holistics SQL for the wrong warehouse. + """ + available = dialects_of(expression) + if AQL_DIALECT in available: + return AQL_DIALECT, available[AQL_DIALECT] + if self.sql_dialect in available: + return self.sql_dialect, available[self.sql_dialect] + + source = next((d for d in PORTABLE_DIALECTS if d in available), None) + if source is not None: + try: + rendered = sqlrefs.transpile(available[source], source, self.sql_dialect) + except sqlrefs.UnparseableSQL as exc: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"the {source} body cannot be rendered as {self.sql_dialect} ({exc}), " + f"so it is written unchanged and may not run" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} expression, or fix the {source} one", + ) + return source, available[source] + self.issues.add( + code=ISSUE_TRANSPILED, + severity=Severity.WARNING, + message=( + f"no {self.sql_dialect} expression, so the {source} one was rendered " + f"into it: {available[source]!r} became {rendered!r}" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} expression to keep the body exact", + ) + return self.sql_dialect, rendered + + fallback = next(iter(available)) + self.issues.add( + code=ISSUE_NO_USABLE_DIALECT, + severity=Severity.ERROR, + message=( + f"the expression offers {', '.join(sorted(available))} and the connection " + f"reads {self.sql_dialect}, with no portable body to render from, so the " + f"{fallback} body is written unchanged and will not run" + ), + object_ref=scope, + remedy=f"add a {self.sql_dialect} or ANSI_SQL expression", + ) + return fallback, available[fallback] + + def definition(self, block: Block, expression: Any, model: Model, scope: str) -> None: + """Write a `definition` property for a field or a measure.""" + dialect, text = self.pick_body(expression, scope) + if dialect == AQL_DIALECT: + block.heredoc("definition", "aql", text) + return + block.heredoc("definition", "sql", self.to_aml_sql(text, dialect, model, scope)) + + def to_aml_sql(self, expression: str, dialect: str, model: Model, scope: str) -> str: + """An Ossie SQL expression with its references written as AML interpolation. + + A bare column name does compile as AML, but AQL reads the interpolation + to learn which field the body depends on. A body without it loses the + query rewriting and the optimisation that link drives, so this restores + every reference rather than passing the SQL through. + """ + if dialect in sqlrefs.NOT_SQL or not sqlrefs.AVAILABLE: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"the body is tagged {dialect!r}, which is not Ossie SQL this converter can " + f"read, so its field references cannot be restored and AQL cannot see them" + ), + object_ref=scope, + remedy="supply an ANSI_SQL or warehouse-SQL dialect for this expression", + ) + return expression + grouping = sqlrefs.window_grouping(expression, dialect) + if grouping: + self.issues.add( + code=ISSUE_WINDOW_NEEDS_GROUPING, + severity=Severity.WARNING, + message=( + f"the body has a window function whose frame reads " + f"{', '.join(grouping)}. AML carries it as written, and the warehouse SQL " + f"it generates is valid only for a query that groups by those. A query " + f"that does not, a grand total for one, produces warehouse SQL the " + f"database rejects, and Holistics does not check this at conversion time" + ), + object_ref=scope, + remedy="query it only alongside those fields, or rewrite the body in AQL", + ) + try: + return sqlrefs.rewrite( + expression, dialect, lambda column: self.to_interpolation(column, model, scope) + ) + except sqlrefs.UnparseableSQL as exc: + self.issues.add( + code=ISSUE_UNPARSEABLE_SQL, + severity=Severity.ERROR, + message=( + f"sqlglot cannot read the body as {dialect} ({exc}), so its field " + f"references cannot be restored and AQL cannot see them" + ), + object_ref=scope, + remedy="fix the expression, or convert it by hand", + ) + return expression + + def to_interpolation(self, column: sqlrefs.ColumnReference, model: Model, scope: str) -> str: + """One Ossie column reference as the AML interpolation it came from.""" + if not column.table: + if column.name in self.metric_aml_name: + return self._measure_reference(column.name, model, scope) + return interpolation.Source(column.name).to_aml() + + target = self.by_ossie_name.get(column.table) + if target is None: + self.issues.add( + code=ISSUE_UNKNOWN_QUALIFIER, + severity=Severity.ERROR, + message=( + f"the body qualifies {column.name!r} with {column.table!r}, which is not " + f"a dataset in this document, so it is left unresolved" + ), + object_ref=scope, + remedy="qualify the reference with a dataset name, or drop the field", + ) + return f"{column.table}.{column.name}" + if target is model: + # A qualifier naming this model usually names one of its fields. It + # can also name a metric this converter put somewhere else, and a + # sibling reference to that spells a field the model does not have. + if column.name in self.metric_aml_name and self.metric_owner.get(column.name) is not model: + return self._measure_reference(column.name, model, scope) + return interpolation.Sibling(self.name_in_aml(column.name, scope)).to_aml() + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body names {column.table}.{column.name}, a field of another dataset. A `@sql` " + f"body resolves inside one model, and Holistics rejects this at SQL generation " + f"with \"Cannot refer to external field in TableModel/QueryModel\" even though " + f"`holistics aml validate` accepts the file" + ), + object_ref=scope, + remedy="move the reference into a dimension on its own model, or rewrite the body in AQL", + ) + return interpolation.ModelField(target.aml_fqn, self.name_in_aml(column.name, scope)).to_aml() + + def _measure_reference(self, metric_name: str, model: Model, scope: str) -> str: + owner = self.metric_owner.get(metric_name) + aml_name = self.metric_aml_name[metric_name] + if owner is model: + return interpolation.Sibling(self.name_in_aml(aml_name, scope)).to_aml() + if owner is None: + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body references the dataset-level metric {metric_name!r}. A `@sql` " + f"body resolves inside one model, so there is no AML spelling for it" + ), + object_ref=scope, + remedy="rewrite the expression in AQL, which does span models", + ) + return metric_name + self.issues.add( + code=ISSUE_CROSS_MODEL_SQL, + severity=Severity.ERROR, + message=( + f"the body references {metric_name!r}, a measure of model {owner.aml_fqn!r}. A " + f"`@sql` body resolves inside one model, and Holistics rejects this at SQL " + f"generation with \"Cannot refer to external field in TableModel/QueryModel\" " + f"even though `holistics aml validate` accepts the file" + ), + object_ref=scope, + remedy="rewrite the expression in AQL, which does span models", + ) + return interpolation.ModelField(owner.aml_fqn, self.name_in_aml(aml_name, scope)).to_aml() + + # -- fields ----------------------------------------------------------- + + def aml_type( + self, payload: dict[str, Any], data: dict[str, Any], scope: str, model: Model | None = None + ) -> str: + stashed = data.get("type") + if stashed: + return stashed + datatype = payload.get("datatype") + mapped = OSSIE_TO_AML_TYPE.get(datatype) if datatype else None + if mapped: + return mapped + if model is not None and payload["name"] in model.numeric: + # A metric aggregates this field with `sum` or `avg`, so it is a + # number whatever the document failed to say. + return "number" + self.issues.add( + code=ISSUE_ASSUMED_TYPE, + severity=Severity.WARNING, + message=( + f"AML requires a type and the Ossie datatype is {datatype!r}, " + f"so {FALLBACK_TYPE!r} is written" + ), + object_ref=scope, + remedy="set a datatype on the field, or edit the AML type after conversion", + ) + return FALLBACK_TYPE + + def common_properties( + self, + block: Block, + payload: dict[str, Any], + data: dict[str, Any], + scope: str, + model: Model | None = None, + ) -> None: + block.text("label", data.get("label") or titleize(payload["name"])) + block.property("type", quote(self.aml_type(payload, data, scope, model))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + + def dimension(self, model: Model, payload: dict[str, Any], primary_key: set[str]) -> Block: + name = payload["name"] + scope = f"{model.ossie_name}.{name}" + data = stash.read(payload) + block = Block(f"dimension {self.name_in_aml(name, scope)}") + self.common_properties(block, payload, data, scope, model) + self.definition(block, payload.get("expression"), model, scope) + block.text("format", data.get("format")) + block.flag("primary_key", name in primary_key) + return block + + def dataset_dimension(self, model: Model, payload: dict[str, Any]) -> Block: + """A field the forward path took from a dataset-level AML dimension.""" + name = payload["name"] + scope = f"{model.ossie_name}.{name}" + data = stash.read(payload) + block = Block(f"dimension {self.name_in_aml(name, scope)}") + block.property("model", model.reference) + self.common_properties(block, payload, data, scope, model) + self.definition(block, payload.get("expression"), model, scope) + block.text("format", data.get("format")) + return block + + def measure(self, model: Model, payload: dict[str, Any]) -> Block: + """One Ossie metric as a `measure` on its owning model. + + `aggregation_type` and the body compose into one Ossie expression on the + way out, so this splits them back apart. When the stashed aggregation + does not wrap the whole expression, the expression was edited after the + forward path wrote it, and `custom` keeps it whole rather than letting + the measure aggregate twice. + """ + metric_name = payload["name"] + aml_name = self.metric_aml_name[metric_name] + scope = f"{model.ossie_name}.{aml_name}" + data = stash.read(payload) + block = Block(f"measure {self.name_in_aml(aml_name, scope)}") + block.text("label", data.get("label") or titleize(aml_name)) + block.property("type", quote(self.aml_type(payload, data, scope))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + + dialect, text = self.pick_body(payload.get("expression"), scope) + aggregation = data.get("aggregation_type") or CUSTOM_AGGREGATION + if dialect == AQL_DIALECT: + block.heredoc("definition", "aql", text) + else: + if aggregation in AGGREGATIONS: + inner = strip_aggregation(aggregation, text) + if inner is None: + self.issues.add( + code=ISSUE_AGGREGATION_NOT_STRIPPED, + severity=Severity.WARNING, + message=( + f"the stashed aggregation {aggregation!r} does not wrap the whole " + f"expression, so the measure is written as 'custom' with the " + f"expression kept whole" + ), + object_ref=scope, + remedy="check the expression against the aggregation", + ) + aggregation = CUSTOM_AGGREGATION + else: + text = inner + block.heredoc("definition", "sql", self.to_aml_sql(text, dialect, model, scope)) + block.property("aggregation_type", quote(aggregation)) + block.text("format", data.get("format")) + return block + + def as_aql(self, text: str, dialect: str, scope: str) -> str | None: + """A dataset-level metric's Ossie SQL body rendered as AQL, or None if it cannot be. + + AML accepts only `@aql` here, so an Ossie SQL body either becomes AQL or has no + AML form at all. `aql.translate` is a name map over functions this + converter has paired with an AQL equivalent. Anything outside that table + raises `Untranslatable`, and the metric is dropped with an ERROR. + """ + try: + rendered, numeric = aql.translate( + text, dialect, lambda table, column: self.aql_reference(table, column, scope) + ) + except aql.Untranslatable as exc: + self.issues.add( + code=ISSUE_METRIC_NOT_TRANSLATED, + severity=Severity.ERROR, + message=( + f"the metric sits on the dataset, where AML accepts only an AQL body. The " + f"converter did not translate this {dialect} body, and the metric is " + f"dropped. {exc}" + ), + object_ref=scope, + remedy="rewrite the metric in AQL, or scope it to a single dataset", + ) + return None + self.issues.add( + code=ISSUE_RENDERED_AS_AQL, + severity=Severity.WARNING, + message=( + f"the metric sits on the dataset, where AML accepts only an AQL body. The " + f"{dialect} expression was rendered as AQL: {text!r} became {rendered!r}" + ), + object_ref=scope, + remedy="check the AQL against the Ossie SQL it came from", + ) + for table, column in numeric: + target = self.by_ossie_name.get(table) + if target is not None: + target.numeric.add(self.name_in_aml(column, scope)) + return rendered + + def aql_reference(self, table: str, column: str, scope: str) -> str: + """One column of an Ossie SQL metric body as the AQL reference it becomes. + + AQL names a field `model.field`, which is the shape the Ossie expression + already uses, so this maps the Ossie dataset name onto the AML model + name and keeps the field name. + + An Ossie SQL body may name a warehouse column the document does not declare as + a field. `SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id)` + in `converters/nvidia/tests/fixtures/sales.ossie.yaml` is one: `orders` + declares no `subtotal`. Ossie SQL resolves that against the table, and AQL + does not, answering `Field `subtotal` not found in model `orders``. So + the column becomes a hidden dimension on that model, and the AQL + reference points at it. + """ + if not table: + # An unqualified name that is a metric is a metric reference, which + # AQL takes: a dataset-level one by its bare name, and a measure by + # the model that holds it. + if column in self.metric_aml_name: + referenced = self.name_in_aml(self.metric_aml_name[column], scope) + owner = self.metric_owner.get(column) + return referenced if owner is None else f"{owner.aml_fqn}.{referenced}" + raise aql.Untranslatable( + f"{column!r} names no dataset, and a metric spanning models has no single " + f"model to resolve it against." + ) + target = self.by_ossie_name.get(table) + if target is None: + raise aql.Untranslatable(f"{table!r} is not a dataset in this document.") + name = self.name_in_aml(column, scope) + declared = { + field["name"] for field in target.payload.get("fields") or [] + } | {self.metric_aml_name[m["name"]] for m in target.measures} + if name not in declared and name not in target.borrowed: + target.borrowed.append(name) + self.issues.add( + code=ISSUE_BORROWED_COLUMN, + severity=Severity.WARNING, + message=( + f"the metric reads {table}.{column}, which the dataset does not declare " + f"as a field. AQL resolves a reference against declared fields only, so " + f"{column!r} is added to model {target.aml_fqn!r} as a hidden dimension " + f"over the column of that name" + ), + object_ref=scope, + remedy=f"declare {column!r} as a field of {table!r} in Ossie", + ) + return f"{target.aml_fqn}.{name}" + + def dataset_metric(self, payload: dict[str, Any]) -> Block | None: + """One Ossie metric as a dataset-level AML `metric`. + + AML accepts only an `@aql` body here. The compiler rejects `@sql` with + "Incompatible property 'definition' of 'Metric'", so a cross-model + metric carrying Ossie SQL has no AML form at all. + """ + name = payload["name"] + scope = name + data = stash.read(payload) + dialect, text = self.pick_body(payload.get("expression"), scope) + if dialect != AQL_DIALECT: + rendered = self.as_aql(text, dialect, scope) + if rendered is None: + return None + text = rendered + block = Block(f"metric {self.name_in_aml(name, scope)}") + block.text("label", data.get("label") or titleize(name)) + block.property("type", quote(self.aml_type(payload, data, scope))) + block.text("description", payload.get("description")) + block.flag("hidden", bool(data.get("hidden"))) + block.heredoc("definition", "aql", text) + block.text("format", data.get("format")) + return block + + # -- models ----------------------------------------------------------- + + def model_file(self, model: Model) -> str: + payload = model.payload + data = model.stash + scope = model.ossie_name + is_query = data.get("aml_type") == "QueryModel" or ( + "aml_type" not in data and bool(interpolation.query_forms_used(payload.get("source", ""))) + ) + source = payload.get("source", "") + if "aml_type" not in data and not is_query: + is_query = _looks_like_query(source) + + block = Block(f"Model {model.aml_name}") + block.property("type", quote("query" if is_query else "table")) + block.text("label", data.get("label") or titleize(model.aml_name)) + block.text("description", payload.get("description")) + block.text("data_source_name", data.get("data_source_name") or self.data_source_name) + if is_query: + referenced = _models_referenced(source) + if referenced: + block.property("models", f"[{', '.join(sorted(referenced))}]") + block.blank() + block.heredoc("query", "sql", source) + else: + block.text("table_name", data.get("table_name") or source) + + for param in data.get("param") or []: + block.blank() + param_block = block.block(f"param {self.name_in_aml(param['name'], scope)}") + param_block.text("label", param.get("label") or titleize(param["name"])) + param_block.property("type", quote(param.get("type") or FALLBACK_TYPE)) + + persistence = data.get("persistence") + if persistence: + block.blank() + _write_persistence(block, persistence) + + primary_key = set(payload.get("primary_key") or []) + for field in payload.get("fields") or []: + if stash.read(field).get("scope") == "dataset": + continue + block.blank() + block.lines.append(self.dimension(model, field, primary_key)) + + for metric in model.measures: + block.blank() + block.lines.append(self.measure(model, metric)) + + for column in model.borrowed: + block.blank() + borrowed = block.block(f"dimension {column}") + borrowed.text("label", titleize(column)) + borrowed.property("type", quote(FALLBACK_TYPE_FOR_BORROWED)) + borrowed.flag("hidden", True) + borrowed.heredoc("definition", "sql", interpolation.Source(column).to_aml()) + + owner = data.get("owner") or self.root_stash.get("owner") + if owner: + block.blank() + block.text("owner", owner) + return amlgen.document(block, license_header=True) + + # -- dataset ---------------------------------------------------------- + + def relationship(self, payload: dict[str, Any]) -> str: + data = stash.read(payload) + kind = data.get("type", "many_to_one") + active = "true" if data.get("active", True) else "false" + from_model = self.by_ossie_name.get(payload.get("from")) + to_model = self.by_ossie_name.get(payload.get("to")) + if from_model is None or to_model is None: + missing = payload.get("from") if from_model is None else payload.get("to") + raise ConversionError( + f"relationship {payload.get('name')!r} names dataset {missing!r}, " + f"which is not in this document" + ) + from_columns = payload.get("from_columns") or [] + to_columns = payload.get("to_columns") or [] + + if len(from_columns) == 1 and kind in ("many_to_one", "one_to_one"): + operator = ">" if kind == "many_to_one" else "-" + return ( + f"relationship({from_model.reference}.{from_columns[0]} {operator} " + f"{to_model.reference}.{to_columns[0]}, {active})" + ) + + config = Block("RelationshipConfig") + rel = config.block("EqualityRelationship") + rel.header = "rel: EqualityRelationship" + rel.property("type", quote(kind)) + rel.property("from", _field_ref(from_model.aml_name, from_columns[0])) + rel.property("to", _field_ref(to_model.aml_name, to_columns[0])) + for from_column, to_column in zip(from_columns[1:], to_columns[1:]): + on = rel.block("on") + on.property("from", _field_ref(from_model.aml_name, from_column)) + on.property("to", _field_ref(to_model.aml_name, to_column)) + config.property("active", active) + return config.render(2).lstrip() + + def dataset_file(self) -> tuple[str, str]: + name = self.dataset_name + block = Block(f"Dataset {name}") + block.text( + "label", + self.root_stash.get("label") or titleize(self.document.get("name") or name), + ) + block.text("description", self.document.get("description")) + block.text("data_source_name", self.data_source_name) + block.blank() + block.property("models", amlgen.array([m.reference for m in self.models])) + block.blank() + block.property( + "relationships", + amlgen.array( + [self.relationship(r) for r in self.document.get("relationships") or []] + ), + ) + + for model in self.models: + for field in model.payload.get("fields") or []: + if stash.read(field).get("scope") == "dataset": + block.blank() + block.lines.append(self.dataset_dimension(model, field)) + + for metric in self.dataset_metrics: + written = self.dataset_metric(metric) + if written is not None: + block.blank() + block.lines.append(written) + + owner = self.root_stash.get("owner") + if owner: + block.blank() + block.text("owner", owner) + return f"{name}.dataset.aml", amlgen.document(block, license_header=True) + + def run(self) -> Result: + # The dataset file is rendered first because its metrics decide which + # columns each model has to borrow, and a model file has to carry those + # as dimensions. The output order is unchanged: models, then dataset. + dataset = self.dataset_file() + files = [(m.path, self.model_file(m)) for m in self.models] + files.append(dataset) + return Result(files, self.issues) + + +def _field_ref(model_name: str, column: str) -> str: + return f"FieldRef {{ model: {quote(model_name)}, field: {quote(column)} }}" + + +def _write_persistence(block: Block, persistence: dict[str, Any]) -> None: + """The stashed persistence object as its AML block. + + `__type__` names the block, and every key the compiler added for its own + bookkeeping starts and ends with `__`, so those are skipped. + """ + kind = persistence.get("__type__") or "FullPersistence" + child = block.block(f"persistence: {kind}") + for key, value in persistence.items(): + if key.startswith("__"): + continue + child.property(key, quote(value) if isinstance(value, str) else str(value).lower()) + + +def _models_referenced(query: str) -> set[str]: + """Every model a query body names, so the AML `models:` list can be rebuilt. + + A `QueryModel` must declare the models its body references. The forward path + carries the body verbatim, so the references are still in the text. + """ + found = set() + for pattern in ("model_relation", "model_columns"): + for match in interpolation.QUERY_FORMS[pattern].finditer(query): + inner = match.group(0).strip("{} ").lstrip("#") + found.add(inner[:-2] if inner.endswith(".*") else inner) + return found + + +def _looks_like_query(source: str) -> bool: + """Whether an Ossie `source` holds a query rather than a table reference. + + Only reached for a document with no stashed `aml_type`, meaning one this + converter did not write. `source` is a single string, so the shape is read + from the text. This takes the permissive reading, counting a leading + `SELECT` or `WITH`, or any whitespace outside a quoted identifier. + """ + text = source.strip() + if text[:6].upper() in ("SELECT",) or text[:4].upper() == "WITH": + return True + return any(character.isspace() for character in text.strip("`\"[]")) + + +def convert( + document: Any, sql_dialect: str, data_source_name: str | None = None +) -> Result: + """One Ossie semantic model to AML source files. + + `sql_dialect` names the warehouse the Holistics connection reads. Every + `@sql` body written is in that dialect, rendered from a portable expression + where the document carries no body in it already. + + `data_source_name` names the connection itself. AML requires it on a dataset + and Ossie has no field for it, so a document this converter did not write + needs it supplied. + """ + if not isinstance(document, dict): + raise ConversionError("the Ossie document is not a mapping") + return _Reverse(document, data_source_name, sql_dialect).run() diff --git a/converters/holistics/src/ossie_holistics/sqlrefs.py b/converters/holistics/src/ossie_holistics/sqlrefs.py new file mode 100644 index 000000000..d03917a74 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/sqlrefs.py @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Finding the column references in an Ossie SQL expression. + +The reverse path has to turn `CAST(created_at AS DATE)` back into +`CAST({{ #SOURCE.created_at }} AS DATE)`. A bare column name does survive +`holistics aml validate`, but it is not equivalent: AQL reads the interpolation +to learn which field a body depends on, and a body with none loses the query +rewriting and the optimisation that depend on that link. So the reference has to +be restored, which means telling a column apart from a keyword, a function name +and a date part. + +sqlglot already makes that distinction, and it reports the character offsets of +the identifiers it parsed. In BigQuery's `DATE_TRUNC(created_at, month)` it +reads `created_at` as a column and `month` as a date part, and in +`CASE WHEN status = 'created_at' THEN created_at END` it reads the second +`created_at` and not the one inside the string literal. + +`rewrite` uses those offsets to splice replacement text into the original string. +The SQL is never re-rendered through sqlglot. `DATE_TRUNC(created_at, month)` +written back out through sqlglot's default dialect returns +`DATE_TRUNC(created_at, MONTH)`, which that same default dialect then re-reads +with the unit and the expression exchanged, so a round trip through the writer +silently changes the meaning. +""" +from __future__ import annotations + +from dataclasses import dataclass + +try: + import sqlglot + from sqlglot import exp + from sqlglot.errors import ParseError, TokenError + + AVAILABLE = True +except ImportError: # pragma: no cover - exercised by the packaging test + AVAILABLE = False + +#: Ossie dialect to the sqlglot dialect that parses it. Mirrors `DIALECT_MAP` in +#: `validation/validate.py`. None means sqlglot's default. +SQLGLOT_DIALECT: dict[str, str | None] = { + "ANSI_SQL": None, + "OSSIE_SQL_2026": None, + "SNOWFLAKE": "snowflake", + "DATABRICKS": "databricks", + "BIGQUERY": "bigquery", +} + +#: Dialects sqlglot cannot parse. A body in one of these is not SQL. +NOT_SQL = frozenset({"MDX", "TABLEAU", "MAQL", "SIGMA", "THOUGHTSPOT", "DAX", "HOLISTICS_AQL"}) + +#: Every expression is parsed as `SELECT `, because a bare column +#: name is not a statement on its own. Offsets shift back by this much. +_PREFIX = "SELECT " + + +class UnparseableSQL(Exception): + """Raised when sqlglot cannot read an expression at the dialect given.""" + + +@dataclass(frozen=True) +class ColumnReference: + """One column in the expression, with the span it occupies in the text.""" + + table: str + name: str + start: int + #: Exclusive, so `text[start:end]` is the reference as written. + end: int + + +def columns(expression: str, dialect: str) -> list[ColumnReference]: + """Every column reference in `expression`, in the order it appears.""" + if not AVAILABLE: + raise UnparseableSQL("sqlglot is not installed") + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(dialect)) + except (ParseError, TokenError, RecursionError) as exc: + raise UnparseableSQL(str(exc).split("\n")[0]) from exc + + found = [] + for column in tree.find_all(exp.Column): + name_meta = column.this.meta + table = column.args.get("table") + if "start" not in name_meta: + continue + start = table.meta["start"] if table is not None and "start" in table.meta else name_meta["start"] + found.append( + ColumnReference( + table=column.table, + name=column.name, + start=start - len(_PREFIX), + end=name_meta["end"] + 1 - len(_PREFIX), + ) + ) + return sorted(found, key=lambda c: c.start) + + +def transpile(expression: str, read: str, write: str) -> str: + """`expression` read as `read` and written as `write`. + + Used only to render a portable body into the warehouse the AML output is + aimed at. Both dialects are named members of `SQLGLOT_DIALECT`. + """ + if not AVAILABLE: + raise UnparseableSQL("sqlglot is not installed") + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(read)) + except (ParseError, TokenError, RecursionError) as exc: + raise UnparseableSQL(str(exc).split("\n")[0]) from exc + rendered = tree.sql(dialect=SQLGLOT_DIALECT.get(write)) + if not rendered.upper().startswith(_PREFIX.strip()): + raise UnparseableSQL(f"sqlglot wrote an unexpected shape: {rendered!r}") + return rendered[len(_PREFIX) :] + + +def window_grouping(expression: str, dialect: str) -> list[str]: + """The columns an OVER clause partitions or orders by, in order. + + A window function survives into an AML `@sql` body and compiles, and the + warehouse SQL it produces is only valid when the query groups by these columns. Holistics + does not check that, so the caller reports it. + """ + if not AVAILABLE: + return [] + try: + tree = sqlglot.parse_one(_PREFIX + expression, read=SQLGLOT_DIALECT.get(dialect)) + except (ParseError, TokenError, RecursionError): + return [] + seen: list[str] = [] + for window in tree.find_all(exp.Window): + # `window.this` is the function being windowed, whose columns the + # aggregate consumes rather than groups by. Only the frame decides the + # grouping, so only PARTITION BY and ORDER BY are read. + frame = [window.args.get("partition_by"), window.args.get("order")] + for part in frame: + for item in part if isinstance(part, list) else [part]: + if item is None: + continue + # A column inside an aggregate within the frame is consumed by + # that aggregate, not grouped by. `ORDER BY SUM(t.v)` orders on + # the aggregate and puts no requirement on `t.v`. + aggregated = { + id(column) + for aggregate in item.find_all(exp.AggFunc) + for column in aggregate.find_all(exp.Column) + } + for column in item.find_all(exp.Column): + if id(column) in aggregated: + continue + name = f"{column.table}.{column.name}" if column.table else column.name + if name not in seen: + seen.append(name) + return seen + + +def rewrite(expression: str, dialect: str, replace) -> str: + """`expression` with each column replaced by `replace(column_reference)`. + + Every character outside a replaced span is kept exactly as it was. + """ + out = expression + for column in reversed(columns(expression, dialect)): + out = out[: column.start] + replace(column) + out[column.end :] + return out diff --git a/converters/holistics/src/ossie_holistics/stash.py b/converters/holistics/src/ossie_holistics/stash.py new file mode 100644 index 000000000..e05a2d710 --- /dev/null +++ b/converters/holistics/src/ossie_holistics/stash.py @@ -0,0 +1,94 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The `HOLISTICS` entry in `custom_extensions`. + +Every AML property the core specification has no field for travels here. The +entry's `data` is a single JSON-encoded string, never a nested object, because +the specification requires that: + + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "table_name": "`ecommerce`.`orders`"}' + +`_v` is the shape version. `read` fails on a version it does not know rather +than misreading a shape it has never seen. + +A foreign vendor's entry on the same object passes through untouched, and an +object with nothing to stash gets no `HOLISTICS` entry at all. +""" +from __future__ import annotations + +import json +from typing import Any + +from .errors import ConversionError + +VENDOR = "HOLISTICS" + +#: The only shape this converter writes, and the only one it reads. +VERSION = 1 + + +def entry(data: dict[str, Any]) -> dict[str, str] | None: + """One `custom_extensions` entry, or None when there is nothing to carry.""" + if not data: + return None + payload = {"_v": VERSION} + payload.update(data) + return {"vendor_name": VENDOR, "data": json.dumps(payload, sort_keys=True)} + + +def attach(target: dict[str, Any], data: dict[str, Any]) -> None: + """Append the `HOLISTICS` entry to `target['custom_extensions']`, if any.""" + stashed = entry(data) + if stashed is None: + return + target.setdefault("custom_extensions", []).append(stashed) + + +def read(obj: Any) -> dict[str, Any]: + """The stashed payload on an Ossie object, `{}` when there is none. + + `_v` is stripped from the result, so a caller reads AML properties by their + own names and never sees the bookkeeping key. + """ + if not isinstance(obj, dict): + return {} + extensions = obj.get("custom_extensions") + if not isinstance(extensions, list): + return {} + for extension in extensions: + if not isinstance(extension, dict) or extension.get("vendor_name") != VENDOR: + continue + raw = extension.get("data") + if not isinstance(raw, str): + raise ConversionError(f"{VENDOR} custom_extensions data is not a string") + try: + payload = json.loads(raw) + except json.JSONDecodeError as exc: + raise ConversionError(f"{VENDOR} custom_extensions data is not valid JSON: {exc}") from exc + if not isinstance(payload, dict): + raise ConversionError(f"{VENDOR} custom_extensions data is not a JSON object") + version = payload.pop("_v", None) + if version != VERSION: + raise ConversionError( + f"{VENDOR} custom_extensions data has _v={version!r}, " + f"and this converter writes and reads _v={VERSION}" + ) + return payload + return {} diff --git a/converters/holistics/tests/conftest.py b/converters/holistics/tests/conftest.py new file mode 100644 index 000000000..7151f0d8c --- /dev/null +++ b/converters/holistics/tests/conftest.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tests")) diff --git a/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml new file mode 100644 index 000000000..68fffe803 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml.issues.yaml @@ -0,0 +1,86 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) / COUNT(DISTINCT users.id)' became 'sum(items.quantity) + / count_distinct(users.id)' + object_ref: quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'ROUND(SAFE_DIVIDE(SUM(items.quantity), ABS(COUNT(users.id))), 2)' became 'round(safe_divide(sum(items.quantity), + abs(count(users.id))), 2)' + object_ref: rounded_ratio + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.product_id) + COUNT(users.id)' became 'sum(items.product_id) + count(users.id)' + object_ref: untyped_total + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_UNDECLARED_COLUMN_ADDED_AS_DIMENSION + severity: WARNING + message: >- + the metric reads items.sale_price, which the dataset does not declare as a field. AQL resolves + a reference against declared fields only, so 'sale_price' is added to model 'items' as a hidden + dimension over the column of that name + object_ref: borrowed_column_total + remedy: declare 'sale_price' as a field of 'items' in Ossie +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.sale_price) + COUNT(users.id)' became 'sum(items.sale_price) + count(users.id)' + object_ref: borrowed_column_total + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) + MY_WAREHOUSE_UDF(COUNT(users.id), 3)' became "sum(items.quantity) + + sql_number('MY_WAREHOUSE_UDF', count(users.id), 3)" + object_ref: passthrough_call + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: windowed_rank + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. LOG is not in this converter's AQL function table, + and its call cannot be read back from the source text, so there is no faithful way to pass it + through. + object_ref: unmapped_function + remedy: rewrite the metric in AQL, or scope it to a single dataset diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml b/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml new file mode 100644 index 000000000..163297527 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/aql_translation.dataset.aml @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset aql_translation { + label: 'Aql Translation' + description: 'Metrics that exercise each path through the SQL to AQL translation' + data_source_name: 'bigquery_demo' + + models: [ + orders, + items, + users, + ] + + relationships: [ + relationship(items.order_id > orders.id, true), + relationship(orders.user_id > users.id, true), + ] + + metric quantity_per_customer { + label: 'Quantity Per Customer' + type: 'number' + description: 'The plain path. SUM and COUNT DISTINCT both have AQL names' + definition: @aql sum(items.quantity) / count_distinct(users.id);; + } + + metric rounded_ratio { + label: 'Rounded Ratio' + type: 'number' + description: 'Functions AQL has. ROUND, SAFE_DIVIDE and ABS are written natively' + definition: @aql round(safe_divide(sum(items.quantity), abs(count(users.id))), 2);; + } + + metric untyped_total { + label: 'Untyped Total' + type: 'number' + description: 'A field with no datatype, summed, so the metric is what types it' + definition: @aql sum(items.product_id) + count(users.id);; + } + + metric borrowed_column_total { + label: 'Borrowed Column Total' + type: 'number' + description: 'A column no dataset declares as a field, so one is added for it' + definition: @aql sum(items.sale_price) + count(users.id);; + } + + metric passthrough_call { + label: 'Passthrough Call' + type: 'number' + description: 'A function sqlglot does not model, carried by AQL\'s raw-SQL call' + definition: @aql sum(items.quantity) + sql_number('MY_WAREHOUSE_UDF', count(users.id), 3);; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml new file mode 100644 index 000000000..17e988fbf --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/items.model.aml @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model items { + type: 'table' + label: 'Items' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension sale_price { + label: 'Sale Price' + type: 'number' + hidden: true + definition: @sql {{ #SOURCE.sale_price }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml new file mode 100644 index 000000000..83bc83752 --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/orders.model.aml @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model orders { + type: 'table' + label: 'Orders' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml b/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml new file mode 100644 index 000000000..a20a34bfa --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/expected_aml/users.model.aml @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model users { + type: 'table' + label: 'Users' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`users`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } +} diff --git a/converters/holistics/tests/fixtures/aql/translation.ossie.yaml b/converters/holistics/tests/fixtures/aql/translation.ossie.yaml new file mode 100644 index 000000000..f8b9ffdca --- /dev/null +++ b/converters/holistics/tests/fixtures/aql/translation.ossie.yaml @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie document written to exercise `to-aml`'s SQL to AQL translation. +# +# Every metric here spans two datasets, which is what forces AML to take an +# `@aql` body and so forces the translation. Each one covers a different path, +# named in its description. The ecommerce tables are the ones the Holistics +# BigQuery demo connection carries, so the generated AML compiles against a real +# schema rather than an invented one. + +version: "0.2.0.dev0" +name: aql_translation +description: Metrics that exercise each path through the SQL to AQL translation + +datasets: + - name: orders + source: "`ecommerce`.`orders`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: user_id + expression: {dialects: [{dialect: ANSI_SQL, expression: user_id}]} + datatype: Decimal + - name: status + expression: {dialects: [{dialect: ANSI_SQL, expression: status}]} + datatype: String + + - name: items + source: "`ecommerce`.`order_items`" + fields: + - name: order_id + expression: {dialects: [{dialect: ANSI_SQL, expression: order_id}]} + datatype: Decimal + - name: quantity + expression: {dialects: [{dialect: ANSI_SQL, expression: quantity}]} + datatype: Decimal + # No datatype. A metric sums it, which is the only evidence that it is a + # number, and `text` would make AQL refuse the aggregate. + - name: product_id + expression: {dialects: [{dialect: ANSI_SQL, expression: product_id}]} + + - name: users + source: "`ecommerce`.`users`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: full_name + expression: {dialects: [{dialect: ANSI_SQL, expression: full_name}]} + datatype: String + +relationships: + - name: items_to_orders + from: items + to: orders + from_columns: [order_id] + to_columns: [id] + - name: orders_to_users + from: orders + to: users + from_columns: [user_id] + to_columns: [id] + +metrics: + - name: quantity_per_customer + description: The plain path. SUM and COUNT DISTINCT both have AQL names + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) / COUNT(DISTINCT users.id) + + - name: rounded_ratio + description: Functions AQL has. ROUND, SAFE_DIVIDE and ABS are written natively + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: ROUND(SAFE_DIVIDE(SUM(items.quantity), ABS(COUNT(users.id))), 2) + + - name: untyped_total + description: A field with no datatype, summed, so the metric is what types it + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.product_id) + COUNT(users.id) + + - name: borrowed_column_total + description: A column no dataset declares as a field, so one is added for it + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.sale_price) + COUNT(users.id) + + - name: passthrough_call + description: A function sqlglot does not model, carried by AQL's raw-SQL call + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + MY_WAREHOUSE_UDF(COUNT(users.id), 3) + + - name: windowed_rank + description: A window function, which the converter does not translate + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: RANK() OVER (PARTITION BY users.id ORDER BY SUM(items.quantity) DESC) + + - name: unmapped_function + description: A function sqlglot models but the AQL table omits + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + LOG10(COUNT(users.id)) diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml new file mode 100644 index 000000000..a3701fb56 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/cities.model.aml @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model cities { + type: 'table' + label: 'Cities' + description: 'One row per city a merchant can operate in.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`cities`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + dimension in_vietnam { + label: 'In Vietnam' + type: 'truefalse' + definition: @aql cities.country_code == 'VN';; + } + + // upstream_measure and downstream_measure form a measure-to-measure chain. + // Resolving downstream_measure yields COUNT(id) + 1, so a converter has to + // follow the chain and detect a cycle rather than recurse forever. + measure upstream_measure { + label: 'Upstream Measure' + type: 'number' + definition: @sql COUNT({{ id }});; + } + + measure downstream_measure { + label: 'Downstream Measure' + type: 'number' + definition: @sql {{ upstream_measure }} + 1;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml new file mode 100644 index 000000000..7fb44ff20 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/delivered_orders.model.aml @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model delivered_orders { + type: 'query' + label: 'Delivered Orders' + description: 'Orders that reached the customer. Derived from ecommerce_orders rather than from a table.' + data_source_name: 'bigquery_demo' + + // The query body is not plain SQL. It names another model, and the compiler + // does not resolve the reference: it passes the body through and inlines the + // referenced model here instead. `{{ #ecommerce_orders }}` is this file's + // equivalent of dbt's `ref()`. + models: [ecommerce_orders] + + query: @sql + select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} + ;; + + // A filter block expands to a predicate only when a value is supplied at + // query time, so no conversion-time rewrite of the body above is correct. + param min_delivery_attempts { + label: 'Minimum Delivery Attempts' + type: 'number' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension delivery_attempts { + label: 'Delivery Attempts' + type: 'number' + definition: @sql {{ #SOURCE.delivery_attempts }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml new file mode 100644 index 000000000..eb828dc73 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset ecommerce { + label: 'Ecommerce' + description: 'A marketplace model: orders, line items, products, merchants and cities.' + data_source_name: 'bigquery_demo' + + models: [ + ecommerce_orders, + ecommerce_users, + order_items, + products, + merchants, + cities, + reporting.calendar, + delivered_orders, + ] + + relationships: [ + relationship(order_items.order_id > ecommerce_orders.id, true), + relationship(order_items.product_id > products.id, true), + relationship(ecommerce_orders.user_id > ecommerce_users.id, true), + relationship(products.merchant_id > merchants.id, true), + // Composite. The `relationship(...)` shorthand carries one column pair, so a + // second pair needs the long form with an `on` block. Blocks combine with AND. + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: FieldRef { model: 'merchants', field: 'city_id' } + to: FieldRef { model: 'cities', field: 'id' } + on { + from: FieldRef { model: 'merchants', field: 'country_code' } + to: FieldRef { model: 'cities', field: 'country_code' } + } + } + active: true + }, + relationship(delivered_orders.user_id > ecommerce_users.id, true), + + // One-to-one. Ossie has no marker for this, so the kind goes to the stash. + relationship(merchants.admin_id - ecommerce_users.id, true), + + // Inactive, and its target sits in a module. Holistics does not join across + // this edge by default, and the compiled endpoint carries no __refFqn__. + relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + ] + + // A dataset-level dimension names the model it attaches to. The body spans + // two models, so it needs the relationship graph to resolve. + dimension item_value { + model: order_items + + label: 'Item Value' + type: 'number' + definition: @aql order_items.quantity * products.price;; + } + + metric total_revenue { + label: 'Total Revenue' + type: 'number' + definition: @aql sum(order_items.item_value);; + format: '#,###[$$]' + } + + metric count_orders { + label: 'Count Orders' + type: 'number' + definition: @aql count(order_items.order_id);; + } + + // of_all() rewrites the grain of the whole query. No SQL rendering of this + // body is correct without the dataset's relationship graph, which is why the + // converter carries AQL under HOLISTICS_AQL instead of translating it. + metric revenue_share_of_all_merchants { + label: 'Revenue Share Of All Merchants' + type: 'number' + definition: @aql + sum(order_items.item_value) + / (sum(order_items.item_value) | of_all(merchants)) + ;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml new file mode 100644 index 000000000..7f981f167 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/merchants.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model merchants { + type: 'table' + label: 'Merchants' + description: 'One row per seller on the marketplace.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`merchants`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + // Each merchant has at most one admin user, so this side is one-to-one. + dimension admin_id { + label: 'Admin Id' + type: 'number' + definition: @sql {{ #SOURCE.admin_id }};; + } + + dimension city_id { + label: 'City Id' + type: 'number' + definition: @sql {{ #SOURCE.city_id }};; + } + + // City ids are unique within a country, so the join to cities needs both + // columns. That makes it a composite relationship. + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml new file mode 100644 index 000000000..93c70b537 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/modules/reporting/calendar.model.aml @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model calendar { + type: 'table' + label: 'Calendar' + description: 'A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`calendar`' + + dimension date { + label: 'Date' + type: 'date' + definition: @sql {{ #SOURCE.date }};; + primary_key: true + } + + dimension year { + label: 'Year' + type: 'number' + definition: @sql {{ #SOURCE.year }};; + } + + dimension is_weekend { + label: 'Is Weekend' + type: 'truefalse' + definition: @sql {{ #SOURCE.is_weekend }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml new file mode 100644 index 000000000..696dbb357 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/order_items.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model order_items { + type: 'table' + label: 'Order Items' + description: 'One row per line item on an order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + // A named aggregation over a bare source column. This is the one measure + // shape that converts to portable ANSI SQL: SUM(quantity). + measure total_quantity { + label: 'Total Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + aggregation_type: 'sum' + } + + // The aggregate sits inside the body instead, so aggregation_type is 'custom'. + measure custom_total_quantity { + label: 'Custom Total Quantity' + type: 'number' + definition: @sql sum({{ #SOURCE.quantity }});; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml new file mode 100644 index 000000000..959c74361 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/orders.model.aml @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_orders { + type: 'table' + label: 'Orders' + description: 'One row per customer order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + // No definition: the dimension name is the column name. + dimension id { + label: 'Id' + type: 'number' + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension created_at { + label: 'Created At' + type: 'datetime' + definition: @sql {{ #SOURCE.created_at }};; + } + + // Warehouse SQL, not portable ANSI. + dimension created_date { + label: 'Created Date' + type: 'date' + definition: @sql CAST({{ #SOURCE.created_at }} AS DATE);; + } + + // BigQuery argument order; a converter must not rewrite this. + dimension created_month { + label: 'Created Month' + type: 'date' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + format: 'dd LLL, yyyy' + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } + + dimension delivered { + label: 'Delivered' + type: 'truefalse' + definition: @aql ecommerce_orders.status == 'delivered';; + } + + // Named aggregation over a sibling field, not a source column. + measure last_created_at { + label: 'Last Created At' + type: 'datetime' + definition: @sql {{ created_at }};; + aggregation_type: 'max' + } + + measure count_orders { + label: 'Count Orders' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count' + } + + // AQL: the filter resolves through the dataset relationship graph. + measure delivered_orders { + label: 'Delivered Orders' + type: 'number' + hidden: true + definition: @aql count(ecommerce_orders.id) | where(ecommerce_orders.delivered);; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml new file mode 100644 index 000000000..8fdbabf81 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/products.model.aml @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model products { + type: 'table' + label: 'Products' + description: 'One row per sellable product.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`products`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension merchant_id { + label: 'Merchant Id' + type: 'number' + definition: @sql {{ #SOURCE.merchant_id }};; + } + + dimension price { + label: 'Price' + type: 'number' + definition: @sql {{ #SOURCE.price }};; + format: '#,###[$$]' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml b/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml new file mode 100644 index 000000000..7f2cbfa01 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/aml/users.model.aml @@ -0,0 +1,98 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_users { + type: 'query' + label: 'Users' + description: 'One row per registered customer.' + data_source_name: 'bigquery_demo' + + query: @sql + select * from `ecommerce`.`users` + ;; + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } + + // No definition. AML defaults the body to `{{ #SOURCE.gender }}`, the source + // column that shares the dimension's name. Only a dimension may omit it. + dimension gender { + label: 'Gender' + type: 'text' + } + + dimension birth_date { + label: 'Birth Date' + type: 'date' + definition: @sql {{ #SOURCE.birth_date }};; + } + + // Sibling reference inside warehouse SQL. + dimension age { + label: 'Age' + type: 'number' + definition: @sql extract(year from age({{ birth_date }}));; + } + + // A sibling reference to a field that is itself computed, so resolution + // follows a chain: age_bucket reads age, which reads birth_date. + dimension age_bucket { + label: 'Age Bucket' + type: 'number' + hidden: true + definition: @sql FLOOR({{ age }} / 10);; + } + + dimension signup_month { + label: 'Signup Month' + type: 'datetime' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + } + + measure count_users { + label: 'Count Users' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count distinct' + } + + param reporting_currency { + label: 'Reporting Currency' + type: 'text' + } + + param minimum_age { + label: 'Minimum Age' + type: 'number' + } + + persistence: FullPersistence { + schema: 'persisted' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json new file mode 100644 index 000000000..d0624b91a --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json @@ -0,0 +1,1437 @@ +{ + "name": "ecommerce", + "__fqn__": "ecommerce", + "label": "Ecommerce", + "description": "A marketplace model: orders, line items, products, merchants and cities.", + "owner": "dev@ossie.apache.org", + "distinct_rows": true, + "dimension": [ + { + "name": "item_value", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "order_items.quantity * products.price" + }, + "model": { + "__fqn__": "order_items", + "name": "order_items", + "__doc__": "", + "label": "Order Items", + "description": "One row per line item on an order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "order_id": { + "name": "order_id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.order_id }}" + }, + "primary_key": false, + "label": "Order Id", + "type": "number", + "__type__": "Dimension" + }, + "product_id": { + "name": "product_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.product_id }}" + }, + "primary_key": false, + "label": "Product Id", + "type": "number", + "__type__": "Dimension" + }, + "quantity": { + "name": "quantity", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "primary_key": false, + "label": "Quantity", + "type": "number", + "__type__": "Dimension" + } + }, + "measure": { + "total_quantity": { + "name": "total_quantity", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "aggregation_type": "sum", + "label": "Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "__type__": "Measure" + }, + "custom_total_quantity": { + "name": "custom_total_quantity", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Custom Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "sum({{ #SOURCE.quantity }})" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`order_items`", + "__type__": "TableModel" + }, + "label": "Item Value", + "type": "number", + "__type__": "DatasetDimension" + } + ], + "metric": { + "total_revenue": { + "name": "total_revenue", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "format": "#,###[$$]", + "label": "Total Revenue", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "sum(order_items.item_value)" + }, + "__type__": "Metric" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(order_items.order_id)" + }, + "__type__": "Metric" + }, + "revenue_share_of_all_merchants": { + "name": "revenue_share_of_all_merchants", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "label": "Revenue Share Of All Merchants", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "sum(order_items.item_value)\n/ (sum(order_items.item_value) | of_all(merchants))" + }, + "__type__": "Metric" + } + }, + "data_source_name": "bigquery_demo", + "models": [ + { + "name": "ecommerce_orders", + "__fqn__": "ecommerce_orders", + "label": "Orders", + "description": "One row per customer order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "created_at": { + "name": "created_at", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.created_at }}" + }, + "primary_key": false, + "label": "Created At", + "type": "datetime", + "__type__": "Dimension" + }, + "created_date": { + "name": "created_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "CAST({{ #SOURCE.created_at }} AS DATE)" + }, + "primary_key": false, + "label": "Created Date", + "type": "date", + "__type__": "Dimension" + }, + "created_month": { + "name": "created_month", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "format": "dd LLL, yyyy", + "primary_key": false, + "label": "Created Month", + "type": "date", + "__type__": "Dimension" + }, + "status": { + "name": "status", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.status }}" + }, + "primary_key": false, + "label": "Status", + "type": "text", + "__type__": "Dimension" + }, + "delivered": { + "name": "delivered", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.status == 'delivered'" + }, + "primary_key": false, + "label": "Delivered", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "last_created_at": { + "name": "last_created_at", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "max", + "label": "Last Created At", + "type": "datetime", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ created_at }}" + }, + "__type__": "Measure" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "aggregation_type": "count", + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + }, + "delivered_orders": { + "name": "delivered_orders", + "__doc__": "", + "__childIdx__": 14, + "hidden": true, + "aggregation_type": "custom", + "label": "Delivered Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(ecommerce_orders.id) | where(ecommerce_orders.delivered)" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`orders`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "ecommerce_users", + "__fqn__": "ecommerce_users", + "label": "Users", + "description": "One row per registered customer.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "models": [], + "persistence": { + "__doc__": "", + "on_cascade": "rebuild", + "schema": "persisted", + "__type__": "FullPersistence" + }, + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "full_name": { + "name": "full_name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.full_name }}" + }, + "primary_key": false, + "label": "Full Name", + "type": "text", + "__type__": "Dimension" + }, + "gender": { + "name": "gender", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "primary_key": false, + "label": "Gender", + "type": "text", + "__type__": "Dimension" + }, + "birth_date": { + "name": "birth_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.birth_date }}" + }, + "primary_key": false, + "label": "Birth Date", + "type": "date", + "__type__": "Dimension" + }, + "age": { + "name": "age", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "extract(year from age({{ birth_date }}))" + }, + "primary_key": false, + "label": "Age", + "type": "number", + "__type__": "Dimension" + }, + "age_bucket": { + "name": "age_bucket", + "__doc__": "", + "__childIdx__": 10, + "hidden": true, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "FLOOR({{ age }} / 10)" + }, + "primary_key": false, + "label": "Age Bucket", + "type": "number", + "__type__": "Dimension" + }, + "signup_month": { + "name": "signup_month", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "primary_key": false, + "label": "Signup Month", + "type": "datetime", + "__type__": "Dimension" + } + }, + "measure": { + "count_users": { + "name": "count_users", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "count distinct", + "label": "Count Users", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + } + }, + "param": { + "reporting_currency": { + "name": "reporting_currency", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "label": "Reporting Currency", + "type": "text", + "__type__": "Param" + }, + "minimum_age": { + "name": "minimum_age", + "__doc__": "", + "__childIdx__": 14, + "hidden": false, + "label": "Minimum Age", + "type": "number", + "__type__": "Param" + } + }, + "type": "query", + "query": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "select * from `ecommerce`.`users`" + }, + "__doc__": "", + "__type__": "QueryModel" + }, + { + "name": "order_items", + "__fqn__": "order_items", + "label": "Order Items", + "description": "One row per line item on an order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "order_id": { + "name": "order_id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.order_id }}" + }, + "primary_key": false, + "label": "Order Id", + "type": "number", + "__type__": "Dimension" + }, + "product_id": { + "name": "product_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.product_id }}" + }, + "primary_key": false, + "label": "Product Id", + "type": "number", + "__type__": "Dimension" + }, + "quantity": { + "name": "quantity", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "primary_key": false, + "label": "Quantity", + "type": "number", + "__type__": "Dimension" + } + }, + "measure": { + "total_quantity": { + "name": "total_quantity", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "aggregation_type": "sum", + "label": "Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.quantity }}" + }, + "__type__": "Measure" + }, + "custom_total_quantity": { + "name": "custom_total_quantity", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Custom Total Quantity", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "sum({{ #SOURCE.quantity }})" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`order_items`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "products", + "__fqn__": "products", + "label": "Products", + "description": "One row per sellable product.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "merchant_id": { + "name": "merchant_id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.merchant_id }}" + }, + "primary_key": false, + "label": "Merchant Id", + "type": "number", + "__type__": "Dimension" + }, + "price": { + "name": "price", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.price }}" + }, + "format": "#,###[$$]", + "primary_key": false, + "label": "Price", + "type": "number", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`products`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "merchants", + "__fqn__": "merchants", + "label": "Merchants", + "description": "One row per seller on the marketplace.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "admin_id": { + "name": "admin_id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.admin_id }}" + }, + "primary_key": false, + "label": "Admin Id", + "type": "number", + "__type__": "Dimension" + }, + "city_id": { + "name": "city_id", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.city_id }}" + }, + "primary_key": false, + "label": "City Id", + "type": "number", + "__type__": "Dimension" + }, + "country_code": { + "name": "country_code", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.country_code }}" + }, + "primary_key": false, + "label": "Country Code", + "type": "text", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`merchants`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "cities", + "__fqn__": "cities", + "label": "Cities", + "description": "One row per city a merchant can operate in.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "name": { + "name": "name", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.name }}" + }, + "primary_key": false, + "label": "Name", + "type": "text", + "__type__": "Dimension" + }, + "country_code": { + "name": "country_code", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.country_code }}" + }, + "primary_key": false, + "label": "Country Code", + "type": "text", + "__type__": "Dimension" + }, + "in_vietnam": { + "name": "in_vietnam", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "cities.country_code == 'VN'" + }, + "primary_key": false, + "label": "In Vietnam", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "upstream_measure": { + "name": "upstream_measure", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "aggregation_type": "custom", + "label": "Upstream Measure", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "COUNT({{ id }})" + }, + "__type__": "Measure" + }, + "downstream_measure": { + "name": "downstream_measure", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "aggregation_type": "custom", + "label": "Downstream Measure", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ upstream_measure }} + 1" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`cities`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "calendar", + "__fqn__": "reporting::calendar", + "label": "Calendar", + "description": "A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "date": { + "name": "date", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.date }}" + }, + "primary_key": true, + "label": "Date", + "type": "date", + "__type__": "Dimension" + }, + "year": { + "name": "year", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.year }}" + }, + "primary_key": false, + "label": "Year", + "type": "number", + "__type__": "Dimension" + }, + "is_weekend": { + "name": "is_weekend", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.is_weekend }}" + }, + "primary_key": false, + "label": "Is Weekend", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "type": "table", + "table_name": "`ecommerce`.`calendar`", + "__doc__": "", + "__type__": "TableModel" + }, + { + "name": "delivered_orders", + "__fqn__": "delivered_orders", + "label": "Delivered Orders", + "description": "Orders that reached the customer. Derived from ecommerce_orders rather than from a table.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "models": [ + { + "name": "ecommerce_orders", + "__fqn__": "ecommerce_orders", + "label": "Orders", + "description": "One row per customer order.", + "owner": "dev@ossie.apache.org", + "data_source_name": "bigquery_demo", + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 5, + "hidden": false, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "created_at": { + "name": "created_at", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.created_at }}" + }, + "primary_key": false, + "label": "Created At", + "type": "datetime", + "__type__": "Dimension" + }, + "created_date": { + "name": "created_date", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "CAST({{ #SOURCE.created_at }} AS DATE)" + }, + "primary_key": false, + "label": "Created Date", + "type": "date", + "__type__": "Dimension" + }, + "created_month": { + "name": "created_month", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "DATE_TRUNC({{ #SOURCE.created_at }}, month)" + }, + "format": "dd LLL, yyyy", + "primary_key": false, + "label": "Created Month", + "type": "date", + "__type__": "Dimension" + }, + "status": { + "name": "status", + "__doc__": "", + "__childIdx__": 10, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.status }}" + }, + "primary_key": false, + "label": "Status", + "type": "text", + "__type__": "Dimension" + }, + "delivered": { + "name": "delivered", + "__doc__": "", + "__childIdx__": 11, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.status == 'delivered'" + }, + "primary_key": false, + "label": "Delivered", + "type": "truefalse", + "__type__": "Dimension" + } + }, + "measure": { + "last_created_at": { + "name": "last_created_at", + "__doc__": "", + "__childIdx__": 12, + "hidden": false, + "aggregation_type": "max", + "label": "Last Created At", + "type": "datetime", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ created_at }}" + }, + "__type__": "Measure" + }, + "count_orders": { + "name": "count_orders", + "__doc__": "", + "__childIdx__": 13, + "hidden": false, + "aggregation_type": "count", + "label": "Count Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "__type__": "Measure" + }, + "delivered_orders": { + "name": "delivered_orders", + "__doc__": "", + "__childIdx__": 14, + "hidden": true, + "aggregation_type": "custom", + "label": "Delivered Orders", + "type": "number", + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "count(ecommerce_orders.id) | where(ecommerce_orders.delivered)" + }, + "__type__": "Measure" + } + }, + "type": "table", + "table_name": "`ecommerce`.`orders`", + "__doc__": "", + "__type__": "TableModel" + } + ], + "dimension": { + "id": { + "name": "id", + "__doc__": "", + "__childIdx__": 7, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.id }}" + }, + "primary_key": true, + "label": "Id", + "type": "number", + "__type__": "Dimension" + }, + "user_id": { + "name": "user_id", + "__doc__": "", + "__childIdx__": 8, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.user_id }}" + }, + "primary_key": false, + "label": "User Id", + "type": "number", + "__type__": "Dimension" + }, + "delivery_attempts": { + "name": "delivery_attempts", + "__doc__": "", + "__childIdx__": 9, + "hidden": false, + "definition": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "{{ #SOURCE.delivery_attempts }}" + }, + "primary_key": false, + "label": "Delivery Attempts", + "type": "number", + "__type__": "Dimension" + } + }, + "param": { + "min_delivery_attempts": { + "name": "min_delivery_attempts", + "__doc__": "", + "__childIdx__": 6, + "hidden": false, + "label": "Minimum Delivery Attempts", + "type": "number", + "__type__": "Param" + } + }, + "type": "query", + "query": { + "__doc__": "", + "__type__": "Heredoc", + "name": "sql", + "content": "select {{ #ecommerce_orders.* }}\nfrom {{ #ecommerce_orders }}\nwhere {% filter(min_delivery_attempts) %} delivery_attempts {% end %}" + }, + "__doc__": "", + "__type__": "QueryModel" + } + ], + "relationships": [ + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "order_items", + "field": "order_id", + "__refFqn__": "order_items", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "order_items", + "field": "product_id", + "__refFqn__": "order_items", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "products", + "field": "id", + "__refFqn__": "products", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "user_id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "products", + "field": "merchant_id", + "__refFqn__": "products", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "merchants", + "field": "id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "on": [ + { + "__doc__": "", + "__childIdx__": 3, + "from": { + "__doc__": "", + "model": "merchants", + "field": "country_code", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "cities", + "field": "country_code", + "__type__": "FieldRef" + }, + "__type__": "RelationshipKey" + } + ], + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "merchants", + "field": "city_id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "cities", + "field": "id", + "__refFqn__": "cities", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "delivered_orders", + "field": "user_id", + "__refFqn__": "delivered_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "one_to_one", + "from": { + "__doc__": "", + "model": "merchants", + "field": "admin_id", + "__refFqn__": "merchants", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "ecommerce_users", + "field": "id", + "__refFqn__": "ecommerce_users", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__doc__": "", + "type": "many_to_one", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "created_date", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "reporting.calendar", + "field": "date", + "__type__": "FieldRef" + }, + "__type__": "EqualityRelationship" + }, + "active": false, + "__type__": "RelationshipConfig" + } + ], + "__doc__": "", + "__type__": "Dataset" +} \ No newline at end of file diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml new file mode 100644 index 000000000..a86517049 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce_orders.delivered + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_QUERY_SOURCE + severity: WARNING + message: >- + a query model becomes a query in `source`, which is one string, so a consumer reads the text to + tell it from a table reference + object_ref: ecommerce_users + remedy: materialise the query as a table in Holistics before converting +- code: HOLISTICS_PARAM_DROPPED + severity: WARNING + message: >- + 2 param(s) have no Ossie equivalent and are stashed. A body that reads one cannot be resolved + statically + object_ref: ecommerce_users + remedy: supply the param value in Holistics before converting +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: order_items.item_value + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: cities.in_vietnam + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_QUERY_SOURCE + severity: WARNING + message: >- + a query model becomes a query in `source`, which is one string, so a consumer reads the text to + tell it from a table reference + object_ref: delivered_orders + remedy: materialise the query as a table in Holistics before converting +- code: HOLISTICS_UNRESOLVED_QUERY_TEMPLATE + severity: ERROR + message: >- + the query body uses Holistics template syntax (model_columns, model_relation, filter_block) and + is carried unresolved. `source` is not runnable SQL + object_ref: delivered_orders + remedy: resolve the query through Holistics, or drop the model +- code: HOLISTICS_PARAM_DROPPED + severity: WARNING + message: >- + 1 param(s) have no Ossie equivalent and are stashed. A body that reads one cannot be resolved + statically + object_ref: delivered_orders + remedy: supply the param value in Holistics before converting +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce_orders.delivered_orders + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.total_revenue + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.count_orders + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_AQL_NOT_TRANSLATED + severity: INFO + message: >- + body is AQL, carried verbatim under HOLISTICS_AQL. A consumer that does not speak AQL can read + and round-trip it but cannot evaluate it + object_ref: ecommerce.revenue_share_of_all_merchants + remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_INACTIVE_RELATIONSHIP + severity: WARNING + message: >- + the relationship is inactive in AML, and Ossie has no inactive marker. A consumer joins across + it where Holistics leaves it out by default + object_ref: ecommerce_orders_to_reporting__calendar + remedy: drop the relationship, or activate it in AML diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml new file mode 100644 index 000000000..9f4da5c7b --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml @@ -0,0 +1,651 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: ecommerce +description: 'A marketplace model: orders, line items, products, merchants and cities.' +datasets: +- name: ecommerce_orders + source: '`ecommerce`.`orders`' + primary_key: + - id + description: One row per customer order. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: BIGQUERY + expression: user_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "User Id", "type": "number"}' + - name: created_at + expression: + dialects: + - dialect: BIGQUERY + expression: created_at + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Created At", "type": "datetime"}' + - name: created_date + expression: + dialects: + - dialect: BIGQUERY + expression: CAST(created_at AS DATE) + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Created Date", "type": "date"}' + - name: created_month + expression: + dialects: + - dialect: BIGQUERY + expression: DATE_TRUNC(created_at, month) + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "dd LLL, yyyy", "label": "Created Month", "type": + "date"}' + - name: status + expression: + dialects: + - dialect: BIGQUERY + expression: status + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Status", "type": "text"}' + - name: delivered + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: ecommerce_orders.status == 'delivered' + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Delivered", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Orders", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`orders`"}' +- name: ecommerce_users + source: select * from `ecommerce`.`users` + primary_key: + - id + description: One row per registered customer. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: full_name + expression: + dialects: + - dialect: BIGQUERY + expression: full_name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Full Name", "type": "text"}' + - name: gender + expression: + dialects: + - dialect: BIGQUERY + expression: gender + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Gender", "type": "text"}' + - name: birth_date + expression: + dialects: + - dialect: BIGQUERY + expression: birth_date + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Birth Date", "type": "date"}' + - name: age + expression: + dialects: + - dialect: BIGQUERY + expression: extract(year from age(ecommerce_users.birth_date)) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Age", "type": "number"}' + - name: age_bucket + expression: + dialects: + - dialect: BIGQUERY + expression: FLOOR(ecommerce_users.age / 10) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "hidden": true, "label": "Age Bucket", "type": "number"}' + - name: signup_month + expression: + dialects: + - dialect: BIGQUERY + expression: DATE_TRUNC(created_at, month) + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Signup Month", "type": "datetime"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "QueryModel", "data_source_name": "bigquery_demo", + "label": "Users", "owner": "dev@ossie.apache.org", "param": [{"label": "Reporting + Currency", "name": "reporting_currency", "type": "text"}, {"label": "Minimum + Age", "name": "minimum_age", "type": "number"}], "persistence": {"__doc__": + "", "__type__": "FullPersistence", "on_cascade": "rebuild", "schema": "persisted"}}' +- name: order_items + source: '`ecommerce`.`order_items`' + description: One row per line item on an order. + fields: + - name: order_id + expression: + dialects: + - dialect: BIGQUERY + expression: order_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Order Id", "type": "number"}' + - name: product_id + expression: + dialects: + - dialect: BIGQUERY + expression: product_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Product Id", "type": "number"}' + - name: quantity + expression: + dialects: + - dialect: BIGQUERY + expression: quantity + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Quantity", "type": "number"}' + - name: item_value + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: order_items.quantity * products.price + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Item Value", "scope": "dataset", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Order Items", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`order_items`"}' +- name: products + source: '`ecommerce`.`products`' + primary_key: + - id + description: One row per sellable product. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: merchant_id + expression: + dialects: + - dialect: BIGQUERY + expression: merchant_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Merchant Id", "type": "number"}' + - name: price + expression: + dialects: + - dialect: BIGQUERY + expression: price + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "#,###[$$]", "label": "Price", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Products", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`products`"}' +- name: merchants + source: '`ecommerce`.`merchants`' + primary_key: + - id + description: One row per seller on the marketplace. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: admin_id + expression: + dialects: + - dialect: BIGQUERY + expression: admin_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Admin Id", "type": "number"}' + - name: city_id + expression: + dialects: + - dialect: BIGQUERY + expression: city_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "City Id", "type": "number"}' + - name: country_code + expression: + dialects: + - dialect: BIGQUERY + expression: country_code + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Country Code", "type": "text"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Merchants", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`merchants`"}' +- name: cities + source: '`ecommerce`.`cities`' + primary_key: + - id + description: One row per city a merchant can operate in. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: name + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Name", "type": "text"}' + - name: country_code + expression: + dialects: + - dialect: BIGQUERY + expression: country_code + datatype: String + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Country Code", "type": "text"}' + - name: in_vietnam + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: cities.country_code == 'VN' + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "In Vietnam", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "label": "Cities", "owner": "dev@ossie.apache.org", "table_name": "`ecommerce`.`cities`"}' +- name: reporting__calendar + source: '`ecommerce`.`calendar`' + primary_key: + - date + description: A date spine. Lives in the reporting module, so its compiled name is + reporting::calendar. + fields: + - name: date + expression: + dialects: + - dialect: BIGQUERY + expression: date + datatype: Date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Date", "type": "date"}' + - name: year + expression: + dialects: + - dialect: BIGQUERY + expression: year + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Year", "type": "number"}' + - name: is_weekend + expression: + dialects: + - dialect: BIGQUERY + expression: is_weekend + datatype: Boolean + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Is Weekend", "type": "truefalse"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "TableModel", "data_source_name": "bigquery_demo", + "fqn": "reporting::calendar", "label": "Calendar", "owner": "dev@ossie.apache.org", + "table_name": "`ecommerce`.`calendar`"}' +- name: delivered_orders + source: 'select {{ #ecommerce_orders.* }} + + from {{ #ecommerce_orders }} + + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %}' + primary_key: + - id + description: Orders that reached the customer. Derived from ecommerce_orders rather + than from a table. + fields: + - name: id + expression: + dialects: + - dialect: BIGQUERY + expression: id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: BIGQUERY + expression: user_id + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "User Id", "type": "number"}' + - name: delivery_attempts + expression: + dialects: + - dialect: BIGQUERY + expression: delivery_attempts + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Delivery Attempts", "type": "number"}' + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "QueryModel", "data_source_name": "bigquery_demo", + "label": "Delivered Orders", "owner": "dev@ossie.apache.org", "param": [{"label": + "Minimum Delivery Attempts", "name": "min_delivery_attempts", "type": "number"}], + "query_forms": ["model_columns", "model_relation", "filter_block"]}' +relationships: +- name: order_items_to_ecommerce_orders + from: order_items + to: ecommerce_orders + from_columns: + - order_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: order_items_to_products + from: order_items + to: products + from_columns: + - product_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: ecommerce_orders_to_ecommerce_users + from: ecommerce_orders + to: ecommerce_users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: products_to_merchants + from: products + to: merchants + from_columns: + - merchant_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: merchants_to_cities + from: merchants + to: cities + from_columns: + - city_id + - country_code + to_columns: + - id + - country_code + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: delivered_orders_to_ecommerce_users + from: delivered_orders + to: ecommerce_users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +- name: merchants_to_ecommerce_users + from: merchants + to: ecommerce_users + from_columns: + - admin_id + to_columns: + - id + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "one_to_one"}' +- name: ecommerce_orders_to_reporting__calendar + from: ecommerce_orders + to: reporting__calendar + from_columns: + - created_date + to_columns: + - date + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "active": false, "direction": "two_way", "nullable": true, "rlp_propagation": + "inherit", "type": "many_to_one"}' +metrics: +- name: ecommerce_orders__last_created_at + expression: + dialects: + - dialect: BIGQUERY + expression: MAX(ecommerce_orders.created_at) + datatype: DateTime + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "max", "aml_name": "last_created_at", "label": + "Last Created At", "model": "ecommerce_orders", "type": "datetime"}' +- name: ecommerce_orders__count_orders + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "count", "aml_name": "count_orders", "label": + "Count Orders", "model": "ecommerce_orders", "type": "number"}' +- name: ecommerce_orders__delivered_orders + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: count(ecommerce_orders.id) | where(ecommerce_orders.delivered) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "delivered_orders", + "hidden": true, "label": "Delivered Orders", "model": "ecommerce_orders", "type": + "number"}' +- name: ecommerce_users__count_users + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(DISTINCT id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "count distinct", "aml_name": "count_users", + "label": "Count Users", "model": "ecommerce_users", "type": "number"}' +- name: order_items__total_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(quantity) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "sum", "aml_name": "total_quantity", "label": + "Total Quantity", "model": "order_items", "type": "number"}' +- name: order_items__custom_total_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: sum(quantity) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "custom_total_quantity", + "label": "Custom Total Quantity", "model": "order_items", "type": "number"}' +- name: cities__upstream_measure + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(cities.id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "upstream_measure", + "label": "Upstream Measure", "model": "cities", "type": "number"}' +- name: cities__downstream_measure + expression: + dialects: + - dialect: BIGQUERY + expression: cities__upstream_measure + 1 + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "aggregation_type": "custom", "aml_name": "downstream_measure", + "label": "Downstream Measure", "model": "cities", "type": "number"}' +- name: total_revenue + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: sum(order_items.item_value) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "format": "#,###[$$]", "label": "Total Revenue", "type": "number"}' +- name: count_orders + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: count(order_items.order_id) + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Count Orders", "type": "number"}' +- name: revenue_share_of_all_merchants + expression: + dialects: + - dialect: HOLISTICS_AQL + expression: 'sum(order_items.item_value) + + / (sum(order_items.item_value) | of_all(merchants))' + datatype: Decimal + custom_extensions: + - vendor_name: HOLISTICS + data: '{"_v": 1, "label": "Revenue Share Of All Merchants", "type": "number"}' +custom_extensions: +- vendor_name: HOLISTICS + data: '{"_v": 1, "aml_type": "Dataset", "data_source_name": "bigquery_demo", "label": + "Ecommerce", "owner": "dev@ossie.apache.org"}' diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml new file mode 100644 index 000000000..505757ed9 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml.issues.yaml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[] diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml new file mode 100644 index 000000000..ea3906566 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/cities.model.aml @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model cities { + type: 'table' + label: 'Cities' + description: 'One row per city a merchant can operate in.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`cities`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + dimension in_vietnam { + label: 'In Vietnam' + type: 'truefalse' + definition: @aql cities.country_code == 'VN';; + } + + measure upstream_measure { + label: 'Upstream Measure' + type: 'number' + definition: @sql COUNT({{ id }});; + aggregation_type: 'custom' + } + + measure downstream_measure { + label: 'Downstream Measure' + type: 'number' + definition: @sql {{ upstream_measure }} + 1;; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml new file mode 100644 index 000000000..765d76fac --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/delivered_orders.model.aml @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model delivered_orders { + type: 'query' + label: 'Delivered Orders' + description: 'Orders that reached the customer. Derived from ecommerce_orders rather than from a table.' + data_source_name: 'bigquery_demo' + models: [ecommerce_orders] + + query: @sql + select {{ #ecommerce_orders.* }} + from {{ #ecommerce_orders }} + where {% filter(min_delivery_attempts) %} delivery_attempts {% end %} + ;; + + param min_delivery_attempts { + label: 'Minimum Delivery Attempts' + type: 'number' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension delivery_attempts { + label: 'Delivery Attempts' + type: 'number' + definition: @sql {{ #SOURCE.delivery_attempts }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml new file mode 100644 index 000000000..10120a784 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset ecommerce { + label: 'Ecommerce' + description: 'A marketplace model: orders, line items, products, merchants and cities.' + data_source_name: 'bigquery_demo' + + models: [ + ecommerce_orders, + ecommerce_users, + order_items, + products, + merchants, + cities, + reporting.calendar, + delivered_orders, + ] + + relationships: [ + relationship(order_items.order_id > ecommerce_orders.id, true), + relationship(order_items.product_id > products.id, true), + relationship(ecommerce_orders.user_id > ecommerce_users.id, true), + relationship(products.merchant_id > merchants.id, true), + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: FieldRef { model: 'merchants', field: 'city_id' } + to: FieldRef { model: 'cities', field: 'id' } + on { + from: FieldRef { model: 'merchants', field: 'country_code' } + to: FieldRef { model: 'cities', field: 'country_code' } + } + } + active: true + }, + relationship(delivered_orders.user_id > ecommerce_users.id, true), + relationship(merchants.admin_id - ecommerce_users.id, true), + relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + ] + + dimension item_value { + model: order_items + label: 'Item Value' + type: 'number' + definition: @aql order_items.quantity * products.price;; + } + + metric total_revenue { + label: 'Total Revenue' + type: 'number' + definition: @aql sum(order_items.item_value);; + format: '#,###[$$]' + } + + metric count_orders { + label: 'Count Orders' + type: 'number' + definition: @aql count(order_items.order_id);; + } + + metric revenue_share_of_all_merchants { + label: 'Revenue Share Of All Merchants' + type: 'number' + definition: @aql + sum(order_items.item_value) + / (sum(order_items.item_value) | of_all(merchants)) + ;; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml new file mode 100644 index 000000000..edb11d78f --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_orders.model.aml @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_orders { + type: 'table' + label: 'Orders' + description: 'One row per customer order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } + + dimension created_at { + label: 'Created At' + type: 'datetime' + definition: @sql {{ #SOURCE.created_at }};; + } + + dimension created_date { + label: 'Created Date' + type: 'date' + definition: @sql CAST({{ #SOURCE.created_at }} AS DATE);; + } + + dimension created_month { + label: 'Created Month' + type: 'date' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + format: 'dd LLL, yyyy' + } + + dimension status { + label: 'Status' + type: 'text' + definition: @sql {{ #SOURCE.status }};; + } + + dimension delivered { + label: 'Delivered' + type: 'truefalse' + definition: @aql ecommerce_orders.status == 'delivered';; + } + + measure last_created_at { + label: 'Last Created At' + type: 'datetime' + definition: @sql {{ created_at }};; + aggregation_type: 'max' + } + + measure count_orders { + label: 'Count Orders' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count' + } + + measure delivered_orders { + label: 'Delivered Orders' + type: 'number' + hidden: true + definition: @aql count(ecommerce_orders.id) | where(ecommerce_orders.delivered);; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml new file mode 100644 index 000000000..f104518dc --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce_users.model.aml @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model ecommerce_users { + type: 'query' + label: 'Users' + description: 'One row per registered customer.' + data_source_name: 'bigquery_demo' + + query: @sql select * from `ecommerce`.`users`;; + + param reporting_currency { + label: 'Reporting Currency' + type: 'text' + } + + param minimum_age { + label: 'Minimum Age' + type: 'number' + } + + persistence: FullPersistence { + on_cascade: 'rebuild' + schema: 'persisted' + } + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension full_name { + label: 'Full Name' + type: 'text' + definition: @sql {{ #SOURCE.full_name }};; + } + + dimension gender { + label: 'Gender' + type: 'text' + definition: @sql {{ #SOURCE.gender }};; + } + + dimension birth_date { + label: 'Birth Date' + type: 'date' + definition: @sql {{ #SOURCE.birth_date }};; + } + + dimension age { + label: 'Age' + type: 'number' + definition: @sql extract(year from age({{ birth_date }}));; + } + + dimension age_bucket { + label: 'Age Bucket' + type: 'number' + hidden: true + definition: @sql FLOOR({{ age }} / 10);; + } + + dimension signup_month { + label: 'Signup Month' + type: 'datetime' + definition: @sql DATE_TRUNC({{ #SOURCE.created_at }}, month);; + } + + measure count_users { + label: 'Count Users' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + aggregation_type: 'count distinct' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml new file mode 100644 index 000000000..6132530f6 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/merchants.model.aml @@ -0,0 +1,57 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model merchants { + type: 'table' + label: 'Merchants' + description: 'One row per seller on the marketplace.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`merchants`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension admin_id { + label: 'Admin Id' + type: 'number' + definition: @sql {{ #SOURCE.admin_id }};; + } + + dimension city_id { + label: 'City Id' + type: 'number' + definition: @sql {{ #SOURCE.city_id }};; + } + + dimension country_code { + label: 'Country Code' + type: 'text' + definition: @sql {{ #SOURCE.country_code }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml new file mode 100644 index 000000000..93c70b537 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/modules/reporting/calendar.model.aml @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model calendar { + type: 'table' + label: 'Calendar' + description: 'A date spine. Lives in the reporting module, so its compiled name is reporting::calendar.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`calendar`' + + dimension date { + label: 'Date' + type: 'date' + definition: @sql {{ #SOURCE.date }};; + primary_key: true + } + + dimension year { + label: 'Year' + type: 'number' + definition: @sql {{ #SOURCE.year }};; + } + + dimension is_weekend { + label: 'Is Weekend' + type: 'truefalse' + definition: @sql {{ #SOURCE.is_weekend }};; + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml new file mode 100644 index 000000000..e78339343 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/order_items.model.aml @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model order_items { + type: 'table' + label: 'Order Items' + description: 'One row per line item on an order.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension product_id { + label: 'Product Id' + type: 'number' + definition: @sql {{ #SOURCE.product_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + measure total_quantity { + label: 'Total Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + aggregation_type: 'sum' + } + + measure custom_total_quantity { + label: 'Custom Total Quantity' + type: 'number' + definition: @sql sum({{ #SOURCE.quantity }});; + aggregation_type: 'custom' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml new file mode 100644 index 000000000..8fdbabf81 --- /dev/null +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/products.model.aml @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model products { + type: 'table' + label: 'Products' + description: 'One row per sellable product.' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`products`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension name { + label: 'Name' + type: 'text' + definition: @sql {{ #SOURCE.name }};; + } + + dimension merchant_id { + label: 'Merchant Id' + type: 'number' + definition: @sql {{ #SOURCE.merchant_id }};; + } + + dimension price { + label: 'Price' + type: 'number' + definition: @sql {{ #SOURCE.price }};; + format: '#,###[$$]' + } + + owner: 'dev@ossie.apache.org' +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml new file mode 100644 index 000000000..6f50c40ae --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml.issues.yaml @@ -0,0 +1,45 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) / COUNT(users.id)' became 'sum(items.quantity) / count(users.id)' + object_ref: quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'quantity_per_customer / 2' became 'quantity_per_customer / 2' + object_ref: half_quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'items_quantity + COUNT(users.id)' became 'items.items_quantity + count(users.id)' + object_ref: quantity_and_customers + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(items.quantity) * quantity_per_customer' became 'sum(items.quantity) * quantity_per_customer' + object_ref: scaled_quantity_per_customer + remedy: check the AQL against the Ossie SQL it came from diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml new file mode 100644 index 000000000..1ec2a360c --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/items.model.aml @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model items { + type: 'table' + label: 'Items' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`order_items`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension order_id { + label: 'Order Id' + type: 'number' + definition: @sql {{ #SOURCE.order_id }};; + } + + dimension quantity { + label: 'Quantity' + type: 'number' + definition: @sql {{ #SOURCE.quantity }};; + } + + measure items_quantity { + label: 'Items Quantity' + type: 'number' + description: 'A plain measure on one model, which the metrics below reference' + definition: @sql SUM({{ quantity }});; + aggregation_type: 'custom' + } + + measure quantity_plus_one { + label: 'Quantity Plus One' + type: 'number' + description: 'A body naming only another metric, so placement follows the reference' + definition: @sql {{ items_quantity }} + 1;; + aggregation_type: 'custom' + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml new file mode 100644 index 000000000..9105caf8a --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/metric_references.dataset.aml @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset metric_references { + label: 'Metric References' + description: 'Metrics that name other metrics, across both AML metric homes' + data_source_name: 'bigquery_demo' + + models: [ + orders, + items, + users, + ] + + relationships: [ + relationship(items.order_id > orders.id, true), + relationship(orders.user_id > users.id, true), + ] + + metric quantity_per_customer { + label: 'Quantity Per Customer' + type: 'number' + description: 'A body spanning two datasets, so AML takes it only as AQL' + definition: @aql sum(items.quantity) / count(users.id);; + } + + metric half_quantity_per_customer { + label: 'Half Quantity Per Customer' + type: 'number' + description: 'An AQL metric naming another AQL metric, which AQL takes by bare name' + definition: @aql quantity_per_customer / 2;; + } + + metric quantity_and_customers { + label: 'Quantity And Customers' + type: 'number' + description: 'An AQL metric naming a measure, which AQL takes qualified by its model' + definition: @aql items.items_quantity + count(users.id);; + } + + metric scaled_quantity_per_customer { + label: 'Scaled Quantity Per Customer' + type: 'number' + description: 'A SQL body naming an AQL metric, so the body becomes AQL as well' + definition: @aql sum(items.quantity) * quantity_per_customer;; + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml new file mode 100644 index 000000000..d2c854294 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/orders.model.aml @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model orders { + type: 'table' + label: 'Orders' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`orders`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } + + dimension user_id { + label: 'User Id' + type: 'number' + definition: @sql {{ #SOURCE.user_id }};; + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml b/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml new file mode 100644 index 000000000..0a5d0d9b3 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/expected_aml/users.model.aml @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model users { + type: 'table' + label: 'Users' + data_source_name: 'bigquery_demo' + table_name: '`ecommerce`.`users`' + + dimension id { + label: 'Id' + type: 'number' + definition: @sql {{ #SOURCE.id }};; + primary_key: true + } +} diff --git a/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml b/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml new file mode 100644 index 000000000..8f0d7c9b4 --- /dev/null +++ b/converters/holistics/tests/fixtures/metric_refs/references.ossie.yaml @@ -0,0 +1,128 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie document written to exercise a metric that names another metric. +# +# AML gives a metric two homes, and the two take different bodies. A model-level +# `measure` takes `@sql`, which resolves inside one model. A dataset-level +# `metric` takes `@aql`, which spans models. A reference across that line only +# works one way: AQL names a measure, and a `@sql` body has no spelling for a +# dataset-level metric. +# +# So the referenced metric decides where the referencing one can go, and these +# metrics cover each way that lands. The ecommerce tables are the ones the +# Holistics BigQuery demo connection carries, so the generated AML compiles +# against a real schema rather than an invented one. + +version: "0.2.0.dev0" +name: metric_references +description: Metrics that name other metrics, across both AML metric homes + +datasets: + - name: orders + source: "`ecommerce`.`orders`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: user_id + expression: {dialects: [{dialect: ANSI_SQL, expression: user_id}]} + datatype: Decimal + + - name: items + source: "`ecommerce`.`order_items`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + - name: order_id + expression: {dialects: [{dialect: ANSI_SQL, expression: order_id}]} + datatype: Decimal + - name: quantity + expression: {dialects: [{dialect: ANSI_SQL, expression: quantity}]} + datatype: Decimal + + - name: users + source: "`ecommerce`.`users`" + primary_key: [id] + fields: + - name: id + expression: {dialects: [{dialect: ANSI_SQL, expression: id}]} + datatype: Decimal + +relationships: + - name: items_to_orders + from: items + to: orders + from_columns: [order_id] + to_columns: [id] + - name: orders_to_users + from: orders + to: users + from_columns: [user_id] + to_columns: [id] + +metrics: + - name: items_quantity + description: A plain measure on one model, which the metrics below reference + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) + + - name: quantity_plus_one + description: A body naming only another metric, so placement follows the reference + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: items_quantity + 1 + + - name: quantity_per_customer + description: A body spanning two datasets, so AML takes it only as AQL + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) / COUNT(users.id) + + - name: half_quantity_per_customer + description: An AQL metric naming another AQL metric, which AQL takes by bare name + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: quantity_per_customer / 2 + + - name: quantity_and_customers + description: An AQL metric naming a measure, which AQL takes qualified by its model + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: items_quantity + COUNT(users.id) + + - name: scaled_quantity_per_customer + description: A SQL body naming an AQL metric, so the body becomes AQL as well + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(items.quantity) * quantity_per_customer diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml b/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml new file mode 100644 index 000000000..7140f485e --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml.issues.yaml @@ -0,0 +1,85 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk)' + became 'sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk)' + object_ref: customer_lifetime_value + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_RENDERED_AS_AQL + severity: WARNING + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The ANSI_SQL expression was + rendered as AQL: 'SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), + 0)' became 'sum(store_sales.ss_ext_sales_price) / nullif(sum(store.s_number_employees), 0)' + object_ref: store_productivity + remedy: check the AQL against the Ossie SQL it came from +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: cumulative_sales + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: brand_rank_in_store + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_METRIC_SQL_NOT_TRANSLATED_TO_AQL + severity: ERROR + message: >- + the metric sits on the dataset, where AML accepts only an AQL body. The converter did not translate + this ANSI_SQL body, and the metric is dropped. It is a window function. Even though AQL has window + functions, this converter cannot output one. Ossie SQL carries the partition and the ordering + in the OVER clause, while AQL takes both from the grain of the query being run. Translating one + into the other means deciding which query grain this OVER clause corresponds to, and the Ossie + specification defines no semantics for how a window function behaves when queried, so nothing + says which grain that is. See https://docs.holistics.io/reference/aql/functions for AQL's window + functions and https://docs.holistics.io/reference/aql/rank for one in full, then write the AQL + by hand. + object_ref: monthly_sales_change + remedy: rewrite the metric in AQL, or scope it to a single dataset +- code: HOLISTICS_ASSUMED_TYPE + severity: WARNING + message: AML requires a type and the Ossie datatype is None, so 'text' is written + object_ref: date_dim.d_quarter_name + remedy: set a datatype on the field, or edit the AML type after conversion +- code: HOLISTICS_ASSUMED_TYPE + severity: WARNING + message: AML requires a type and the Ossie datatype is None, so 'text' is written + object_ref: date_dim.d_moy + remedy: set a datatype on the field, or edit the AML type after conversion diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml new file mode 100644 index 000000000..d1da6bbbc --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/customer.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model customer { + type: 'table' + label: 'Customer' + description: 'Customer dimension with demographic information' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.customer' + + dimension c_customer_sk { + label: 'C Customer Sk' + type: 'number' + description: 'Surrogate key for customer' + definition: @sql {{ #SOURCE.c_customer_sk }};; + primary_key: true + } + + dimension c_customer_id { + label: 'C Customer Id' + type: 'text' + description: 'Business key for customer' + definition: @sql {{ #SOURCE.c_customer_id }};; + } + + dimension c_first_name { + label: 'C First Name' + type: 'text' + description: 'Customer first name' + definition: @sql {{ #SOURCE.c_first_name }};; + } + + dimension c_last_name { + label: 'C Last Name' + type: 'text' + description: 'Customer last name' + definition: @sql {{ #SOURCE.c_last_name }};; + } + + dimension customer_full_name { + label: 'Customer Full Name' + type: 'text' + description: 'Customer full name (computed field)' + definition: @sql {{ #SOURCE.c_first_name }} || ' ' || {{ #SOURCE.c_last_name }};; + } + + dimension c_email_address { + label: 'C Email Address' + type: 'text' + description: 'Customer email address' + definition: @sql {{ #SOURCE.c_email_address }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml new file mode 100644 index 000000000..74a67672c --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/date_dim.model.aml @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model date_dim { + type: 'table' + label: 'Date Dim' + description: 'Date dimension with calendar attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.date_dim' + + dimension d_date_sk { + label: 'D Date Sk' + type: 'number' + description: 'Surrogate key for date' + definition: @sql {{ #SOURCE.d_date_sk }};; + primary_key: true + } + + dimension d_date { + label: 'D Date' + type: 'date' + description: 'Actual date value' + definition: @sql {{ #SOURCE.d_date }};; + } + + dimension d_year { + label: 'D Year' + type: 'number' + description: 'Year' + definition: @sql {{ #SOURCE.d_year }};; + } + + dimension d_quarter_name { + label: 'D Quarter Name' + type: 'text' + description: 'Quarter name (e.g., 2024Q1)' + definition: @sql {{ #SOURCE.d_quarter_name }};; + } + + dimension d_moy { + label: 'D Moy' + type: 'text' + description: 'Month of year (1-12)' + definition: @sql {{ #SOURCE.d_moy }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml new file mode 100644 index 000000000..1b21bf48e --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/item.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model item { + type: 'table' + label: 'Item' + description: 'Item/Product dimension with product attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.item' + + dimension i_item_sk { + label: 'I Item Sk' + type: 'number' + description: 'Surrogate key for item' + definition: @sql {{ #SOURCE.i_item_sk }};; + primary_key: true + } + + dimension i_item_id { + label: 'I Item Id' + type: 'text' + description: 'Business key for item' + definition: @sql {{ #SOURCE.i_item_id }};; + } + + dimension i_item_desc { + label: 'I Item Desc' + type: 'text' + description: 'Item description' + definition: @sql {{ #SOURCE.i_item_desc }};; + } + + dimension i_brand { + label: 'I Brand' + type: 'text' + description: 'Brand name' + definition: @sql {{ #SOURCE.i_brand }};; + } + + dimension i_category { + label: 'I Category' + type: 'text' + description: 'Item category' + definition: @sql {{ #SOURCE.i_category }};; + } + + dimension i_current_price { + label: 'I Current Price' + type: 'number' + description: 'Current price of the item' + definition: @sql {{ #SOURCE.i_current_price }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml new file mode 100644 index 000000000..a51a24997 --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/store.model.aml @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model store { + type: 'table' + label: 'Store' + description: 'Store dimension with location and store attributes' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.store' + + dimension s_store_sk { + label: 'S Store Sk' + type: 'number' + description: 'Surrogate key for store' + definition: @sql {{ #SOURCE.s_store_sk }};; + primary_key: true + } + + dimension s_store_id { + label: 'S Store Id' + type: 'text' + description: 'Business key for store' + definition: @sql {{ #SOURCE.s_store_id }};; + } + + dimension s_store_name { + label: 'S Store Name' + type: 'text' + description: 'Store name' + definition: @sql {{ #SOURCE.s_store_name }};; + } + + dimension s_city { + label: 'S City' + type: 'text' + description: 'City where store is located' + definition: @sql {{ #SOURCE.s_city }};; + } + + dimension s_state { + label: 'S State' + type: 'text' + description: 'State where store is located' + definition: @sql {{ #SOURCE.s_state }};; + } + + dimension s_number_employees { + label: 'S Number Employees' + type: 'number' + description: 'Number of employees at the store' + definition: @sql {{ #SOURCE.s_number_employees }};; + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml new file mode 100644 index 000000000..8b8c6265a --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/store_sales.model.aml @@ -0,0 +1,105 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Model store_sales { + type: 'table' + label: 'Store Sales' + description: 'Fact table containing all store sales transactions' + data_source_name: 'tpcds_warehouse' + table_name: 'tpcds.public.store_sales' + + dimension ss_sold_date_sk { + label: 'Ss Sold Date Sk' + type: 'number' + description: 'Foreign key to date dimension' + definition: @sql {{ #SOURCE.ss_sold_date_sk }};; + } + + dimension ss_item_sk { + label: 'Ss Item Sk' + type: 'number' + description: 'Foreign key to item dimension' + definition: @sql {{ #SOURCE.ss_item_sk }};; + primary_key: true + } + + dimension ss_customer_sk { + label: 'Ss Customer Sk' + type: 'number' + description: 'Foreign key to customer dimension' + definition: @sql {{ #SOURCE.ss_customer_sk }};; + } + + dimension ss_store_sk { + label: 'Ss Store Sk' + type: 'number' + description: 'Foreign key to store dimension' + definition: @sql {{ #SOURCE.ss_store_sk }};; + } + + dimension ss_quantity { + label: 'Ss Quantity' + type: 'number' + description: 'Quantity of items sold' + definition: @sql {{ #SOURCE.ss_quantity }};; + } + + dimension ss_sales_price { + label: 'Ss Sales Price' + type: 'number' + description: 'Sales price per unit' + definition: @sql {{ #SOURCE.ss_sales_price }};; + } + + dimension ss_ext_sales_price { + label: 'Ss Ext Sales Price' + type: 'number' + description: 'Extended sales price (quantity * price)' + definition: @sql {{ #SOURCE.ss_ext_sales_price }};; + } + + dimension ss_net_profit { + label: 'Ss Net Profit' + type: 'number' + description: 'Net profit from the sale' + definition: @sql {{ #SOURCE.ss_net_profit }};; + } + + measure total_sales { + label: 'Total Sales' + type: 'number' + description: 'Total sales revenue across all transactions' + definition: @sql SUM({{ ss_ext_sales_price }});; + aggregation_type: 'custom' + } + + measure total_profit { + label: 'Total Profit' + type: 'number' + description: 'Total net profit from store sales' + definition: @sql SUM({{ ss_net_profit }});; + aggregation_type: 'custom' + } + + measure sales_by_brand { + label: 'Sales By Brand' + type: 'number' + description: 'Total sales by brand (requires grouping by item.i_brand)' + definition: @sql SUM({{ ss_ext_sales_price }});; + aggregation_type: 'custom' + } +} diff --git a/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml b/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml new file mode 100644 index 000000000..f1eed69c9 --- /dev/null +++ b/converters/holistics/tests/fixtures/tpcds/expected_aml/tpcds_retail_model.dataset.aml @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Dataset tpcds_retail_model { + label: 'Tpcds Retail Model' + description: 'TPC-DS retail semantic model for sales and customer analytics' + data_source_name: 'tpcds_warehouse' + + models: [ + store_sales, + date_dim, + customer, + item, + store, + ] + + relationships: [ + relationship(store_sales.ss_sold_date_sk > date_dim.d_date_sk, true), + relationship(store_sales.ss_customer_sk > customer.c_customer_sk, true), + relationship(store_sales.ss_item_sk > item.i_item_sk, true), + relationship(store_sales.ss_store_sk > store.s_store_sk, true), + ] + + metric customer_lifetime_value { + label: 'Customer Lifetime Value' + type: 'number' + description: 'Average lifetime sales value per customer' + definition: @aql sum(store_sales.ss_ext_sales_price) / count_distinct(customer.c_customer_sk);; + } + + metric store_productivity { + label: 'Store Productivity' + type: 'number' + description: 'Sales per employee across stores' + definition: @aql sum(store_sales.ss_ext_sales_price) / nullif(sum(store.s_number_employees), 0);; + } +} diff --git a/converters/holistics/tests/snapshots.py b/converters/holistics/tests/snapshots.py new file mode 100644 index 000000000..3a53b133f --- /dev/null +++ b/converters/holistics/tests/snapshots.py @@ -0,0 +1,152 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""What every snapshot holds, computed once and shared. + +`tests/test_snapshots.py` compares these against the committed files, and +`tools/update_snapshots.py` writes them. One definition, so a snapshot can never +disagree with the test that checks it. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from ossie_holistics import _yaml, aml_to_ossie, ossie_to_aml +from ossie_holistics.issues import IssueLog + +REPO_ROOT = Path(__file__).resolve().parents[3] +CONVERTER_ROOT = Path(__file__).resolve().parents[1] +FIXTURES = CONVERTER_ROOT / "tests" / "fixtures" + +ECOMMERCE_COMPILED = FIXTURES / "ecommerce" / "ecommerce.dataset.aml.json" +TPCDS_OSSIE = REPO_ROOT / "examples" / "tpcds_semantic_model.yaml" +AQL_TRANSLATION = FIXTURES / "aql" / "translation.ossie.yaml" +METRIC_REFERENCES = FIXTURES / "metric_refs" / "references.ossie.yaml" + +#: The fixture is a BigQuery dataset: `DATE_TRUNC(created_at, month)` is BigQuery +#: argument order, not ANSI SQL. +SQL_DIALECT = "BIGQUERY" + +#: TPC-DS carries no HOLISTICS stash, so the AML-required connection name has to +#: come from the caller. Its expressions are all `ANSI_SQL`, so naming that as +#: the target warehouse writes every body as it stands. +TPCDS_DATA_SOURCE = "tpcds_warehouse" +TPCDS_SQL_DIALECT = "ANSI_SQL" + +LICENSE_HEADER = """\ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" + + +def _issues(log: IssueLog) -> str: + return LICENSE_HEADER + _yaml.dump_issues(log.as_dicts()) + + +def forward() -> tuple[dict[str, str], aml_to_ossie.Result]: + """Compiled AML to Ossie: one YAML document and its issue log.""" + payload = json.loads(ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + result = aml_to_ossie.convert(payload, sql_dialect=SQL_DIALECT) + return { + "ecommerce/expected.ossie.yaml": LICENSE_HEADER + _yaml.dump(result.model), + "ecommerce/expected.issues.yaml": _issues(result.issues), + }, result + + +def reverse_ecommerce() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie back to AML, starting from this converter's own forward output. + + Reading the forward result rather than the committed snapshot keeps the two + directions from drifting apart between snapshot updates. + """ + _, forward_result = forward() + result = ossie_to_aml.convert(forward_result.model, SQL_DIALECT) + files = {f"ecommerce/expected_aml/{name}": text for name, text in result.files} + files["ecommerce/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_tpcds() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML, starting from a document this converter did not write. + + `examples/tpcds_semantic_model.yaml` carries no HOLISTICS stash, so every + AML property the core specification has no field for is derived or supplied. + """ + document = _yaml.load(TPCDS_OSSIE.read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, TPCDS_SQL_DIALECT, data_source_name=TPCDS_DATA_SOURCE + ) + files = {f"tpcds/expected_aml/{name}": text for name, text in result.files} + files["tpcds/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_aql() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML for a document written to exercise the SQL to AQL translation. + + Every metric in it spans two datasets, which is what makes AML take an + `@aql` body, and each covers a different path through `aql.translate`. + """ + document = _yaml.load(AQL_TRANSLATION.read_text(encoding="utf-8")) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="bigquery_demo") + files = {f"aql/expected_aml/{name}": text for name, text in result.files} + files["aql/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def reverse_metric_references() -> tuple[dict[str, str], ossie_to_aml.Result]: + """Ossie to AML for metrics that name other metrics. + + Which AML home a metric lands in decides how it can name another one, so + these cover a reference into each home, and the placement that a body + naming only another metric depends on. + """ + document = _yaml.load(METRIC_REFERENCES.read_text(encoding="utf-8")) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="bigquery_demo") + files = {f"metric_refs/expected_aml/{name}": text for name, text in result.files} + files["metric_refs/expected_aml.issues.yaml"] = _issues(result.issues) + return files, result + + +def all_snapshots() -> dict[str, str]: + """Every committed snapshot, keyed by its path under `tests/fixtures/`.""" + files: dict[str, str] = {} + for produce in ( + forward, + reverse_ecommerce, + reverse_tpcds, + reverse_aql, + reverse_metric_references, + ): + files.update(produce()[0]) + return files diff --git a/converters/holistics/tests/test_cross_vendor.py b/converters/holistics/tests/test_cross_vendor.py new file mode 100644 index 000000000..3627fe6c1 --- /dev/null +++ b/converters/holistics/tests/test_cross_vendor.py @@ -0,0 +1,137 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Every Ossie document in this repository, converted to AML. + +The reverse path's whole job is reading documents this converter did not write, +and the snapshot fixtures cannot cover that: one of them this converter produced +and the other is a single example. The sweep found two defects the fixtures +missed, both of them crashes rather than bad output. + +`examples/flights.semantic_model.yaml` is named `Flights semantic model`. Ossie +allows any non-empty string, and AML allows letters, digits and underscore, so +the conversion raised instead of renaming. + +Three fixtures use one name for the semantic model and for a dataset inside it, +including `converters/cube/tests/fixtures/databricks_ossie.yaml`, where both are +`orders`. Ossie keeps the two in separate namespaces and AML does not, so the +generated AML failed to compile with "Duplicated name 'orders'". + +Adding a fixture elsewhere in the repository extends this test with no edit +here, which is the point. apache/ossie#352 asked the Solid converter for the +same sweep. +""" +from __future__ import annotations + +import subprocess +from collections import Counter +from pathlib import Path + +import pytest + +import snapshots +from ossie_holistics import _yaml, ossie_to_aml, sqlrefs +from test_snapshots import HOLISTICS_CLI, needs_cli + +#: Any `data_source_name`. These documents have no Holistics connection behind +#: them, and the value only has to be present for the AML to compile. +PROBE_CONNECTION = "probe" + + +def _target_dialect(document) -> str: + """The SQL dialect most of a document's expressions already use. + + An operator converting one of these would name the warehouse the Holistics + connection reads. Picking the document's own majority dialect is the closest + stand-in, and it keeps the sweep testing the conversion rather than testing + sqlglot's rendering. + """ + counts = Counter() + for dataset in document.get("datasets") or []: + for field in dataset.get("fields") or []: + for entry in (field.get("expression") or {}).get("dialects") or []: + counts[entry.get("dialect")] += 1 + for name, _ in counts.most_common(): + if name in sqlrefs.SQLGLOT_DIALECT: + return name + return "ANSI_SQL" + + +def _foreign_ossie_documents() -> list[Path]: + """Every flat Ossie document outside this converter's own directory. + + A flat document carries `version` and `datasets` at the root. That is the + shape `main` moved to, and the heuristic apache/ossie#352 settled on after + the wrapped `semantic_model:` list went away. + """ + found = [] + roots = [snapshots.REPO_ROOT / "converters", snapshots.REPO_ROOT / "examples"] + for root in roots: + for path in sorted(root.rglob("*.yaml")): + if snapshots.CONVERTER_ROOT in path.parents or ".venv" in path.parts: + continue + try: + document = _yaml.load(path.read_text(encoding="utf-8")) + except Exception: # noqa: BLE001 - a malformed sibling fixture is not our failure + continue + if isinstance(document, dict) and document.get("version") and "datasets" in document: + found.append(path) + return found + + +FOREIGN = _foreign_ossie_documents() + + +def _ids(paths: list[Path]) -> list[str]: + return [str(p.relative_to(snapshots.REPO_ROOT)) for p in paths] + + +def test_the_sweep_finds_documents(): + """A heuristic that silently matches nothing would pass every test below.""" + assert len(FOREIGN) >= 20, _ids(FOREIGN) + + +@pytest.mark.parametrize("path", FOREIGN, ids=_ids(FOREIGN)) +def test_every_ossie_document_converts(path): + document = _yaml.load(path.read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, _target_dialect(document), data_source_name=PROBE_CONNECTION + ) + assert result.files + assert sum(1 for name, _ in result.files if name.endswith(".dataset.aml")) == 1 + + +@needs_cli +@pytest.mark.parametrize("path", FOREIGN, ids=_ids(FOREIGN)) +def test_every_converted_document_compiles(path, tmp_path): + document = _yaml.load(path.read_text(encoding="utf-8")) + result = ossie_to_aml.convert( + document, _target_dialect(document), data_source_name=PROBE_CONNECTION + ) + for name, text in result.files: + target = tmp_path / name + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(text, encoding="utf-8") + dataset_file = next(name for name, _ in result.files if name.endswith(".dataset.aml")) + completed = subprocess.run( + [HOLISTICS_CLI, "aml", "validate", dataset_file, "-r", "."], + cwd=tmp_path, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "No validation errors found" in completed.stdout diff --git a/converters/holistics/tests/test_issue_codes.py b/converters/holistics/tests/test_issue_codes.py new file mode 100644 index 000000000..3b7d64176 --- /dev/null +++ b/converters/holistics/tests/test_issue_codes.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""The issue-code table in the documentation matches the source. + +A reader who sees `HOLISTICS_WINDOW_FUNCTION_NEEDS_A_GROUPING` in an issue log +looks it up in `docs/limitations.md`. A code the converter raises and the table +omits leaves them with nothing to find, and a code the table lists and the +converter never raises sends them looking for a cause that cannot occur. +""" +from __future__ import annotations + +import ast +import re +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LIMITATIONS = (ROOT / "docs" / "limitations.md").read_text(encoding="utf-8") + + +def raised_codes() -> dict[str, set[str]]: + """Each issue code the converter can raise, and the severities it uses.""" + found: dict[str, set[str]] = {} + for module in ("aml_to_ossie", "ossie_to_aml"): + text = (ROOT / "src" / "ossie_holistics" / f"{module}.py").read_text(encoding="utf-8") + constants = dict(re.findall(r'^(ISSUE_\w+) = "(\w+)"', text, re.M)) + for node in ast.walk(ast.parse(text)): + if not (isinstance(node, ast.Call) and getattr(node.func, "attr", "") == "add"): + continue + keywords = {k.arg: k.value for k in node.keywords} + code, severity = keywords.get("code"), keywords.get("severity") + if isinstance(code, ast.Name) and isinstance(severity, ast.Attribute): + found.setdefault(constants[code.id], set()).add(severity.attr) + return found + + +def documented_codes() -> dict[str, str]: + return dict(re.findall(r"^\| `(HOLISTICS_\w+)` \| (\w+) \|", LIMITATIONS, re.M)) + + +def test_every_raised_code_is_documented(): + assert set(raised_codes()) - set(documented_codes()) == set() + + +def test_every_documented_code_is_raised(): + assert set(documented_codes()) - set(raised_codes()) == set() + + +def test_the_documented_severity_matches_the_source(): + documented = documented_codes() + mismatched = { + code: (documented[code], sorted(severities)) + for code, severities in raised_codes().items() + if {documented[code]} != severities + } + assert not mismatched diff --git a/converters/holistics/tests/test_malformed_input.py b/converters/holistics/tests/test_malformed_input.py new file mode 100644 index 000000000..b0cdd6616 --- /dev/null +++ b/converters/holistics/tests/test_malformed_input.py @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""A malformed Ossie document is reported, not crashed on. + +`to-aml` reads documents this converter did not write, so its input is a +boundary rather than a known shape. The CLI catches `ConversionError`, `OSError` +and `UnicodeDecodeError`, so anything else reaches the user as a traceback. + +Two sibling reviews asked for exactly this. apache/ossie#407 found a converter +that checked `datasets` was present but not that it was a list of mappings, and +apache/ossie#297 found several direct `element["id"]` reads that raised +`KeyError` on a hand-edited document. +""" +from __future__ import annotations + +import pytest + +from ossie_holistics import ossie_to_aml +from ossie_holistics.errors import ConversionError + +VERSION = "0.2.0.dev0" +EXPRESSION = {"dialects": [{"dialect": "ANSI_SQL", "expression": "x"}]} + + +def document(**overrides): + base = {"version": VERSION, "name": "m", "datasets": [{"name": "d", "source": "s"}]} + base.update(overrides) + return base + + +MALFORMED = { + "datasets is a mapping": document(datasets={"a": {}}), + "datasets is a string": document(datasets="nope"), + "datasets is empty": document(datasets=[]), + "datasets is absent": {"version": VERSION, "name": "m"}, + "a dataset is a string": document(datasets=["nope"]), + "a dataset has no name": document(datasets=[{"source": "s"}]), + "a dataset name is not a string": document(datasets=[{"name": 7, "source": "s"}]), + "fields is a mapping": document(datasets=[{"name": "d", "source": "s", "fields": {}}]), + "a field has no name": document( + datasets=[{"name": "d", "source": "s", "fields": [{"expression": EXPRESSION}]}] + ), + "a metric has no name": document(metrics=[{"expression": EXPRESSION}]), + "a metric is a string": document(metrics=["nope"]), + "the document is a list": [], + "the document has no name": {"version": VERSION, "datasets": [{"name": "d", "source": "s"}]}, +} + + +@pytest.mark.parametrize("payload", MALFORMED.values(), ids=list(MALFORMED)) +def test_a_malformed_document_raises_conversion_error(payload): + with pytest.raises(ConversionError): + ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py new file mode 100644 index 000000000..cd7038263 --- /dev/null +++ b/converters/holistics/tests/test_snapshots.py @@ -0,0 +1,538 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Snapshot tests over the two fixtures, plus the external checks that confirm +the snapshots are not merely stable but correct. + +A snapshot test proves the converter still produces what it produced. It cannot +prove the output is right. Three checks here do that, each against the real +artifact rather than a proxy: + +`test_the_converted_document_passes_the_ossie_schema` runs the forward output +through `core-spec/ossie-schema.json`. + +`test_the_generated_aml_compiles` runs `holistics aml validate` over every +generated file, which parses the AML and type-checks its AQL. + +`test_the_round_trip_returns_the_same_ossie_document` compiles the generated AML +back to JSON, converts that to Ossie, and compares it with the document the +reverse path started from. That closes the loop AML to Ossie to AML to Ossie. + +Those that need the Holistics CLI skip when it is absent, so the suite runs +without it. CI installs it, so they do not skip there. +""" +from __future__ import annotations + +import json +import re +import shutil +import subprocess +from pathlib import Path + +import pytest + +import snapshots +from ossie_holistics import _yaml, aml_to_ossie, aql, ossie_to_aml +from ossie_holistics.issues import Severity + +HOLISTICS_CLI = shutil.which("holistics") or str(Path.home() / ".holistics" / "bin" / "holistics") + +needs_cli = pytest.mark.skipif( + not Path(HOLISTICS_CLI).exists(), + reason="the Holistics CLI is not installed (https://docs.holistics.io/docs/cli)", +) + + +@pytest.mark.parametrize("relative", sorted(snapshots.all_snapshots())) +def test_the_snapshot_matches_what_the_converter_produces(relative): + expected = snapshots.FIXTURES / relative + assert expected.exists(), ( + f"{relative} is missing. Run python3 tools/update_snapshots.py" + ) + assert snapshots.all_snapshots()[relative] == expected.read_text(encoding="utf-8"), ( + f"{relative} is stale. Read the diff, then run python3 tools/update_snapshots.py" + ) + + +def test_no_committed_snapshot_is_orphaned(): + """Every file under `tests/fixtures/expected*` is one a snapshot produces. + + Renaming a model leaves its old file behind, and a stale file that no test + reads looks like coverage it is not. + """ + produced = {snapshots.FIXTURES / name for name in snapshots.all_snapshots()} + committed = { + path + for path in snapshots.FIXTURES.rglob("expected*") + if path.is_file() + } | { + path + for path in snapshots.FIXTURES.rglob("expected_aml/**/*") + if path.is_file() + } + assert committed - produced == set() + + +def test_the_converted_document_passes_the_ossie_schema(): + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads( + (snapshots.REPO_ROOT / "core-spec" / "ossie-schema.json").read_text(encoding="utf-8") + ) + _, result = snapshots.forward() + jsonschema.validate(result.model, schema) + + +def test_the_forward_direction_reports_the_losses_it_declares(): + """Every limitation `docs/limitations.md` lists for this fixture is raised. + + Asserted on the codes rather than the count, so adding an unrelated issue + does not fail this and removing a declared one does. + """ + _, result = snapshots.forward() + raised = {issue.code for issue in result.issues.issues} + assert raised == { + aml_to_ossie.ISSUE_AQL_NOT_TRANSLATED, + aml_to_ossie.ISSUE_INACTIVE_RELATIONSHIP, + aml_to_ossie.ISSUE_PARAM_DROPPED, + aml_to_ossie.ISSUE_QUERY_SOURCE, + aml_to_ossie.ISSUE_UNRESOLVED_QUERY, + } + + +def test_every_qualified_reference_resolves(): + """A `.` reference names a field the document declares. + + apache/ossie#459 found 211 of 612 references in one converter's output where + the qualifier was the Ossie dataset name and the other half was the + warehouse column, so the pair resolved in neither namespace. Upstream's + validator cannot see it, because `Dim_Customer.Customer_Name` parses as SQL. + """ + _, result = snapshots.forward() + model = result.model + declared = { + dataset["name"]: {field["name"] for field in dataset.get("fields") or []} + for dataset in model["datasets"] + } + metrics = {metric["name"] for metric in model.get("metrics") or []} + + def sql_expressions(): + for dataset in model["datasets"]: + for field in dataset.get("fields") or []: + for entry in field["expression"]["dialects"]: + yield f"field {dataset['name']}.{field['name']}", entry + for metric in model.get("metrics") or []: + for entry in metric["expression"]["dialects"]: + yield f"metric {metric['name']}", entry + + unresolved = [] + for where, entry in sql_expressions(): + if entry["dialect"] == aml_to_ossie.AQL_DIALECT: + continue + for qualifier, name in re.findall(r"\b([A-Za-z_]\w*)\.(\w+)", entry["expression"]): + if name not in declared.get(qualifier, ()): + unresolved.append(f"{where}: {qualifier}.{name}") + for bare in re.findall(r"(? dict: + """Two joined datasets and the metrics given, as one Ossie document.""" + + def column(name: str) -> dict: + return { + "name": name, + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}, + } + + return { + "version": aml_to_ossie.SPEC_VERSION, + "name": "m", + "datasets": [ + { + "name": "orders", + "source": "db.orders", + "primary_key": ["id"], + "fields": [column("id"), column("amount")], + }, + { + "name": "users", + "source": "db.users", + "primary_key": ["id"], + "fields": [column("id"), column("order_id")], + }, + ], + "relationships": [ + { + "name": "users_to_orders", + "from": "users", + "to": "orders", + "from_columns": ["order_id"], + "to_columns": ["id"], + } + ], + "metrics": metrics, + } + + +def _metric(name: str, sql: str, **stashed) -> dict: + payload = { + "name": name, + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql}]}, + } + if stashed: + payload["custom_extensions"] = [ + {"vendor_name": "HOLISTICS", "data": json.dumps({"_v": 1, **stashed})} + ] + return payload + + +def test_a_sql_measure_naming_a_dataset_level_metric_is_an_error(): + """A `@sql` body has no spelling for a metric that lives on the dataset. + + `orders.spanning` qualifies a dataset-level metric with a model, and the + sibling reference that reads like names a field `orders` does not have. + Holistics accepts the file and fails at query time, so the converter has to + catch it. + """ + document = _two_dataset_document( + [ + _metric("spanning", "SUM(orders.amount) / COUNT(users.id)"), + _metric("doubled", "SUM(orders.amount) * orders.spanning"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_CROSS_MODEL_SQL] + assert [i.object_ref for i in raised] == ["orders.doubled"] + assert raised[0].severity is Severity.ERROR + assert "dataset-level metric 'spanning'" in raised[0].message + + +def test_a_sql_measure_naming_another_models_measure_is_an_error(): + """A measure stashed onto one model, naming a measure of another. + + Without the stash the body's two models send the metric to the dataset, + where AQL resolves it. The stash pins it to one model, and the `@sql` body + that leaves cannot name the other, failing SQL generation with + `Cannot refer to external field in TableModel/QueryModel`. + """ + document = _two_dataset_document( + [ + _metric("revenue", "SUM(orders.amount)", model="orders", aml_name="revenue"), + _metric("per_user", "COUNT(users.id) + revenue", model="users", aml_name="per_user"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_CROSS_MODEL_SQL] + assert [i.object_ref for i in raised] == ["users.per_user"] + assert raised[0].severity is Severity.ERROR + assert "a measure of model 'orders'" in raised[0].message + + +def test_a_metric_reference_cycle_is_an_error(): + """Placement follows metric references, so a cycle has to end the walk. + + `holistics aml validate` accepts a cycle, and Holistics inlines one metric + into the next, so the pair has no SQL. + """ + document = _two_dataset_document( + [ + _metric("ping", "pong + 1"), + _metric("pong", "ping * 2"), + ] + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_METRIC_CYCLE] + assert [i.object_ref for i in raised] == ["ping"] + assert raised[0].severity is Severity.ERROR + assert result.issues.has_errors() + + +#: sqlglot names a node after the shape it parsed, so several SQL spellings +#: share one node. The passthrough reads the original name back out of the +#: source using the offset sqlglot recorded for the name token. +SOURCE_NAMES = { + "LPAD(a.s, 5)": "sql_number('LPAD', a.s, 5)", + "RPAD(a.s, 5)": "sql_number('RPAD', a.s, 5)", + "LTRIM(a.s)": "sql_number('LTRIM', a.s)", + "RTRIM(a.s)": "sql_number('RTRIM', a.s)", + "STRPOS(a.s, 'z')": "sql_number('STRPOS', a.s, 'z')", + "MY_UDF(a.x)": "sql_number('MY_UDF', a.x)", +} + +#: Calls sqlglot rewrote, so an argument carries no offset and the original +#: cannot be rebuilt. `LOG10` gains a base the source never had, and +#: `DATE_TRUNC` has its unit normalised, and an absent offset does not say which. +REWRITTEN_CALLS = ["LOG10(a.x)", "DATE_TRUNC('month', a.d)", "SUBSTRING(a.s, 1, 3)"] + + +@pytest.mark.parametrize("sql,expected", SOURCE_NAMES.items(), ids=list(SOURCE_NAMES)) +def test_the_passthrough_uses_the_name_the_source_wrote(sql, expected): + rendered, _ = aql.translate(sql, "ANSI_SQL", lambda table, column: f"{table}.{column}") + assert rendered == expected + + +@pytest.mark.parametrize("sql", REWRITTEN_CALLS) +def test_a_call_sqlglot_rewrote_raises_untranslatable(sql): + with pytest.raises(aql.Untranslatable): + aql.translate(sql, "ANSI_SQL", lambda table, column: f"{table}.{column}") + + +def _write(files: dict[str, str], directory: Path) -> None: + for name, text in files.items(): + path = directory / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + + +def _validate_aml(directory: Path, dataset_file: str) -> subprocess.CompletedProcess: + return subprocess.run( + [HOLISTICS_CLI, "aml", "validate", dataset_file, "-r", "."], + cwd=directory, + capture_output=True, + text=True, + ) + + +@needs_cli +@pytest.mark.parametrize( + "produce,dataset_file", + [ + (snapshots.reverse_ecommerce, "ecommerce.dataset.aml"), + (snapshots.reverse_tpcds, "tpcds_retail_model.dataset.aml"), + (snapshots.reverse_aql, "aql_translation.dataset.aml"), + (snapshots.reverse_metric_references, "metric_references.dataset.aml"), + ], + ids=["ecommerce", "tpcds", "aql", "metric_refs"], +) +def test_the_generated_aml_compiles(produce, dataset_file, tmp_path): + _, result = produce() + _write(dict(result.files), tmp_path) + completed = _validate_aml(tmp_path, dataset_file) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert "No validation errors found" in completed.stdout + + +@needs_cli +def test_the_source_fixture_still_compiles_to_the_committed_json(tmp_path): + """The committed JSON is what the CLI produces from the committed AML. + + The fixture ships both, and an edit to the AML that forgets the recompile + would leave every other test reading the old payload. + """ + source = snapshots.FIXTURES / "ecommerce" / "aml" + completed = subprocess.run( + [HOLISTICS_CLI, "aml", "compile", "ecommerce.dataset.aml", "-r", ".", "-o", str(tmp_path)], + cwd=source, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + produced = json.loads((tmp_path / "ecommerce.dataset.aml.json").read_text(encoding="utf-8")) + committed = json.loads(snapshots.ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + assert produced == committed + + +@needs_cli +def test_the_round_trip_returns_the_same_ossie_document(tmp_path): + """AML to Ossie to AML to Ossie returns the document it started from. + + Compiling the generated AML is the only way to check the reverse path + against the thing that actually reads AML, rather than against this + converter's own idea of it. + """ + _, forward_result = snapshots.forward() + _, reverse_result = snapshots.reverse_ecommerce() + + generated = tmp_path / "aml" + _write(dict(reverse_result.files), generated) + compiled = tmp_path / "compiled" + completed = subprocess.run( + [HOLISTICS_CLI, "aml", "compile", "ecommerce.dataset.aml", "-r", ".", "-o", str(compiled)], + cwd=generated, + capture_output=True, + text=True, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + payload = json.loads((compiled / "ecommerce.dataset.aml.json").read_text(encoding="utf-8")) + again = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) + assert _yaml.dump(again.model) == _yaml.dump(forward_result.model) diff --git a/converters/holistics/tools/check_links.py b/converters/holistics/tools/check_links.py new file mode 100644 index 000000000..d41f1ba35 --- /dev/null +++ b/converters/holistics/tools/check_links.py @@ -0,0 +1,92 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Resolve every relative link and heading anchor across the documents. + +The mapping is split over several files, so a link that was a same-page anchor +can quietly become a dangling cross-file reference. This resolves each one +against the files and headings that actually exist. External URLs are not +fetched; only their shape is checked. + +Dev tooling, not part of the wheel. Run it with ``python3 tools/check_links.py``. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent.parent + +LINK = re.compile(r"\[(?P[^\]]+)\]\((?P[^)]+)\)") +HEADING = re.compile(r"(?m)^#{1,6}\s+(?P.+?)\s*$") + + +def slug(title: str) -> str: + """GitHub's heading anchor: lowercase, punctuation dropped, spaces hyphenated.""" + s = title.strip().lower() + s = re.sub(r"`([^`]*)`", r"\1", s) + s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s) + s = re.sub(r"[^\w\s-]", "", s) + return re.sub(r"\s+", "-", s).strip("-") + + +def anchors(path: Path) -> set[str]: + return {slug(m.group("title")) for m in HEADING.finditer(path.read_text())} + + +def main() -> int: + docs = [HERE / "README.md"] + sorted((HERE / "docs").glob("*.md")) + failures: list[str] = [] + checked = 0 + + for doc in docs: + text = doc.read_text() + for match in LINK.finditer(text): + target = match.group("target").strip() + where = f"{doc.relative_to(HERE).as_posix()}: [{match.group('text')}]({target})" + checked += 1 + + if target.startswith(("http://", "https://")): + if " " in target: + failures.append(f"{where} -> malformed URL") + continue + if target.startswith("mailto:"): + continue + + file_part, _, anchor = target.partition("#") + resolved = doc.parent / file_part if file_part else doc + if not resolved.exists(): + failures.append(f"{where} -> no such file {resolved.relative_to(HERE).as_posix()}") + continue + if anchor and anchor not in anchors(resolved): + failures.append( + f"{where} -> {resolved.name} has no heading anchored '{anchor}'" + ) + + if failures: + print(f"{len(failures)} of {checked} links do not resolve:\n", file=sys.stderr) + for f in failures: + print(f" {f}", file=sys.stderr) + return 1 + print(f"all {checked} links across {len(docs)} documents resolve") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/holistics/tools/update_snapshots.py b/converters/holistics/tools/update_snapshots.py new file mode 100644 index 000000000..9e5dda508 --- /dev/null +++ b/converters/holistics/tools/update_snapshots.py @@ -0,0 +1,50 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Rewrite every committed snapshot under `tests/fixtures/`. + +Run after a deliberate change to the converter, then read the diff. The same +`tests/snapshots.py` definitions drive `tests/test_snapshots.py`, so what this +writes is exactly what that compares. + + python3 tools/update_snapshots.py +""" +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "tests")) + +from snapshots import FIXTURES, all_snapshots # noqa: E402 + + +def main() -> int: + written = 0 + for relative, text in sorted(all_snapshots().items()): + path = FIXTURES / relative + path.parent.mkdir(parents=True, exist_ok=True) + if path.exists() and path.read_text(encoding="utf-8") == text: + continue + path.write_text(text, encoding="utf-8") + print(f"wrote {path.relative_to(ROOT)}") + written += 1 + print(f"{written} snapshot(s) changed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/holistics/uv.lock b/converters/holistics/uv.lock new file mode 100644 index 000000000..d28986e83 --- /dev/null +++ b/converters/holistics/uv.lock @@ -0,0 +1,560 @@ +version = 1 +revision = 5 +requires-python = ">=3.10" +resolution-markers = [ + "python_full_version >= '3.11'", + "python_full_version < '3.11'", +] + +[[package]] +name = "apache-ossie-holistics" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.dev-dependencies] +dev = [ + { name = "jsonschema" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=30.12.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py", version = "0.30.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "rpds-py", version = "2026.9.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/20/af/3f2f423103f1113b36230496629986e0ef7e199d2aa8392452b484b38ced/rpds_py-0.30.0.tar.gz", hash = "sha256:dd8ff7cf90014af0c0f787eea34794ebf6415242ee1d6fa91eaba725cc441e84", size = 69469, upload-time = "2025-11-30T20:24:38.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/0c/0c411a0ec64ccb6d104dcabe0e713e05e153a9a2c3c2bd2b32ce412166fe/rpds_py-0.30.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:679ae98e00c0e8d68a7fda324e16b90fd5260945b45d3b824c892cec9eea3288", size = 370490, upload-time = "2025-11-30T20:21:33.256Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/4ba3d0fb7297ebae71171822554abe48d7cab29c28b8f9f2c04b79988c05/rpds_py-0.30.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4cc2206b76b4f576934f0ed374b10d7ca5f457858b157ca52064bdfc26b9fc00", size = 359751, upload-time = "2025-11-30T20:21:34.591Z" }, + { url = "https://files.pythonhosted.org/packages/cd/7c/e4933565ef7f7a0818985d87c15d9d273f1a649afa6a52ea35ad011195ea/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:389a2d49eded1896c3d48b0136ead37c48e221b391c052fba3f4055c367f60a6", size = 389696, upload-time = "2025-11-30T20:21:36.122Z" }, + { url = "https://files.pythonhosted.org/packages/5e/01/6271a2511ad0815f00f7ed4390cf2567bec1d4b1da39e2c27a41e6e3b4de/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:32c8528634e1bf7121f3de08fa85b138f4e0dc47657866630611b03967f041d7", size = 403136, upload-time = "2025-11-30T20:21:37.728Z" }, + { url = "https://files.pythonhosted.org/packages/55/64/c857eb7cd7541e9b4eee9d49c196e833128a55b89a9850a9c9ac33ccf897/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f207f69853edd6f6700b86efb84999651baf3789e78a466431df1331608e5324", size = 524699, upload-time = "2025-11-30T20:21:38.92Z" }, + { url = "https://files.pythonhosted.org/packages/9c/ed/94816543404078af9ab26159c44f9e98e20fe47e2126d5d32c9d9948d10a/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:67b02ec25ba7a9e8fa74c63b6ca44cf5707f2fbfadae3ee8e7494297d56aa9df", size = 412022, upload-time = "2025-11-30T20:21:40.407Z" }, + { url = "https://files.pythonhosted.org/packages/61/b5/707f6cf0066a6412aacc11d17920ea2e19e5b2f04081c64526eb35b5c6e7/rpds_py-0.30.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0c0e95f6819a19965ff420f65578bacb0b00f251fefe2c8b23347c37174271f3", size = 390522, upload-time = "2025-11-30T20:21:42.17Z" }, + { url = "https://files.pythonhosted.org/packages/13/4e/57a85fda37a229ff4226f8cbcf09f2a455d1ed20e802ce5b2b4a7f5ed053/rpds_py-0.30.0-cp310-cp310-manylinux_2_31_riscv64.whl", hash = "sha256:a452763cc5198f2f98898eb98f7569649fe5da666c2dc6b5ddb10fde5a574221", size = 404579, upload-time = "2025-11-30T20:21:43.769Z" }, + { url = "https://files.pythonhosted.org/packages/f9/da/c9339293513ec680a721e0e16bf2bac3db6e5d7e922488de471308349bba/rpds_py-0.30.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e0b65193a413ccc930671c55153a03ee57cecb49e6227204b04fae512eb657a7", size = 421305, upload-time = "2025-11-30T20:21:44.994Z" }, + { url = "https://files.pythonhosted.org/packages/f9/be/522cb84751114f4ad9d822ff5a1aa3c98006341895d5f084779b99596e5c/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:858738e9c32147f78b3ac24dc0edb6610000e56dc0f700fd5f651d0a0f0eb9ff", size = 572503, upload-time = "2025-11-30T20:21:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9b/de879f7e7ceddc973ea6e4629e9b380213a6938a249e94b0cdbcc325bb66/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:da279aa314f00acbb803da1e76fa18666778e8a8f83484fba94526da5de2cba7", size = 598322, upload-time = "2025-11-30T20:21:48.709Z" }, + { url = "https://files.pythonhosted.org/packages/48/ac/f01fc22efec3f37d8a914fc1b2fb9bcafd56a299edbe96406f3053edea5a/rpds_py-0.30.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7c64d38fb49b6cdeda16ab49e35fe0da2e1e9b34bc38bd78386530f218b37139", size = 560792, upload-time = "2025-11-30T20:21:50.024Z" }, + { url = "https://files.pythonhosted.org/packages/e2/da/4e2b19d0f131f35b6146425f846563d0ce036763e38913d917187307a671/rpds_py-0.30.0-cp310-cp310-win32.whl", hash = "sha256:6de2a32a1665b93233cde140ff8b3467bdb9e2af2b91079f0333a0974d12d464", size = 221901, upload-time = "2025-11-30T20:21:51.32Z" }, + { url = "https://files.pythonhosted.org/packages/96/cb/156d7a5cf4f78a7cc571465d8aec7a3c447c94f6749c5123f08438bcf7bc/rpds_py-0.30.0-cp310-cp310-win_amd64.whl", hash = "sha256:1726859cd0de969f88dc8673bdd954185b9104e05806be64bcd87badbe313169", size = 235823, upload-time = "2025-11-30T20:21:52.505Z" }, + { url = "https://files.pythonhosted.org/packages/4d/6e/f964e88b3d2abee2a82c1ac8366da848fce1c6d834dc2132c3fda3970290/rpds_py-0.30.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:a2bffea6a4ca9f01b3f8e548302470306689684e61602aa3d141e34da06cf425", size = 370157, upload-time = "2025-11-30T20:21:53.789Z" }, + { url = "https://files.pythonhosted.org/packages/94/ba/24e5ebb7c1c82e74c4e4f33b2112a5573ddc703915b13a073737b59b86e0/rpds_py-0.30.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dc4f992dfe1e2bc3ebc7444f6c7051b4bc13cd8e33e43511e8ffd13bf407010d", size = 359676, upload-time = "2025-11-30T20:21:55.475Z" }, + { url = "https://files.pythonhosted.org/packages/84/86/04dbba1b087227747d64d80c3b74df946b986c57af0a9f0c98726d4d7a3b/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:422c3cb9856d80b09d30d2eb255d0754b23e090034e1deb4083f8004bd0761e4", size = 389938, upload-time = "2025-11-30T20:21:57.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/bb/1463f0b1722b7f45431bdd468301991d1328b16cffe0b1c2918eba2c4eee/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:07ae8a593e1c3c6b82ca3292efbe73c30b61332fd612e05abee07c79359f292f", size = 402932, upload-time = "2025-11-30T20:21:58.47Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/2520700a5c1f2d76631f948b0736cdf9b0acb25abd0ca8e889b5c62ac2e3/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f90dd7557b6bd57f40abe7747e81e0c0b119bef015ea7726e69fe550e394a4", size = 525830, upload-time = "2025-11-30T20:21:59.699Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ad/bd0331f740f5705cc555a5e17fdf334671262160270962e69a2bdef3bf76/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:99b47d6ad9a6da00bec6aabe5a6279ecd3c06a329d4aa4771034a21e335c3a97", size = 412033, upload-time = "2025-11-30T20:22:00.991Z" }, + { url = "https://files.pythonhosted.org/packages/f8/1e/372195d326549bb51f0ba0f2ecb9874579906b97e08880e7a65c3bef1a99/rpds_py-0.30.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:33f559f3104504506a44bb666b93a33f5d33133765b0c216a5bf2f1e1503af89", size = 390828, upload-time = "2025-11-30T20:22:02.723Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2b/d88bb33294e3e0c76bc8f351a3721212713629ffca1700fa94979cb3eae8/rpds_py-0.30.0-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:946fe926af6e44f3697abbc305ea168c2c31d3e3ef1058cf68f379bf0335a78d", size = 404683, upload-time = "2025-11-30T20:22:04.367Z" }, + { url = "https://files.pythonhosted.org/packages/50/32/c759a8d42bcb5289c1fac697cd92f6fe01a018dd937e62ae77e0e7f15702/rpds_py-0.30.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:495aeca4b93d465efde585977365187149e75383ad2684f81519f504f5c13038", size = 421583, upload-time = "2025-11-30T20:22:05.814Z" }, + { url = "https://files.pythonhosted.org/packages/2b/81/e729761dbd55ddf5d84ec4ff1f47857f4374b0f19bdabfcf929164da3e24/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:d9a0ca5da0386dee0655b4ccdf46119df60e0f10da268d04fe7cc87886872ba7", size = 572496, upload-time = "2025-11-30T20:22:07.713Z" }, + { url = "https://files.pythonhosted.org/packages/14/f6/69066a924c3557c9c30baa6ec3a0aa07526305684c6f86c696b08860726c/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:8d6d1cc13664ec13c1b84241204ff3b12f9bb82464b8ad6e7a5d3486975c2eed", size = 598669, upload-time = "2025-11-30T20:22:09.312Z" }, + { url = "https://files.pythonhosted.org/packages/5f/48/905896b1eb8a05630d20333d1d8ffd162394127b74ce0b0784ae04498d32/rpds_py-0.30.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3896fa1be39912cf0757753826bc8bdc8ca331a28a7c4ae46b7a21280b06bb85", size = 561011, upload-time = "2025-11-30T20:22:11.309Z" }, + { url = "https://files.pythonhosted.org/packages/22/16/cd3027c7e279d22e5eb431dd3c0fbc677bed58797fe7581e148f3f68818b/rpds_py-0.30.0-cp311-cp311-win32.whl", hash = "sha256:55f66022632205940f1827effeff17c4fa7ae1953d2b74a8581baaefb7d16f8c", size = 221406, upload-time = "2025-11-30T20:22:13.101Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5b/e7b7aa136f28462b344e652ee010d4de26ee9fd16f1bfd5811f5153ccf89/rpds_py-0.30.0-cp311-cp311-win_amd64.whl", hash = "sha256:a51033ff701fca756439d641c0ad09a41d9242fa69121c7d8769604a0a629825", size = 236024, upload-time = "2025-11-30T20:22:14.853Z" }, + { url = "https://files.pythonhosted.org/packages/14/a6/364bba985e4c13658edb156640608f2c9e1d3ea3c81b27aa9d889fff0e31/rpds_py-0.30.0-cp311-cp311-win_arm64.whl", hash = "sha256:47b0ef6231c58f506ef0b74d44e330405caa8428e770fec25329ed2cb971a229", size = 229069, upload-time = "2025-11-30T20:22:16.577Z" }, + { url = "https://files.pythonhosted.org/packages/03/e7/98a2f4ac921d82f33e03f3835f5bf3a4a40aa1bfdc57975e74a97b2b4bdd/rpds_py-0.30.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a161f20d9a43006833cd7068375a94d035714d73a172b681d8881820600abfad", size = 375086, upload-time = "2025-11-30T20:22:17.93Z" }, + { url = "https://files.pythonhosted.org/packages/4d/a1/bca7fd3d452b272e13335db8d6b0b3ecde0f90ad6f16f3328c6fb150c889/rpds_py-0.30.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6abc8880d9d036ecaafe709079969f56e876fcf107f7a8e9920ba6d5a3878d05", size = 359053, upload-time = "2025-11-30T20:22:19.297Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/ae157e83a6357eceff62ba7e52113e3ec4834a84cfe07fa4b0757a7d105f/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ca28829ae5f5d569bb62a79512c842a03a12576375d5ece7d2cadf8abe96ec28", size = 390763, upload-time = "2025-11-30T20:22:21.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/36/eb2eb8515e2ad24c0bd43c3ee9cd74c33f7ca6430755ccdb240fd3144c44/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a1010ed9524c73b94d15919ca4d41d8780980e1765babf85f9a2f90d247153dd", size = 408951, upload-time = "2025-11-30T20:22:23.408Z" }, + { url = "https://files.pythonhosted.org/packages/d6/65/ad8dc1784a331fabbd740ef6f71ce2198c7ed0890dab595adb9ea2d775a1/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f8d1736cfb49381ba528cd5baa46f82fdc65c06e843dab24dd70b63d09121b3f", size = 514622, upload-time = "2025-11-30T20:22:25.16Z" }, + { url = "https://files.pythonhosted.org/packages/63/8e/0cfa7ae158e15e143fe03993b5bcd743a59f541f5952e1546b1ac1b5fd45/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d948b135c4693daff7bc2dcfc4ec57237a29bd37e60c2fabf5aff2bbacf3e2f1", size = 414492, upload-time = "2025-11-30T20:22:26.505Z" }, + { url = "https://files.pythonhosted.org/packages/60/1b/6f8f29f3f995c7ffdde46a626ddccd7c63aefc0efae881dc13b6e5d5bb16/rpds_py-0.30.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47f236970bccb2233267d89173d3ad2703cd36a0e2a6e92d0560d333871a3d23", size = 394080, upload-time = "2025-11-30T20:22:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/6d/d5/a266341051a7a3ca2f4b750a3aa4abc986378431fc2da508c5034d081b70/rpds_py-0.30.0-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:2e6ecb5a5bcacf59c3f912155044479af1d0b6681280048b338b28e364aca1f6", size = 408680, upload-time = "2025-11-30T20:22:29.341Z" }, + { url = "https://files.pythonhosted.org/packages/10/3b/71b725851df9ab7a7a4e33cf36d241933da66040d195a84781f49c50490c/rpds_py-0.30.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8fa71a2e078c527c3e9dc9fc5a98c9db40bcc8a92b4e8858e36d329f8684b51", size = 423589, upload-time = "2025-11-30T20:22:31.469Z" }, + { url = "https://files.pythonhosted.org/packages/00/2b/e59e58c544dc9bd8bd8384ecdb8ea91f6727f0e37a7131baeff8d6f51661/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:73c67f2db7bc334e518d097c6d1e6fed021bbc9b7d678d6cc433478365d1d5f5", size = 573289, upload-time = "2025-11-30T20:22:32.997Z" }, + { url = "https://files.pythonhosted.org/packages/da/3e/a18e6f5b460893172a7d6a680e86d3b6bc87a54c1f0b03446a3c8c7b588f/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5ba103fb455be00f3b1c2076c9d4264bfcb037c976167a6047ed82f23153f02e", size = 599737, upload-time = "2025-11-30T20:22:34.419Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e2/714694e4b87b85a18e2c243614974413c60aa107fd815b8cbc42b873d1d7/rpds_py-0.30.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7cee9c752c0364588353e627da8a7e808a66873672bcb5f52890c33fd965b394", size = 563120, upload-time = "2025-11-30T20:22:35.903Z" }, + { url = "https://files.pythonhosted.org/packages/6f/ab/d5d5e3bcedb0a77f4f613706b750e50a5a3ba1c15ccd3665ecc636c968fd/rpds_py-0.30.0-cp312-cp312-win32.whl", hash = "sha256:1ab5b83dbcf55acc8b08fc62b796ef672c457b17dbd7820a11d6c52c06839bdf", size = 223782, upload-time = "2025-11-30T20:22:37.271Z" }, + { url = "https://files.pythonhosted.org/packages/39/3b/f786af9957306fdc38a74cef405b7b93180f481fb48453a114bb6465744a/rpds_py-0.30.0-cp312-cp312-win_amd64.whl", hash = "sha256:a090322ca841abd453d43456ac34db46e8b05fd9b3b4ac0c78bcde8b089f959b", size = 240463, upload-time = "2025-11-30T20:22:39.021Z" }, + { url = "https://files.pythonhosted.org/packages/f3/d2/b91dc748126c1559042cfe41990deb92c4ee3e2b415f6b5234969ffaf0cc/rpds_py-0.30.0-cp312-cp312-win_arm64.whl", hash = "sha256:669b1805bd639dd2989b281be2cfd951c6121b65e729d9b843e9639ef1fd555e", size = 230868, upload-time = "2025-11-30T20:22:40.493Z" }, + { url = "https://files.pythonhosted.org/packages/ed/dc/d61221eb88ff410de3c49143407f6f3147acf2538c86f2ab7ce65ae7d5f9/rpds_py-0.30.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:f83424d738204d9770830d35290ff3273fbb02b41f919870479fab14b9d303b2", size = 374887, upload-time = "2025-11-30T20:22:41.812Z" }, + { url = "https://files.pythonhosted.org/packages/fd/32/55fb50ae104061dbc564ef15cc43c013dc4a9f4527a1f4d99baddf56fe5f/rpds_py-0.30.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:e7536cd91353c5273434b4e003cbda89034d67e7710eab8761fd918ec6c69cf8", size = 358904, upload-time = "2025-11-30T20:22:43.479Z" }, + { url = "https://files.pythonhosted.org/packages/58/70/faed8186300e3b9bdd138d0273109784eea2396c68458ed580f885dfe7ad/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2771c6c15973347f50fece41fc447c054b7ac2ae0502388ce3b6738cd366e3d4", size = 389945, upload-time = "2025-11-30T20:22:44.819Z" }, + { url = "https://files.pythonhosted.org/packages/bd/a8/073cac3ed2c6387df38f71296d002ab43496a96b92c823e76f46b8af0543/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0a59119fc6e3f460315fe9d08149f8102aa322299deaa5cab5b40092345c2136", size = 407783, upload-time = "2025-11-30T20:22:46.103Z" }, + { url = "https://files.pythonhosted.org/packages/77/57/5999eb8c58671f1c11eba084115e77a8899d6e694d2a18f69f0ba471ec8b/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:76fec018282b4ead0364022e3c54b60bf368b9d926877957a8624b58419169b7", size = 515021, upload-time = "2025-11-30T20:22:47.458Z" }, + { url = "https://files.pythonhosted.org/packages/e0/af/5ab4833eadc36c0a8ed2bc5c0de0493c04f6c06de223170bd0798ff98ced/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:692bef75a5525db97318e8cd061542b5a79812d711ea03dbc1f6f8dbb0c5f0d2", size = 414589, upload-time = "2025-11-30T20:22:48.872Z" }, + { url = "https://files.pythonhosted.org/packages/b7/de/f7192e12b21b9e9a68a6d0f249b4af3fdcdff8418be0767a627564afa1f1/rpds_py-0.30.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9027da1ce107104c50c81383cae773ef5c24d296dd11c99e2629dbd7967a20c6", size = 394025, upload-time = "2025-11-30T20:22:50.196Z" }, + { url = "https://files.pythonhosted.org/packages/91/c4/fc70cd0249496493500e7cc2de87504f5aa6509de1e88623431fec76d4b6/rpds_py-0.30.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:9cf69cdda1f5968a30a359aba2f7f9aa648a9ce4b580d6826437f2b291cfc86e", size = 408895, upload-time = "2025-11-30T20:22:51.87Z" }, + { url = "https://files.pythonhosted.org/packages/58/95/d9275b05ab96556fefff73a385813eb66032e4c99f411d0795372d9abcea/rpds_py-0.30.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a4796a717bf12b9da9d3ad002519a86063dcac8988b030e405704ef7d74d2d9d", size = 422799, upload-time = "2025-11-30T20:22:53.341Z" }, + { url = "https://files.pythonhosted.org/packages/06/c1/3088fc04b6624eb12a57eb814f0d4997a44b0d208d6cace713033ff1a6ba/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5d4c2aa7c50ad4728a094ebd5eb46c452e9cb7edbfdb18f9e1221f597a73e1e7", size = 572731, upload-time = "2025-11-30T20:22:54.778Z" }, + { url = "https://files.pythonhosted.org/packages/d8/42/c612a833183b39774e8ac8fecae81263a68b9583ee343db33ab571a7ce55/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ba81a9203d07805435eb06f536d95a266c21e5b2dfbf6517748ca40c98d19e31", size = 599027, upload-time = "2025-11-30T20:22:56.212Z" }, + { url = "https://files.pythonhosted.org/packages/5f/60/525a50f45b01d70005403ae0e25f43c0384369ad24ffe46e8d9068b50086/rpds_py-0.30.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:945dccface01af02675628334f7cf49c2af4c1c904748efc5cf7bbdf0b579f95", size = 563020, upload-time = "2025-11-30T20:22:58.2Z" }, + { url = "https://files.pythonhosted.org/packages/0b/5d/47c4655e9bcd5ca907148535c10e7d489044243cc9941c16ed7cd53be91d/rpds_py-0.30.0-cp313-cp313-win32.whl", hash = "sha256:b40fb160a2db369a194cb27943582b38f79fc4887291417685f3ad693c5a1d5d", size = 223139, upload-time = "2025-11-30T20:23:00.209Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e1/485132437d20aa4d3e1d8b3fb5a5e65aa8139f1e097080c2a8443201742c/rpds_py-0.30.0-cp313-cp313-win_amd64.whl", hash = "sha256:806f36b1b605e2d6a72716f321f20036b9489d29c51c91f4dd29a3e3afb73b15", size = 240224, upload-time = "2025-11-30T20:23:02.008Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/ffd128ed1146a153d928617b0ef673960130be0009c77d8fbf0abe306713/rpds_py-0.30.0-cp313-cp313-win_arm64.whl", hash = "sha256:d96c2086587c7c30d44f31f42eae4eac89b60dabbac18c7669be3700f13c3ce1", size = 230645, upload-time = "2025-11-30T20:23:03.43Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1b/b10de890a0def2a319a2626334a7f0ae388215eb60914dbac8a3bae54435/rpds_py-0.30.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:eb0b93f2e5c2189ee831ee43f156ed34e2a89a78a66b98cadad955972548be5a", size = 364443, upload-time = "2025-11-30T20:23:04.878Z" }, + { url = "https://files.pythonhosted.org/packages/0d/bf/27e39f5971dc4f305a4fb9c672ca06f290f7c4e261c568f3dea16a410d47/rpds_py-0.30.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:922e10f31f303c7c920da8981051ff6d8c1a56207dbdf330d9047f6d30b70e5e", size = 353375, upload-time = "2025-11-30T20:23:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/40/58/442ada3bba6e8e6615fc00483135c14a7538d2ffac30e2d933ccf6852232/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdc62c8286ba9bf7f47befdcea13ea0e26bf294bda99758fd90535cbaf408000", size = 383850, upload-time = "2025-11-30T20:23:07.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/14/f59b0127409a33c6ef6f5c1ebd5ad8e32d7861c9c7adfa9a624fc3889f6c/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:47f9a91efc418b54fb8190a6b4aa7813a23fb79c51f4bb84e418f5476c38b8db", size = 392812, upload-time = "2025-11-30T20:23:09.228Z" }, + { url = "https://files.pythonhosted.org/packages/b3/66/e0be3e162ac299b3a22527e8913767d869e6cc75c46bd844aa43fb81ab62/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1f3587eb9b17f3789ad50824084fa6f81921bbf9a795826570bda82cb3ed91f2", size = 517841, upload-time = "2025-11-30T20:23:11.186Z" }, + { url = "https://files.pythonhosted.org/packages/3d/55/fa3b9cf31d0c963ecf1ba777f7cf4b2a2c976795ac430d24a1f43d25a6ba/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:39c02563fc592411c2c61d26b6c5fe1e51eaa44a75aa2c8735ca88b0d9599daa", size = 408149, upload-time = "2025-11-30T20:23:12.864Z" }, + { url = "https://files.pythonhosted.org/packages/60/ca/780cf3b1a32b18c0f05c441958d3758f02544f1d613abf9488cd78876378/rpds_py-0.30.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a1234d8febafdfd33a42d97da7a43f5dcb120c1060e352a3fbc0c6d36e2083", size = 383843, upload-time = "2025-11-30T20:23:14.638Z" }, + { url = "https://files.pythonhosted.org/packages/82/86/d5f2e04f2aa6247c613da0c1dd87fcd08fa17107e858193566048a1e2f0a/rpds_py-0.30.0-cp313-cp313t-manylinux_2_31_riscv64.whl", hash = "sha256:eb2c4071ab598733724c08221091e8d80e89064cd472819285a9ab0f24bcedb9", size = 396507, upload-time = "2025-11-30T20:23:16.105Z" }, + { url = "https://files.pythonhosted.org/packages/4b/9a/453255d2f769fe44e07ea9785c8347edaf867f7026872e76c1ad9f7bed92/rpds_py-0.30.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6bdfdb946967d816e6adf9a3d8201bfad269c67efe6cefd7093ef959683c8de0", size = 414949, upload-time = "2025-11-30T20:23:17.539Z" }, + { url = "https://files.pythonhosted.org/packages/a3/31/622a86cdc0c45d6df0e9ccb6becdba5074735e7033c20e401a6d9d0e2ca0/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c77afbd5f5250bf27bf516c7c4a016813eb2d3e116139aed0096940c5982da94", size = 565790, upload-time = "2025-11-30T20:23:19.029Z" }, + { url = "https://files.pythonhosted.org/packages/1c/5d/15bbf0fb4a3f58a3b1c67855ec1efcc4ceaef4e86644665fff03e1b66d8d/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:61046904275472a76c8c90c9ccee9013d70a6d0f73eecefd38c1ae7c39045a08", size = 590217, upload-time = "2025-11-30T20:23:20.885Z" }, + { url = "https://files.pythonhosted.org/packages/6d/61/21b8c41f68e60c8cc3b2e25644f0e3681926020f11d06ab0b78e3c6bbff1/rpds_py-0.30.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4c5f36a861bc4b7da6516dbdf302c55313afa09b81931e8280361a4f6c9a2d27", size = 555806, upload-time = "2025-11-30T20:23:22.488Z" }, + { url = "https://files.pythonhosted.org/packages/f9/39/7e067bb06c31de48de3eb200f9fc7c58982a4d3db44b07e73963e10d3be9/rpds_py-0.30.0-cp313-cp313t-win32.whl", hash = "sha256:3d4a69de7a3e50ffc214ae16d79d8fbb0922972da0356dcf4d0fdca2878559c6", size = 211341, upload-time = "2025-11-30T20:23:24.449Z" }, + { url = "https://files.pythonhosted.org/packages/0a/4d/222ef0b46443cf4cf46764d9c630f3fe4abaa7245be9417e56e9f52b8f65/rpds_py-0.30.0-cp313-cp313t-win_amd64.whl", hash = "sha256:f14fc5df50a716f7ece6a80b6c78bb35ea2ca47c499e422aa4463455dd96d56d", size = 225768, upload-time = "2025-11-30T20:23:25.908Z" }, + { url = "https://files.pythonhosted.org/packages/86/81/dad16382ebbd3d0e0328776d8fd7ca94220e4fa0798d1dc5e7da48cb3201/rpds_py-0.30.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:68f19c879420aa08f61203801423f6cd5ac5f0ac4ac82a2368a9fcd6a9a075e0", size = 362099, upload-time = "2025-11-30T20:23:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/2b/60/19f7884db5d5603edf3c6bce35408f45ad3e97e10007df0e17dd57af18f8/rpds_py-0.30.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ec7c4490c672c1a0389d319b3a9cfcd098dcdc4783991553c332a15acf7249be", size = 353192, upload-time = "2025-11-30T20:23:29.151Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c4/76eb0e1e72d1a9c4703c69607cec123c29028bff28ce41588792417098ac/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f251c812357a3fed308d684a5079ddfb9d933860fc6de89f2b7ab00da481e65f", size = 384080, upload-time = "2025-11-30T20:23:30.785Z" }, + { url = "https://files.pythonhosted.org/packages/72/87/87ea665e92f3298d1b26d78814721dc39ed8d2c74b86e83348d6b48a6f31/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ac98b175585ecf4c0348fd7b29c3864bda53b805c773cbf7bfdaffc8070c976f", size = 394841, upload-time = "2025-11-30T20:23:32.209Z" }, + { url = "https://files.pythonhosted.org/packages/77/ad/7783a89ca0587c15dcbf139b4a8364a872a25f861bdb88ed99f9b0dec985/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3e62880792319dbeb7eb866547f2e35973289e7d5696c6e295476448f5b63c87", size = 516670, upload-time = "2025-11-30T20:23:33.742Z" }, + { url = "https://files.pythonhosted.org/packages/5b/3c/2882bdac942bd2172f3da574eab16f309ae10a3925644e969536553cb4ee/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4e7fc54e0900ab35d041b0601431b0a0eb495f0851a0639b6ef90f7741b39a18", size = 408005, upload-time = "2025-11-30T20:23:35.253Z" }, + { url = "https://files.pythonhosted.org/packages/ce/81/9a91c0111ce1758c92516a3e44776920b579d9a7c09b2b06b642d4de3f0f/rpds_py-0.30.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47e77dc9822d3ad616c3d5759ea5631a75e5809d5a28707744ef79d7a1bcfcad", size = 382112, upload-time = "2025-11-30T20:23:36.842Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/1da49d4a107027e5fbc64daeab96a0706361a2918da10cb41769244b805d/rpds_py-0.30.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:b4dc1a6ff022ff85ecafef7979a2c6eb423430e05f1165d6688234e62ba99a07", size = 399049, upload-time = "2025-11-30T20:23:38.343Z" }, + { url = "https://files.pythonhosted.org/packages/df/5a/7ee239b1aa48a127570ec03becbb29c9d5a9eb092febbd1699d567cae859/rpds_py-0.30.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4559c972db3a360808309e06a74628b95eaccbf961c335c8fe0d590cf587456f", size = 415661, upload-time = "2025-11-30T20:23:40.263Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/caa143cf6b772f823bc7929a45da1fa83569ee49b11d18d0ada7f5ee6fd6/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0ed177ed9bded28f8deb6ab40c183cd1192aa0de40c12f38be4d59cd33cb5c65", size = 565606, upload-time = "2025-11-30T20:23:42.186Z" }, + { url = "https://files.pythonhosted.org/packages/64/91/ac20ba2d69303f961ad8cf55bf7dbdb4763f627291ba3d0d7d67333cced9/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ad1fa8db769b76ea911cb4e10f049d80bf518c104f15b3edb2371cc65375c46f", size = 591126, upload-time = "2025-11-30T20:23:44.086Z" }, + { url = "https://files.pythonhosted.org/packages/21/20/7ff5f3c8b00c8a95f75985128c26ba44503fb35b8e0259d812766ea966c7/rpds_py-0.30.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:46e83c697b1f1c72b50e5ee5adb4353eef7406fb3f2043d64c33f20ad1c2fc53", size = 553371, upload-time = "2025-11-30T20:23:46.004Z" }, + { url = "https://files.pythonhosted.org/packages/72/c7/81dadd7b27c8ee391c132a6b192111ca58d866577ce2d9b0ca157552cce0/rpds_py-0.30.0-cp314-cp314-win32.whl", hash = "sha256:ee454b2a007d57363c2dfd5b6ca4a5d7e2c518938f8ed3b706e37e5d470801ed", size = 215298, upload-time = "2025-11-30T20:23:47.696Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d2/1aaac33287e8cfb07aab2e6b8ac1deca62f6f65411344f1433c55e6f3eb8/rpds_py-0.30.0-cp314-cp314-win_amd64.whl", hash = "sha256:95f0802447ac2d10bcc69f6dc28fe95fdf17940367b21d34e34c737870758950", size = 228604, upload-time = "2025-11-30T20:23:49.501Z" }, + { url = "https://files.pythonhosted.org/packages/e8/95/ab005315818cc519ad074cb7784dae60d939163108bd2b394e60dc7b5461/rpds_py-0.30.0-cp314-cp314-win_arm64.whl", hash = "sha256:613aa4771c99f03346e54c3f038e4cc574ac09a3ddfb0e8878487335e96dead6", size = 222391, upload-time = "2025-11-30T20:23:50.96Z" }, + { url = "https://files.pythonhosted.org/packages/9e/68/154fe0194d83b973cdedcdcc88947a2752411165930182ae41d983dcefa6/rpds_py-0.30.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:7e6ecfcb62edfd632e56983964e6884851786443739dbfe3582947e87274f7cb", size = 364868, upload-time = "2025-11-30T20:23:52.494Z" }, + { url = "https://files.pythonhosted.org/packages/83/69/8bbc8b07ec854d92a8b75668c24d2abcb1719ebf890f5604c61c9369a16f/rpds_py-0.30.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a1d0bc22a7cdc173fedebb73ef81e07faef93692b8c1ad3733b67e31e1b6e1b8", size = 353747, upload-time = "2025-11-30T20:23:54.036Z" }, + { url = "https://files.pythonhosted.org/packages/ab/00/ba2e50183dbd9abcce9497fa5149c62b4ff3e22d338a30d690f9af970561/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d08f00679177226c4cb8c5265012eea897c8ca3b93f429e546600c971bcbae7", size = 383795, upload-time = "2025-11-30T20:23:55.556Z" }, + { url = "https://files.pythonhosted.org/packages/05/6f/86f0272b84926bcb0e4c972262f54223e8ecc556b3224d281e6598fc9268/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5965af57d5848192c13534f90f9dd16464f3c37aaf166cc1da1cae1fd5a34898", size = 393330, upload-time = "2025-11-30T20:23:57.033Z" }, + { url = "https://files.pythonhosted.org/packages/cb/e9/0e02bb2e6dc63d212641da45df2b0bf29699d01715913e0d0f017ee29438/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9a4e86e34e9ab6b667c27f3211ca48f73dba7cd3d90f8d5b11be56e5dbc3fb4e", size = 518194, upload-time = "2025-11-30T20:23:58.637Z" }, + { url = "https://files.pythonhosted.org/packages/ee/ca/be7bca14cf21513bdf9c0606aba17d1f389ea2b6987035eb4f62bd923f25/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e5d3e6b26f2c785d65cc25ef1e5267ccbe1b069c5c21b8cc724efee290554419", size = 408340, upload-time = "2025-11-30T20:24:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c7/736e00ebf39ed81d75544c0da6ef7b0998f8201b369acf842f9a90dc8fce/rpds_py-0.30.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:626a7433c34566535b6e56a1b39a7b17ba961e97ce3b80ec62e6f1312c025551", size = 383765, upload-time = "2025-11-30T20:24:01.759Z" }, + { url = "https://files.pythonhosted.org/packages/4a/3f/da50dfde9956aaf365c4adc9533b100008ed31aea635f2b8d7b627e25b49/rpds_py-0.30.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:acd7eb3f4471577b9b5a41baf02a978e8bdeb08b4b355273994f8b87032000a8", size = 396834, upload-time = "2025-11-30T20:24:03.687Z" }, + { url = "https://files.pythonhosted.org/packages/4e/00/34bcc2565b6020eab2623349efbdec810676ad571995911f1abdae62a3a0/rpds_py-0.30.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe5fa731a1fa8a0a56b0977413f8cacac1768dad38d16b3a296712709476fbd5", size = 415470, upload-time = "2025-11-30T20:24:05.232Z" }, + { url = "https://files.pythonhosted.org/packages/8c/28/882e72b5b3e6f718d5453bd4d0d9cf8df36fddeb4ddbbab17869d5868616/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:74a3243a411126362712ee1524dfc90c650a503502f135d54d1b352bd01f2404", size = 565630, upload-time = "2025-11-30T20:24:06.878Z" }, + { url = "https://files.pythonhosted.org/packages/3b/97/04a65539c17692de5b85c6e293520fd01317fd878ea1995f0367d4532fb1/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3e8eeb0544f2eb0d2581774be4c3410356eba189529a6b3e36bbbf9696175856", size = 591148, upload-time = "2025-11-30T20:24:08.445Z" }, + { url = "https://files.pythonhosted.org/packages/85/70/92482ccffb96f5441aab93e26c4d66489eb599efdcf96fad90c14bbfb976/rpds_py-0.30.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:dbd936cde57abfee19ab3213cf9c26be06d60750e60a8e4dd85d1ab12c8b1f40", size = 556030, upload-time = "2025-11-30T20:24:10.956Z" }, + { url = "https://files.pythonhosted.org/packages/20/53/7c7e784abfa500a2b6b583b147ee4bb5a2b3747a9166bab52fec4b5b5e7d/rpds_py-0.30.0-cp314-cp314t-win32.whl", hash = "sha256:dc824125c72246d924f7f796b4f63c1e9dc810c7d9e2355864b3c3a73d59ade0", size = 211570, upload-time = "2025-11-30T20:24:12.735Z" }, + { url = "https://files.pythonhosted.org/packages/d0/02/fa464cdfbe6b26e0600b62c528b72d8608f5cc49f96b8d6e38c95d60c676/rpds_py-0.30.0-cp314-cp314t-win_amd64.whl", hash = "sha256:27f4b0e92de5bfbc6f86e43959e6edd1425c33b5e69aab0984a72047f2bcf1e3", size = 226532, upload-time = "2025-11-30T20:24:14.634Z" }, + { url = "https://files.pythonhosted.org/packages/69/71/3f34339ee70521864411f8b6992e7ab13ac30d8e4e3309e07c7361767d91/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:c2262bdba0ad4fc6fb5545660673925c2d2a5d9e2e0fb603aad545427be0fc58", size = 372292, upload-time = "2025-11-30T20:24:16.537Z" }, + { url = "https://files.pythonhosted.org/packages/57/09/f183df9b8f2d66720d2ef71075c59f7e1b336bec7ee4c48f0a2b06857653/rpds_py-0.30.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:ee6af14263f25eedc3bb918a3c04245106a42dfd4f5c2285ea6f997b1fc3f89a", size = 362128, upload-time = "2025-11-30T20:24:18.086Z" }, + { url = "https://files.pythonhosted.org/packages/7a/68/5c2594e937253457342e078f0cc1ded3dd7b2ad59afdbf2d354869110a02/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3adbb8179ce342d235c31ab8ec511e66c73faa27a47e076ccc92421add53e2bb", size = 391542, upload-time = "2025-11-30T20:24:20.092Z" }, + { url = "https://files.pythonhosted.org/packages/49/5c/31ef1afd70b4b4fbdb2800249f34c57c64beb687495b10aec0365f53dfc4/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:250fa00e9543ac9b97ac258bd37367ff5256666122c2d0f2bc97577c60a1818c", size = 404004, upload-time = "2025-11-30T20:24:22.231Z" }, + { url = "https://files.pythonhosted.org/packages/e3/63/0cfbea38d05756f3440ce6534d51a491d26176ac045e2707adc99bb6e60a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9854cf4f488b3d57b9aaeb105f06d78e5529d3145b1e4a41750167e8c213c6d3", size = 527063, upload-time = "2025-11-30T20:24:24.302Z" }, + { url = "https://files.pythonhosted.org/packages/42/e6/01e1f72a2456678b0f618fc9a1a13f882061690893c192fcad9f2926553a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:993914b8e560023bc0a8bf742c5f303551992dcb85e247b1e5c7f4a7d145bda5", size = 413099, upload-time = "2025-11-30T20:24:25.916Z" }, + { url = "https://files.pythonhosted.org/packages/b8/25/8df56677f209003dcbb180765520c544525e3ef21ea72279c98b9aa7c7fb/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:58edca431fb9b29950807e301826586e5bbf24163677732429770a697ffe6738", size = 392177, upload-time = "2025-11-30T20:24:27.834Z" }, + { url = "https://files.pythonhosted.org/packages/4a/b4/0a771378c5f16f8115f796d1f437950158679bcd2a7c68cf251cfb00ed5b/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:dea5b552272a944763b34394d04577cf0f9bd013207bc32323b5a89a53cf9c2f", size = 406015, upload-time = "2025-11-30T20:24:29.457Z" }, + { url = "https://files.pythonhosted.org/packages/36/d8/456dbba0af75049dc6f63ff295a2f92766b9d521fa00de67a2bd6427d57a/rpds_py-0.30.0-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ba3af48635eb83d03f6c9735dfb21785303e73d22ad03d489e88adae6eab8877", size = 423736, upload-time = "2025-11-30T20:24:31.22Z" }, + { url = "https://files.pythonhosted.org/packages/13/64/b4d76f227d5c45a7e0b796c674fd81b0a6c4fbd48dc29271857d8219571c/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:dff13836529b921e22f15cb099751209a60009731a68519630a24d61f0b1b30a", size = 573981, upload-time = "2025-11-30T20:24:32.934Z" }, + { url = "https://files.pythonhosted.org/packages/20/91/092bacadeda3edf92bf743cc96a7be133e13a39cdbfd7b5082e7ab638406/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:1b151685b23929ab7beec71080a8889d4d6d9fa9a983d213f07121205d48e2c4", size = 599782, upload-time = "2025-11-30T20:24:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b7/b95708304cd49b7b6f82fdd039f1748b66ec2b21d6a45180910802f1abf1/rpds_py-0.30.0-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:ac37f9f516c51e5753f27dfdef11a88330f04de2d564be3991384b2f3535d02e", size = 562191, upload-time = "2025-11-30T20:24:36.853Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.11'", +] +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/6d/ddb8dfcf41892ba9c672d27988592c038d30db61708230313831b6780a1b/rpds_py-2026.9.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:2711d29b653b3bce48a63d18b9c6b53274669e6d6c4094dddeb4d9a0e45128b2", size = 344947, upload-time = "2026-10-04T16:28:55.207Z" }, + { url = "https://files.pythonhosted.org/packages/9e/b2/632c3d034961149eff606e3e357547620a9ab37b849ae5815f67838753fd/rpds_py-2026.9.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3231c4c0e521dafa5be0c9f114ee2c2ad46650836f2d72caa86801950c3e7044", size = 346770, upload-time = "2026-10-04T16:28:56.884Z" }, + { url = "https://files.pythonhosted.org/packages/6d/45/4df346410bb873d03ec60c3b891d1fceb8cabde1971d29844bbb625cc296/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e01b3c878c8641913e688edd1b3f08658c6783d29cf6b826bd3c0d1ae7a1ffaa", size = 377795, upload-time = "2026-10-04T16:28:58.193Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b8/a70f302207e4f29f873c01a986fcac5def49b9c8224328cef1ad7da6b7a9/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3e524c7874ac72884d28e16dd5b8d839fd09e0fe76b020d3fbca23212a7b8c52", size = 383137, upload-time = "2026-10-04T16:28:59.411Z" }, + { url = "https://files.pythonhosted.org/packages/13/e3/3aa28a48a01169d0e20728bf0e5ebbea3ae2774285b40853ae49b3ef332a/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:761fdae6728ceb99ab182fad2f0cc1e262f610834dc891aea1d1a2a2e634776f", size = 496789, upload-time = "2026-10-04T16:29:01.011Z" }, + { url = "https://files.pythonhosted.org/packages/3e/a7/414e8d088602d66ad6f47c65bf07fe5090992a8574cdd4571c1a83b647ed/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b723eb406dec5bc9ec516c73ab9c3239a3284e017f7eb89ee2b3258bd504fb7", size = 395929, upload-time = "2026-10-04T16:29:02.465Z" }, + { url = "https://files.pythonhosted.org/packages/2d/5c/aac3b960fd55d37b49f77c0eebbe043eaf36762ebd48e7b46951e59c1c05/rpds_py-2026.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:136a1c3fe4402b7008bc81cb62ee538481795b61a7e83df88dff3b3f02b726ff", size = 375426, upload-time = "2026-10-04T16:29:03.944Z" }, + { url = "https://files.pythonhosted.org/packages/d3/80/77987a41c40be7202a78d9aed4a650bfb05792237cfd3b9e22638d990fe5/rpds_py-2026.9.1-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:839dde845559254f34885267c6878f60d61d5205180226d976fe488d45fa128e", size = 381497, upload-time = "2026-10-04T16:29:05.308Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d0/640b02eb1ecd71a913c1178b6235287b31006266bca9a26862e80e497b54/rpds_py-2026.9.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:56cd8b3f77d7b6812f533b662186a1f28316931166ddc00fb893b1b0db7e9888", size = 405740, upload-time = "2026-10-04T16:29:06.578Z" }, + { url = "https://files.pythonhosted.org/packages/3e/8c/d034195d45d215625a23f2ae85dbbdb86a49a93a51272061c97c80b9d316/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:074a4d198bc34d9a8ea425114fc3ded6d11ec01f6a314a8db67454a5152d8834", size = 555850, upload-time = "2026-10-04T16:29:08.115Z" }, + { url = "https://files.pythonhosted.org/packages/de/f8/532f1addfc86207bc3cf23749638bb0d457bce891595012ed8a08d0543b8/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:6beb738155fe8ab8091afdfa5a3226b21c2b1593f1e50ebb90eb25b44dbc0391", size = 618661, upload-time = "2026-10-04T16:29:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/50/5b/b0f2c9833a36a5b1f6ac5cb0e635fabacf713266f46ceefba2102da0bf51/rpds_py-2026.9.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:57492a550a1d88d29d003247e5f78dd8cf04a701fac0e4c8db8745a6d2504e0a", size = 583530, upload-time = "2026-10-04T16:29:11.072Z" }, + { url = "https://files.pythonhosted.org/packages/bf/20/464ed4b98d3c8c6a334f3b5e24d3886de04b05658a95fc40a37e0e6e4775/rpds_py-2026.9.1-cp311-cp311-win32.whl", hash = "sha256:d95a354e02393eada6d7351184671aced9d4cce109dabf927cb7aa99624352a1", size = 208125, upload-time = "2026-10-04T16:29:12.408Z" }, + { url = "https://files.pythonhosted.org/packages/df/c9/384a9b62f8cd89c8e3e609b8c9052c140e8c01d807261e0517c904b8bf29/rpds_py-2026.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:ce4d4f52e2a4324396caddbd45a97d8d7be5f42edd25d2355282a9c34f9b2f7f", size = 226045, upload-time = "2026-10-04T16:29:13.751Z" }, + { url = "https://files.pythonhosted.org/packages/4f/3b/0a691f0dcb2301fa9c736346da2d0d4991778373283301c5c12b8f952f3d/rpds_py-2026.9.1-cp311-cp311-win_arm64.whl", hash = "sha256:fdcd198979b4ecffcc1beba366a7fbcf4eb41243691a82fe52ceb0b902f09c12", size = 222636, upload-time = "2026-10-04T16:29:14.989Z" }, + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/83/ea/ee88fd9e756ff93fb6b1182a47ec09504a242620e33ce1d20679efefe841/rpds_py-2026.9.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a36b70596407634ca82d4b989a3729074a008537a0522e4c8046a67c729103e9", size = 346229, upload-time = "2026-10-04T16:29:38.82Z" }, + { url = "https://files.pythonhosted.org/packages/57/71/a097d6552f837500fc36e6b23d09cfb9890c3cc47531f9ca64e149799615/rpds_py-2026.9.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eba5d173f7d5708b22a93815017a4611873ed54db9f268077c0dd1ed99cfc858", size = 340514, upload-time = "2026-10-04T16:29:40.405Z" }, + { url = "https://files.pythonhosted.org/packages/bd/b7/497e85768bf4e0d8ddbaa096a4cac31d1509251dee2728a8490aa367e0b5/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:457866b85daf5034296666168b84a69e0b2e89dc4f1af102b46f6448a60b9063", size = 372611, upload-time = "2026-10-04T16:29:41.778Z" }, + { url = "https://files.pythonhosted.org/packages/52/4b/74ab4108916250b6e198e0d3af05bc6835eb046315f22f7a0ceb49667c5a/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a3a52a3ba86436ab3aef510fbe21512abc2ddd1993005dfe50514bd2284ef025", size = 377712, upload-time = "2026-10-04T16:29:43.242Z" }, + { url = "https://files.pythonhosted.org/packages/0c/8e/067e77d9d7b3cc793c9d909b7e97e7aadbbb1fb094093b6876902cc96d38/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d7841166b7fa64c9c56404617ae4341448847482d45933b13135d26c130519e5", size = 488106, upload-time = "2026-10-04T16:29:44.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b4/c5aae6c2dde269bf955f6b7d35065c655a57e47750d9668052ad67e74dda/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:926bdd3e3b5998ddf70cc64bc8cf57209571f9044542913afb673799fec77dd0", size = 390795, upload-time = "2026-10-04T16:29:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/a0/36/76fab39973ee11e7f9f357c55138197bb01c86f6502cb76487e3b4f42db0/rpds_py-2026.9.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7868b85224291c6cb6759f9b5adb9745f486d226f62b16a614dd5a2a5ab2b35b", size = 370794, upload-time = "2026-10-04T16:29:47.603Z" }, + { url = "https://files.pythonhosted.org/packages/3d/fe/cd2a80e6d7b871937a60e935c5d507aa390d143f4ff3636f640b9733d5df/rpds_py-2026.9.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3cd182d7291d29b92c521a0069d9c01ba6193628a9a105531d11b40a6d731a33", size = 375673, upload-time = "2026-10-04T16:29:49.223Z" }, + { url = "https://files.pythonhosted.org/packages/6c/18/7464a9953724e55a3b3206062fa0ffdeaa519584c6aabf65d3956d94f131/rpds_py-2026.9.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e6ea1cda8d8c688278430e4268a42f5e5da3bdd74578dfadc0820c3f1766ce83", size = 398758, upload-time = "2026-10-04T16:29:50.601Z" }, + { url = "https://files.pythonhosted.org/packages/c0/86/1534b436700fd49ff411063b7c4d7e938adfabf90895b6cf1622d5a7d1f6/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:5943980471829f6de242a20b109de3111ba6b77e3af0ffc587028ac854b05e6c", size = 550370, upload-time = "2026-10-04T16:29:52.002Z" }, + { url = "https://files.pythonhosted.org/packages/57/1c/e1fa82a8a01e3c5820f3ba98a8b2673f642128eb368fa88871b01dd2c909/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:76d3af9732d2dab69f28179b40ba2d87e2f1d5824b4a694780aa787d685e8f36", size = 613106, upload-time = "2026-10-04T16:29:53.655Z" }, + { url = "https://files.pythonhosted.org/packages/29/55/b20b8c4c3dde8755bfcd5b08492a02d0cd2e26929aedf6199ca2a377d42a/rpds_py-2026.9.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:78326f4cb4427a56ba4996c0762b63be45f06b85f086526420d2b3a66e40f84d", size = 578235, upload-time = "2026-10-04T16:29:55.157Z" }, + { url = "https://files.pythonhosted.org/packages/55/42/df3f7bbc3f7ab37a8a9db8d6c2ff2c985422899f1f7926afbf7ec3c0b8b4/rpds_py-2026.9.1-cp313-cp313-win32.whl", hash = "sha256:172e47169583f46ce118cbec68e6795d0da0f4606b488b6434f8276bca0a058c", size = 205293, upload-time = "2026-10-04T16:29:56.669Z" }, + { url = "https://files.pythonhosted.org/packages/31/9c/ba5a9569d719bfdd6ce863df4133ac6a1658cf1b07cc3534c31db729fbbc/rpds_py-2026.9.1-cp313-cp313-win_amd64.whl", hash = "sha256:3e93b2cd69a9830be33e03945cd7cda940a0a8bfcfbff41d6144f0cb0d3d8bd9", size = 222409, upload-time = "2026-10-04T16:29:58.049Z" }, + { url = "https://files.pythonhosted.org/packages/35/72/f28ca566f6c23c35bbf7445f65eb0364577b25a026305995e24f78b83d94/rpds_py-2026.9.1-cp313-cp313-win_arm64.whl", hash = "sha256:d151e148117294133bf8af7eeace085e7e87432db15ab6adf640330298a47f6f", size = 217681, upload-time = "2026-10-04T16:29:59.449Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b7/a662f367d4896dd0a10cef2fc91f10b7f08af1c10858e287e019563f338d/rpds_py-2026.9.1-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:1d77b649e6f7cdf12ca5c2a98dad0ad37f9ea9b6f960408a92f0cb12bb3d04d9", size = 347463, upload-time = "2026-10-04T16:30:48.203Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3f/ad45d03df4f84ebae5439577ee81f3999d182711e82235c8037b6528890e/rpds_py-2026.9.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:00ba2d8c7dd4ee537978ddf4b3fbd712bef2d8751603f7f3146b3f4287768e25", size = 342051, upload-time = "2026-10-04T16:30:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/7e/31/3dcd68c13d4bcc59c1f7eb33ac8e80698f06f3eb0d8a1e06419836071c20/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec450527cbf485e13c8d3602a54f428ab0432fdade0ede75efd74b735421c871", size = 374143, upload-time = "2026-10-04T16:30:51.508Z" }, + { url = "https://files.pythonhosted.org/packages/cf/0d/68c1f058a250fbd1380ebda9fc227cbf50117adf8ffe8161ef383ea79f68/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:306ee1850d8105b5baf977e78d45fcadd12c1a54678d614c9baf217708446e91", size = 378604, upload-time = "2026-10-04T16:30:53.206Z" }, + { url = "https://files.pythonhosted.org/packages/d7/d6/2d4c59b85397cb4800594fadf692688ccf5ce556adc930e7a5bf21061a5e/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ef6b65b03247c54692ad4fd9ee97cb772781927db72e3cb05e70b3db6d1ff14f", size = 490790, upload-time = "2026-10-04T16:30:54.925Z" }, + { url = "https://files.pythonhosted.org/packages/da/04/7e05dc3aebaf52f4e026766bd668fdd09a9d0e23f64a14686b36b3501892/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a575404ebc9cf2e91edd32eaf570ec1430eb900d4f56724ba7dd4bc1fc9c176d", size = 393146, upload-time = "2026-10-04T16:30:56.625Z" }, + { url = "https://files.pythonhosted.org/packages/57/ca/e2e9a0a46a74ed51a0498ba1fe10f4ea6b2d9a155f372a2f91e51f18cf10/rpds_py-2026.9.1-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c16ab111bc27c646ba8aa005d0527754edc538ebb636f0b1bf8e244b48d1945", size = 372022, upload-time = "2026-10-04T16:30:58.295Z" }, + { url = "https://files.pythonhosted.org/packages/ba/cb/8f8774df5134e23424372838bcc5c7ed4127d723e1ff52f7bebd4dcb2563/rpds_py-2026.9.1-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:7664419f27db41d4f1c43a78dccda7dd6e8ef2428df3ee01d0c2a07a6b071297", size = 377092, upload-time = "2026-10-04T16:30:59.984Z" }, + { url = "https://files.pythonhosted.org/packages/90/02/8d7095d73bf9114219be40230baa5df00611e0a82ed9517779ff9c19f82b/rpds_py-2026.9.1-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4b26b03d9d2658ee2fa234f8f4f19f38a09773fe5261028025032e26d4d35af0", size = 400830, upload-time = "2026-10-04T16:31:01.721Z" }, + { url = "https://files.pythonhosted.org/packages/ec/02/8206856f8f363cd042a8315dc86b3912f5dcb6d3c66bbb24b69c2bfb0775/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:be3e47e2d91aa3942ff9bf4077a505226005abfc39b6f7554a91c1b9393986b9", size = 551478, upload-time = "2026-10-04T16:31:03.472Z" }, + { url = "https://files.pythonhosted.org/packages/41/6b/36211f1bb1f0b0313f496d92f5905b74ea107f27cb16fa3355a82f04575e/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:6307a0da524939decb8ca4a3933b8ab62525794411d6984fca6726e732804af6", size = 615173, upload-time = "2026-10-04T16:31:05.281Z" }, + { url = "https://files.pythonhosted.org/packages/35/77/cda0c4a6f055446b692f0ed5692f73707cafd3dff82672ede31b1a9b59de/rpds_py-2026.9.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:159a7aab5c5e8b112c8830f54717ce56da1252ebdbb526f5be2df2309280b9e7", size = 579653, upload-time = "2026-10-04T16:31:07.065Z" }, + { url = "https://files.pythonhosted.org/packages/74/ec/d8385f446240aed643b9e92a5055cff3015cc04a13c61f73b2883c478ed5/rpds_py-2026.9.1-cp315-cp315-win32.whl", hash = "sha256:dbc2673f9223d420c91145599b3ba45a8a50c207d1976908e5fb5ddb0c9b9429", size = 206498, upload-time = "2026-10-04T16:31:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/6d/a5/71b5cd00e0521e3b6b81828baea368c62b6b700ebdd9554cd7d41ddf12fa/rpds_py-2026.9.1-cp315-cp315-win_amd64.whl", hash = "sha256:75c38c50ab9aca840225d9a9a3810bf11d04bd5c1f186cabbb8aee56db3e9b15", size = 223121, upload-time = "2026-10-04T16:31:10.84Z" }, + { url = "https://files.pythonhosted.org/packages/33/58/dba857c3bc8221b31b62eb170a3080f4f191de79f047200389b7ed1b06a7/rpds_py-2026.9.1-cp315-cp315-win_arm64.whl", hash = "sha256:a431156bb41865fc14cd5d79bb9d7bbed83110b0159e34e62ae30951f96c0009", size = 218741, upload-time = "2026-10-04T16:31:12.592Z" }, + { url = "https://files.pythonhosted.org/packages/5b/d0/320ab28ccc1415eeb509d68682b0014fb74690cd49f1c2d29a232475af50/rpds_py-2026.9.1-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ef0d8c843e2827d6c120ab4687e9423fb1d893db1df27b7c1506615bcb9734a0", size = 349719, upload-time = "2026-10-04T16:31:14.48Z" }, + { url = "https://files.pythonhosted.org/packages/56/88/f5b12f1358f443c08b7ce3cc8391d82d335f4872580e2e097847fd36087a/rpds_py-2026.9.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:45bc6bccf78b20fd834237d18db64965d7ee68ba7f60440a26c7ab71e7b8d51a", size = 343267, upload-time = "2026-10-04T16:31:16.827Z" }, + { url = "https://files.pythonhosted.org/packages/f7/0c/c765b0059d532acb3b9c45d781ccc22f15a96dbe443d00903f643ba9df10/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d55198263bb51f557550c6ed2e6d1cb6a6fed6eb5c9120b741c5926bef8a45d", size = 374555, upload-time = "2026-10-04T16:31:18.931Z" }, + { url = "https://files.pythonhosted.org/packages/6b/8a/cafddfda77564a10cd21184640c3bffda6a2b8d20972fb5dedd5e0166328/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8763f20692da7df39b0afdd1ba3042b004c50a45994f76c2d9a25641f7673db", size = 378346, upload-time = "2026-10-04T16:31:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/b9/01/5e626016eff72c183bf6c96539240cace15d402468647a453ec08415b2fc/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e43d4a1f673e8a1cbd8533e809e02b4bf9d4f2280269bb640436556312121250", size = 492373, upload-time = "2026-10-04T16:31:22.614Z" }, + { url = "https://files.pythonhosted.org/packages/3b/9c/15a2469e9389242f46896b3f0a01d68caea8a5a35c011fcb05ef333aae73/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea394a937f17a54c51239348bdbe2e3518124c8d4a8951ba04a311d3095bd18f", size = 394371, upload-time = "2026-10-04T16:31:24.768Z" }, + { url = "https://files.pythonhosted.org/packages/63/f5/c100ff77e1e6366e947c75969c258fdfc4b7bc5bbfe7351e62ffbf2a1228/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdeaa99ce822dca76cfb1b993e9120c5ea212f2eb66d48950ad63c349668a018", size = 375132, upload-time = "2026-10-04T16:31:26.588Z" }, + { url = "https://files.pythonhosted.org/packages/dd/f4/fe0269c9de253e99c81cabc12b8971a5feaa083debdaff1221e06264d9e3/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:b4f062343e7ad3fa94f2c66e5ae667dee47ee74dd41a9057c4fbe163236a123d", size = 377642, upload-time = "2026-10-04T16:31:28.677Z" }, + { url = "https://files.pythonhosted.org/packages/05/65/b34a7b257baccff8f4a24a722933166d4941d5ebdc9c3f4bc4ffcd5ce4f4/rpds_py-2026.9.1-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22ffd29a63d71fb1b81552c21f2c2b734949b7ac751a9be70675a939a900839b", size = 400855, upload-time = "2026-10-04T16:31:30.802Z" }, + { url = "https://files.pythonhosted.org/packages/98/32/844e54176b6071b90b38a564e6940bc6eb8f97b2890dc709c19db9dec0f4/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:08dae4a4095150a7c4545a1fb40b98e1ab1744fbc2770d92c977b9dadaa49ab6", size = 552573, upload-time = "2026-10-04T16:31:32.709Z" }, + { url = "https://files.pythonhosted.org/packages/17/73/6041d20729dffbfdf155c02d65be58bc225a1c1fb548fd87c23ef306138f/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9a0460d43603d1fd9ef59c30278531e15d78581721ddb538fa560aa7817ea4ad", size = 615840, upload-time = "2026-10-04T16:31:34.565Z" }, + { url = "https://files.pythonhosted.org/packages/af/9e/418094adaee6b056ce199051b255448ed872829051e341b2294c80da0977/rpds_py-2026.9.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:1c2d1f6da5128eabf34e963d7163a818846075a52568250d006c4c953b40f903", size = 582764, upload-time = "2026-10-04T16:31:36.665Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9b/1698ebf6b840ddfe8b472198abbed6ace35c6e398faeecd6c47dae742a4e/rpds_py-2026.9.1-cp315-cp315t-win32.whl", hash = "sha256:5c6ee90dee3e85e055ddfd502d611643d9b0fd94c818220bda84ec3dacd9b27b", size = 203795, upload-time = "2026-10-04T16:31:38.53Z" }, + { url = "https://files.pythonhosted.org/packages/8a/e2/91f70d804c61f8eac39a417e82aa1024f655ab9e8bdd7393b196242bc41b/rpds_py-2026.9.1-cp315-cp315t-win_amd64.whl", hash = "sha256:fe5ad0664ec772b02c45859041aa17655709cced7a31005817fbbbd988c25567", size = 223037, upload-time = "2026-10-04T16:31:40.468Z" }, + { url = "https://files.pythonhosted.org/packages/f3/ad/f0eba548e297987964ef8088403db875b333963bf8189f3f563b6936b20e/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c0d2cb595a420b34d5086db0add011e26e2c09d6a024afbac4228bf8f863a30", size = 384112, upload-time = "2026-10-04T16:31:42.643Z" }, + { url = "https://files.pythonhosted.org/packages/33/89/891fa4d81f1eae7fe262ecd0aba2c51de94a116c7f3ccfb034fa75f4c537/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3d6ed6a98cfd19155996605474982cc470d7601746a6439078f1a5a3fa8b050", size = 389662, upload-time = "2026-10-04T16:31:44.642Z" }, + { url = "https://files.pythonhosted.org/packages/4f/46/912cf623b2263f9c9c2d04e8508fc38dbe89983faeba5fb36ca4558ae550/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8171b44a054e5c67fd748ada04187f1250bf35b95f85e52ab64bcf3331a923bb", size = 504637, upload-time = "2026-10-04T16:31:46.667Z" }, + { url = "https://files.pythonhosted.org/packages/9b/5c/7f3cfa7c5f01765c9f47c8398210ed0c1eb686c0e99c223cb1c6776d3fb9/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fda1d96e542c37b6c804547dbf489c129fe7c97183a76a5ec275909ba1a063df", size = 402607, upload-time = "2026-10-04T16:31:49.068Z" }, + { url = "https://files.pythonhosted.org/packages/f0/76/2e915d4115ee453c33e6af17bc1df7e22ddc52974e178eeeb92a78c09983/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:270bdcdaac5d5b6f73c5e22e7e135c7f2a50e789f71d9e241d5be8d90026e19a", size = 382762, upload-time = "2026-10-04T16:31:51.24Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e5/844382925d7b84a4074ad79fc2b9d05eb6231ea243b34f247e68e519b606/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:84a6ecc0c940169190d2c23bd969debd48c94dbc855acd60188a68d71d421608", size = 389563, upload-time = "2026-10-04T16:31:53.549Z" }, + { url = "https://files.pythonhosted.org/packages/81/82/d6e3951602dc2e24790ef9d299aa24bf5e2493df5043f2c6da00480cc447/rpds_py-2026.9.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab4b2fda7c2b542f7f9d886cc6a838c5079d2b76f72e6081411faba11adde2c9", size = 416842, upload-time = "2026-10-04T16:31:55.76Z" }, + { url = "https://files.pythonhosted.org/packages/ea/1c/8d248e4ea671d525ab7daa0703ad81cc2175076fa0ccd7ef088facbd2b01/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:44b32a7c4f0da3d28af31c259e38ddcff096f855e205ed0671d02fcf44f1ea1c", size = 562434, upload-time = "2026-10-04T16:31:57.874Z" }, + { url = "https://files.pythonhosted.org/packages/ad/38/e7ac0db898120d15dea25e3661b1242a40cb3b23d1f4db86c15a01fe0ba1/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:a3dbc5ed9514908d5046107d7b1346bde71eea61de6e0e4919c19354f97e769f", size = 627909, upload-time = "2026-10-04T16:31:59.977Z" }, + { url = "https://files.pythonhosted.org/packages/03/c8/37e555514932cb1ee36c75349187aa882a98dee5d5654c761699ed8a0d27/rpds_py-2026.9.1-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6eae33003518fd4cb4f83a218d5371469dd3001aa3b87128c005b07762f7fe5e", size = 593658, upload-time = "2026-10-04T16:32:02.14Z" }, + { url = "https://files.pythonhosted.org/packages/12/5c/5f69a91add94a8a33acea68a2b034a464946a654ff7296af9d957ef0d6a5/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_10_12_x86_64.whl", hash = "sha256:0483515261947e4e8b8e1375bf7463e7eb6ccfb3d86e7b554d90cd5285f20f32", size = 351769, upload-time = "2026-10-04T16:32:04.675Z" }, + { url = "https://files.pythonhosted.org/packages/41/22/35ea8bf3b2682b90ab687c74e4828418ff12298a5b0b0a0c033f61c86a30/rpds_py-2026.9.1-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:4cfaf02209061880210819934de2f4f6aa83dc04dafe6770276acc240a56da31", size = 351719, upload-time = "2026-10-04T16:32:06.834Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/11/c0/cd761f018601bf954c9018420eb6707e3675292dfa514cfda44d06aff2ee/sqlglot-30.21.0.tar.gz", hash = "sha256:6787ac202a3adc0e25434f1512370a655e51cb3de8c792544de5f6ff9c618d25", size = 6064227, upload-time = "2026-09-30T17:35:09.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/93/acaa0862f841051e490326c1fc8c61d618d2df26f9e7aed26c331419ab15/sqlglot-30.21.0-py3-none-any.whl", hash = "sha256:816d1a4815b7562b3976efcc0b561c928a743f976efd0fb8ad6db7a5b96c069e", size = 763892, upload-time = "2026-09-30T17:35:07.976Z" }, +] + +[[package]] +name = "tomli" +version = "2.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/22/de/48c59722572767841493b26183a0d1cc411d54fd759c5607c4590b6563a6/tomli-2.4.1.tar.gz", hash = "sha256:7c7e1a961a0b2f2472c1ac5b69affa0ae1132c39adcb67aba98568702b9cc23f", size = 17543, upload-time = "2026-03-25T20:22:03.828Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/11/db3d5885d8528263d8adc260bb2d28ebf1270b96e98f0e0268d32b8d9900/tomli-2.4.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f8f0fc26ec2cc2b965b7a3b87cd19c5c6b8c5e5f436b984e85f486d652285c30", size = 154704, upload-time = "2026-03-25T20:21:10.473Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f7/675db52c7e46064a9aa928885a9b20f4124ecb9bc2e1ce74c9106648d202/tomli-2.4.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ab97e64ccda8756376892c53a72bd1f964e519c77236368527f758fbc36a53a", size = 149454, upload-time = "2026-03-25T20:21:12.036Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/81c50943cf953efa35bce7646caab3cf457a7d8c030b27cfb40d7235f9ee/tomli-2.4.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96481a5786729fd470164b47cdb3e0e58062a496f455ee41b4403be77cb5a076", size = 237561, upload-time = "2026-03-25T20:21:13.098Z" }, + { url = "https://files.pythonhosted.org/packages/48/c1/f41d9cb618acccca7df82aaf682f9b49013c9397212cb9f53219e3abac37/tomli-2.4.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a881ab208c0baf688221f8cecc5401bd291d67e38a1ac884d6736cbcd8247e9", size = 243824, upload-time = "2026-03-25T20:21:14.569Z" }, + { url = "https://files.pythonhosted.org/packages/22/e4/5a816ecdd1f8ca51fb756ef684b90f2780afc52fc67f987e3c61d800a46d/tomli-2.4.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:47149d5bd38761ac8be13a84864bf0b7b70bc051806bc3669ab1cbc56216b23c", size = 242227, upload-time = "2026-03-25T20:21:15.712Z" }, + { url = "https://files.pythonhosted.org/packages/6b/49/2b2a0ef529aa6eec245d25f0c703e020a73955ad7edf73e7f54ddc608aa5/tomli-2.4.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ec9bfaf3ad2df51ace80688143a6a4ebc09a248f6ff781a9945e51937008fcbc", size = 247859, upload-time = "2026-03-25T20:21:17.001Z" }, + { url = "https://files.pythonhosted.org/packages/83/bd/6c1a630eaca337e1e78c5903104f831bda934c426f9231429396ce3c3467/tomli-2.4.1-cp311-cp311-win32.whl", hash = "sha256:ff2983983d34813c1aeb0fa89091e76c3a22889ee83ab27c5eeb45100560c049", size = 97204, upload-time = "2026-03-25T20:21:18.079Z" }, + { url = "https://files.pythonhosted.org/packages/42/59/71461df1a885647e10b6bb7802d0b8e66480c61f3f43079e0dcd315b3954/tomli-2.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:5ee18d9ebdb417e384b58fe414e8d6af9f4e7a0ae761519fb50f721de398dd4e", size = 108084, upload-time = "2026-03-25T20:21:18.978Z" }, + { url = "https://files.pythonhosted.org/packages/b8/83/dceca96142499c069475b790e7913b1044c1a4337e700751f48ed723f883/tomli-2.4.1-cp311-cp311-win_arm64.whl", hash = "sha256:c2541745709bad0264b7d4705ad453b76ccd191e64aa6f0fc66b69a293a45ece", size = 95285, upload-time = "2026-03-25T20:21:20.309Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ba/42f134a3fe2b370f555f44b1d72feebb94debcab01676bf918d0cb70e9aa/tomli-2.4.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c742f741d58a28940ce01d58f0ab2ea3ced8b12402f162f4d534dfe18ba1cd6a", size = 155924, upload-time = "2026-03-25T20:21:21.626Z" }, + { url = "https://files.pythonhosted.org/packages/dc/c7/62d7a17c26487ade21c5422b646110f2162f1fcc95980ef7f63e73c68f14/tomli-2.4.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:7f86fd587c4ed9dd76f318225e7d9b29cfc5a9d43de44e5754db8d1128487085", size = 150018, upload-time = "2026-03-25T20:21:23.002Z" }, + { url = "https://files.pythonhosted.org/packages/5c/05/79d13d7c15f13bdef410bdd49a6485b1c37d28968314eabee452c22a7fda/tomli-2.4.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff18e6a727ee0ab0388507b89d1bc6a22b138d1e2fa56d1ad494586d61d2eae9", size = 244948, upload-time = "2026-03-25T20:21:24.04Z" }, + { url = "https://files.pythonhosted.org/packages/10/90/d62ce007a1c80d0b2c93e02cab211224756240884751b94ca72df8a875ca/tomli-2.4.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:136443dbd7e1dee43c68ac2694fde36b2849865fa258d39bf822c10e8068eac5", size = 253341, upload-time = "2026-03-25T20:21:25.177Z" }, + { url = "https://files.pythonhosted.org/packages/1a/7e/caf6496d60152ad4ed09282c1885cca4eea150bfd007da84aea07bcc0a3e/tomli-2.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5e262d41726bc187e69af7825504c933b6794dc3fbd5945e41a79bb14c31f585", size = 248159, upload-time = "2026-03-25T20:21:26.364Z" }, + { url = "https://files.pythonhosted.org/packages/99/e7/c6f69c3120de34bbd882c6fba7975f3d7a746e9218e56ab46a1bc4b42552/tomli-2.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5cb41aa38891e073ee49d55fbc7839cfdb2bc0e600add13874d048c94aadddd1", size = 253290, upload-time = "2026-03-25T20:21:27.46Z" }, + { url = "https://files.pythonhosted.org/packages/d6/2f/4a3c322f22c5c66c4b836ec58211641a4067364f5dcdd7b974b4c5da300c/tomli-2.4.1-cp312-cp312-win32.whl", hash = "sha256:da25dc3563bff5965356133435b757a795a17b17d01dbc0f42fb32447ddfd917", size = 98141, upload-time = "2026-03-25T20:21:28.492Z" }, + { url = "https://files.pythonhosted.org/packages/24/22/4daacd05391b92c55759d55eaee21e1dfaea86ce5c571f10083360adf534/tomli-2.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:52c8ef851d9a240f11a88c003eacb03c31fc1c9c4ec64a99a0f922b93874fda9", size = 108847, upload-time = "2026-03-25T20:21:29.386Z" }, + { url = "https://files.pythonhosted.org/packages/68/fd/70e768887666ddd9e9f5d85129e84910f2db2796f9096aa02b721a53098d/tomli-2.4.1-cp312-cp312-win_arm64.whl", hash = "sha256:f758f1b9299d059cc3f6546ae2af89670cb1c4d48ea29c3cacc4fe7de3058257", size = 95088, upload-time = "2026-03-25T20:21:30.677Z" }, + { url = "https://files.pythonhosted.org/packages/07/06/b823a7e818c756d9a7123ba2cda7d07bc2dd32835648d1a7b7b7a05d848d/tomli-2.4.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:36d2bd2ad5fb9eaddba5226aa02c8ec3fa4f192631e347b3ed28186d43be6b54", size = 155866, upload-time = "2026-03-25T20:21:31.65Z" }, + { url = "https://files.pythonhosted.org/packages/14/6f/12645cf7f08e1a20c7eb8c297c6f11d31c1b50f316a7e7e1e1de6e2e7b7e/tomli-2.4.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:eb0dc4e38e6a1fd579e5d50369aa2e10acfc9cace504579b2faabb478e76941a", size = 149887, upload-time = "2026-03-25T20:21:33.028Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/90637574e5e7212c09099c67ad349b04ec4d6020324539297b634a0192b0/tomli-2.4.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7f2c7f2b9ca6bdeef8f0fa897f8e05085923eb091721675170254cbc5b02897", size = 243704, upload-time = "2026-03-25T20:21:34.51Z" }, + { url = "https://files.pythonhosted.org/packages/10/8f/d3ddb16c5a4befdf31a23307f72828686ab2096f068eaf56631e136c1fdd/tomli-2.4.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f3c6818a1a86dd6dca7ddcaaf76947d5ba31aecc28cb1b67009a5877c9a64f3f", size = 251628, upload-time = "2026-03-25T20:21:36.012Z" }, + { url = "https://files.pythonhosted.org/packages/e3/f1/dbeeb9116715abee2485bf0a12d07a8f31af94d71608c171c45f64c0469d/tomli-2.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d312ef37c91508b0ab2cee7da26ec0b3ed2f03ce12bd87a588d771ae15dcf82d", size = 247180, upload-time = "2026-03-25T20:21:37.136Z" }, + { url = "https://files.pythonhosted.org/packages/d3/74/16336ffd19ed4da28a70959f92f506233bd7cfc2332b20bdb01591e8b1d1/tomli-2.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:51529d40e3ca50046d7606fa99ce3956a617f9b36380da3b7f0dd3dd28e68cb5", size = 251674, upload-time = "2026-03-25T20:21:38.298Z" }, + { url = "https://files.pythonhosted.org/packages/16/f9/229fa3434c590ddf6c0aa9af64d3af4b752540686cace29e6281e3458469/tomli-2.4.1-cp313-cp313-win32.whl", hash = "sha256:2190f2e9dd7508d2a90ded5ed369255980a1bcdd58e52f7fe24b8162bf9fedbd", size = 97976, upload-time = "2026-03-25T20:21:39.316Z" }, + { url = "https://files.pythonhosted.org/packages/6a/1e/71dfd96bcc1c775420cb8befe7a9d35f2e5b1309798f009dca17b7708c1e/tomli-2.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:8d65a2fbf9d2f8352685bc1364177ee3923d6baf5e7f43ea4959d7d8bc326a36", size = 108755, upload-time = "2026-03-25T20:21:40.248Z" }, + { url = "https://files.pythonhosted.org/packages/83/7a/d34f422a021d62420b78f5c538e5b102f62bea616d1d75a13f0a88acb04a/tomli-2.4.1-cp313-cp313-win_arm64.whl", hash = "sha256:4b605484e43cdc43f0954ddae319fb75f04cc10dd80d830540060ee7cd0243cd", size = 95265, upload-time = "2026-03-25T20:21:41.219Z" }, + { url = "https://files.pythonhosted.org/packages/3c/fb/9a5c8d27dbab540869f7c1f8eb0abb3244189ce780ba9cd73f3770662072/tomli-2.4.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:fd0409a3653af6c147209d267a0e4243f0ae46b011aa978b1080359fddc9b6cf", size = 155726, upload-time = "2026-03-25T20:21:42.23Z" }, + { url = "https://files.pythonhosted.org/packages/62/05/d2f816630cc771ad836af54f5001f47a6f611d2d39535364f148b6a92d6b/tomli-2.4.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a120733b01c45e9a0c34aeef92bf0cf1d56cfe81ed9d47d562f9ed591a9828ac", size = 149859, upload-time = "2026-03-25T20:21:43.386Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/66341bdb858ad9bd0ceab5a86f90eddab127cf8b046418009f2125630ecb/tomli-2.4.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:559db847dc486944896521f68d8190be1c9e719fced785720d2216fe7022b662", size = 244713, upload-time = "2026-03-25T20:21:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/df/6d/c5fad00d82b3c7a3ab6189bd4b10e60466f22cfe8a08a9394185c8a8111c/tomli-2.4.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:01f520d4f53ef97964a240a035ec2a869fe1a37dde002b57ebc4417a27ccd853", size = 252084, upload-time = "2026-03-25T20:21:45.62Z" }, + { url = "https://files.pythonhosted.org/packages/00/71/3a69e86f3eafe8c7a59d008d245888051005bd657760e96d5fbfb0b740c2/tomli-2.4.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f94b27a62cfad8496c8d2513e1a222dd446f095fca8987fceef261225538a15", size = 247973, upload-time = "2026-03-25T20:21:46.937Z" }, + { url = "https://files.pythonhosted.org/packages/67/50/361e986652847fec4bd5e4a0208752fbe64689c603c7ae5ea7cb16b1c0ca/tomli-2.4.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ede3e6487c5ef5d28634ba3f31f989030ad6af71edfb0055cbbd14189ff240ba", size = 256223, upload-time = "2026-03-25T20:21:48.467Z" }, + { url = "https://files.pythonhosted.org/packages/8c/9a/b4173689a9203472e5467217e0154b00e260621caa227b6fa01feab16998/tomli-2.4.1-cp314-cp314-win32.whl", hash = "sha256:3d48a93ee1c9b79c04bb38772ee1b64dcf18ff43085896ea460ca8dec96f35f6", size = 98973, upload-time = "2026-03-25T20:21:49.526Z" }, + { url = "https://files.pythonhosted.org/packages/14/58/640ac93bf230cd27d002462c9af0d837779f8773bc03dee06b5835208214/tomli-2.4.1-cp314-cp314-win_amd64.whl", hash = "sha256:88dceee75c2c63af144e456745e10101eb67361050196b0b6af5d717254dddf7", size = 109082, upload-time = "2026-03-25T20:21:50.506Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2f/702d5e05b227401c1068f0d386d79a589bb12bf64c3d2c72ce0631e3bc49/tomli-2.4.1-cp314-cp314-win_arm64.whl", hash = "sha256:b8c198f8c1805dc42708689ed6864951fd2494f924149d3e4bce7710f8eb5232", size = 96490, upload-time = "2026-03-25T20:21:51.474Z" }, + { url = "https://files.pythonhosted.org/packages/45/4b/b877b05c8ba62927d9865dd980e34a755de541eb65fffba52b4cc495d4d2/tomli-2.4.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:d4d8fe59808a54658fcc0160ecfb1b30f9089906c50b23bcb4c69eddc19ec2b4", size = 164263, upload-time = "2026-03-25T20:21:52.543Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/6ab420d37a270b89f7195dec5448f79400d9e9c1826df982f3f8e97b24fd/tomli-2.4.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7008df2e7655c495dd12d2a4ad038ff878d4ca4b81fccaf82b714e07eae4402c", size = 160736, upload-time = "2026-03-25T20:21:53.674Z" }, + { url = "https://files.pythonhosted.org/packages/02/e0/3630057d8eb170310785723ed5adcdfb7d50cb7e6455f85ba8a3deed642b/tomli-2.4.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1d8591993e228b0c930c4bb0db464bdad97b3289fb981255d6c9a41aedc84b2d", size = 270717, upload-time = "2026-03-25T20:21:55.129Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b4/1613716072e544d1a7891f548d8f9ec6ce2faf42ca65acae01d76ea06bb0/tomli-2.4.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:734e20b57ba95624ecf1841e72b53f6e186355e216e5412de414e3c51e5e3c41", size = 278461, upload-time = "2026-03-25T20:21:56.228Z" }, + { url = "https://files.pythonhosted.org/packages/05/38/30f541baf6a3f6df77b3df16b01ba319221389e2da59427e221ef417ac0c/tomli-2.4.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8a650c2dbafa08d42e51ba0b62740dae4ecb9338eefa093aa5c78ceb546fcd5c", size = 274855, upload-time = "2026-03-25T20:21:57.653Z" }, + { url = "https://files.pythonhosted.org/packages/77/a3/ec9dd4fd2c38e98de34223b995a3b34813e6bdadf86c75314c928350ed14/tomli-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:504aa796fe0569bb43171066009ead363de03675276d2d121ac1a4572397870f", size = 283144, upload-time = "2026-03-25T20:21:59.089Z" }, + { url = "https://files.pythonhosted.org/packages/ef/be/605a6261cac79fba2ec0c9827e986e00323a1945700969b8ee0b30d85453/tomli-2.4.1-cp314-cp314t-win32.whl", hash = "sha256:b1d22e6e9387bf4739fbe23bfa80e93f6b0373a7f1b96c6227c32bef95a4d7a8", size = 108683, upload-time = "2026-03-25T20:22:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/12/64/da524626d3b9cc40c168a13da8335fe1c51be12c0a63685cc6db7308daae/tomli-2.4.1-cp314-cp314t-win_amd64.whl", hash = "sha256:2c1c351919aca02858f740c6d33adea0c5deea37f9ecca1cc1ef9e884a619d26", size = 121196, upload-time = "2026-03-25T20:22:01.169Z" }, + { url = "https://files.pythonhosted.org/packages/5a/cd/e80b62269fc78fc36c9af5a6b89c835baa8af28ff5ad28c7028d60860320/tomli-2.4.1-cp314-cp314t-win_arm64.whl", hash = "sha256:eab21f45c7f66c13f2a9e0e1535309cee140182a9cdae1e041d02e47291e8396", size = 100393, upload-time = "2026-03-25T20:22:02.137Z" }, + { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From a90bc6d0bc22ec692b7eb46e63cc71ece3738078 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Thu, 8 Oct 2026 12:46:32 +0700 Subject: [PATCH 03/12] chore(ci): label pull requests that touch converters/holistics `.github/labeler.yml` gives each converter directory its own label, and the new one had no entry, so a pull request touching it got only the catch-all `converter` label. Kept out of the converter's own commit, which touches nothing outside `converters/holistics/` so that it can move to apache/ossie-converters by dropping one path prefix. This entry stays with the repository that runs the labeler. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .github/labeler.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/labeler.yml b/.github/labeler.yml index 513dc7b31..14693687c 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -77,6 +77,10 @@ converter/ontology: - changed-files: - any-glob-to-any-file: 'converters/ontology/**' +converter/holistics: + - changed-files: + - any-glob-to-any-file: 'converters/holistics/**' + # Other areas cli: - changed-files: From 11d898ebc76d4507e9f03db0fe903f1a4d45fbc7 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:32:21 +0700 Subject: [PATCH 04/12] fixup! feat(holistics): add the Holistics AML converter dialects_of validates the shape it is handed. `(expression or {}).get(...)` assumed a mapping, so an `expression` written as a bare string, or a `dialects` entry that is not a mapping, raised `AttributeError` instead of a message. The CLI catches `ConversionError` alone, so that reached the user as a traceback. `_named_objects` already guarded the other collections. Three more cases join the malformed-input table. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 34 +++++++++++++++---- .../holistics/tests/test_malformed_input.py | 21 ++++++++++++ 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 91eea8397..1c2afd0c8 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -147,10 +147,8 @@ def _named_objects(container: Any, where: str) -> list[dict[str, Any]]: """`container` as a list of named mappings, or a `ConversionError` saying why. A document this converter did not write is parsed at the boundary rather - than indexed into. Without this, `datasets` given as a mapping raised - `AttributeError: 'str' object has no attribute 'get'` and a field with no - `name` raised `KeyError`, neither of which the CLI catches, so a malformed - document produced a traceback instead of a message. apache/ossie#297 and + than indexed into. The CLI catches `ConversionError` alone, so any other + exception reaches the user as a traceback. apache/ossie#297 and apache/ossie#407 both asked a sibling converter for the same guard. """ if container is None: @@ -171,10 +169,34 @@ def _named_objects(container: Any, where: str) -> list[dict[str, Any]]: def dialects_of(expression: Any) -> dict[str, str]: - dialects = (expression or {}).get("dialects") or [] + """Every `{dialect: expression}` pair, validated at the boundary. + + A document this converter did not write arrives unchecked, and the CLI + catches `ConversionError` alone, so the shape is tested rather than assumed. + """ + if expression is None: + raise ConversionError("expression is missing, and the specification requires one") + if not isinstance(expression, dict): + raise ConversionError( + f"expression is {type(expression).__name__}, and the specification defines it " + f"as a mapping holding a 'dialects' list" + ) + dialects = expression.get("dialects") or [] if not dialects: raise ConversionError("expression has no dialects") - return {entry.get("dialect", "ANSI_SQL"): entry.get("expression", "") for entry in dialects} + if not isinstance(dialects, list): + raise ConversionError( + f"expression.dialects is {type(dialects).__name__}, and must be a list" + ) + available: dict[str, str] = {} + for index, entry in enumerate(dialects): + if not isinstance(entry, dict): + raise ConversionError( + f"expression.dialects[{index}] is {type(entry).__name__}, and must be a " + f"mapping holding 'dialect' and 'expression'" + ) + available[entry.get("dialect", "ANSI_SQL")] = entry.get("expression", "") + return available class _Reverse: diff --git a/converters/holistics/tests/test_malformed_input.py b/converters/holistics/tests/test_malformed_input.py index b0cdd6616..bd8a75102 100644 --- a/converters/holistics/tests/test_malformed_input.py +++ b/converters/holistics/tests/test_malformed_input.py @@ -59,6 +59,27 @@ def document(**overrides): "a metric is a string": document(metrics=["nope"]), "the document is a list": [], "the document has no name": {"version": VERSION, "datasets": [{"name": "d", "source": "s"}]}, + "an expression is a string": document( + datasets=[{"name": "d", "source": "s", "fields": [{"name": "x", "expression": "SUM(x)"}]}] + ), + "a dialect entry is a string": document( + datasets=[ + { + "name": "d", + "source": "s", + "fields": [{"name": "x", "expression": {"dialects": ["ANSI_SQL"]}}], + } + ] + ), + "dialects is a string": document( + datasets=[ + { + "name": "d", + "source": "s", + "fields": [{"name": "x", "expression": {"dialects": "ANSI_SQL"}}], + } + ] + ), } From 69373066fbbbe3ed222ce824025ca2d17cfe7f08 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:33:48 +0700 Subject: [PATCH 05/12] fixup! feat(holistics): add the Holistics AML converter A stashed value is spelled explicitly rather than through str().lower(). `str(value).lower()` is right for a bool and a number and wrong for the rest: a list came out as Python repr text (`['a']`) and `None` as `none`, neither of which AML parses. bool, number, string and list each have a spelling now. A nested object carrying a `__type__` becomes a nested block under its own key, and a type with no spelling stops the run rather than writing a body that fails to compile later and further from its cause. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 56 +++++++++++++++---- converters/holistics/tests/test_snapshots.py | 37 ++++++++++++ 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 1c2afd0c8..091c5fc58 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -888,7 +888,7 @@ def model_file(self, model: Model) -> str: persistence = data.get("persistence") if persistence: block.blank() - _write_persistence(block, persistence) + _write_persistence(block, persistence, scope) primary_key = set(payload.get("primary_key") or []) for field in payload.get("fields") or []: @@ -1003,18 +1003,54 @@ def _field_ref(model_name: str, column: str) -> str: return f"FieldRef {{ model: {quote(model_name)}, field: {quote(column)} }}" -def _write_persistence(block: Block, persistence: dict[str, Any]) -> None: - """The stashed persistence object as its AML block. +def _aml_value(value: Any, where: str) -> str: + """One stashed value as the AML source text that spells it. - `__type__` names the block, and every key the compiler added for its own - bookkeeping starts and ends with `__`, so those are skipped. + A type with no spelling here stops the run, because a body AML cannot parse + fails later and further from its cause. """ - kind = persistence.get("__type__") or "FullPersistence" - child = block.block(f"persistence: {kind}") - for key, value in persistence.items(): - if key.startswith("__"): + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (int, float)): + return str(value) + if isinstance(value, str): + return quote(value) + if isinstance(value, list): + return "[" + ", ".join(_aml_value(member, where) for member in value) + "]" + raise ConversionError( + f"{where}: the stash holds {type(value).__name__}, which this converter cannot " + f"write as AML. Drop the key from custom_extensions, or add a spelling for it" + ) + + +def _write_stashed_object(block: Block, key: str, payload: dict[str, Any], where: str) -> None: + """One stashed object as `key: <Type> { ... }`. + + `__type__` names the AML type, and every key the compiler added for its own + bookkeeping starts and ends with `__`, so those are skipped. A nested object + carries its own `__type__` and becomes a nested block under its own key. + """ + kind = payload.get("__type__") + if not kind: + raise ConversionError( + f"{where}: {key} is an object with no __type__, so there is no AML type " + f"name to write it under" + ) + child = block.block(f"{key}: {kind}") + for name, value in payload.items(): + if name.startswith("__"): continue - child.property(key, quote(value) if isinstance(value, str) else str(value).lower()) + if isinstance(value, dict): + _write_stashed_object(child, name, value, f"{where}.{name}") + continue + child.property(name, _aml_value(value, f"{where}.{name}")) + + +def _write_persistence(block: Block, persistence: dict[str, Any], where: str) -> None: + """The stashed persistence object as its AML block.""" + payload = dict(persistence) + payload.setdefault("__type__", "FullPersistence") + _write_stashed_object(block, "persistence", payload, f"{where}.persistence") def _models_referenced(query: str) -> set[str]: diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py index cd7038263..d67004149 100644 --- a/converters/holistics/tests/test_snapshots.py +++ b/converters/holistics/tests/test_snapshots.py @@ -47,6 +47,7 @@ import snapshots from ossie_holistics import _yaml, aml_to_ossie, aql, ossie_to_aml +from ossie_holistics.errors import ConversionError from ossie_holistics.issues import Severity HOLISTICS_CLI = shutil.which("holistics") or str(Path.home() / ".holistics" / "bin" / "holistics") @@ -536,3 +537,39 @@ def test_the_round_trip_returns_the_same_ossie_document(tmp_path): payload = json.loads((compiled / "ecommerce.dataset.aml.json").read_text(encoding="utf-8")) again = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) assert _yaml.dump(again.model) == _yaml.dump(forward_result.model) + + +STASH_VALUES = { + "a list": (["a", "b"], "opt: ['a', 'b']"), + "a bool": (True, "opt: true"), + "a number": (3, "opt: 3"), + "a string": ("x", "opt: 'x'"), +} + + +def _with_persistence(value): + document = _two_dataset_document([]) + document["datasets"][0]["custom_extensions"] = [ + { + "vendor_name": "HOLISTICS", + "data": json.dumps( + {"_v": 1, "persistence": {"__type__": "FullPersistence", "opt": value}} + ), + } + ] + return document + + +@pytest.mark.parametrize("value,expected", STASH_VALUES.values(), ids=list(STASH_VALUES)) +def test_a_stashed_persistence_value_is_written_as_aml(value, expected): + """Each type has its own spelling, so none arrives as Python repr text.""" + result = ossie_to_aml.convert(_with_persistence(value), "ANSI_SQL", data_source_name="probe") + orders = next(text for name, text in result.files if name.endswith("orders.model.aml")) + assert expected in orders + + +@pytest.mark.parametrize("value", [None, {"no": "type"}], ids=["none", "untyped mapping"]) +def test_a_stashed_value_with_no_aml_spelling_stops_the_run(value): + """Writing a body AML cannot parse would fail later and further away.""" + with pytest.raises(ConversionError): + ossie_to_aml.convert(_with_persistence(value), "ANSI_SQL", data_source_name="probe") From e35d98e2336cd3e736b777a57e1e4a295e36e996 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:34:35 +0700 Subject: [PATCH 06/12] fixup! feat(holistics): add the Holistics AML converter aql_reference compares declared names on the AML spelling. `declared` held Ossie names and `self.metric_aml_name[...]`, and was tested against `self.name_in_aml(column, scope)`, which is sanitized. A field whose Ossie name is not a legal AML identifier therefore read as undeclared. That costs more than the spurious BORROWED_COLUMN warning in the report: the metric borrows a hidden dimension under the sanitized name, which the model already declares under that same name, and Holistics rejects the file as a duplicate identifier. `sanitize` rather than `name_in_aml`, which logs a rename warning per call. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 6 +++-- converters/holistics/tests/test_snapshots.py | 25 +++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 091c5fc58..87f5f3737 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -808,9 +808,11 @@ def aql_reference(self, table: str, column: str, scope: str) -> str: if target is None: raise aql.Untranslatable(f"{table!r} is not a dataset in this document.") name = self.name_in_aml(column, scope) + # Compared on the AML spelling, because `name` is already sanitized. + # `sanitize` rather than `name_in_aml`, which logs a rename per call. declared = { - field["name"] for field in target.payload.get("fields") or [] - } | {self.metric_aml_name[m["name"]] for m in target.measures} + sanitize(field["name"]) for field in target.payload.get("fields") or [] + } | {sanitize(self.metric_aml_name[m["name"]]) for m in target.measures} if name not in declared and name not in target.borrowed: target.borrowed.append(name) self.issues.add( diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py index d67004149..1e01f4797 100644 --- a/converters/holistics/tests/test_snapshots.py +++ b/converters/holistics/tests/test_snapshots.py @@ -573,3 +573,28 @@ def test_a_stashed_value_with_no_aml_spelling_stops_the_run(value): """Writing a body AML cannot parse would fail later and further away.""" with pytest.raises(ConversionError): ossie_to_aml.convert(_with_persistence(value), "ANSI_SQL", data_source_name="probe") + + +def test_a_field_name_aml_cannot_spell_is_still_declared(): + """`declared` holds the AML spelling, which is what a reference carries. + + An Ossie field named `my field` is written `my_field`. Matched on the Ossie + name it reads as undeclared, and the metric borrows a hidden dimension of a + name the model already has, which Holistics rejects as a duplicate. + """ + document = _two_dataset_document( + [_metric("spanning", 'SUM(orders."my field") + COUNT(users.id)')] + ) + document["datasets"][0]["fields"].append( + { + "name": "my field", + "datatype": "Decimal", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "my field"}]}, + } + ) + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + + borrowed = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_BORROWED_COLUMN] + assert borrowed == [] + orders = next(text for name, text in result.files if name.endswith("orders.model.aml")) + assert orders.count("dimension my_field {") == 1 From 62d626d10b4d2d77aa3fb42c487920292899f79a Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:35:23 +0700 Subject: [PATCH 07/12] fixup! feat(holistics): add the Holistics AML converter Relationship column lists are checked before either one is indexed. An absent or empty `from_columns` fails `len(from_columns) == 1`, falls through to `from_columns[0]` and raises `IndexError`. The CLI catches `ConversionError`, `OSError` and `UnicodeDecodeError`, so `to-aml` ends in a traceback. Unequal lengths are worse, because they produce output rather than an error: `zip(from_columns[1:], to_columns[1:])` stops at the shorter list, so a three-column key against two target columns emits a two-column join that compiles and joins on a condition nobody wrote. Both now raise `ConversionError` naming the relationship and both counts. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 7 ++++ .../holistics/tests/test_malformed_input.py | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 87f5f3737..27bceff87 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -931,8 +931,15 @@ def relationship(self, payload: dict[str, Any]) -> str: f"relationship {payload.get('name')!r} names dataset {missing!r}, " f"which is not in this document" ) + name = payload.get("name") from_columns = payload.get("from_columns") or [] to_columns = payload.get("to_columns") or [] + if not from_columns or len(from_columns) != len(to_columns): + raise ConversionError( + f"relationship {name!r} has {len(from_columns)} from_columns and " + f"{len(to_columns)} to_columns. AML joins a column to a column, so the " + f"two lists must be the same length and must not be empty" + ) if len(from_columns) == 1 and kind in ("many_to_one", "one_to_one"): operator = ">" if kind == "many_to_one" else "-" diff --git a/converters/holistics/tests/test_malformed_input.py b/converters/holistics/tests/test_malformed_input.py index bd8a75102..a8f09f7e6 100644 --- a/converters/holistics/tests/test_malformed_input.py +++ b/converters/holistics/tests/test_malformed_input.py @@ -83,7 +83,44 @@ def document(**overrides): } +def joined(from_columns, to_columns): + """Two datasets and one relationship pairing the column lists given.""" + column = {"name": "id", "expression": EXPRESSION} + return document( + datasets=[ + {"name": "orders", "source": "db.orders", "fields": [column]}, + {"name": "users", "source": "db.users", "fields": [column]}, + ], + relationships=[ + { + "name": "r", + "from": "orders", + "to": "users", + "from_columns": from_columns, + "to_columns": to_columns, + } + ], + ) + + @pytest.mark.parametrize("payload", MALFORMED.values(), ids=list(MALFORMED)) def test_a_malformed_document_raises_conversion_error(payload): with pytest.raises(ConversionError): ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + +@pytest.mark.parametrize( + "from_columns,to_columns", + [([], []), (["id"], []), ([], ["id"]), (["a", "b", "c"], ["a", "b"])], + ids=["both empty", "no to_columns", "no from_columns", "unequal lengths"], +) +def test_relationship_columns_must_pair_up(from_columns, to_columns): + """AML joins a column to a column, so the two lists have to line up. + + An unequal pair has no reading: a three-column key against two target + columns would join on a condition nobody wrote. + """ + with pytest.raises(ConversionError, match="from_columns"): + ossie_to_aml.convert( + joined(from_columns, to_columns), "ANSI_SQL", data_source_name="probe" + ) From 17c2dbfcd8e240f372b29efbfa7d26ea508bd9df Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:37:36 +0700 Subject: [PATCH 08/12] fixup! feat(holistics): add the Holistics AML converter The relationship stash round trips. `aml_to_ossie` stashes `direction`, `nullable`, `rlp_propagation` and `where`, and the reverse path read only `type` and `active`, so a Holistics -> Ossie -> Holistics trip dropped all four. `where` mattered most: the join filter disappears and the join silently widens. All four are settable. `direction`, `nullable` and `rlp_propagation` are properties of `RelationshipConfig`, the last taking an enumeration whose members include `'inherit'` and `'one_way'`. Each is written only when it differs from the compiler's default, so a relationship sitting at its defaults keeps the short `relationship(...)` form rather than expanding into a block that means the same thing. `where` belongs to the relationship itself and takes a block of `from` and `to` AQL predicates. A relationship written inline in `relationships:` has nowhere to put one, so a relationship carrying a filter is declared above the `Dataset` block under a name and the list holds `RelationshipConfig { rel: <name>, ... }`. It is written as the `Relationship` supertype and the compiler narrows it from `type`, which the compiled payload confirms by tagging it `EqualityRelationship`. Two more that the long form needed: `FieldRef { model: ..., field: ... }` is now `ref(...)`, aml-std's constructor for the same value and how AML is normally written, and a module model is named dotted the way the short form names it rather than by its last segment, which did not resolve. Verified against `holistics aml compile`: the filter returns under `rel.where.from` with its content unchanged, and `direction: 'one_way'` survives. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 129 +++++++++++++++++- .../expected_aml/ecommerce.dataset.aml | 8 +- converters/holistics/tests/test_snapshots.py | 58 ++++++++ 3 files changed, 184 insertions(+), 11 deletions(-) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 27bceff87..3640b4cf7 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -142,6 +142,16 @@ def unflatten(name: str) -> str: #: Snowflake's `_extract_expression` and Cube's `pick_expression`. PORTABLE_DIALECTS = ("ANSI_SQL", "OSSIE_SQL_2026") +#: What the compiler fills in for a relationship that declares none of them. +#: Every compiled payload carries all three, so writing them back +#: unconditionally would turn every short `relationship(...)` into a +#: `RelationshipConfig` block that means exactly the same thing. +RELATIONSHIP_DEFAULTS: dict[str, Any] = { + "direction": "two_way", + "nullable": True, + "rlp_propagation": "inherit", +} + def _named_objects(container: Any, where: str) -> list[dict[str, Any]]: """`container` as a list of named mappings, or a `ConversionError` saying why. @@ -257,6 +267,9 @@ def __init__( self.metric_owner: dict[str, Model] = {} self.metric_aml_name: dict[str, str] = {} self.dataset_metrics: list[dict[str, Any]] = [] + #: Relationships that carry a `where` filter, declared above the + #: Dataset block because only a named relationship can hold one. + self.relationship_declarations: list[str] = [] self._metric_payloads: dict[str, dict[str, Any]] = {} self._metric_datasets: dict[str, Any] = {} self._cyclic_metrics: set[str] = set() @@ -941,7 +954,14 @@ def relationship(self, payload: dict[str, Any]) -> str: f"two lists must be the same length and must not be empty" ) - if len(from_columns) == 1 and kind in ("many_to_one", "one_to_one"): + extra = self._relationship_properties(data, name) + where = data.get("where") + if where is not None: + return self._named_relationship( + payload, kind, active, from_model, to_model, + from_columns, to_columns, extra, where, + ) + if len(from_columns) == 1 and kind in ("many_to_one", "one_to_one") and not extra: operator = ">" if kind == "many_to_one" else "-" return ( f"relationship({from_model.reference}.{from_columns[0]} {operator} " @@ -952,15 +972,105 @@ def relationship(self, payload: dict[str, Any]) -> str: rel = config.block("EqualityRelationship") rel.header = "rel: EqualityRelationship" rel.property("type", quote(kind)) - rel.property("from", _field_ref(from_model.aml_name, from_columns[0])) - rel.property("to", _field_ref(to_model.aml_name, to_columns[0])) + rel.property("from", _field_ref(from_model.reference, from_columns[0])) + rel.property("to", _field_ref(to_model.reference, to_columns[0])) for from_column, to_column in zip(from_columns[1:], to_columns[1:]): on = rel.block("on") - on.property("from", _field_ref(from_model.aml_name, from_column)) - on.property("to", _field_ref(to_model.aml_name, to_column)) + on.property("from", _field_ref(from_model.reference, from_column)) + on.property("to", _field_ref(to_model.reference, to_column)) + config.property("active", active) + for key, value in extra: + config.property(key, value) + return config.render(2).lstrip() + + def _named_relationship( + self, + payload: dict[str, Any], + kind: str, + active: str, + from_model: Model, + to_model: Model, + from_columns: list[str], + to_columns: list[str], + extra: list[tuple[str, str]], + where: Any, + ) -> str: + """A relationship carrying a `where` filter, as a named declaration. + + A `where` block belongs to the relationship, and one written inline in + `relationships:` has nowhere to put it. So the relationship is declared + above the `Dataset` block under a name, and the list holds + `RelationshipConfig { rel: <name>, ... }`. + + `Relationship` rather than `EqualityRelationship`, because it is the + supertype and the compiler narrows it from `type`. The compiled payload + comes back tagged `EqualityRelationship` either way. + """ + scope = f"relationship {payload.get('name')}" + rel_name = self.name_in_aml(str(payload.get("name")), scope) + declaration = Block(f"Relationship {rel_name}") + declaration.property("type", quote(kind)) + declaration.property("from", _field_ref(from_model.reference, from_columns[0])) + declaration.property("to", _field_ref(to_model.reference, to_columns[0])) + for from_column, to_column in zip(from_columns[1:], to_columns[1:]): + on = declaration.block("on") + on.property("from", _field_ref(from_model.reference, from_column)) + on.property("to", _field_ref(to_model.reference, to_column)) + self._write_where(declaration, where, scope) + self.relationship_declarations.append(amlgen.document(declaration, license_header=False)) + + config = Block("RelationshipConfig") + config.property("rel", rel_name) config.property("active", active) + for key, value in extra: + config.property(key, value) return config.render(2).lstrip() + def _write_where(self, block: Block, where: Any, scope: str) -> None: + """The stashed `RelationshipFilter` as a `where { ... }` block. + + Each side is an AQL predicate narrowing the join on that end, and + either may be absent. `comment_photo` in the Holistics AMQL test corpus + carries only `from`. + """ + if not isinstance(where, dict): + raise ConversionError( + f"{scope}: the stashed `where` is {type(where).__name__}, and AML writes " + f"it as a block holding a `from` and a `to` predicate" + ) + child = block.block("where") + written = False + for side in ("from", "to"): + heredoc = where.get(side) + if heredoc is None: + continue + if not isinstance(heredoc, dict) or "content" not in heredoc: + raise ConversionError( + f"{scope}: the stashed `where.{side}` is not a heredoc object, so " + f"there is no predicate to write" + ) + child.heredoc(side, heredoc.get("name") or "aql", heredoc["content"].strip()) + written = True + if not written: + raise ConversionError( + f"{scope}: the stashed `where` names neither a `from` nor a `to` " + f"predicate, so it narrows nothing" + ) + + def _relationship_properties(self, data: dict[str, Any], name: Any) -> list[tuple[str, str]]: + """The stashed `RelationshipConfig` properties worth writing back. + + `direction`, `nullable` and `rlp_propagation` are all settable on a + `RelationshipConfig`. Each carries a compiler default on every payload, + so only a value that differs from it is written. + """ + extra: list[tuple[str, str]] = [] + for key, default in RELATIONSHIP_DEFAULTS.items(): + value = data.get(key) + if value is not None and value != default: + extra.append((key, _aml_value(value, f"relationship {name}"))) + return extra + def dataset_file(self) -> tuple[str, str]: name = self.dataset_name block = Block(f"Dataset {name}") @@ -996,7 +1106,11 @@ def dataset_file(self) -> tuple[str, str]: if owner: block.blank() block.text("owner", owner) - return f"{name}.dataset.aml", amlgen.document(block, license_header=True) + body = amlgen.document(block, license_header=True) + if self.relationship_declarations: + head, separator, rest = body.partition(f"Dataset {name} {{") + body = head + "".join(self.relationship_declarations) + "\n" + separator + rest + return f"{name}.dataset.aml", body def run(self) -> Result: # The dataset file is rendered first because its metrics decide which @@ -1009,7 +1123,8 @@ def run(self) -> Result: def _field_ref(model_name: str, column: str) -> str: - return f"FieldRef {{ model: {quote(model_name)}, field: {quote(column)} }}" + """`ref()` is aml-std's constructor for a FieldRef, and is how AML spells one.""" + return f"ref({quote(model_name)}, {quote(column)})" def _aml_value(value: Any, where: str) -> str: diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml index 10120a784..5ed24121a 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml @@ -39,11 +39,11 @@ Dataset ecommerce { RelationshipConfig { rel: EqualityRelationship { type: 'many_to_one' - from: FieldRef { model: 'merchants', field: 'city_id' } - to: FieldRef { model: 'cities', field: 'id' } + from: ref('merchants', 'city_id') + to: ref('cities', 'id') on { - from: FieldRef { model: 'merchants', field: 'country_code' } - to: FieldRef { model: 'cities', field: 'country_code' } + from: ref('merchants', 'country_code') + to: ref('cities', 'country_code') } } active: true diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py index 1e01f4797..60826679c 100644 --- a/converters/holistics/tests/test_snapshots.py +++ b/converters/holistics/tests/test_snapshots.py @@ -598,3 +598,61 @@ def test_a_field_name_aml_cannot_spell_is_still_declared(): assert borrowed == [] orders = next(text for name, text in result.files if name.endswith("orders.model.aml")) assert orders.count("dimension my_field {") == 1 + + +def _dataset_text(document): + result = ossie_to_aml.convert(document, "ANSI_SQL", data_source_name="probe") + return next(text for name, text in result.files if name.endswith(".dataset.aml")) + + +def _with_relationship_stash(**stashed): + document = _two_dataset_document([]) + document["relationships"][0]["custom_extensions"] = [ + { + "vendor_name": "HOLISTICS", + "data": json.dumps({"_v": 1, "type": "many_to_one", "active": True, **stashed}), + } + ] + return document + + +def test_a_stashed_relationship_filter_is_written_back(): + """A `where` filter belongs to the relationship, not to the config. + + One written inline in `relationships:` has nowhere to put it, so a + relationship carrying a filter is declared above the `Dataset` block under + a name and the list holds `RelationshipConfig { rel: <name>, ... }`. + """ + text = _dataset_text( + _with_relationship_stash( + where={ + "__type__": "RelationshipFilter", + "from": {"__type__": "Heredoc", "name": "aql", "content": "users.id > 0"}, + } + ) + ) + assert "Relationship users_to_orders {" in text + assert "from: ref('users', 'order_id')" in text + assert "to: ref('orders', 'id')" in text + assert "where {" in text + assert "from: @aql users.id > 0;;" in text + assert "rel: users_to_orders" in text + assert text.index("Relationship users_to_orders {") < text.index("Dataset m {") + + +def test_a_non_default_relationship_property_is_written_back(): + """Only a value that differs from the compiler's default is written. + + Every compiled payload carries all three, so writing them unconditionally + would turn every short `relationship(...)` into a block meaning the same. + """ + assert "relationship(users.order_id > orders.id, true)" in _dataset_text( + _two_dataset_document([]) + ) + + text = _dataset_text( + _with_relationship_stash(direction="one_way", nullable=False, rlp_propagation="one_way") + ) + assert "direction: 'one_way'" in text + assert "nullable: false" in text + assert "rlp_propagation: 'one_way'" in text From adfe5b98ea55dc411372e8cb329b936c0c9155f0 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 10:51:25 +0700 Subject: [PATCH 09/12] fixup! feat(holistics): add the Holistics AML converter Cover the relationship round trip in the fixture rather than only in assertions. The ecommerce fixture carried the short form, a composite key, a one-to-one, an inactive edge and a module-path target, and none of the properties the previous commit restores. Two relationships now cover the rest. `delivered_to_users` is declared by name and carries a `where` filter on both ends, and `products -> merchants` sets `direction`, `nullable` and `rlp_propagation` away from their defaults. That puts them through the whole pipeline: `holistics aml compile`, the forward path, the reverse path, `holistics aml validate`, and the round trip back to Ossie, which stays byte-identical. Doing so caught one defect the inline assertions could not. `_write_where` stripped the heredoc content, and the compiler preserves it, so a predicate written `@aql x > 0 ;;` came back a character shorter and the round trip no longer matched. The content is now written verbatim. A range and a many-to-many relationship carry a `match` predicate rather than column pairs, so neither has an Ossie form and both are dropped with an ERROR. That was declared in limitations.md and never asserted; a test now injects each kind into the compiled payload and checks the issue and the drop. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../src/ossie_holistics/ossie_to_aml.py | 4 +- .../ecommerce/aml/ecommerce.dataset.aml | 27 ++++++++++- .../ecommerce/ecommerce.dataset.aml.json | 24 ++++++++-- .../fixtures/ecommerce/expected.issues.yaml | 7 +++ .../fixtures/ecommerce/expected.ossie.yaml | 9 ++-- .../expected_aml/ecommerce.dataset.aml | 27 ++++++++++- converters/holistics/tests/test_snapshots.py | 47 +++++++++++++++++++ 7 files changed, 134 insertions(+), 11 deletions(-) diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index 3640b4cf7..b7453e0ff 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -1049,7 +1049,9 @@ def _write_where(self, block: Block, where: Any, scope: str) -> None: f"{scope}: the stashed `where.{side}` is not a heredoc object, so " f"there is no predicate to write" ) - child.heredoc(side, heredoc.get("name") or "aql", heredoc["content"].strip()) + # Verbatim, including trailing space: the round trip compares the + # compiled payload, and the compiler keeps what the heredoc held. + child.heredoc(side, heredoc.get("name") or "aql", heredoc["content"]) written = True if not written: raise ConversionError( diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml index eb828dc73..a07373061 100644 --- a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml +++ b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml @@ -15,6 +15,19 @@ // specific language governing permissions and limitations // under the License. +// A join filter narrowing each end. `where` belongs to the relationship, so a +// relationship carrying one is declared here and the dataset references it by +// name. `Relationship` is the supertype; the compiler narrows it from `type`. +Relationship delivered_to_users { + type: 'many_to_one' + from: ref('delivered_orders', 'user_id') + to: ref('ecommerce_users', 'id') + where { + from: @aql delivered_orders.delivery_attempts > 0 ;; + to: @aql ecommerce_users.age > 18 ;; + } +} + Dataset ecommerce { label: 'Ecommerce' description: 'A marketplace model: orders, line items, products, merchants and cities.' @@ -35,7 +48,17 @@ Dataset ecommerce { relationship(order_items.order_id > ecommerce_orders.id, true), relationship(order_items.product_id > products.id, true), relationship(ecommerce_orders.user_id > ecommerce_users.id, true), - relationship(products.merchant_id > merchants.id, true), + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: ref('products', 'merchant_id') + to: ref('merchants', 'id') + } + active: true + direction: 'one_way' + nullable: false + rlp_propagation: 'one_way' + }, // Composite. The `relationship(...)` shorthand carries one column pair, so a // second pair needs the long form with an `on` block. Blocks combine with AND. RelationshipConfig { @@ -50,7 +73,7 @@ Dataset ecommerce { } active: true }, - relationship(delivered_orders.user_id > ecommerce_users.id, true), + RelationshipConfig { rel: delivered_to_users, active: true }, // One-to-one. Ossie has no marker for this, so the kind goes to the stash. relationship(merchants.admin_id - ecommerce_users.id, true), diff --git a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json index d0624b91a..c4e1125a9 100644 --- a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json +++ b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json @@ -1284,9 +1284,9 @@ "__type__": "RelationshipConfig" }, { - "direction": "two_way", - "rlp_propagation": "inherit", - "nullable": true, + "direction": "one_way", + "rlp_propagation": "one_way", + "nullable": false, "rel": { "__doc__": "", "type": "many_to_one", @@ -1359,7 +1359,25 @@ "rlp_propagation": "inherit", "nullable": true, "rel": { + "__fqn__": "delivered_to_users", + "name": "delivered_to_users", "__doc__": "", + "where": { + "__doc__": "", + "from": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "delivered_orders.delivery_attempts > 0 " + }, + "to": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_users.age > 18 " + }, + "__type__": "RelationshipFilter" + }, "type": "many_to_one", "from": { "__doc__": "", diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml index a86517049..b577e1b2b 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml +++ b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml @@ -99,6 +99,13 @@ and round-trip it but cannot evaluate it object_ref: ecommerce.revenue_share_of_all_merchants remedy: evaluate through Holistics, or rewrite the body as Holistics SQL +- code: HOLISTICS_RELATIONSHIP_FILTER + severity: WARNING + message: >- + the relationship carries a filter that narrows the join, and Ossie has no equivalent. The join + widens to every matching row + object_ref: delivered_orders_to_ecommerce_users + remedy: move the condition into a dimension in AML - code: HOLISTICS_INACTIVE_RELATIONSHIP severity: WARNING message: >- diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml index 9f4da5c7b..cf746ba3d 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml +++ b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml @@ -486,8 +486,8 @@ relationships: - id custom_extensions: - vendor_name: HOLISTICS - data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": - "inherit", "type": "many_to_one"}' + data: '{"_v": 1, "active": true, "direction": "one_way", "nullable": false, "rlp_propagation": + "one_way", "type": "many_to_one"}' - name: merchants_to_cities from: merchants to: cities @@ -511,7 +511,10 @@ relationships: custom_extensions: - vendor_name: HOLISTICS data: '{"_v": 1, "active": true, "direction": "two_way", "nullable": true, "rlp_propagation": - "inherit", "type": "many_to_one"}' + "inherit", "type": "many_to_one", "where": {"__doc__": "", "__type__": "RelationshipFilter", + "from": {"__doc__": "", "__type__": "Heredoc", "content": "delivered_orders.delivery_attempts + > 0 ", "name": "aql"}, "to": {"__doc__": "", "__type__": "Heredoc", "content": + "ecommerce_users.age > 18 ", "name": "aql"}}}' - name: merchants_to_ecommerce_users from: merchants to: ecommerce_users diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml index 5ed24121a..a58317586 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml @@ -15,6 +15,16 @@ // specific language governing permissions and limitations // under the License. +Relationship delivered_orders_to_ecommerce_users { + type: 'many_to_one' + from: ref('delivered_orders', 'user_id') + to: ref('ecommerce_users', 'id') + where { + from: @aql delivered_orders.delivery_attempts > 0 ;; + to: @aql ecommerce_users.age > 18 ;; + } +} + Dataset ecommerce { label: 'Ecommerce' description: 'A marketplace model: orders, line items, products, merchants and cities.' @@ -35,7 +45,17 @@ Dataset ecommerce { relationship(order_items.order_id > ecommerce_orders.id, true), relationship(order_items.product_id > products.id, true), relationship(ecommerce_orders.user_id > ecommerce_users.id, true), - relationship(products.merchant_id > merchants.id, true), + RelationshipConfig { + rel: EqualityRelationship { + type: 'many_to_one' + from: ref('products', 'merchant_id') + to: ref('merchants', 'id') + } + active: true + direction: 'one_way' + nullable: false + rlp_propagation: 'one_way' + }, RelationshipConfig { rel: EqualityRelationship { type: 'many_to_one' @@ -48,7 +68,10 @@ Dataset ecommerce { } active: true }, - relationship(delivered_orders.user_id > ecommerce_users.id, true), + RelationshipConfig { + rel: delivered_orders_to_ecommerce_users + active: true + }, relationship(merchants.admin_id - ecommerce_users.id, true), relationship(ecommerce_orders.created_date > reporting.calendar.date, false), ] diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py index 60826679c..335f229e6 100644 --- a/converters/holistics/tests/test_snapshots.py +++ b/converters/holistics/tests/test_snapshots.py @@ -111,6 +111,7 @@ def test_the_forward_direction_reports_the_losses_it_declares(): aml_to_ossie.ISSUE_PARAM_DROPPED, aml_to_ossie.ISSUE_QUERY_SOURCE, aml_to_ossie.ISSUE_UNRESOLVED_QUERY, + aml_to_ossie.ISSUE_RELATIONSHIP_FILTER, } @@ -656,3 +657,49 @@ def test_a_non_default_relationship_property_is_written_back(): assert "direction: 'one_way'" in text assert "nullable: false" in text assert "rlp_propagation: 'one_way'" in text + + +MATCH_RELATIONSHIPS = { + "many_to_many": "ManyToManyRelationship", + "range": "RangeRelationship", +} + + +@pytest.mark.parametrize("kind,aml_type", MATCH_RELATIONSHIPS.items(), ids=list(MATCH_RELATIONSHIPS)) +def test_a_match_relationship_is_dropped_with_an_error(kind, aml_type): + """A range or many-to-many join carries `match`, not column pairs. + + Ossie encodes a relationship as `from_columns` and `to_columns`, and an AQL + predicate spanning both models has no form there, so the edge is dropped + and the two datasets read as unrelated. + """ + payload = json.loads(snapshots.ECOMMERCE_COMPILED.read_text(encoding="utf-8")) + before = len(payload["relationships"]) + payload["relationships"].append( + { + "__type__": "RelationshipConfig", + "active": True, + "rel": { + "__type__": aml_type, + "type": kind, + "from": {"__type__": "FieldRef", "model": "order_items", "field": "id"}, + "to": {"__type__": "FieldRef", "model": "products", "field": "id"}, + "match": { + "__type__": "Heredoc", + "name": "aql", + "content": "order_items.id >= products.id", + }, + }, + } + ) + result = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) + + raised = [ + i + for i in result.issues.issues + if i.code == aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP + ] + assert len(raised) == 1 + assert kind in raised[0].message + assert raised[0].severity is Severity.ERROR + assert len(result.model["relationships"]) == before From da64d258d0b6287f5c6af982ca0902587657b86a Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 11:03:42 +0700 Subject: [PATCH 10/12] fixup! feat(holistics): add the Holistics AML converter Carry a match relationship in the stash rather than dropping it. A range or many-to-many join states its whole condition as one AQL predicate over both models. Ossie encodes a relationship as `from_columns` and `to_columns`, which cannot hold that, so the edge had no `relationships[]` entry and nothing else kept it either. The compiler's payload now travels on the document stash under `match_relationships`, and the reverse path rebuilds the declaration from it: the predicate, the endpoints and the kind all come back as written. Same treatment `param` already gets, and the same reason. That makes the issue a WARNING rather than an ERROR. A consumer reading Ossie still sees the two datasets as unrelated, which is what the message says, but the join itself survives a Holistics -> Ossie -> Holistics trip. `aml.Relationship` keeps the payload it was parsed from, so the stash is the compiler's own object rather than one rebuilt from the parsed fields. The ecommerce fixture gains a `ManyToManyRelationship`, which puts it through `holistics aml compile`, both directions, `holistics aml validate` and the round trip. `range` has no natural pair in that schema, so a test reaches it by injecting one into the compiled payload. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- converters/holistics/docs/limitations.md | 4 +- .../holistics/src/ossie_holistics/aml.py | 4 ++ .../src/ossie_holistics/aml_to_ossie.py | 12 +++- .../src/ossie_holistics/ossie_to_aml.py | 56 +++++++++++++++++-- .../ecommerce/aml/ecommerce.dataset.aml | 10 ++++ .../ecommerce/ecommerce.dataset.aml.json | 33 +++++++++++ .../fixtures/ecommerce/expected.issues.yaml | 8 +++ .../fixtures/ecommerce/expected.ossie.yaml | 10 +++- .../expected_aml/ecommerce.dataset.aml | 10 ++++ converters/holistics/tests/test_snapshots.py | 36 +++++++----- 10 files changed, 158 insertions(+), 25 deletions(-) diff --git a/converters/holistics/docs/limitations.md b/converters/holistics/docs/limitations.md index c0cd4fde9..02be4779f 100644 --- a/converters/holistics/docs/limitations.md +++ b/converters/holistics/docs/limitations.md @@ -25,7 +25,7 @@ structured issue at conversion time, listed under [Issue codes](#issue-codes). | Warehouse SQL portability | Emitted under `--sql-dialect`, never rewritten | A document converted for one warehouse does not run on another | | Dashboards, reports and canvases | Out of scope. Ossie models semantics, not presentation | No loss to the semantic model | | A query body using Holistics template syntax | Detected and carried unresolved, never resolved. **ERROR severity** | The dataset's `source` is not runnable SQL. A consumer must resolve it through Holistics or drop the dataset | -| `RangeRelationship` and `ManyToManyRelationship` | Neither has an Ossie form. **ERROR severity**, one issue per relationship | A range or many-to-many join is dropped, so a consumer sees the two datasets as unrelated. Both carry an AQL `match` rather than column pairs, which is why no column mapping exists | +| `RangeRelationship` and `ManyToManyRelationship` | Neither has an Ossie form. **WARNING severity**, one issue per relationship | Both carry an AQL `match` rather than column pairs, which is why no column mapping exists. The payload is stashed on the document and the reverse path rebuilds it, so a consumer reading Ossie sees the two datasets as unrelated while the round trip keeps the join | | A relationship `where` filter | A `RelationshipFilter` restricts the join condition. No Ossie equivalent | The join widens to every matching row, so a consumer counts rows Holistics excludes. **WARNING severity** | | A `QueryModel` becoming a query in `source` | `source` is one string, so a consumer reads the text to tell a query from a table reference. **WARNING severity** | Whether a given target reads it as a query depends on that target. See below | @@ -61,9 +61,9 @@ reasoning behind the ones that need it. | `HOLISTICS_PARAM_DROPPED` | WARNING | A model declares params, stashed rather than carried | | `HOLISTICS_QUERY_SOURCE` | WARNING | A `QueryModel` becomes a query in `source` | | `HOLISTICS_RELATIONSHIP_FILTER` | WARNING | A relationship carries a `where` filter | +| `HOLISTICS_UNSUPPORTED_RELATIONSHIP` | WARNING | A range or many-to-many relationship, stashed on the document | | `HOLISTICS_UNKNOWN_DATATYPE` | WARNING | An AML type outside the data type table | | `HOLISTICS_UNRESOLVED_QUERY_TEMPLATE` | ERROR | A query body uses Holistics template syntax | -| `HOLISTICS_UNSUPPORTED_RELATIONSHIP` | ERROR | A range or many-to-many relationship | | `HOLISTICS_UNKNOWN_AGGREGATION` | ERROR | An `aggregation_type` outside the aggregation table | | `HOLISTICS_UNRESOLVED_REFERENCE` | ERROR | An interpolation naming nothing in the dataset | diff --git a/converters/holistics/src/ossie_holistics/aml.py b/converters/holistics/src/ossie_holistics/aml.py index 94b5e3d1b..fb4b18562 100644 --- a/converters/holistics/src/ossie_holistics/aml.py +++ b/converters/holistics/src/ossie_holistics/aml.py @@ -136,6 +136,9 @@ class Relationship: rlp_propagation: str | None where: Any | None aml_type: str + #: The payload as the compiler wrote it. A kind with no Ossie form is + #: carried in the stash verbatim, so the reverse path can rebuild it. + raw: dict[str, Any] @dataclass(frozen=True) @@ -284,6 +287,7 @@ def _relationship(payload: dict[str, Any], index: int) -> Relationship: rlp_propagation=payload.get("rlp_propagation"), where=rel.get("where"), aml_type=rel.get("__type__", ""), + raw=payload, ) diff --git a/converters/holistics/src/ossie_holistics/aml_to_ossie.py b/converters/holistics/src/ossie_holistics/aml_to_ossie.py index 1609a2e6c..07f780f4a 100644 --- a/converters/holistics/src/ossie_holistics/aml_to_ossie.py +++ b/converters/holistics/src/ossie_holistics/aml_to_ossie.py @@ -83,6 +83,9 @@ def __init__(self, dataset: aml.Dataset, sql_dialect: str): self.sql_dialect = sql_dialect self.issues = IssueLog() + #: Relationships whose condition is a `match` predicate. Ossie encodes a + #: relationship as column pairs, so these travel in the document stash. + self.match_relationships: list[dict[str, Any]] = [] self.models_by_fqn = dataset.model_by_fqn() self.dataset_name: dict[str, str] = {} claimed_by: dict[str, str] = {} @@ -473,14 +476,17 @@ def convert_relationship( if relationship.kind not in aml.EQUALITY_KINDS: self.issues.add( code=ISSUE_UNSUPPORTED_RELATIONSHIP, - severity=Severity.ERROR, + severity=Severity.WARNING, message=( f"a {relationship.kind.value} relationship carries an AQL match condition " - f"rather than column pairs, so it has no Ossie form and is dropped" + f"rather than column pairs, so it has no `relationships[]` entry and is " + f"stashed on the document instead. A consumer reading Ossie sees the two " + f"datasets as unrelated" ), object_ref=scope, remedy="model the join as an equality relationship in AML", ) + self.match_relationships.append(relationship.raw) return None from_fqn = self.resolve_model(relationship.pairs[0][0].model) @@ -579,6 +585,8 @@ def run(self) -> Result: model["metrics"] = metrics data: dict[str, Any] = {"aml_type": "Dataset"} + if self.match_relationships: + data["match_relationships"] = self.match_relationships if self.dataset.fqn != self.dataset.name: data["fqn"] = self.dataset.fqn if self.dataset.label: diff --git a/converters/holistics/src/ossie_holistics/ossie_to_aml.py b/converters/holistics/src/ossie_holistics/ossie_to_aml.py index b7453e0ff..37f9465e5 100644 --- a/converters/holistics/src/ossie_holistics/ossie_to_aml.py +++ b/converters/holistics/src/ossie_holistics/ossie_to_aml.py @@ -1059,6 +1059,53 @@ def _write_where(self, block: Block, where: Any, scope: str) -> None: f"predicate, so it narrows nothing" ) + def match_relationship(self, payload: dict[str, Any]) -> str: + """A stashed `match` relationship, as its declaration and config entry. + + A range or many-to-many join states its whole condition as one AQL + predicate over both models, which Ossie's column pairs cannot hold, so + the forward path carries the compiler's payload verbatim. Rebuilding it + needs no interpretation: the predicate, the endpoints and the kind all + travel as written. + """ + rel = payload.get("rel") + if not isinstance(rel, dict): + raise ConversionError( + "a stashed match relationship has no `rel` object to rebuild from" + ) + name = rel.get("name") or rel.get("__fqn__") + if not name: + raise ConversionError( + "a stashed match relationship has no name, and AML can only carry a " + "`match` predicate on a named relationship" + ) + scope = f"relationship {name}" + aml_name = self.name_in_aml(str(name), scope) + declaration = Block(f"{rel.get('__type__') or 'Relationship'} {aml_name}") + declaration.property("type", quote(rel.get("type") or "many_to_many")) + for side in ("from", "to"): + endpoint = rel.get(side) + if not isinstance(endpoint, dict) or "model" not in endpoint: + raise ConversionError(f"{scope}: the stashed `{side}` is not a field reference") + declaration.property(side, _field_ref(endpoint["model"], endpoint.get("field", ""))) + match = rel.get("match") + if not isinstance(match, dict) or "content" not in match: + raise ConversionError( + f"{scope}: the stashed `match` is not a heredoc, so there is no predicate " + f"to write and the join has no condition" + ) + declaration.heredoc("match", match.get("name") or "aql", match["content"]) + self.relationship_declarations.append( + amlgen.document(declaration, license_header=False) + ) + + config = Block("RelationshipConfig") + config.property("rel", aml_name) + config.property("active", "true" if payload.get("active", True) else "false") + for key, value in self._relationship_properties(payload, name): + config.property(key, value) + return config.render(2).lstrip() + def _relationship_properties(self, data: dict[str, Any], name: Any) -> list[tuple[str, str]]: """The stashed `RelationshipConfig` properties worth writing back. @@ -1085,12 +1132,9 @@ def dataset_file(self) -> tuple[str, str]: block.blank() block.property("models", amlgen.array([m.reference for m in self.models])) block.blank() - block.property( - "relationships", - amlgen.array( - [self.relationship(r) for r in self.document.get("relationships") or []] - ), - ) + entries = [self.relationship(r) for r in self.document.get("relationships") or []] + entries.extend(self.match_relationship(r) for r in self.root_stash.get("match_relationships") or []) + block.property("relationships", amlgen.array(entries)) for model in self.models: for field in model.payload.get("fields") or []: diff --git a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml index a07373061..30bd2068b 100644 --- a/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml +++ b/converters/holistics/tests/fixtures/ecommerce/aml/ecommerce.dataset.aml @@ -28,6 +28,15 @@ Relationship delivered_to_users { } } +// A many-to-many join. Its whole condition is a `match` expression rather than +// column pairs, so Ossie has no form for it and the converter stashes it. +ManyToManyRelationship orders_in_year { + type: 'many_to_many' + from: ref('ecommerce_orders', 'id') + to: ref('reporting.calendar', 'date') + match: @aql ecommerce_orders.created_date == reporting.calendar.date ;; +} + Dataset ecommerce { label: 'Ecommerce' description: 'A marketplace model: orders, line items, products, merchants and cities.' @@ -81,6 +90,7 @@ Dataset ecommerce { // Inactive, and its target sits in a module. Holistics does not join across // this edge by default, and the compiled endpoint carries no __refFqn__. relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + RelationshipConfig { rel: orders_in_year, active: true }, ] // A dataset-level dimension names the model it attaches to. The body spans diff --git a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json index c4e1125a9..32878f5a2 100644 --- a/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json +++ b/converters/holistics/tests/fixtures/ecommerce/ecommerce.dataset.aml.json @@ -1448,6 +1448,39 @@ }, "active": false, "__type__": "RelationshipConfig" + }, + { + "direction": "two_way", + "rlp_propagation": "inherit", + "nullable": true, + "rel": { + "__fqn__": "orders_in_year", + "name": "orders_in_year", + "__doc__": "", + "type": "many_to_many", + "from": { + "__doc__": "", + "model": "ecommerce_orders", + "field": "id", + "__refFqn__": "ecommerce_orders", + "__type__": "FieldRef" + }, + "to": { + "__doc__": "", + "model": "reporting.calendar", + "field": "date", + "__type__": "FieldRef" + }, + "match": { + "__doc__": "", + "__type__": "Heredoc", + "name": "aql", + "content": "ecommerce_orders.created_date == reporting.calendar.date " + }, + "__type__": "ManyToManyRelationship" + }, + "active": true, + "__type__": "RelationshipConfig" } ], "__doc__": "", diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml index b577e1b2b..496f697bf 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml +++ b/converters/holistics/tests/fixtures/ecommerce/expected.issues.yaml @@ -113,3 +113,11 @@ it where Holistics leaves it out by default object_ref: ecommerce_orders_to_reporting__calendar remedy: drop the relationship, or activate it in AML +- code: HOLISTICS_UNSUPPORTED_RELATIONSHIP + severity: WARNING + message: >- + a many_to_many relationship carries an AQL match condition rather than column pairs, so it has + no `relationships[]` entry and is stashed on the document instead. A consumer reading Ossie sees + the two datasets as unrelated + object_ref: relationship 8 + remedy: model the join as an equality relationship in AML diff --git a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml index cf746ba3d..6748f8b65 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml +++ b/converters/holistics/tests/fixtures/ecommerce/expected.ossie.yaml @@ -651,4 +651,12 @@ metrics: custom_extensions: - vendor_name: HOLISTICS data: '{"_v": 1, "aml_type": "Dataset", "data_source_name": "bigquery_demo", "label": - "Ecommerce", "owner": "dev@ossie.apache.org"}' + "Ecommerce", "match_relationships": [{"__type__": "RelationshipConfig", "active": + true, "direction": "two_way", "nullable": true, "rel": {"__doc__": "", "__fqn__": + "orders_in_year", "__type__": "ManyToManyRelationship", "from": {"__doc__": "", + "__refFqn__": "ecommerce_orders", "__type__": "FieldRef", "field": "id", "model": + "ecommerce_orders"}, "match": {"__doc__": "", "__type__": "Heredoc", "content": + "ecommerce_orders.created_date == reporting.calendar.date ", "name": "aql"}, "name": + "orders_in_year", "to": {"__doc__": "", "__type__": "FieldRef", "field": "date", + "model": "reporting.calendar"}, "type": "many_to_many"}, "rlp_propagation": "inherit"}], + "owner": "dev@ossie.apache.org"}' diff --git a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml index a58317586..45e669bf9 100644 --- a/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml +++ b/converters/holistics/tests/fixtures/ecommerce/expected_aml/ecommerce.dataset.aml @@ -24,6 +24,12 @@ Relationship delivered_orders_to_ecommerce_users { to: @aql ecommerce_users.age > 18 ;; } } +ManyToManyRelationship orders_in_year { + type: 'many_to_many' + from: ref('ecommerce_orders', 'id') + to: ref('reporting.calendar', 'date') + match: @aql ecommerce_orders.created_date == reporting.calendar.date ;; +} Dataset ecommerce { label: 'Ecommerce' @@ -74,6 +80,10 @@ Dataset ecommerce { }, relationship(merchants.admin_id - ecommerce_users.id, true), relationship(ecommerce_orders.created_date > reporting.calendar.date, false), + RelationshipConfig { + rel: orders_in_year + active: true + }, ] dimension item_value { diff --git a/converters/holistics/tests/test_snapshots.py b/converters/holistics/tests/test_snapshots.py index 335f229e6..bb728a695 100644 --- a/converters/holistics/tests/test_snapshots.py +++ b/converters/holistics/tests/test_snapshots.py @@ -112,6 +112,7 @@ def test_the_forward_direction_reports_the_losses_it_declares(): aml_to_ossie.ISSUE_QUERY_SOURCE, aml_to_ossie.ISSUE_UNRESOLVED_QUERY, aml_to_ossie.ISSUE_RELATIONSHIP_FILTER, + aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP, } @@ -666,21 +667,23 @@ def test_a_non_default_relationship_property_is_written_back(): @pytest.mark.parametrize("kind,aml_type", MATCH_RELATIONSHIPS.items(), ids=list(MATCH_RELATIONSHIPS)) -def test_a_match_relationship_is_dropped_with_an_error(kind, aml_type): - """A range or many-to-many join carries `match`, not column pairs. +def test_a_match_relationship_is_stashed_and_written_back(kind, aml_type): + """A range or many-to-many join states its condition as one AQL predicate. - Ossie encodes a relationship as `from_columns` and `to_columns`, and an AQL - predicate spanning both models has no form there, so the edge is dropped - and the two datasets read as unrelated. + Ossie encodes a relationship as `from_columns` and `to_columns`, which + cannot hold that, so the payload travels in the document stash and the + reverse path rebuilds the declaration from it. The ecommerce fixture covers + many-to-many end to end; this reaches `range` as well. """ payload = json.loads(snapshots.ECOMMERCE_COMPILED.read_text(encoding="utf-8")) - before = len(payload["relationships"]) - payload["relationships"].append( + payload["relationships"] = [ { "__type__": "RelationshipConfig", "active": True, "rel": { "__type__": aml_type, + "__fqn__": "items_near_products", + "name": "items_near_products", "type": kind, "from": {"__type__": "FieldRef", "model": "order_items", "field": "id"}, "to": {"__type__": "FieldRef", "model": "products", "field": "id"}, @@ -691,15 +694,20 @@ def test_a_match_relationship_is_dropped_with_an_error(kind, aml_type): }, }, } - ) - result = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) + ] + forward = aml_to_ossie.convert(payload, sql_dialect=snapshots.SQL_DIALECT) raised = [ - i - for i in result.issues.issues - if i.code == aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP + i for i in forward.issues.issues if i.code == aml_to_ossie.ISSUE_UNSUPPORTED_RELATIONSHIP ] assert len(raised) == 1 assert kind in raised[0].message - assert raised[0].severity is Severity.ERROR - assert len(result.model["relationships"]) == before + assert raised[0].severity is Severity.WARNING + assert not forward.model.get("relationships") + + reverse = ossie_to_aml.convert(forward.model, snapshots.SQL_DIALECT) + text = next(t for name, t in reverse.files if name.endswith(".dataset.aml")) + assert f"{aml_type} items_near_products {{" in text + assert f"type: '{kind}'" in text + assert "match: @aql order_items.id >= products.id;;" in text + assert "rel: items_near_products" in text From b4e8dd84030e5a2f7a4798d0e9cd3228dfb60484 Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 11:10:34 +0700 Subject: [PATCH 11/12] fixup! feat(holistics): add the Holistics AML converter Drop the cross-vendor sweep. It converted every Ossie document under `converters/` and `examples/` and compiled each result, which tied this suite to fixtures nobody here owns. Upstream folded `converters/hex` into `converters/databricks/python` and the sweep's floor failed, with nothing in this converter having changed. Its value was in what it found rather than in running forever, and both findings now have their own test: a dataset sharing the semantic model's name, which AML rejects as a duplicate, and an `aggregation_type` whose wrapper does not enclose the whole expression, where stripping it would leave a fragment. Neither was reachable from the fixtures, so both would have gone untested when the file went. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- converters/holistics/README.md | 2 - converters/holistics/docs/reverse.md | 7 +- .../holistics/tests/test_cross_vendor.py | 137 ------------------ .../holistics/tests/test_malformed_input.py | 56 +++++++ 4 files changed, 59 insertions(+), 143 deletions(-) delete mode 100644 converters/holistics/tests/test_cross_vendor.py diff --git a/converters/holistics/README.md b/converters/holistics/README.md index 3d25c8d35..2f2da9403 100644 --- a/converters/holistics/README.md +++ b/converters/holistics/README.md @@ -151,8 +151,6 @@ These run the Holistics CLI: - the committed compiled JSON against what `holistics aml compile` produces from the committed AML source - a round trip: compile the generated AML, convert that back to Ossie, and compare it with the document `to-aml` started from. For `ecommerce` the two are byte-identical. -- `to-aml` over every other Ossie document in this repository, each one compiled. See - `tests/test_cross_vendor.py`, which found two crashes the two fixtures missed. These need nothing beyond the package: diff --git a/converters/holistics/docs/reverse.md b/converters/holistics/docs/reverse.md index 7331aa3dd..a7a18b09d 100644 --- a/converters/holistics/docs/reverse.md +++ b/converters/holistics/docs/reverse.md @@ -194,10 +194,9 @@ The generated AML is checkable without a Holistics account: holistics aml validate <out-dir>/<name>.dataset.aml -r <out-dir> ``` -That parses every generated file and type-checks the AQL. `tests/test_snapshots.py` runs it over both -fixtures and asserts a clean exit, and `tests/test_cross_vendor.py` runs it over every other Ossie document in -this repository. "The output is valid AML" is measured rather than asserted. CI installs the CLI, so those -tests run there too. +That parses every generated file and type-checks the AQL. `tests/test_snapshots.py` runs it over every +fixture and asserts a clean exit, so "the output is valid AML" is measured rather than asserted. CI installs +the CLI, so those tests run there too. Compiling goes further than validating. `test_the_round_trip_returns_the_same_ossie_document` runs `holistics aml compile` over the generated AML and converts the result back to Ossie, then compares it with diff --git a/converters/holistics/tests/test_cross_vendor.py b/converters/holistics/tests/test_cross_vendor.py deleted file mode 100644 index 3627fe6c1..000000000 --- a/converters/holistics/tests/test_cross_vendor.py +++ /dev/null @@ -1,137 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Every Ossie document in this repository, converted to AML. - -The reverse path's whole job is reading documents this converter did not write, -and the snapshot fixtures cannot cover that: one of them this converter produced -and the other is a single example. The sweep found two defects the fixtures -missed, both of them crashes rather than bad output. - -`examples/flights.semantic_model.yaml` is named `Flights semantic model`. Ossie -allows any non-empty string, and AML allows letters, digits and underscore, so -the conversion raised instead of renaming. - -Three fixtures use one name for the semantic model and for a dataset inside it, -including `converters/cube/tests/fixtures/databricks_ossie.yaml`, where both are -`orders`. Ossie keeps the two in separate namespaces and AML does not, so the -generated AML failed to compile with "Duplicated name 'orders'". - -Adding a fixture elsewhere in the repository extends this test with no edit -here, which is the point. apache/ossie#352 asked the Solid converter for the -same sweep. -""" -from __future__ import annotations - -import subprocess -from collections import Counter -from pathlib import Path - -import pytest - -import snapshots -from ossie_holistics import _yaml, ossie_to_aml, sqlrefs -from test_snapshots import HOLISTICS_CLI, needs_cli - -#: Any `data_source_name`. These documents have no Holistics connection behind -#: them, and the value only has to be present for the AML to compile. -PROBE_CONNECTION = "probe" - - -def _target_dialect(document) -> str: - """The SQL dialect most of a document's expressions already use. - - An operator converting one of these would name the warehouse the Holistics - connection reads. Picking the document's own majority dialect is the closest - stand-in, and it keeps the sweep testing the conversion rather than testing - sqlglot's rendering. - """ - counts = Counter() - for dataset in document.get("datasets") or []: - for field in dataset.get("fields") or []: - for entry in (field.get("expression") or {}).get("dialects") or []: - counts[entry.get("dialect")] += 1 - for name, _ in counts.most_common(): - if name in sqlrefs.SQLGLOT_DIALECT: - return name - return "ANSI_SQL" - - -def _foreign_ossie_documents() -> list[Path]: - """Every flat Ossie document outside this converter's own directory. - - A flat document carries `version` and `datasets` at the root. That is the - shape `main` moved to, and the heuristic apache/ossie#352 settled on after - the wrapped `semantic_model:` list went away. - """ - found = [] - roots = [snapshots.REPO_ROOT / "converters", snapshots.REPO_ROOT / "examples"] - for root in roots: - for path in sorted(root.rglob("*.yaml")): - if snapshots.CONVERTER_ROOT in path.parents or ".venv" in path.parts: - continue - try: - document = _yaml.load(path.read_text(encoding="utf-8")) - except Exception: # noqa: BLE001 - a malformed sibling fixture is not our failure - continue - if isinstance(document, dict) and document.get("version") and "datasets" in document: - found.append(path) - return found - - -FOREIGN = _foreign_ossie_documents() - - -def _ids(paths: list[Path]) -> list[str]: - return [str(p.relative_to(snapshots.REPO_ROOT)) for p in paths] - - -def test_the_sweep_finds_documents(): - """A heuristic that silently matches nothing would pass every test below.""" - assert len(FOREIGN) >= 20, _ids(FOREIGN) - - -@pytest.mark.parametrize("path", FOREIGN, ids=_ids(FOREIGN)) -def test_every_ossie_document_converts(path): - document = _yaml.load(path.read_text(encoding="utf-8")) - result = ossie_to_aml.convert( - document, _target_dialect(document), data_source_name=PROBE_CONNECTION - ) - assert result.files - assert sum(1 for name, _ in result.files if name.endswith(".dataset.aml")) == 1 - - -@needs_cli -@pytest.mark.parametrize("path", FOREIGN, ids=_ids(FOREIGN)) -def test_every_converted_document_compiles(path, tmp_path): - document = _yaml.load(path.read_text(encoding="utf-8")) - result = ossie_to_aml.convert( - document, _target_dialect(document), data_source_name=PROBE_CONNECTION - ) - for name, text in result.files: - target = tmp_path / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(text, encoding="utf-8") - dataset_file = next(name for name, _ in result.files if name.endswith(".dataset.aml")) - completed = subprocess.run( - [HOLISTICS_CLI, "aml", "validate", dataset_file, "-r", "."], - cwd=tmp_path, - capture_output=True, - text=True, - ) - assert completed.returncode == 0, completed.stdout + completed.stderr - assert "No validation errors found" in completed.stdout diff --git a/converters/holistics/tests/test_malformed_input.py b/converters/holistics/tests/test_malformed_input.py index a8f09f7e6..3752c5b78 100644 --- a/converters/holistics/tests/test_malformed_input.py +++ b/converters/holistics/tests/test_malformed_input.py @@ -124,3 +124,59 @@ def test_relationship_columns_must_pair_up(from_columns, to_columns): ossie_to_aml.convert( joined(from_columns, to_columns), "ANSI_SQL", data_source_name="probe" ) + + +def test_a_dataset_sharing_the_model_name_is_moved_aside(): + """Ossie keeps the two in separate namespaces and AML has one. + + A document naming both the semantic model and a dataset `orders` would + emit `Dataset orders` beside `Model orders`, which AML rejects as a + duplicate. The dataset is the safe side to move, since nothing references + it by name. + """ + payload = document( + name="orders", + datasets=[{"name": "orders", "source": "db.orders", "fields": [ + {"name": "id", "expression": EXPRESSION} + ]}], + ) + result = ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + raised = [i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_DATASET_RENAMED] + assert len(raised) == 1 + text = next(t for name, t in result.files if name.endswith(".dataset.aml")) + assert "Dataset orders_dataset {" in text + assert "Model orders {" in next(t for name, t in result.files if name.endswith("orders.model.aml")) + + +def test_an_aggregation_that_does_not_wrap_the_whole_expression_is_kept_whole(): + """`SUM(a) + SUM(b)` starts with `SUM(` and ends with `)` and is not a sum. + + Stripping the wrapper would leave `a) + SUM(b` and re-aggregating it would + double the aggregate, so the expression is carried whole under `custom`. + """ + payload = document( + datasets=[{"name": "orders", "source": "db.orders", "fields": [ + {"name": "a", "expression": EXPRESSION}, + {"name": "b", "expression": EXPRESSION}, + ]}], + metrics=[{ + "name": "total", + "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(orders.a) + SUM(orders.b)"} + ]}, + "custom_extensions": [{ + "vendor_name": "HOLISTICS", + "data": '{"_v": 1, "model": "orders", "aml_name": "total", "aggregation_type": "sum"}', + }], + }], + ) + result = ossie_to_aml.convert(payload, "ANSI_SQL", data_source_name="probe") + + raised = [ + i for i in result.issues.issues if i.code == ossie_to_aml.ISSUE_AGGREGATION_NOT_STRIPPED + ] + assert len(raised) == 1 + text = next(t for name, t in result.files if name.endswith("orders.model.aml")) + assert "aggregation_type: 'custom'" in text + assert "SUM({{ a }}) + SUM({{ b }})" in text From 848471a38ab321d4290668c58f8b9e4e4c9a855a Mon Sep 17 00:00:00 2001 From: Huynh Tan <danhuynhdev@gmail.com> Date: Fri, 9 Oct 2026 11:21:57 +0700 Subject: [PATCH 12/12] fixup! feat(core-spec): register HOLISTICS_AQL as a dialect Teach the Hex converter the new dialect member. `converters/hex` matches exhaustively over `OssieDialect` and closes with `assert_never`, so a new member is a type error rather than a silent fallthrough. `ty check` reports it at `load_ossie_dialect_expression.py:118`. AQL resolves through a dataset's relationship graph and sqlglot cannot parse it, so it joins `MDX`, `MAQL`, `THOUGHTSPOT`, `SIGMA` and `DAX` as a language with no engine to read it with. `_DIALECT_NAME_MAP` in `convert_ossie_dialect.py` is typed over the whole enum and gets the same treatment. The `--dialect` CLI choices come from the enum, so the export CLI's snapshot gains `holistics_aql` between `dax` and `ossie_sql_2026`, where the member sits. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> --- .../hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py | 1 + .../src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py | 1 + converters/hex/tests/cli/test_cli_export.py | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py b/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py index 122d138b8..0557a4fa6 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/convert_ossie_dialect.py @@ -36,6 +36,7 @@ OssieDialect.SIGMA: None, OssieDialect.THOUGHTSPOT: None, OssieDialect.DAX: None, + OssieDialect.HOLISTICS_AQL: None, } diff --git a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py index 18b521267..65f33d404 100644 --- a/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py +++ b/converters/hex/src/ossie_hex/ossie_to_hex/load_ossie_dialect_expression.py @@ -111,6 +111,7 @@ def _to_sqlglot_dialect(ossie_dialect: OssieDialect) -> SQLGlotDialect | None: OssieDialect.THOUGHTSPOT, OssieDialect.SIGMA, OssieDialect.DAX, + OssieDialect.HOLISTICS_AQL, ): # There's no parallel for these read = None diff --git a/converters/hex/tests/cli/test_cli_export.py b/converters/hex/tests/cli/test_cli_export.py index dbcc425f0..4f37b1c08 100644 --- a/converters/hex/tests/cli/test_cli_export.py +++ b/converters/hex/tests/cli/test_cli_export.py @@ -225,5 +225,5 @@ def test_invalid_dialect(capsys: pytest.CaptureFixture[str], tmp_path: Path) -> message = message + separator + choices.replace("'", "") assert message == snapshot("""\ usage: ossie-hex export [-h] -i INPUT [-o OUTPUT] [-d DIALECT] [-v] -ossie-hex export: error: argument -d/--dialect: invalid choice: 'invalid' (choose from ansi_sql, snowflake, mdx, maql, tableau, databricks, bigquery, sigma, thoughtspot, dax, ossie_sql_2026) +ossie-hex export: error: argument -d/--dialect: invalid choice: 'invalid' (choose from ansi_sql, snowflake, mdx, maql, tableau, databricks, bigquery, sigma, thoughtspot, dax, holistics_aql, ossie_sql_2026) """)