diff --git a/.github/labeler.yml b/.github/labeler.yml index 14693687..c3e5fa4a 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -61,6 +61,10 @@ converter/wisdom: - changed-files: - any-glob-to-any-file: 'converters/wisdom/**' +converter/sap-csn: + - changed-files: + - any-glob-to-any-file: 'converters/sap-csn/**' + converter/sigma: - changed-files: - any-glob-to-any-file: 'converters/sigma/**' diff --git a/.github/workflows/converter-sap-csn-ci.yml b/.github/workflows/converter-sap-csn-ci.yml new file mode 100644 index 00000000..e5413c0e --- /dev/null +++ b/.github/workflows/converter-sap-csn-ci.yml @@ -0,0 +1,73 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters SAP CSN CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/sap-csn/**' + - 'core-spec/**' + - 'examples/**' + - 'python/**' + - '.github/workflows/converter-sap-csn-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/sap-csn/**' + - 'core-spec/**' + - 'examples/**' + - 'python/**' + - '.github/workflows/converter-sap-csn-ci.yml' + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + version: "0.12.19" + + - name: Sync dependencies + working-directory: converters/sap-csn + run: | + uv sync + + - name: Unit Tests + working-directory: converters/sap-csn + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 8c76aded..83f16234 100644 --- a/converters/README.md +++ b/converters/README.md @@ -91,6 +91,7 @@ The Ossie specification currently defines extensions for the following vendors: | `CUBE` | Cube data model | | `HEX` | Hex semantic project | | `HOLISTICS` | Holistics AML dataset | +| `SAP_CSN` | SAP CSN (Core Schema Notation) document | [`wisdom-semantic-view`](wisdom-semantic-view/README.md) is a separate, fixture-scoped spoke. It reads one Snowflake semantic view, writes one Ossie document, writes a Wisdom domain-export JSON (format `1.0`), and reverses that path. It keeps metric synonyms and `module_custom_instructions.sql_generation`. It does not call `converters/snowflake` (Cortex Analyst export only) or `converters/wisdom` (that spoke drops synonyms and field or metric `ai_context`). diff --git a/converters/sap-csn/README.md b/converters/sap-csn/README.md new file mode 100644 index 00000000..70acd62e --- /dev/null +++ b/converters/sap-csn/README.md @@ -0,0 +1,113 @@ + + +# Ossie ⇄ SAP CSN converter + +Converts between a SAP **CSN** document (Core Schema Notation, the JSON that +`cds compile --to csn` writes and SAP Datasphere exchanges) and an Apache Ossie +semantic model, in both directions. It works offline on files: it does not call +SAP. + +``` +ossie-sap-csn csn-to-ossie -i model.csn.json -o semantic_model.yaml [--name NAME] +ossie-sap-csn ossie-to-csn -i semantic_model.yaml -o model.csn.json +``` + +Warnings (information the target cannot hold, or that was set aside) go to stderr, +one per line, as `[WARNING] : — `. Input that cannot be +converted at all exits with status 1 and a message. + +``` +uv sync +uv run pytest +``` + +## CSN → Ossie + +| CSN | Ossie | +|-----|-------| +| `entity` (table or view) | dataset; `source` is `@cds.persistence.name`, else the CSN name; the name drops the namespace unless two entities would collide | +| element | field; `type` resolves through user-defined types to a CDS built-in, then to an Ossie `datatype` | +| `key: true` | `primary_key` | +| `@assert.unique.*` | `unique_keys` | +| calculated element (`value`) | field expression, translated from CXN to ANSI SQL | +| `Association` / `Composition` with a target in the document | relationship, from the owning entity to the target | +| managed association | relationship, plus a field for each generated foreign-key column (`_`) | +| `@Aggregation.default` (`SUM`, `MIN`, `MAX`, `AVG`, `COUNT`, `COUNT_DISTINCT`) | metric over the element | +| `@Aggregation.default` + `@Ossie.metric: true` | metric over the element's expression; no field | +| `@EndUserText.label`, `@Common.Label` | `label` | +| `doc`, `@EndUserText.quickInfo`, `@Common.QuickInfo` | `description` | +| a temporal type, or `@Semantics.businessDate` / `calendar` / `fiscal` | `dimension.is_time: true` | + +Everything else — remaining annotations, `length`/`precision`/`scale`, the `query` +or `projection` of a view, types, services, actions and the root keys — is kept in a +`SAP_CSN` custom extension so the document exports back unchanged. + +## Ossie → CSN + +| Ossie | CSN | +|-------|-----| +| dataset | `entity` named by the `SAP_CSN` extension, else the dataset name | +| field | element; `datatype` → `cds.String`, `Integer`, `Decimal`, `Double`, `Boolean`, `Date`, `Time`, `Timestamp` | +| field expression other than its own column | calculated element (`value`), translated from SQL to CXN | +| relationship | association on the `from` entity: managed (`keys`) when its foreign-key fields were generated by one, otherwise an `on` condition. A composite key becomes an `and` of equalities | +| metric, one aggregate over a column of the same name | `@Aggregation.default` on that element | +| metric, one aggregate over an expression | derived measure element (`value` + `@Aggregation.default` + `@Ossie.metric`) | +| `unique_keys` | `@assert.unique.` | + +A derived measure may reach other datasets through relationships (`SUM(orders.qty * book.price)`), +because a relationship runs from the many side to the one side and so cannot fan out the rows +being aggregated. + +### What CSN has no place for + +* **Metrics CSN cannot express** — `COUNT(*)`, ratios of two aggregates, window functions, + expressions over datasets no relationship connects — are written to `meta.ossie.metrics` on the + CSN root and reported as `METRIC_NOT_MODELED`. CSN consumers ignore `meta.ossie`; this converter + reads it back. +* **AI context and other vendors' custom extensions** ride in `@Ossie.aiContext` and + `@Ossie.customExtensions` annotations. Model-level ones go to `meta.ossie.model`. +* **A field expression CXN cannot hold** (a window function, a subquery) is exported as a plain + column with the original text in `@Ossie.sql`, and reported as `EXPRESSION_NOT_TRANSLATABLE`. +* **A key naming a column the dataset does not declare** is carried whole in `@Ossie.primaryKey` + or `@Ossie.uniqueKeys`, and reported. +* **A field with no `datatype`** is exported as `cds.String`, because CSN requires a type, and + reported as `ASSUMED_DATATYPE`. + +Only the `ANSI_SQL` dialect is read. A field or metric that has only another dialect is reported +as `DIALECT_UNAVAILABLE` rather than guessed at. + +## Limitations + +* **CAP and Datasphere CSN.** The converter reads the annotations both write for analytics + (`@Aggregation.default`, `@Analytics.dataCategory`, `@EndUserText.label`). It does not interpret + Datasphere's analytic-model objects (fact sources, attribute and measure groups, variables) as + such; they arrive as plain entities with their annotations preserved. +* **Not translated:** structured and array-typed elements, virtual elements, `$self` backlink + associations (the forward association already is the relationship), associations whose target is + not in the document, `on` conditions other than a conjunction of column equalities, and + expressions with subqueries, `$session` or `$user`. Each is kept in the `SAP_CSN` extension and + reported, never dropped silently. +* **Views.** A view's elements become fields and its `query` is preserved, but the converter does + not interpret the query; the dataset `source` is the view's own name. +* **Export validity.** The converter does not run the CDS compiler. A CSN it writes follows the + CSN schema but has not been compiled with `cds`. +* **Conversion is not a CDS or SQL migration.** Expressions are translated between SQL and CXN + only for the constructs both express: references, literals, arithmetic, comparison, `AND`/`OR`/`NOT`, + `IN`, `LIKE`, `BETWEEN`, `IS NULL`, `CASE`, `CAST` and plain function calls. diff --git a/converters/sap-csn/pyproject.toml b/converters/sap-csn/pyproject.toml new file mode 100644 index 00000000..3ce1e785 --- /dev/null +++ b/converters/sap-csn/pyproject.toml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "pytest>=8.0", + # So the core-spec schema validation in the tests runs rather than skipping; + # the converter itself does not need it. + "jsonschema>=4.0", +] + +[project] +name = "apache-ossie-sap-csn" +version = "0.2.0.dev0" +description = "SAP CSN (Core Schema Notation) <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "SAP", + "CSN", + "CDS", + "Datasphere", +] +dependencies = [ + "apache-ossie>=0.2.0.dev0", + "PyYAML>=6.0", + # Translating calculated-element and measure expressions between SQL and CXN. + "sqlglot>=20.0", +] + +[project.scripts] +ossie-sap-csn = "ossie_sap_csn.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_sap_csn"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] + +# apache-ossie is not yet published to PyPI; resolve it from the in-repo +# package for now. Remove this block once apache-ossie is published to PyPI. +[tool.uv.sources] +apache-ossie = { path = "../../python", editable = true } diff --git a/converters/sap-csn/src/ossie_sap_csn/__init__.py b/converters/sap-csn/src/ossie_sap_csn/__init__.py new file mode 100644 index 00000000..17c82750 --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/__init__.py @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from ossie_sap_csn.csn_to_ossie import CsnToOssieConverter +from ossie_sap_csn.ossie_to_csn import OssieToCsnConverter + +__all__ = ["CsnToOssieConverter", "OssieToCsnConverter"] diff --git a/converters/sap-csn/src/ossie_sap_csn/_common.py b/converters/sap-csn/src/ossie_sap_csn/_common.py new file mode 100644 index 00000000..b63b5f62 --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/_common.py @@ -0,0 +1,108 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Names and helpers shared by both conversion directions.""" + +import json +from typing import Any, Optional + +from ossie import OssieCustomExtension + +from ossie_sap_csn.converter_issues import ConverterError + +VENDOR = "SAP_CSN" + +# Standard SAP annotations this converter reads and writes. +LABEL_KEYS = ("@EndUserText.label", "@Common.Label") +QUICKINFO_KEYS = ("@EndUserText.quickInfo", "@Common.QuickInfo") +AGGREGATION_KEY = "@Aggregation.default" +DATA_CATEGORY_KEY = "@Analytics.dataCategory" +UNIQUE_PREFIX = "@assert.unique." +TEMPORAL_PREFIXES = ("@Semantics.businessDate", "@Semantics.calendar", "@Semantics.fiscal") + +# Annotations this converter invents, so Ossie content that CSN has no home for +# survives a round trip. CSN allows arbitrary annotations; consumers ignore these. +OSSIE_AI_CONTEXT = "@Ossie.aiContext" +OSSIE_EXTENSIONS = "@Ossie.customExtensions" +OSSIE_IS_TIME = "@Ossie.isTime" +OSSIE_METRIC = "@Ossie.metric" +OSSIE_SQL = "@Ossie.sql" +# The full key lists, written only when a key names a column CSN has no element for. +OSSIE_PRIMARY_KEY = "@Ossie.primaryKey" +OSSIE_UNIQUE_KEYS = "@Ossie.uniqueKeys" +OSSIE_ANNOTATIONS = frozenset( + { + OSSIE_AI_CONTEXT, + OSSIE_EXTENSIONS, + OSSIE_IS_TIME, + OSSIE_METRIC, + OSSIE_SQL, + OSSIE_PRIMARY_KEY, + OSSIE_UNIQUE_KEYS, + } +) + +# `meta.ossie` on the CSN root holds what has no element-level home. +META_KEY = "ossie" + +# Aggregations `@Aggregation.default` can carry that Ossie metrics express. +AGGREGATIONS = ("SUM", "MIN", "MAX", "AVG", "COUNT", "COUNT_DISTINCT") + + +def enum_value(value: Any) -> Optional[str]: + """The symbol of a CSN enum annotation value (`{"#": "SUM"}` or `"#SUM"`).""" + if isinstance(value, dict) and isinstance(value.get("#"), str): + return value["#"] + if isinstance(value, str) and value.startswith("#"): + return value[1:] + return None + + +def enum_annotation(symbol: str) -> dict[str, str]: + return {"#": symbol} + + +def vendor_extension(data: dict[str, Any]) -> OssieCustomExtension: + return OssieCustomExtension(vendor_name=VENDOR, data=json.dumps(data, sort_keys=True)) + + +def read_vendor_extension(extensions: Optional[list[OssieCustomExtension]], where: str) -> dict[str, Any]: + """The decoded SAP_CSN extension payload, or ``{}`` when there is none.""" + for extension in extensions or []: + if extension.vendor_name == VENDOR: + try: + data = json.loads(extension.data) + except json.JSONDecodeError as exc: + raise ConverterError(f"{where}: SAP_CSN custom extension is not valid JSON: {exc}") from exc + if not isinstance(data, dict): + raise ConverterError(f"{where}: SAP_CSN custom extension must be a JSON object") + return data + return {} + + +def foreign_extensions(extensions: Optional[list[OssieCustomExtension]]) -> list[dict[str, str]]: + """Extensions that belong to other vendors, to carry through untouched.""" + return [e.model_dump() for e in extensions or [] if e.vendor_name != VENDOR] + + +def split_annotations(obj: dict[str, Any], consumed: set[str]) -> dict[str, Any]: + """Annotations on ``obj`` that the converter did not map to an Ossie property.""" + return { + k: v + for k, v in obj.items() + if k.startswith("@") and k not in consumed and k not in OSSIE_ANNOTATIONS + } diff --git a/converters/sap-csn/src/ossie_sap_csn/cli.py b/converters/sap-csn/src/ossie_sap_csn/cli.py new file mode 100644 index 00000000..c505a12f --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/cli.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""CLI entry point for the ossie-sap-csn converter. + +Usage: + ossie-sap-csn csn-to-ossie -i model.csn.json -o semantic_model.yaml [--name NAME] + ossie-sap-csn ossie-to-csn -i semantic_model.yaml -o model.csn.json +""" + +import argparse +import json +import sys +from pathlib import Path + +import yaml +from pydantic import ValidationError + +from ossie import OssieDocument +from ossie_sap_csn.converter_issues import ConverterError +from ossie_sap_csn.csn_to_ossie import CsnToOssieConverter +from ossie_sap_csn.ossie_to_csn import OssieToCsnConverter + + +def _report(issues: list) -> None: + for issue in issues: + print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} — {issue.detail}", file=sys.stderr) + + +def _cmd_csn_to_ossie(args: argparse.Namespace) -> None: + try: + csn = json.loads(Path(args.input).read_text()) + except json.JSONDecodeError as exc: + raise ConverterError(f"{args.input} is not valid JSON: {exc}") from exc + result = CsnToOssieConverter().convert(csn, name=args.name) + _report(result.issues) + Path(args.output).write_text(result.output.to_ossie_yaml()) + print(f"Written to {args.output}", file=sys.stderr) + + +def _cmd_ossie_to_csn(args: argparse.Namespace) -> None: + raw = yaml.safe_load(Path(args.input).read_text()) + try: + document = OssieDocument.model_validate(raw) + except ValidationError as exc: + raise ConverterError(f"{args.input} is not a valid Ossie document:\n{exc}") from exc + result = OssieToCsnConverter().convert(document) + _report(result.issues) + Path(args.output).write_text(json.dumps(result.output, indent=2) + "\n") + print(f"Written to {args.output}", file=sys.stderr) + + +def main() -> None: + parser = argparse.ArgumentParser( + prog="ossie-sap-csn", + description="Convert between SAP CSN (Core Schema Notation) JSON and Ossie YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + to_ossie = subparsers.add_parser("csn-to-ossie", help="Convert SAP CSN JSON → Ossie YAML") + to_ossie.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to CSN JSON") + to_ossie.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output Ossie YAML") + to_ossie.add_argument("--name", metavar="NAME", help="Name of the Ossie semantic model (CSN has no model name)") + + to_csn = subparsers.add_parser("ossie-to-csn", help="Convert Ossie YAML → SAP CSN JSON") + to_csn.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to Ossie YAML") + to_csn.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output CSN JSON") + + args = parser.parse_args() + try: + if args.command == "csn-to-ossie": + _cmd_csn_to_ossie(args) + else: + _cmd_ossie_to_csn(args) + except ConverterError as exc: + print(f"error: {exc}", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/converters/sap-csn/src/ossie_sap_csn/converter_issues.py b/converters/sap-csn/src/ossie_sap_csn/converter_issues.py new file mode 100644 index 00000000..31f09e83 --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/converter_issues.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass +from enum import Enum +from typing import Generic, List, TypeVar + + +class ConverterIssueType(Enum): + """Identifies the kind of information loss or uncertainty during conversion.""" + + UNSUPPORTED_DEFINITION_KIND = "UNSUPPORTED_DEFINITION_KIND" + EXPRESSION_NOT_TRANSLATABLE = "EXPRESSION_NOT_TRANSLATABLE" + ELEMENT_NOT_MODELED = "ELEMENT_NOT_MODELED" + RELATIONSHIP_DROPPED = "RELATIONSHIP_DROPPED" + RELATIONSHIP_COLUMN_UNRESOLVED = "RELATIONSHIP_COLUMN_UNRESOLVED" + METRIC_NOT_MODELED = "METRIC_NOT_MODELED" + OPAQUE_DATATYPE = "OPAQUE_DATATYPE" + ASSUMED_DATATYPE = "ASSUMED_DATATYPE" + PRIMARY_KEY_COLUMN_UNRESOLVED = "PRIMARY_KEY_COLUMN_UNRESOLVED" + UNIQUE_KEY_COLUMN_UNRESOLVED = "UNIQUE_KEY_COLUMN_UNRESOLVED" + DIALECT_UNAVAILABLE = "DIALECT_UNAVAILABLE" + + +class ConverterError(Exception): + """Raised when the input cannot be converted at all, as opposed to a partial, + lossy conversion that a :class:`ConverterIssue` can describe.""" + + +@dataclass(frozen=True) +class ConverterIssue: + """Records a single instance of information loss or uncertainty during conversion.""" + + issue_type: ConverterIssueType + element_name: str + detail: str = "" + + +T = TypeVar("T") + + +@dataclass(frozen=True) +class ConverterResult(Generic[T]): + """Return value of a converter's convert() method, pairing the output with any conversion issues.""" + + output: T + issues: List[ConverterIssue] diff --git a/converters/sap-csn/src/ossie_sap_csn/csn_to_ossie.py b/converters/sap-csn/src/ossie_sap_csn/csn_to_ossie.py new file mode 100644 index 00000000..6a5137fb --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/csn_to_ossie.py @@ -0,0 +1,650 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a SAP CSN document (Core Schema Notation, JSON) into an Ossie semantic model. + +Mapping: + +* an ``entity`` (table or view) becomes a dataset, and its elements become fields; +* a ``key`` element becomes part of the primary key; +* an association or composition with a resolvable target becomes a relationship, + and a managed association's generated foreign-key columns become fields; +* an element annotated ``@Aggregation.default`` (``SUM``, ``MIN``, ``MAX``, ``AVG``, + ``COUNT``, ``COUNT_DISTINCT``) becomes a metric; +* ``@EndUserText.label`` and ``doc`` become the label and description. + +Whatever has no Ossie equivalent travels in ``custom_extensions`` under the +``SAP_CSN`` vendor, so the document can be exported back without loss. +""" + +from typing import Any, Optional + +from ossie import ( + OssieCustomExtension, + OssieDataset, + OssieDialect, + OssieDialectExpression, + OssieDimension, + OssieDocument, + OssieExpression, + OssieField, + OssieMetric, + OssieRelationship, +) +from sqlglot import exp + +from ossie_sap_csn import _common as c +from ossie_sap_csn.converter_issues import ( + ConverterError, + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_sap_csn.cxn import UnsupportedExpression, cxn_to_sql, parse_sql, quote_identifier +from ossie_sap_csn.datatypes import ASSOCIATION_TYPES, builtin_type, ossie_datatype + +# Entity keys that map to Ossie properties; every other key travels verbatim. +_MAPPED_ENTITY_KEYS = frozenset({"kind", "elements", "doc"}) +# Element keys that map to Ossie properties or are rebuilt on export. +_MAPPED_ELEMENT_KEYS = frozenset({"type", "key", "value", "doc", "target", "on", "keys"}) +# Definition kinds worth reporting when they are set aside (structural kinds are not). +_REPORTED_KINDS = frozenset({"aspect", "action", "function", "event", "annotation"}) +_STRUCTURAL_KINDS = frozenset({"context", "service", "type"}) + + +def _extensions(own: dict[str, Any], obj: dict[str, Any]) -> Optional[list[Any]]: + extensions = [] + if own: + extensions.append(c.vendor_extension(own)) + for foreign in obj.get(c.OSSIE_EXTENSIONS) or []: + extensions.append(OssieCustomExtension.model_validate(foreign)) + return extensions or None + + +def _first(obj: dict[str, Any], keys: tuple[str, ...]) -> tuple[Optional[str], Optional[str]]: + """First string-valued annotation among ``keys``, as ``(key, value)``.""" + for key in keys: + if isinstance(obj.get(key), str): + return key, obj[key] + return None, None + + +def _qualify(sql: str, table: str) -> str: + """Prefix every unqualified column of ``sql`` with ``table``.""" + tree = parse_sql(sql) + for column in tree.find_all(exp.Column): + if not column.table: + column.set("table", exp.to_identifier(table, quoted=quote_identifier(table) != table)) + return tree.sql() + + +class _Entities: + """Entity names and the datasets they become.""" + + def __init__(self, definitions: dict[str, Any]) -> None: + self.csn_names = [n for n, d in definitions.items() if isinstance(d, dict) and d.get("kind") == "entity"] + shorts = [n.rsplit(".", 1)[-1] for n in self.csn_names] + unique = len(set(shorts)) == len(shorts) + self.dataset_name: dict[str, str] = {} + used: set[str] = set() + for n, s in zip(self.csn_names, shorts): + base = s if unique else n.replace(".", "_") + candidate, suffix = base, 1 + while candidate in used: # flattening can collide too (`foo.bar` and `foo_bar`) + suffix += 1 + candidate = f"{base}_{suffix}" + used.add(candidate) + self.dataset_name[n] = candidate + + def __contains__(self, csn_name: object) -> bool: + return csn_name in self.dataset_name + + +def _key_elements(definitions: dict[str, Any], entity: str) -> list[tuple[str, Optional[str]]]: + """Names and built-in types of an entity's non-association key elements.""" + result = [] + for name, element in (definitions[entity].get("elements") or {}).items(): + if isinstance(element, dict) and element.get("key"): + builtin = builtin_type(element.get("type"), definitions) + if builtin not in ASSOCIATION_TYPES: + result.append((name, builtin)) + return result + + +def _flatten_and(tokens: list[Any]) -> list[Any]: + """Unwrap single nested ``xpr`` groups so an ``on`` condition reads as one token list.""" + while len(tokens) == 1 and isinstance(tokens[0], dict) and "xpr" in tokens[0]: + tokens = tokens[0]["xpr"] + return tokens + + +def _parse_on(assoc: str, on: list[Any]) -> list[tuple[str, str]]: + """Return ``(from_column, to_column)`` pairs for an ``on`` condition of equalities. + + Raises :class:`UnsupportedExpression` for anything other than an AND of + ``assoc.col = col`` terms. + """ + tokens = _flatten_and(on) + terms: list[list[Any]] = [[]] + for token in tokens: + if isinstance(token, str) and token.lower() == "and": + terms.append([]) + else: + terms[-1].append(token) + pairs = [] + for term in terms: + term = _flatten_and(term) + if len(term) != 3 or term[1] != "=": + raise UnsupportedExpression("the on-condition is not a conjunction of equalities") + refs = [] + for side in (term[0], term[2]): + if not (isinstance(side, dict) and "ref" in side and all(isinstance(s, str) for s in side["ref"])): + raise UnsupportedExpression("the on-condition compares something other than columns") + ref = side["ref"] + if ref[0] in ("$self", "$projection") and len(ref) > 1: + ref = ref[1:] + elif ref == ["$self"]: + raise UnsupportedExpression("a $self backlink has no column pairs of its own") + refs.append(ref) + target_side = [r for r in refs if len(r) == 2 and r[0] == assoc] + local_side = [r for r in refs if len(r) == 1] + if len(target_side) != 1 or len(local_side) != 1: + raise UnsupportedExpression("each equality must pair one association column with one local column") + pairs.append((local_side[0][0], target_side[0][1])) + return pairs + + +class CsnToOssieConverter: + """Converts a parsed CSN document into an :class:`OssieDocument`.""" + + def convert(self, csn: dict[str, Any], name: Optional[str] = None) -> ConverterResult[OssieDocument]: + if not isinstance(csn, dict) or not isinstance(csn.get("definitions"), dict): + raise ConverterError("input is not a CSN document: it has no 'definitions' object") + definitions: dict[str, Any] = csn["definitions"] + issues: list[ConverterIssue] = [] + + entities = _Entities(definitions) + if not entities.csn_names: + raise ConverterError("the CSN document defines no entities; an Ossie model needs at least one dataset") + + meta = csn.get("meta") if isinstance(csn.get("meta"), dict) else {} + ossie_meta = meta.get(c.META_KEY) if isinstance(meta.get(c.META_KEY), dict) else {} + + datasets: list[OssieDataset] = [] + relationships: list[OssieRelationship] = [] + metrics: list[OssieMetric] = [] + measure_candidates: list[dict[str, Any]] = [] + + for csn_name in entities.csn_names: + dataset, entity_relationships, entity_measures = self._entity( + csn_name, definitions[csn_name], definitions, entities, issues + ) + datasets.append(dataset) + relationships.extend(entity_relationships) + measure_candidates.extend(entity_measures) + + # Relationship names are global; prefix the ones two entities both use. + counts: dict[str, int] = {} + for r in relationships: + counts[r.name] = counts.get(r.name, 0) + 1 + if any(n > 1 for n in counts.values()): + relationships = [ + r.model_copy(update={"name": f"{r.from_dataset}_{r.name}"}) if counts[r.name] > 1 else r + for r in relationships + ] + + metrics.extend(self._metrics(measure_candidates)) + for raw in ossie_meta.get("metrics") or []: + metrics.append(OssieMetric.model_validate(raw)) + + other = {n: d for n, d in definitions.items() if n not in entities} + for def_name, definition in other.items(): + kind = definition.get("kind") if isinstance(definition, dict) else None + if kind in _REPORTED_KINDS: + issues.append( + ConverterIssue( + ConverterIssueType.UNSUPPORTED_DEFINITION_KIND, + def_name, + f"CSN definition kind {kind!r} has no equivalent in an Ossie semantic model; " + "preserved verbatim in custom_extensions only.", + ) + ) + + model_data: dict[str, Any] = {} + for key, value in csn.items(): + if key == "definitions": + continue + if key == "meta": + value = {k: v for k, v in value.items() if k != c.META_KEY} if isinstance(value, dict) else value + if not value: + continue + model_data.setdefault("csn", {})[key] = value + if other: + model_data["other_definitions"] = other + + model_info = ossie_meta.get("model") if isinstance(ossie_meta.get("model"), dict) else {} + document = OssieDocument( + name=name or model_info.get("name") or "sap_csn_model", + description=model_info.get("description"), + ai_context=model_info.get("ai_context"), + datasets=datasets, + relationships=relationships or None, + metrics=metrics or None, + custom_extensions=_extensions(model_data, {c.OSSIE_EXTENSIONS: model_info.get("custom_extensions")}), + ) + return ConverterResult(output=document, issues=issues) + + # ---------------------------------------------------------------------------------- + + def _entity( + self, + csn_name: str, + entity: dict[str, Any], + definitions: dict[str, Any], + entities: _Entities, + issues: list[ConverterIssue], + ) -> tuple[OssieDataset, list[OssieRelationship], list[dict[str, Any]]]: + dataset_name = entities.dataset_name[csn_name] + elements: dict[str, Any] = entity.get("elements") or {} + + def resolve_path(path: list[str]) -> str: + head = elements.get(path[0]) + target = head.get("target") if isinstance(head, dict) else None + if len(path) == 2 and target in entities: + return f"{quote_identifier(entities.dataset_name[target])}.{quote_identifier(path[1])}" + raise UnsupportedExpression(f"path expression {'.'.join(path)!r} cannot be resolved") + + fields: list[OssieField] = [] + field_names: set[str] = set() + primary_key: list[str] = [] + unmodeled: dict[str, Any] = {} + measures: list[dict[str, Any]] = [] + assoc_elements: list[tuple[str, dict[str, Any]]] = [] + + for el_name, element in elements.items(): + if not isinstance(element, dict): + continue + builtin = builtin_type(element.get("type"), definitions) + if builtin in ASSOCIATION_TYPES: + assoc_elements.append((el_name, element)) + continue + if element.get("virtual"): + unmodeled[el_name] = element + issues.append( + ConverterIssue( + ConverterIssueType.ELEMENT_NOT_MODELED, + f"{dataset_name}.{el_name}", + "Virtual element is not persisted and so is not a column; preserved in custom_extensions only.", + ) + ) + continue + if builtin is None and ("elements" in element or "items" in element or element.get("type") in definitions): + unmodeled[el_name] = element + issues.append( + ConverterIssue( + ConverterIssueType.ELEMENT_NOT_MODELED, + f"{dataset_name}.{el_name}", + "Structured or array-typed element has no Ossie field equivalent; " + "preserved in custom_extensions only.", + ) + ) + continue + + sql_override = element.get(c.OSSIE_SQL) + if isinstance(sql_override, dict) and "dialect" in sql_override and "expression" in sql_override: + expression_sql: Optional[str] = sql_override["expression"] + dialect = OssieDialect(sql_override["dialect"]) + elif "value" in element: + try: + expression_sql = cxn_to_sql(element["value"], resolve_path) + except UnsupportedExpression as exc: + unmodeled[el_name] = element + issues.append( + ConverterIssue( + ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE, + f"{dataset_name}.{el_name}", + f"Calculated element has no SQL rendering ({exc}); " + "preserved in custom_extensions only.", + ) + ) + continue + dialect = OssieDialect.ANSI_SQL + else: + expression_sql = quote_identifier(el_name) + dialect = OssieDialect.ANSI_SQL + + datatype = ossie_datatype(builtin) + if datatype is None: + datatype = "Opaque" + issues.append( + ConverterIssue( + ConverterIssueType.OPAQUE_DATATYPE, + f"{dataset_name}.{el_name}", + f"CSN type {element.get('type')!r} has no portable Ossie datatype.", + ) + ) + + aggregation = c.enum_value(element.get(c.AGGREGATION_KEY)) + is_measure = aggregation in c.AGGREGATIONS + derived = bool(element.get(c.OSSIE_METRIC)) and is_measure + + consumed: set[str] = set() + label_key, label = _first(element, c.LABEL_KEYS) + if label_key: + consumed.add(label_key) + desc_key, description = ("doc", element["doc"]) if isinstance(element.get("doc"), str) else (None, None) + if description is None: + desc_key, description = _first(element, c.QUICKINFO_KEYS) + if desc_key: + consumed.add(desc_key) + if is_measure: + consumed.add(c.AGGREGATION_KEY) + + own: dict[str, Any] = {"type": element.get("type")} + if label_key and label_key != c.LABEL_KEYS[0]: + own["label_key"] = label_key + if desc_key and desc_key != "doc": + own["description_key"] = desc_key + annotations = c.split_annotations(element, consumed) + if annotations: + own["annotations"] = annotations + native = {k: v for k, v in element.items() if not k.startswith("@") and k not in _MAPPED_ELEMENT_KEYS} + if native: + own["native"] = native + if derived and label: + own["label"] = label + + if is_measure: + measures.append( + { + "dataset": dataset_name, + "entity": csn_name, + "element": el_name, + "aggregation": aggregation, + "aggregation_raw": element[c.AGGREGATION_KEY], + "derived": derived, + "expression": expression_sql, + "datatype": datatype, + "builtin": builtin, + "description": description, + "own": own, + "element_obj": element, + } + ) + if derived: + continue + + is_time = datatype in ("Date", "Time", "DateTime", "DateTimeTz") or any( + k.startswith(c.TEMPORAL_PREFIXES) for k in element + ) + if c.OSSIE_IS_TIME in element: + is_time = bool(element[c.OSSIE_IS_TIME]) + + fields.append( + OssieField( + name=el_name, + expression=OssieExpression(dialects=[OssieDialectExpression(dialect=dialect, expression=expression_sql)]), + dimension=None if is_measure else OssieDimension(is_time=is_time), + label=label, + description=description, + datatype=datatype, + ai_context=element.get(c.OSSIE_AI_CONTEXT), + custom_extensions=_extensions(own, element), + ) + ) + field_names.add(el_name) + if element.get("key"): + primary_key.append(el_name) + + relationships: list[OssieRelationship] = [] + for el_name, element in assoc_elements: + self._association( + dataset_name, csn_name, el_name, element, definitions, entities, fields, field_names, + primary_key, relationships, unmodeled, issues, + ) + + # Keys the entity declares through `key` on an association were added above; + # keep primary-key order as declared on the entity. + declared_order = [n for n in elements if n in primary_key] + [n for n in primary_key if n not in elements] + primary_key = list(dict.fromkeys(declared_order)) + + unique_keys: list[list[str]] = [] + unique_names: list[str] = [] + consumed_unique: set[str] = set() + for key, value in entity.items(): + if not key.startswith(c.UNIQUE_PREFIX): + continue + columns = [i["="] if isinstance(i, dict) and "=" in i else i for i in value] if isinstance(value, list) else [] + if columns and all(isinstance(i, str) and i in field_names for i in columns): + unique_keys.append(columns) + unique_names.append(key[len(c.UNIQUE_PREFIX):]) + consumed_unique.add(key) + else: + issues.append( + ConverterIssue( + ConverterIssueType.UNIQUE_KEY_COLUMN_UNRESOLVED, + f"{dataset_name}.{key[len(c.UNIQUE_PREFIX):]}", + "Unique constraint names something other than modeled columns; " + "preserved in custom_extensions only.", + ) + ) + + # A key that named a column CSN has no element for travels whole; it is the + # authoritative list, and the constraints derived from it above are redundant. + if isinstance(entity.get(c.OSSIE_PRIMARY_KEY), list): + primary_key = list(entity[c.OSSIE_PRIMARY_KEY]) + if isinstance(entity.get(c.OSSIE_UNIQUE_KEYS), list): + unique_keys = [list(k) for k in entity[c.OSSIE_UNIQUE_KEYS]] + + consumed_entity: set[str] = set(consumed_unique) + label_key, label = _first(entity, c.LABEL_KEYS) + description = entity["doc"] if isinstance(entity.get("doc"), str) else None + desc_key: Optional[str] = "doc" if description is not None else None + if description is None: + desc_key, description = _first(entity, c.QUICKINFO_KEYS) + if description is None and label_key: + desc_key, description = label_key, label + if desc_key and desc_key != "doc": + consumed_entity.add(desc_key) + + own_entity: dict[str, Any] = {"entity": csn_name} + if desc_key and desc_key != "doc": + own_entity["description_key"] = desc_key + if unique_names: + own_entity["unique_key_names"] = unique_names + entity_annotations = c.split_annotations(entity, consumed_entity) + if entity_annotations: + own_entity["annotations"] = entity_annotations + native_entity = {k: v for k, v in entity.items() if not k.startswith("@") and k not in _MAPPED_ENTITY_KEYS} + if native_entity: + own_entity["native"] = native_entity + if unmodeled: + own_entity["unmodeled_elements"] = unmodeled + + source = entity.get("@cds.persistence.name") if isinstance(entity.get("@cds.persistence.name"), str) else csn_name + dataset = OssieDataset( + name=dataset_name, + source=source, + primary_key=primary_key or None, + unique_keys=unique_keys or None, + description=description, + ai_context=entity.get(c.OSSIE_AI_CONTEXT), + fields=fields or None, + custom_extensions=_extensions(own_entity, entity), + ) + return dataset, relationships, measures + + def _association( + self, + dataset_name: str, + csn_name: str, + el_name: str, + element: dict[str, Any], + definitions: dict[str, Any], + entities: _Entities, + fields: list[OssieField], + field_names: set[str], + primary_key: list[str], + relationships: list[OssieRelationship], + unmodeled: dict[str, Any], + issues: list[ConverterIssue], + ) -> None: + where = f"{dataset_name}.{el_name}" + + def drop(reason: str) -> None: + unmodeled[el_name] = element + issues.append( + ConverterIssue( + ConverterIssueType.RELATIONSHIP_DROPPED, + where, + f"{reason}; preserved in custom_extensions only.", + ) + ) + + target = element.get("target") + if not isinstance(target, str) or target not in entities: + drop(f"Association target {target!r} is not an entity of this document") + return + + managed = "on" not in element + generated: list[tuple[str, str, Optional[str]]] = [] # (fk_name, to_column, builtin type) + if not managed: + try: + pairs = _parse_on(el_name, element["on"]) + except UnsupportedExpression as exc: + drop(f"The association's on-condition cannot be expressed as column pairs ({exc})") + return + from_columns = [p[0] for p in pairs] + to_columns = [p[1] for p in pairs] + else: + keys = element.get("keys") + target_keys = dict(_key_elements(definitions, target)) + if keys is None: + keys = [{"ref": [k]} for k in target_keys] + if not keys: + drop("Managed association has no keys to join on") + return + from_columns, to_columns = [], [] + for key in keys: + ref = key.get("ref") if isinstance(key, dict) else None + if not ref or not all(isinstance(s, str) for s in ref): + drop("Managed association key is not a plain column reference") + return + to_column = "_".join(ref) + fk = key.get("$generatedFieldName") or f"{el_name}_{key.get('as') or to_column}" + from_columns.append(fk) + to_columns.append(to_column) + generated.append((fk, to_column, target_keys.get(to_column))) + + unresolved = [col for col in from_columns if col not in field_names and col not in {g[0] for g in generated}] + if unresolved: + drop(f"Join columns {unresolved} are not modeled columns of {dataset_name}") + return + + for fk, to_column, builtin in generated: + if fk in field_names: + continue + datatype = ossie_datatype(builtin) + fields.append( + OssieField( + name=fk, + expression=OssieExpression( + dialects=[OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=quote_identifier(fk))] + ), + dimension=OssieDimension(is_time=False), + datatype=datatype, + custom_extensions=[ + c.vendor_extension( + {"generated_foreign_key": True, "association": el_name} + ) + ], + ) + ) + field_names.add(fk) + if element.get("key"): + primary_key.extend(from_columns) + + consumed: set[str] = set() + own: dict[str, Any] = { + "association": el_name, + "entity": csn_name, + "managed": managed, + "type": element.get("type"), + } + annotations = c.split_annotations(element, consumed) + if annotations: + own["annotations"] = annotations + native = {k: v for k, v in element.items() if not k.startswith("@") and k not in _MAPPED_ELEMENT_KEYS} + if native: + own["native"] = native + relationships.append( + OssieRelationship( + name=el_name, + **{"from": dataset_name}, + to=entities.dataset_name[target], + from_columns=from_columns, + to_columns=to_columns, + ai_context=element.get(c.OSSIE_AI_CONTEXT), + custom_extensions=_extensions(own, element), + ) + ) + + def _metrics(self, candidates: list[dict[str, Any]]) -> list[OssieMetric]: + counts: dict[str, int] = {} + for m in candidates: + counts[m["element"]] = counts.get(m["element"], 0) + 1 + metrics = [] + for m in candidates: + name = m["element"] if counts[m["element"]] == 1 else f"{m['dataset']}_{m['element']}" + agg = m["aggregation"] + if m["derived"]: + try: + target = _qualify(m["expression"], m["dataset"]) + except UnsupportedExpression: + target = f"{quote_identifier(m['dataset'])}.{quote_identifier(m['element'])}" + else: + target = f"{quote_identifier(m['dataset'])}.{quote_identifier(m['element'])}" + sql = f"COUNT(DISTINCT {target})" if agg == "COUNT_DISTINCT" else f"{agg}({target})" + + if agg in ("COUNT", "COUNT_DISTINCT"): + datatype: Optional[str] = "Integer" + elif agg == "AVG": + datatype = "Float" if m["builtin"] == "cds.Double" else "Decimal" + else: + datatype = m["datatype"] + + own: dict[str, Any] = { + "entity": m["entity"], + "element": m["element"], + "aggregation": m["aggregation_raw"], + "derived": m["derived"], + } + if m["derived"]: + own["measure"] = m["own"] + metrics.append( + OssieMetric( + name=name, + expression=OssieExpression( + dialects=[OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression=sql)] + ), + description=m["description"] if m["derived"] else None, + datatype=datatype, + ai_context=m["element_obj"].get(c.OSSIE_AI_CONTEXT) if m["derived"] else None, + custom_extensions=_extensions(own, m["element_obj"] if m["derived"] else {}), + ) + ) + return metrics diff --git a/converters/sap-csn/src/ossie_sap_csn/cxn.py b/converters/sap-csn/src/ossie_sap_csn/cxn.py new file mode 100644 index 00000000..fd632493 --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/cxn.py @@ -0,0 +1,303 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Translate between CSN expressions (CXN) and ANSI SQL text. + +CXN is the JSON expression notation CSN uses for calculated elements and view +columns: ``{"ref": [...]}``, ``{"val": ...}``, ``{"func": ..., "args": [...]}`` +and ``{"xpr": [...]}`` (a token list in which a nested ``xpr`` is a +parenthesised group). Only the subset that has a faithful SQL rendering is +handled; anything else raises :class:`UnsupportedExpression` so the caller can +report it instead of guessing. +""" + +import math +import re +from typing import Any, Callable, Optional + +import sqlglot +from sqlglot import exp +from sqlglot.errors import SqlglotError +from sqlglot.tokens import TokenType + +from ossie_sap_csn.datatypes import CDS_TO_SQL_TYPE + + +class UnsupportedExpression(Exception): + """The expression uses a construct this converter does not translate.""" + + +# Resolves a CXN path `["assoc", "col"]` (more than one segment) to SQL text. +PathResolver = Callable[[list[str]], str] + + +def _no_paths(path: list[str]) -> str: + raise UnsupportedExpression(f"path expression {'.'.join(path)!r} cannot be resolved") + + +# -------------------------------------------------------------------------------------- +# CXN -> SQL +# -------------------------------------------------------------------------------------- + + +_PLAIN_IDENTIFIER = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-9;]*m") + + +def quote_identifier(name: str) -> str: + """``name`` as an SQL identifier, double-quoted unless it is a plain non-keyword word.""" + if _PLAIN_IDENTIFIER.fullmatch(name): + try: + tokens = sqlglot.tokenize(name) + except SqlglotError: + tokens = [] + if len(tokens) == 1 and tokens[0].token_type == TokenType.VAR: + return name + return '"' + name.replace('"', '""') + '"' + + +def _literal(value: Any) -> str: + if value is None: + return "NULL" + if isinstance(value, bool): + return "TRUE" if value else "FALSE" + if isinstance(value, float) and not math.isfinite(value): + raise UnsupportedExpression(f"literal {value!r} has no SQL rendering") + if isinstance(value, (int, float)): + return repr(value) + if isinstance(value, str): + return "'" + value.replace("'", "''") + "'" + raise UnsupportedExpression(f"literal {value!r} has no SQL rendering") + + +def _ref_to_sql(ref: list[Any], resolve_path: PathResolver) -> str: + if not ref or not all(isinstance(s, str) for s in ref): + # A segment may be an object carrying a filter or arguments. + raise UnsupportedExpression("reference with a filtered or parameterised segment") + if ref[0] in ("$self", "$projection"): + ref = ref[1:] + if not ref: + raise UnsupportedExpression("bare $self reference") + if ref[0].startswith("$"): + raise UnsupportedExpression(f"{ref[0]!r} has no SQL rendering") + if len(ref) == 1: + return quote_identifier(ref[0]) + return resolve_path(list(ref)) + + +def _tokens(tokens: list[Any], resolve_path: PathResolver) -> str: + parts: list[str] = [] + for token in tokens: + if isinstance(token, str): + parts.append(token.upper() if token.isalpha() else token) + else: + parts.append(_node(token, resolve_path, nested_group=True)) + return " ".join(parts) + + +def _node(node: Any, resolve_path: PathResolver, nested_group: bool = False) -> str: + if not isinstance(node, dict): + raise UnsupportedExpression(f"unexpected expression node {node!r}") + if "cast" in node: + cast_type = (node["cast"] or {}).get("type") + sql_type = CDS_TO_SQL_TYPE.get(cast_type) + if sql_type is None: + raise UnsupportedExpression(f"cast to {cast_type!r}") + inner = {k: v for k, v in node.items() if k != "cast"} + return f"CAST({_node(inner, resolve_path)} AS {sql_type})" + if "ref" in node: + return _ref_to_sql(node["ref"], resolve_path) + if "val" in node: + return _literal(node["val"]) + if "xpr" in node: + body = _tokens(node["xpr"], resolve_path) + return f"({body})" if nested_group else body + if "list" in node: + return "(" + ", ".join(_node(i, resolve_path) for i in node["list"]) + ")" + if "func" in node: + args = node.get("args", []) + if not isinstance(args, list): + raise UnsupportedExpression(f"function {node['func']!r} with named arguments") + rendered = [] + for arg in args: + if arg == "*": + rendered.append("*") + elif isinstance(arg, dict) and "xpr" in arg: + # An argument-level xpr is the argument itself (`DISTINCT x`), not a group. + rendered.append(_tokens(arg["xpr"], resolve_path)) + else: + rendered.append(_node(arg, resolve_path)) + return f"{str(node['func']).upper()}({', '.join(rendered)})" + raise UnsupportedExpression(f"expression node with keys {sorted(node)}") + + +def cxn_to_sql(node: Any, resolve_path: PathResolver = _no_paths) -> str: + """Render one CXN expression as ANSI SQL text.""" + return _node(node, resolve_path) + + +# -------------------------------------------------------------------------------------- +# SQL -> CXN +# -------------------------------------------------------------------------------------- + +_BINARY_OPS: dict[type, str] = { + exp.Add: "+", + exp.Sub: "-", + exp.Mul: "*", + exp.Div: "/", + exp.EQ: "=", + exp.NEQ: "!=", + exp.GT: ">", + exp.GTE: ">=", + exp.LT: "<", + exp.LTE: "<=", + exp.And: "and", + exp.Or: "or", + exp.DPipe: "||", +} + +_SQL_TYPE_TO_CDS: dict[str, str] = { + "VARCHAR": "cds.String", + "TEXT": "cds.String", + "INT": "cds.Integer", + "SMALLINT": "cds.Int16", + "BIGINT": "cds.Int64", + "DECIMAL": "cds.Decimal", + "DOUBLE": "cds.Double", + "BOOLEAN": "cds.Boolean", + "DATE": "cds.Date", + "TIME": "cds.Time", + "TIMESTAMP": "cds.Timestamp", +} + + +def parse_sql(sql: str) -> exp.Expression: + try: + return sqlglot.parse_one(sql) + except SqlglotError as exc: + # sqlglot underlines the error position with ANSI escapes; keep them out of issue text. + raise UnsupportedExpression(f"not parseable as SQL: {_ANSI_ESCAPE.sub('', str(exc))}") from exc + + +# Resolves a SQL column `table.col` to a CXN `ref` list. +ColumnResolver = Callable[[Optional[str], str], list[str]] + + +def _to_cxn(node: exp.Expression, resolve_column: ColumnResolver) -> Any: + if isinstance(node, exp.Paren): + inner = _to_cxn(node.this, resolve_column) + return inner if isinstance(inner, dict) and "xpr" in inner else {"xpr": [inner]} + if isinstance(node, exp.Column): + table = node.table or None + return {"ref": resolve_column(table, node.name)} + if isinstance(node, exp.Null): + return {"val": None} + if isinstance(node, exp.Boolean): + return {"val": bool(node.this)} + if isinstance(node, exp.Literal): + if node.is_string: + return {"val": node.this} + text = node.this + return {"val": int(text) if text.lstrip("-").isdigit() else float(text)} + if isinstance(node, exp.Neg): + return {"xpr": ["-", _to_cxn(node.this, resolve_column)]} + if isinstance(node, exp.Not): + return {"xpr": ["not", _to_cxn(node.this, resolve_column)]} + for node_type, op in _BINARY_OPS.items(): + if type(node) is node_type: + return {"xpr": [_to_cxn(node.left, resolve_column), op, _to_cxn(node.right, resolve_column)]} + if isinstance(node, exp.Is): + right = node.expression + if isinstance(right, exp.Null): + return {"xpr": [_to_cxn(node.this, resolve_column), "is", "null"]} + raise UnsupportedExpression("IS with a non-NULL right-hand side") + if isinstance(node, exp.Like): + return {"xpr": [_to_cxn(node.this, resolve_column), "like", _to_cxn(node.expression, resolve_column)]} + if isinstance(node, exp.In): + if node.args.get("query") is not None or node.args.get("unnest") is not None: + raise UnsupportedExpression("IN with a subquery") + items = [_to_cxn(e, resolve_column) for e in node.expressions] + return {"xpr": [_to_cxn(node.this, resolve_column), "in", {"list": items}]} + if isinstance(node, exp.Between): + return { + "xpr": [ + _to_cxn(node.this, resolve_column), + "between", + _to_cxn(node.args["low"], resolve_column), + "and", + _to_cxn(node.args["high"], resolve_column), + ] + } + if isinstance(node, exp.Case): + tokens: list[Any] = ["case"] + if node.this is not None: + tokens.append(_to_cxn(node.this, resolve_column)) + for branch in node.args.get("ifs") or []: + tokens += ["when", _to_cxn(branch.this, resolve_column), "then", _to_cxn(branch.args["true"], resolve_column)] + if node.args.get("default") is not None: + tokens += ["else", _to_cxn(node.args["default"], resolve_column)] + tokens.append("end") + return {"xpr": tokens} + if isinstance(node, (exp.Cast, exp.TryCast)): + to = node.args["to"] + cds = _SQL_TYPE_TO_CDS.get(to.this.name if isinstance(to.this, exp.DataType.Type) else str(to.this)) + if cds is None: + raise UnsupportedExpression(f"cast to {to.sql()}") + inner = _to_cxn(node.this, resolve_column) + if not isinstance(inner, dict) or "xpr" in inner: + raise UnsupportedExpression("cast of a compound expression") + return {**inner, "cast": {"type": cds}} + if isinstance(node, exp.Count) and isinstance(node.this, exp.Distinct): + distinct_args = [_to_cxn(e, resolve_column) for e in node.this.expressions] + return {"func": "count", "args": [{"xpr": ["distinct", *distinct_args]}]} + if isinstance(node, exp.Star): + return "*" + if isinstance(node, (exp.Subquery, exp.Select, exp.Window)): + raise UnsupportedExpression(f"{type(node).__name__} has no CXN translation here") + if isinstance(node, exp.Func): + args: list[Any] = [] + if isinstance(node, exp.Anonymous): + name = node.name.lower() + operands = list(node.expressions) + else: + name = node.sql_name().lower() + operands = [] + for key in node.arg_types: + value = node.args.get(key) + if value is None: + continue + if isinstance(value, list): + operands.extend(value) + elif isinstance(value, exp.Expression): + operands.append(value) + else: + raise UnsupportedExpression(f"function {name!r} with a non-expression argument") + for operand in operands: + if isinstance(operand, exp.Star): + args.append("*") + else: + args.append(_to_cxn(operand, resolve_column)) + return {"func": name, "args": args} + raise UnsupportedExpression(f"{type(node).__name__} has no CXN translation") + + +def sql_to_cxn(sql: str, resolve_column: ColumnResolver) -> dict[str, Any]: + """Convert ANSI SQL text into one CXN expression (always an object).""" + result = _to_cxn(parse_sql(sql), resolve_column) + if isinstance(result, str): + raise UnsupportedExpression("a bare '*' is not an expression") + return result diff --git a/converters/sap-csn/src/ossie_sap_csn/datatypes.py b/converters/sap-csn/src/ossie_sap_csn/datatypes.py new file mode 100644 index 00000000..d777e3c0 --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/datatypes.py @@ -0,0 +1,127 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Mapping between CDS built-in types and Ossie logical data types.""" + +from typing import Any, Optional + +# CDS built-ins, plus the SAP HANA-specific ones CSN spells `cds.hana.*`. +_CDS_TO_OSSIE: dict[str, str] = { + "cds.String": "String", + "cds.LargeString": "String", + "cds.UUID": "String", + "cds.hana.CHAR": "String", + "cds.hana.NCHAR": "String", + "cds.hana.VARCHAR": "String", + "cds.hana.CLOB": "String", + "cds.Integer": "Integer", + "cds.Int16": "Integer", + "cds.Int32": "Integer", + "cds.Int64": "Integer", + "cds.Integer64": "Integer", + "cds.UInt8": "Integer", + "cds.hana.SMALLINT": "Integer", + "cds.hana.TINYINT": "Integer", + "cds.Decimal": "Decimal", + "cds.DecimalFloat": "Decimal", + "cds.hana.SMALLDECIMAL": "Decimal", + "cds.Double": "Float", + "cds.hana.REAL": "Float", + "cds.Boolean": "Boolean", + "cds.Date": "Date", + "cds.Time": "Time", + "cds.DateTime": "DateTime", + "cds.Timestamp": "DateTime", +} + +# Types with no portable Ossie equivalent: carried as Opaque. +_CDS_OPAQUE = frozenset( + { + "cds.Binary", + "cds.LargeBinary", + "cds.Vector", + "cds.hana.BINARY", + "cds.hana.ST_POINT", + "cds.hana.ST_GEOMETRY", + } +) + +_OSSIE_TO_CDS: dict[str, str] = { + "String": "cds.String", + "Integer": "cds.Integer", + "Decimal": "cds.Decimal", + "Float": "cds.Double", + "Boolean": "cds.Boolean", + "Date": "cds.Date", + "Time": "cds.Time", + "DateTime": "cds.Timestamp", + "DateTimeTz": "cds.Timestamp", +} + +ASSOCIATION_TYPES = frozenset({"cds.Association", "cds.Composition"}) + + +def builtin_type(type_name: Optional[str], definitions: dict[str, Any]) -> Optional[str]: + """Resolve a (possibly user-defined) CSN type name down to a CDS built-in. + + Returns ``None`` when the chain ends in something that is not a known built-in + (an unknown name, a structured type, or a cycle). + """ + seen: set[str] = set() + while type_name is not None and type_name not in seen: + if type_name in _CDS_TO_OSSIE or type_name in _CDS_OPAQUE or type_name in ASSOCIATION_TYPES: + return type_name + seen.add(type_name) + definition = definitions.get(type_name) + if not isinstance(definition, dict): + return None + type_name = definition.get("type") + return None + + +def ossie_datatype(builtin: Optional[str]) -> Optional[str]: + """Ossie datatype for a CDS built-in, ``Opaque`` for a known non-portable one, else ``None``.""" + if builtin is None: + return None + if builtin in _CDS_TO_OSSIE: + return _CDS_TO_OSSIE[builtin] + if builtin in _CDS_OPAQUE: + return "Opaque" + return None + + +def cds_type(datatype: Optional[str]) -> Optional[str]: + """CDS built-in for an Ossie datatype (``None`` for ``Opaque`` or an unset type).""" + return _OSSIE_TO_CDS.get(datatype) if datatype else None + + +# SQL type names used when rendering / parsing CAST in expressions. +CDS_TO_SQL_TYPE: dict[str, str] = { + "cds.String": "VARCHAR", + "cds.LargeString": "VARCHAR", + "cds.Integer": "INT", + "cds.Int16": "SMALLINT", + "cds.Int32": "INT", + "cds.Int64": "BIGINT", + "cds.Decimal": "DECIMAL", + "cds.Double": "DOUBLE", + "cds.Boolean": "BOOLEAN", + "cds.Date": "DATE", + "cds.Time": "TIME", + "cds.DateTime": "TIMESTAMP", + "cds.Timestamp": "TIMESTAMP", +} diff --git a/converters/sap-csn/src/ossie_sap_csn/ossie_to_csn.py b/converters/sap-csn/src/ossie_sap_csn/ossie_to_csn.py new file mode 100644 index 00000000..89dc336f --- /dev/null +++ b/converters/sap-csn/src/ossie_sap_csn/ossie_to_csn.py @@ -0,0 +1,574 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Ossie semantic model into a SAP CSN document (Core Schema Notation, JSON). + +Mapping: + +* a dataset becomes an ``entity``; its fields become elements, and primary-key + fields carry ``key: true``; +* a field whose expression is not just its own column becomes a calculated element + (``value``), with the expression translated from SQL to CXN; +* a relationship becomes an association on the ``from`` entity: managed (``keys``) + when the foreign-key columns are the ones CDS would generate, otherwise with an + ``on`` condition; +* a metric becomes ``@Aggregation.default`` on the element it aggregates, or a + derived measure element when it aggregates an expression of one dataset; +* a metric CSN cannot express (several datasets, ``COUNT(*)``, a ratio, ...) goes + into ``meta.ossie.metrics``, which this converter reads back and other CSN + consumers ignore. + +Ossie content with no CSN equivalent (AI context, other vendors' extensions) +rides in ``@Ossie.*`` annotations, which CSN permits and consumers ignore. +""" + +import re +from typing import Any, Optional + +from ossie import OssieDataset, OssieDocument, OssieField, OssieMetric, OssieRelationship +from sqlglot import exp + +from ossie_sap_csn import _common as c +from ossie_sap_csn.converter_issues import ( + ConverterError, + ConverterIssue, + ConverterIssueType, + ConverterResult, +) +from ossie_sap_csn.cxn import UnsupportedExpression, parse_sql, sql_to_cxn +from ossie_sap_csn.datatypes import builtin_type, cds_type, ossie_datatype + +_PERSISTENCE_NAME = "@cds.persistence.name" +_IDENTIFIER_UNSAFE = re.compile(r"[^A-Za-z0-9_]") + +_AGG_NODES: dict[type, str] = {exp.Sum: "SUM", exp.Min: "MIN", exp.Max: "MAX", exp.Avg: "AVG", exp.Count: "COUNT"} + + +def _select_sql(expression: Any) -> tuple[Optional[str], Optional[str]]: + """The ANSI_SQL expression text, and the first other dialect's text when ANSI_SQL is absent.""" + other = None + for entry in expression.dialects: + if entry.dialect.value == "ANSI_SQL": + return entry.expression, None + other = other or f"{entry.dialect.value}:{entry.expression}" + return None, other + + +class _Metric: + """A metric of the shape ``AGG(expr)``, split into its parts.""" + + def __init__(self, aggregation: str, inner: exp.Expression) -> None: + self.aggregation = aggregation + self.inner = inner + + +def _parse_aggregate(sql: str) -> Optional[_Metric]: + try: + tree = parse_sql(sql) + except UnsupportedExpression: + return None + if isinstance(tree, exp.Count) and isinstance(tree.this, exp.Distinct) and len(tree.this.expressions) == 1: + return _Metric("COUNT_DISTINCT", tree.this.expressions[0]) + for node_type, name in _AGG_NODES.items(): + if type(tree) is node_type: + inner = tree.this + if inner is None or isinstance(inner, (exp.Star, exp.Distinct)): + return None + return _Metric(name, inner) + return None + + +def _identifier(name: str) -> str: + return _IDENTIFIER_UNSAFE.sub("_", name) + + +class OssieToCsnConverter: + """Converts an :class:`OssieDocument` into a CSN document (as a JSON-ready dict).""" + + def convert(self, document: OssieDocument) -> ConverterResult[dict[str, Any]]: + issues: list[ConverterIssue] = [] + model_ext = c.read_vendor_extension(document.custom_extensions, f"model {document.name!r}") + + datasets = {d.name: d for d in document.datasets} + if len(datasets) != len(document.datasets): + raise ConverterError("the document declares two datasets with the same name") + dataset_ext = {n: c.read_vendor_extension(d.custom_extensions, f"dataset {n!r}") for n, d in datasets.items()} + entity_name = {n: dataset_ext[n].get("entity") or n for n in datasets} + if len(set(entity_name.values())) != len(entity_name): + raise ConverterError("two datasets map to the same CSN entity name") + + # Relationships first: they decide which foreign-key fields CDS generates, and + # what a path through an association is called in a calculated expression. + plans: dict[str, list[dict[str, Any]]] = {n: [] for n in datasets} + taken_assoc: dict[str, set[str]] = {n: {f.name for f in (d.fields or [])} for n, d in datasets.items()} + for rel in document.relationships or []: + plan = self._plan_relationship(rel, datasets, taken_assoc, entity_name, issues) + if plan is not None: + plans[rel.from_dataset].append(plan) + generated_fks = { + n: {col for plan in plans[n] if plan["managed"] for col in plan["from_columns"]} for n in datasets + } + + definitions: dict[str, Any] = {} + metric_homes = self._place_metrics(document, datasets, dataset_ext, entity_name, plans, issues) + + for ds_name, dataset in datasets.items(): + definitions[entity_name[ds_name]] = self._entity( + dataset, dataset_ext[ds_name], plans[ds_name], generated_fks[ds_name], + metric_homes, datasets, entity_name, issues, + ) + + for def_name, definition in (model_ext.get("other_definitions") or {}).items(): + definitions.setdefault(def_name, definition) + + csn: dict[str, Any] = {"definitions": definitions} + csn_residue = dict(model_ext.get("csn") or {}) + csn["$version"] = csn_residue.pop("$version", "2.0") + meta = dict(csn_residue.pop("meta", None) or {}) + meta.setdefault("creator", "apache-ossie-sap-csn") + for key, value in csn_residue.items(): + csn[key] = value + + ossie_meta: dict[str, Any] = {} + model_info: dict[str, Any] = {"name": document.name} + if document.description: + model_info["description"] = document.description + if document.ai_context is not None: + model_info["ai_context"] = _dump(document.ai_context) + foreign = c.foreign_extensions(document.custom_extensions) + if foreign: + model_info["custom_extensions"] = foreign + ossie_meta["model"] = model_info + if metric_homes["meta"]: + ossie_meta["metrics"] = [m.model_dump(mode="json", by_alias=True, exclude_none=True) for m in metric_homes["meta"]] + meta[c.META_KEY] = ossie_meta + csn["meta"] = meta + return ConverterResult(output=csn, issues=issues) + + # ---------------------------------------------------------------------------------- + + def _plan_relationship( + self, + rel: OssieRelationship, + datasets: dict[str, OssieDataset], + taken: dict[str, set[str]], + entity_name: dict[str, str], + issues: list[ConverterIssue], + ) -> Optional[dict[str, Any]]: + if rel.from_dataset not in datasets or rel.to not in datasets: + issues.append( + ConverterIssue( + ConverterIssueType.RELATIONSHIP_DROPPED, + rel.name, + f"Relationship joins {rel.from_dataset!r} to {rel.to!r}, and one of them is not a dataset.", + ) + ) + return None + if len(rel.from_columns) != len(rel.to_columns): + issues.append( + ConverterIssue( + ConverterIssueType.RELATIONSHIP_COLUMN_UNRESOLVED, + rel.name, + "from_columns and to_columns differ in length, so no column pairs can be formed.", + ) + ) + return None + ext = c.read_vendor_extension(rel.custom_extensions, f"relationship {rel.name!r}") + name = ext.get("association") or _identifier(rel.name) + if name in taken[rel.from_dataset]: + name = f"{name}_assoc" + taken[rel.from_dataset].add(name) + + fields = {f.name: f for f in datasets[rel.from_dataset].fields or []} + managed = all( + col == f"{name}_{to_col}" + and col in fields + and c.read_vendor_extension(fields[col].custom_extensions, col).get("generated_foreign_key") + and c.read_vendor_extension(fields[col].custom_extensions, col).get("association") == name + for col, to_col in zip(rel.from_columns, rel.to_columns) + ) + return { + "rel": rel, + "ext": ext, + "name": name, + "managed": managed, + "from_columns": list(rel.from_columns), + "to_columns": list(rel.to_columns), + "target": entity_name[rel.to], + } + + def _place_metrics( + self, + document: OssieDocument, + datasets: dict[str, OssieDataset], + dataset_ext: dict[str, dict[str, Any]], + entity_name: dict[str, str], + plans: dict[str, list[dict[str, Any]]], + issues: list[ConverterIssue], + ) -> dict[str, Any]: + """Decide for each metric whether it is an annotation, a derived measure, or ``meta``.""" + annotate: dict[tuple[str, str], tuple[OssieMetric, str]] = {} # (dataset, field) -> (metric, aggregation) + derived: dict[str, list[tuple[OssieMetric, _Metric]]] = {n: [] for n in datasets} + in_meta: list[OssieMetric] = [] + element_names = {n: {f.name for f in (d.fields or [])} for n, d in datasets.items()} + # An association is an element too, so a measure may not take its name. + taken_names = {n: element_names[n] | {p["name"] for p in plans[n]} for n in datasets} + # A dataset reached by two associations has no single path to it from here. + reachable = { + n: {t for t in (p["rel"].to for p in plans[n]) if sum(q["rel"].to == t for q in plans[n]) == 1} + for n in datasets + } + + for metric in document.metrics or []: + sql, other = _select_sql(metric.expression) + if sql is None: + issues.append( + ConverterIssue( + ConverterIssueType.DIALECT_UNAVAILABLE, + metric.name, + f"Metric has no ANSI_SQL expression (only {other.split(':')[0]}); " + "carried in meta.ossie.metrics only.", + ) + ) + in_meta.append(metric) + continue + parsed = _parse_aggregate(sql) + home = self._home(parsed, datasets, reachable) if parsed is not None else None + if home is None: + in_meta.append(metric) + issues.append( + ConverterIssue( + ConverterIssueType.METRIC_NOT_MODELED, + metric.name, + "CSN expresses a measure as one aggregation over one dataset's columns " + "(and those of datasets it reaches through associations); " + f"{sql!r} is not of that shape, so it is carried in meta.ossie.metrics only.", + ) + ) + continue + ds_name, inner = home + plain = isinstance(inner, exp.Column) and not inner.table and inner.name == metric.name and inner.name in element_names[ds_name] + if plain and (ds_name, metric.name) not in annotate: + annotate[(ds_name, metric.name)] = (metric, parsed.aggregation) + elif metric.name in taken_names[ds_name] or any( + m.name == metric.name for lst in derived.values() for m, _ in lst + ): + in_meta.append(metric) + issues.append( + ConverterIssue( + ConverterIssueType.METRIC_NOT_MODELED, + metric.name, + f"A measure element named {metric.name!r} would collide with an existing element " + "of the same name; carried in meta.ossie.metrics only.", + ) + ) + else: + derived[ds_name].append((metric, _Metric(parsed.aggregation, inner))) + return {"annotate": annotate, "derived": derived, "meta": in_meta} + + @staticmethod + def _home( + parsed: _Metric, datasets: dict[str, OssieDataset], reachable: dict[str, set[str]] + ) -> Optional[tuple[str, exp.Expression]]: + """The dataset ``parsed`` aggregates, with its own columns unqualified. + + The home is the one dataset from which every other dataset the expression + names is reachable through an association. Relationships run from the many + side to the one side, so such a path is to-one and cannot fan out the rows + being aggregated. + """ + tables = {col.table for col in parsed.inner.find_all(exp.Column)} + if None in tables or "" in tables or not tables <= set(datasets): + return None + homes = [t for t in tables if tables - {t} <= reachable[t]] + if len(homes) != 1: + return None + inner = parsed.inner.copy() + for col in inner.find_all(exp.Column): + if col.table == homes[0]: + col.set("table", None) + return homes[0], inner + + # ---------------------------------------------------------------------------------- + + def _entity( + self, + dataset: OssieDataset, + ext: dict[str, Any], + plans: list[dict[str, Any]], + generated_fks: set[str], + metric_homes: dict[str, Any], + datasets: dict[str, OssieDataset], + entity_name: dict[str, str], + issues: list[ConverterIssue], + ) -> dict[str, Any]: + entity: dict[str, Any] = {"kind": "entity"} + desc_key = ext.get("description_key", "doc") + if dataset.description: + entity[desc_key] = dataset.description + for key, value in (ext.get("annotations") or {}).items(): + if key != _PERSISTENCE_NAME: # decided by dataset.source below + entity[key] = value + # `source` is the table the entity persists to; CDS defaults to the entity's own name. + if dataset.source != entity_name[dataset.name]: + entity[_PERSISTENCE_NAME] = dataset.source + entity.update(_carry(dataset.ai_context, dataset.custom_extensions)) + + modeled = {f.name for f in dataset.fields or []} | {p["name"] for p in plans} + names = ext.get("unique_key_names") or [] + for i, columns in enumerate(dataset.unique_keys or []): + if all(col in modeled for col in columns): + key_name = names[i] if i < len(names) else f"uk{i + 1}" + entity[c.UNIQUE_PREFIX + key_name] = [{"=": col} for col in columns] + if any(col not in modeled for columns in dataset.unique_keys or [] for col in columns): + entity[c.OSSIE_UNIQUE_KEYS] = [list(columns) for columns in dataset.unique_keys] + issues.append( + ConverterIssue( + ConverterIssueType.UNIQUE_KEY_COLUMN_UNRESOLVED, + dataset.name, + "A unique key names a column that is not a field of the dataset, so CDS cannot " + "declare it; the keys are carried in an @Ossie.uniqueKeys annotation only.", + ) + ) + + # A path through an association (`customer.name`) is how CXN reaches another entity. + assoc_to_dataset: dict[str, list[str]] = {} + for p in plans: + assoc_to_dataset.setdefault(p["rel"].to, []).append(p["name"]) + + def resolve_column(table: Optional[str], column: str) -> list[str]: + if table is None or table == dataset.name: + return [column] + if len(assoc_to_dataset.get(table, [])) > 1: + raise UnsupportedExpression( + f"column {table}.{column} is reached through several associations " + f"({', '.join(assoc_to_dataset[table])}), so the path is ambiguous" + ) + if table in assoc_to_dataset: + return [assoc_to_dataset[table][0], column] + raise UnsupportedExpression(f"column {table}.{column} reaches a dataset with no association from here") + + primary_key = set(dataset.primary_key or []) + if not primary_key <= modeled: + entity[c.OSSIE_PRIMARY_KEY] = list(dataset.primary_key) + issues.append( + ConverterIssue( + ConverterIssueType.PRIMARY_KEY_COLUMN_UNRESOLVED, + dataset.name, + f"Primary key names {sorted(primary_key - modeled)}, which are not fields of the dataset, " + "so CDS cannot mark them `key`; the key is carried in an @Ossie.primaryKey annotation, " + "and the columns that do exist are marked `key`.", + ) + ) + elements: dict[str, Any] = {} + annotate = metric_homes["annotate"] + for field in dataset.fields or []: + fk_ext = c.read_vendor_extension(field.custom_extensions, f"field {dataset.name}.{field.name}") + if field.name in generated_fks and fk_ext.get("generated_foreign_key"): + continue # CDS generates this column from the managed association + elements[field.name] = self._element( + dataset, field, fk_ext, field.name in primary_key, resolve_column, + annotate.get((dataset.name, field.name)), issues, + ) + + for plan in plans: + elements[plan["name"]] = self._association(plan, primary_key) + + measures = metric_homes["derived"][dataset.name] + for metric, parsed in measures: + elements[metric.name] = self._derived_measure(dataset, metric, parsed, resolve_column, issues) + # Only for an entity authored in Ossie: one read from CSN keeps the category it had. + authored_here = not ext + if authored_here and (measures or any(k[0] == dataset.name for k in annotate)): + entity[c.DATA_CATEGORY_KEY] = c.enum_annotation("CUBE") + + for name, raw in (ext.get("unmodeled_elements") or {}).items(): + elements.setdefault(name, raw) + if elements: + entity["elements"] = elements + for key, value in (ext.get("native") or {}).items(): + entity[key] = value + return entity + + def _element( + self, + dataset: OssieDataset, + field: OssieField, + ext: dict[str, Any], + is_key: bool, + resolve_column: Any, + measure: Optional[tuple[OssieMetric, str]], + issues: list[ConverterIssue], + ) -> dict[str, Any]: + where = f"{dataset.name}.{field.name}" + element: dict[str, Any] = {"type": self._type(field.datatype, ext.get("type"), where, issues)} + if is_key: + element["key"] = True + + sql, other = _select_sql(field.expression) + override: Optional[dict[str, str]] = None + if sql is None: + dialect, _, text = other.partition(":") + override = {"dialect": dialect, "expression": text} + issues.append( + ConverterIssue( + ConverterIssueType.DIALECT_UNAVAILABLE, + where, + f"Field has no ANSI_SQL expression (only {dialect}); exported as a plain column " + "with the original expression in an @Ossie.sql annotation.", + ) + ) + else: + try: + cxn = sql_to_cxn(sql, resolve_column) + if cxn != {"ref": [field.name]}: + element["value"] = cxn + except UnsupportedExpression as exc: + override = {"dialect": "ANSI_SQL", "expression": sql} + issues.append( + ConverterIssue( + ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE, + where, + f"Expression {sql!r} has no CXN translation ({exc}); exported as a plain column " + "with the original expression in an @Ossie.sql annotation.", + ) + ) + if override: + element[c.OSSIE_SQL] = override + + if field.label: + element[ext.get("label_key", c.LABEL_KEYS[0])] = field.label + if field.description: + element[ext.get("description_key", "doc")] = field.description + for key, value in (ext.get("annotations") or {}).items(): + element[key] = value + if measure is not None: + element[c.AGGREGATION_KEY] = c.enum_annotation(measure[1]) + element.update(_carry(field.ai_context, field.custom_extensions)) + + default_is_time = field.datatype is not None and field.datatype.value in ("Date", "Time", "DateTime", "DateTimeTz") + if field.dimension is not None and field.dimension.is_time is not None and field.dimension.is_time != default_is_time: + element[c.OSSIE_IS_TIME] = field.dimension.is_time + for key, value in (ext.get("native") or {}).items(): + element[key] = value + return element + + def _derived_measure( + self, + dataset: OssieDataset, + metric: OssieMetric, + parsed: _Metric, + resolve_column: Any, + issues: list[ConverterIssue], + ) -> dict[str, Any]: + where = f"{dataset.name}.{metric.name}" + ext = c.read_vendor_extension(metric.custom_extensions, f"metric {metric.name!r}") + measure_ext = ext.get("measure") or {} + datatype = metric.datatype.value if metric.datatype else None + if datatype is None and parsed.aggregation in ("COUNT", "COUNT_DISTINCT"): + datatype = "Integer" + element: dict[str, Any] = {"type": self._type(datatype, measure_ext.get("type"), where, issues)} + try: + element["value"] = sql_to_cxn(parsed.inner.sql(), resolve_column) + except UnsupportedExpression as exc: + # Unreachable for metrics that passed placement unless a column path is unresolvable. + raise ConverterError(f"{where}: {exc}") from exc + element[c.AGGREGATION_KEY] = c.enum_annotation(parsed.aggregation) + element[c.OSSIE_METRIC] = True + if metric.description: + element["doc"] = metric.description + if measure_ext.get("label"): + element[measure_ext.get("label_key", c.LABEL_KEYS[0])] = measure_ext["label"] + for key, value in (measure_ext.get("annotations") or {}).items(): + element[key] = value + element.update(_carry(metric.ai_context, metric.custom_extensions)) + for key, value in (measure_ext.get("native") or {}).items(): + element[key] = value + return element + + def _association(self, plan: dict[str, Any], primary_key: set[str]) -> dict[str, Any]: + ext = plan["ext"] + element: dict[str, Any] = {"type": ext.get("type") or "cds.Association", "target": plan["target"]} + name = plan["name"] + if plan["managed"]: + element["keys"] = [{"ref": [col]} for col in plan["to_columns"]] + # The generated foreign-key columns are not elements, so `key` can only live here. + if plan["from_columns"] and all(col in primary_key for col in plan["from_columns"]): + element["key"] = True + else: + condition: list[Any] = [] + for from_col, to_col in zip(plan["from_columns"], plan["to_columns"]): + if condition: + condition.append("and") + condition += [{"ref": [name, to_col]}, "=", {"ref": [from_col]}] + element["on"] = condition + for key, value in (ext.get("annotations") or {}).items(): + element[key] = value + rel = plan["rel"] + element.update(_carry(rel.ai_context, rel.custom_extensions)) + for key, value in (ext.get("native") or {}).items(): + element[key] = value + return element + + def _type( + self, + datatype: Optional[str], + original: Optional[str], + where: str, + issues: list[ConverterIssue], + ) -> str: + if original is not None: + # A CSN document is its own best witness for the CDS type it started as. + known = builtin_type(original, {}) + if datatype is None or ossie_datatype(known) in (datatype, None): + return original + mapped = cds_type(datatype) + if mapped is not None: + return mapped + if datatype == "Opaque": + issues.append( + ConverterIssue( + ConverterIssueType.OPAQUE_DATATYPE, + where, + "Opaque datatype has no CDS built-in without a SAP_CSN extension naming one; exported as cds.String.", + ) + ) + else: + issues.append( + ConverterIssue( + ConverterIssueType.ASSUMED_DATATYPE, + where, + "No datatype is declared and CSN requires a type; assumed cds.String.", + ) + ) + return "cds.String" + + +def _dump(value: Any) -> Any: + """JSON-ready form of an Ossie ``ai_context``.""" + return value if isinstance(value, str) else value.model_dump(mode="json", exclude_none=True) + + +def _carry(ai_context: Any, extensions: Any) -> dict[str, Any]: + """`@Ossie.*` annotations that keep AI context and other vendors' extensions.""" + carried: dict[str, Any] = {} + if ai_context is not None: + carried[c.OSSIE_AI_CONTEXT] = _dump(ai_context) + foreign = c.foreign_extensions(extensions) + if foreign: + carried[c.OSSIE_EXTENSIONS] = foreign + return carried diff --git a/converters/sap-csn/tests/__init__.py b/converters/sap-csn/tests/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/sap-csn/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/sap-csn/tests/fixtures/bookshop.csn.json b/converters/sap-csn/tests/fixtures/bookshop.csn.json new file mode 100644 index 00000000..14682ca6 --- /dev/null +++ b/converters/sap-csn/tests/fixtures/bookshop.csn.json @@ -0,0 +1,85 @@ +{ + "$version": "2.0", + "meta": { "creator": "CDS Compiler v5.0.0" }, + "namespace": "sap.capire.bookshop", + "definitions": { + "sap.capire.bookshop": { "kind": "context" }, + "sap.capire.bookshop.Amount": { "kind": "type", "type": "cds.Decimal", "precision": 11, "scale": 2 }, + "sap.capire.bookshop.Authors": { + "kind": "entity", + "doc": "People who wrote books.", + "@EndUserText.label": "Authors", + "elements": { + "ID": { "type": "cds.Integer", "key": true }, + "name": { "type": "cds.String", "length": 111, "@EndUserText.label": "Author name" }, + "dateOfBirth": { "type": "cds.Date" }, + "books": { + "type": "cds.Association", + "cardinality": { "max": "*" }, + "target": "sap.capire.bookshop.Books", + "on": [ { "ref": [ "books", "author" ] }, "=", { "ref": [ "$self" ] } ] + } + } + }, + "sap.capire.bookshop.Books": { + "kind": "entity", + "@Analytics.dataCategory": { "#": "CUBE" }, + "@assert.unique.isbn": [ { "=": "isbn" } ], + "elements": { + "ID": { "type": "cds.Integer", "key": true }, + "title": { "type": "cds.String", "length": 111, "@EndUserText.label": "Title", "@EndUserText.quickInfo": "Title as printed on the cover" }, + "isbn": { "type": "cds.String", "length": 13 }, + "author": { "type": "cds.Association", "target": "sap.capire.bookshop.Authors", "keys": [ { "ref": [ "ID" ] } ] }, + "published": { "type": "cds.Date" }, + "stock": { "type": "cds.Integer", "@Aggregation.default": { "#": "SUM" } }, + "price": { "type": "sap.capire.bookshop.Amount", "@Aggregation.default": { "#": "AVG" }, "@Semantics.amount.currencyCode": "currency_code" }, + "currency_code": { "type": "cds.String", "length": 3 }, + "priceWithTax": { "type": "cds.Decimal", "value": { "xpr": [ { "ref": [ "price" ] }, "*", { "val": 1.19 } ] } }, + "authorName": { "type": "cds.String", "value": { "ref": [ "author", "name" ] } }, + "cover": { "type": "cds.LargeBinary" }, + "descr": { "type": "cds.LargeString", "localized": true } + } + }, + "sap.capire.bookshop.Orders": { + "kind": "entity", + "elements": { + "ID": { "type": "cds.UUID", "key": true }, + "book_ID": { "type": "cds.Integer" }, + "book": { + "type": "cds.Association", + "target": "sap.capire.bookshop.Books", + "on": [ { "ref": [ "book", "ID" ] }, "=", { "ref": [ "book_ID" ] } ] + }, + "orderedAt": { "type": "cds.Timestamp" }, + "quantity": { "type": "cds.Integer" }, + "amount": { + "type": "cds.Decimal", + "precision": 11, + "scale": 2, + "@Aggregation.default": { "#": "SUM" }, + "@Ossie.metric": true, + "value": { "xpr": [ { "ref": [ "quantity" ] }, "*", { "ref": [ "book", "price" ] } ] } + }, + "customers": { "type": "cds.Composition", "target": "sap.capire.bookshop.Orders.Items" }, + "tags": { "items": { "type": "cds.String" } } + } + }, + "sap.capire.bookshop.Orders.Items": { + "kind": "entity", + "elements": { + "ID": { "type": "cds.UUID", "key": true }, + "note": { "type": "cds.String" } + } + }, + "sap.capire.bookshop.ListOfBooks": { + "kind": "entity", + "projection": { "from": { "ref": [ "sap.capire.bookshop.Books" ] } }, + "elements": { + "ID": { "type": "cds.Integer", "key": true }, + "title": { "type": "cds.String", "length": 111 } + } + }, + "CatalogService": { "kind": "service" }, + "sap.capire.bookshop.submitOrder": { "kind": "action", "params": { "book": { "type": "cds.Integer" } } } + } +} diff --git a/converters/sap-csn/tests/helpers.py b/converters/sap-csn/tests/helpers.py new file mode 100644 index 00000000..a5a98d96 --- /dev/null +++ b/converters/sap-csn/tests/helpers.py @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +from pathlib import Path +from typing import Any + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] +FIXTURES_DIR = Path(__file__).parent / "fixtures" +SCHEMA_PATH = REPO_ROOT / "core-spec" / "ossie-schema.json" +TPCDS_PATH = REPO_ROOT / "examples" / "tpcds_semantic_model.yaml" + +ORDERS = "sap.capire.bookshop.Orders" +BOOKS = "sap.capire.bookshop.Books" + + +def load_fixture(name: str) -> dict[str, Any]: + return json.loads((FIXTURES_DIR / name).read_text()) + + +def load_tpcds() -> dict[str, Any]: + return yaml.safe_load(TPCDS_PATH.read_text()) + + +def validate_against_schema(document: Any) -> None: + """Validate an OssieDocument against the core-spec JSON Schema.""" + import jsonschema + + schema = json.loads(SCHEMA_PATH.read_text()) + jsonschema.validate(json.loads(document.to_ossie_json()), schema) + + +def by_name(items: Any) -> dict[str, Any]: + return {i.name: i for i in items or []} + + +def sap_ext(obj: Any) -> dict[str, Any]: + for extension in obj.custom_extensions or []: + if extension.vendor_name == "SAP_CSN": + return json.loads(extension.data) + return {} + + +def sql_of(obj: Any) -> str: + return obj.expression.dialects[0].expression diff --git a/converters/sap-csn/tests/test_cli.py b/converters/sap-csn/tests/test_cli.py new file mode 100644 index 00000000..8b9859ed --- /dev/null +++ b/converters/sap-csn/tests/test_cli.py @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest +import yaml + +from ossie_sap_csn.cli import main + +from .helpers import FIXTURES_DIR, TPCDS_PATH + + +def run(monkeypatch, *argv): + monkeypatch.setattr("sys.argv", ["ossie-sap-csn", *argv]) + main() + + +def test_csn_to_ossie_writes_yaml(tmp_path, monkeypatch, capsys): + out = tmp_path / "model.yaml" + run(monkeypatch, "csn-to-ossie", "-i", str(FIXTURES_DIR / "bookshop.csn.json"), "-o", str(out), "--name", "shop") + doc = yaml.safe_load(out.read_text()) + assert doc["name"] == "shop" + assert doc["version"] == "0.2.0.dev0" + assert "[WARNING] RELATIONSHIP_DROPPED: Authors.books" in capsys.readouterr().err + + +def test_ossie_to_csn_writes_json(tmp_path, monkeypatch, capsys): + out = tmp_path / "model.csn.json" + run(monkeypatch, "ossie-to-csn", "-i", str(TPCDS_PATH), "-o", str(out)) + csn = json.loads(out.read_text()) + assert "store_sales" in csn["definitions"] + assert "[WARNING] METRIC_NOT_MODELED: cumulative_sales" in capsys.readouterr().err + + +def test_a_file_that_is_not_json_exits_with_a_message_not_a_traceback(tmp_path, monkeypatch, capsys): + bad = tmp_path / "bad.json" + bad.write_text("{nope") + with pytest.raises(SystemExit) as exit_info: + run(monkeypatch, "csn-to-ossie", "-i", str(bad), "-o", str(tmp_path / "o.yaml")) + assert exit_info.value.code == 1 + assert "not valid JSON" in capsys.readouterr().err + + +def test_an_invalid_ossie_document_exits_with_a_message(tmp_path, monkeypatch, capsys): + bad = tmp_path / "bad.yaml" + bad.write_text("name: x\n") + with pytest.raises(SystemExit) as exit_info: + run(monkeypatch, "ossie-to-csn", "-i", str(bad), "-o", str(tmp_path / "o.json")) + assert exit_info.value.code == 1 + assert "not a valid Ossie document" in capsys.readouterr().err diff --git a/converters/sap-csn/tests/test_csn_to_ossie.py b/converters/sap-csn/tests/test_csn_to_ossie.py new file mode 100644 index 00000000..f02da3bf --- /dev/null +++ b/converters/sap-csn/tests/test_csn_to_ossie.py @@ -0,0 +1,288 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import copy + +import pytest + +from ossie_sap_csn import CsnToOssieConverter, OssieToCsnConverter +from ossie_sap_csn.converter_issues import ConverterError, ConverterIssueType + +from .helpers import BOOKS, ORDERS, by_name, load_fixture, sap_ext, sql_of, validate_against_schema + + +@pytest.fixture(scope="module") +def result(): + return CsnToOssieConverter().convert(load_fixture("bookshop.csn.json"), name="bookshop") + + +@pytest.fixture(scope="module") +def datasets(result): + return by_name(result.output.datasets) + + +def issues_of(result, kind): + return [i for i in result.issues if i.issue_type is kind] + + +def test_output_validates_against_the_core_schema(result): + validate_against_schema(result.output) + + +def test_every_entity_becomes_a_dataset_named_without_its_namespace(datasets): + assert set(datasets) == {"Authors", "Books", "Orders", "Items", "ListOfBooks"} + assert datasets["Books"].source == BOOKS + assert sap_ext(datasets["Books"])["entity"] == BOOKS + + +def test_colliding_short_names_fall_back_to_the_full_name(): + csn = { + "definitions": { + "a.Item": {"kind": "entity", "elements": {"ID": {"type": "cds.Integer", "key": True}}}, + "b.Item": {"kind": "entity", "elements": {"ID": {"type": "cds.Integer", "key": True}}}, + } + } + names = {d.name for d in CsnToOssieConverter().convert(csn).output.datasets} + assert names == {"a_Item", "b_Item"} + + +def test_persistence_name_wins_over_the_cds_name_as_source(): + csn = {"definitions": {"my.E": {"kind": "entity", "@cds.persistence.name": "MY_E_TABLE", "elements": {}}}} + assert CsnToOssieConverter().convert(csn).output.datasets[0].source == "MY_E_TABLE" + + +def test_keys_and_unique_constraints(datasets): + assert datasets["Books"].primary_key == ["ID"] + assert datasets["Books"].unique_keys == [["isbn"]] + assert sap_ext(datasets["Books"])["unique_key_names"] == ["isbn"] + + +def test_labels_descriptions_and_doc_comments(datasets): + books = by_name(datasets["Books"].fields) + assert books["title"].label == "Title" + assert books["title"].description == "Title as printed on the cover" + assert datasets["Authors"].description == "People who wrote books." + assert by_name(datasets["Authors"].fields)["name"].label == "Author name" + + +def test_datatypes_resolve_through_user_defined_types(datasets): + books = by_name(datasets["Books"].fields) + assert books["price"].datatype.value == "Decimal" # sap.capire.bookshop.Amount -> cds.Decimal + assert books["published"].datatype.value == "Date" + assert books["isbn"].datatype.value == "String" + assert by_name(datasets["Orders"].fields)["orderedAt"].datatype.value == "DateTime" + assert by_name(datasets["Orders"].fields)["ID"].datatype.value == "String" # cds.UUID + + +def test_binary_columns_are_opaque_and_keep_their_cds_type(result, datasets): + cover = by_name(datasets["Books"].fields)["cover"] + assert cover.datatype.value == "Opaque" + assert sap_ext(cover)["type"] == "cds.LargeBinary" + assert issues_of(result, ConverterIssueType.OPAQUE_DATATYPE) == [] # a known type, so nothing is lost + + +def test_unknown_type_is_opaque_and_reported(): + csn = {"definitions": {"E": {"kind": "entity", "elements": {"x": {"type": "vendor.Weird"}}}}} + result = CsnToOssieConverter().convert(csn) + assert result.output.datasets[0].fields[0].datatype.value == "Opaque" + assert [i.element_name for i in issues_of(result, ConverterIssueType.OPAQUE_DATATYPE)] == ["E.x"] + + +def test_temporal_fields_are_time_dimensions(datasets): + books = by_name(datasets["Books"].fields) + assert books["published"].is_time_dimension() + assert not books["title"].is_time_dimension() + + +def test_calculated_elements_become_sql_expressions(datasets): + books = by_name(datasets["Books"].fields) + assert sql_of(books["priceWithTax"]) == "price * 1.19" + assert sql_of(books["authorName"]) == "Authors.name" # a path through the `author` association + + +def test_managed_association_yields_a_relationship_and_its_foreign_key_field(result, datasets): + relationships = by_name(result.output.relationships) + author = relationships["author"] + assert (author.from_dataset, author.to) == ("Books", "Authors") + assert (author.from_columns, author.to_columns) == (["author_ID"], ["ID"]) + fk = by_name(datasets["Books"].fields)["author_ID"] + assert fk.datatype.value == "Integer" + assert sap_ext(fk) == {"association": "author", "generated_foreign_key": True} + + +def test_on_condition_association_uses_the_declared_columns(result): + book = by_name(result.output.relationships)["book"] + assert (book.from_dataset, book.to) == ("Orders", "Books") + assert (book.from_columns, book.to_columns) == (["book_ID"], ["ID"]) + assert sap_ext(book)["managed"] is False + + +def test_composition_is_a_relationship_too(result): + customers = by_name(result.output.relationships)["customers"] + assert customers.to == "Items" + assert sap_ext(customers)["type"] == "cds.Composition" + + +def test_backlink_is_kept_aside_and_reported(result, datasets): + assert "books" in sap_ext(datasets["Authors"])["unmodeled_elements"] + assert [i.element_name for i in issues_of(result, ConverterIssueType.RELATIONSHIP_DROPPED)] == ["Authors.books"] + assert "books" not in by_name(datasets["Authors"].fields) + + +def test_structured_element_is_kept_aside_and_reported(result, datasets): + assert "tags" in sap_ext(datasets["Orders"])["unmodeled_elements"] + assert [i.element_name for i in issues_of(result, ConverterIssueType.ELEMENT_NOT_MODELED)] == ["Orders.tags"] + + +def test_aggregation_annotations_become_metrics(result): + metrics = by_name(result.output.metrics) + assert sql_of(metrics["stock"]) == "SUM(Books.stock)" + assert sql_of(metrics["price"]) == "AVG(Books.price)" + assert metrics["stock"].datatype.value == "Integer" + assert metrics["price"].datatype.value == "Decimal" + + +def test_a_measure_stays_a_field_unless_it_is_derived(datasets): + assert "stock" in by_name(datasets["Books"].fields) + assert "amount" not in by_name(datasets["Orders"].fields) + assert by_name(datasets["Books"].fields)["stock"].dimension is None + + +def test_derived_measure_aggregates_its_expression_across_the_association(result): + assert sql_of(by_name(result.output.metrics)["amount"]) == "SUM(Orders.quantity * Books.price)" + + +def test_annotations_without_an_ossie_property_are_preserved(datasets): + price = sap_ext(by_name(datasets["Books"].fields)["price"]) + assert price["annotations"] == {"@Semantics.amount.currencyCode": "currency_code"} + assert sap_ext(by_name(datasets["Books"].fields)["title"])["native"] == {"length": 111} + assert sap_ext(datasets["Books"])["annotations"] == {"@Analytics.dataCategory": {"#": "CUBE"}} + + +def test_the_view_definition_is_preserved(datasets): + assert sap_ext(datasets["ListOfBooks"])["native"]["projection"]["from"]["ref"] == [BOOKS] + + +def test_other_definitions_and_root_keys_are_preserved(result): + data = sap_ext(result.output) + assert set(data["other_definitions"]) >= {"CatalogService", "sap.capire.bookshop.Amount", "sap.capire.bookshop.submitOrder"} + assert data["csn"]["namespace"] == "sap.capire.bookshop" + assert [i.element_name for i in issues_of(result, ConverterIssueType.UNSUPPORTED_DEFINITION_KIND)] == [ + "sap.capire.bookshop.submitOrder" + ] + + +def test_model_name_defaults_when_none_is_given(): + csn = load_fixture("bookshop.csn.json") + assert CsnToOssieConverter().convert(csn).output.name == "sap_csn_model" + + +def test_same_association_name_on_two_entities_is_disambiguated(): + key = {"type": "cds.Integer", "key": True} + csn = { + "definitions": { + "T": {"kind": "entity", "elements": {"ID": key}}, + "A": {"kind": "entity", "elements": {"ID": key, "t": {"type": "cds.Association", "target": "T"}}}, + "B": {"kind": "entity", "elements": {"ID": key, "t": {"type": "cds.Association", "target": "T"}}}, + } + } + names = {r.name for r in CsnToOssieConverter().convert(csn).output.relationships} + assert names == {"A_t", "B_t"} + + +def test_same_measure_name_on_two_entities_is_disambiguated(): + def entity(): + return {"kind": "entity", "elements": {"total": {"type": "cds.Integer", "@Aggregation.default": "#SUM"}}} + + csn = {"definitions": {"A": entity(), "B": entity()}} + names = {m.name for m in CsnToOssieConverter().convert(csn).output.metrics} + assert names == {"A_total", "B_total"} + + +def test_calculated_element_that_cannot_become_sql_is_kept_aside(): + csn = { + "definitions": { + "E": { + "kind": "entity", + "elements": { + "ID": {"type": "cds.Integer", "key": True}, + "who": {"type": "cds.String", "value": {"ref": ["$session", "user"]}}, + }, + } + } + } + result = CsnToOssieConverter().convert(csn) + assert "who" not in by_name(result.output.datasets[0].fields) + assert "who" in sap_ext(result.output.datasets[0])["unmodeled_elements"] + assert [i.issue_type for i in result.issues] == [ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE] + + +def test_association_to_an_unknown_target_is_kept_aside(): + csn = { + "definitions": { + "E": { + "kind": "entity", + "elements": {"ID": {"type": "cds.Integer", "key": True}, "x": {"type": "cds.Association", "target": "Elsewhere"}}, + } + } + } + result = CsnToOssieConverter().convert(csn) + assert result.output.relationships is None + assert [i.issue_type for i in result.issues] == [ConverterIssueType.RELATIONSHIP_DROPPED] + + +def test_input_is_not_mutated(): + csn = load_fixture("bookshop.csn.json") + before = copy.deepcopy(csn) + CsnToOssieConverter().convert(csn) + assert csn == before + + +@pytest.mark.parametrize( + "bad", + [[], {}, {"definitions": []}, {"definitions": {"T": {"kind": "type", "type": "cds.String"}}}], +) +def test_input_that_is_not_a_csn_model_is_an_error(bad): + with pytest.raises(ConverterError): + CsnToOssieConverter().convert(bad) + + +def test_flattened_names_that_still_collide_are_made_unique(): + entity = {"kind": "entity", "elements": {"ID": {"type": "cds.Integer", "key": True}}} + csn = {"definitions": {"a.Item": entity, "b.Item": entity, "foo.bar": entity, "foo_bar": entity}} + result = CsnToOssieConverter().convert(csn) + names = [d.name for d in result.output.datasets] + assert len(set(names)) == len(names) == 4 + # Export must accept the document import produced. + assert set(OssieToCsnConverter().convert(result.output).output["definitions"]) == set(csn["definitions"]) + + +def test_element_named_like_a_sql_keyword_yields_valid_expressions(): + csn = { + "definitions": { + "Orders": { + "kind": "entity", + "elements": { + "from": {"type": "cds.Integer"}, + "plus": {"type": "cds.Integer", "value": {"xpr": [{"ref": ["from"]}, "+", {"val": 1}]}}, + }, + } + } + } + fields = by_name(CsnToOssieConverter().convert(csn).output.datasets[0].fields) + assert sql_of(fields["from"]) == '"from"' + assert sql_of(fields["plus"]) == '"from" + 1' diff --git a/converters/sap-csn/tests/test_cxn.py b/converters/sap-csn/tests/test_cxn.py new file mode 100644 index 00000000..d89ad77d --- /dev/null +++ b/converters/sap-csn/tests/test_cxn.py @@ -0,0 +1,129 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +from ossie_sap_csn.cxn import UnsupportedExpression, cxn_to_sql, sql_to_cxn + + +def _bare(table, column): + return [column] if table is None else [table, column] + + +@pytest.mark.parametrize( + "cxn, sql", + [ + ({"ref": ["a"]}, "a"), + ({"val": "it's"}, "'it''s'"), + ({"val": 3}, "3"), + ({"val": None}, "NULL"), + ({"val": True}, "TRUE"), + ({"xpr": [{"ref": ["a"]}, "*", {"val": 2}]}, "a * 2"), + ({"xpr": [{"xpr": [{"ref": ["a"]}, "+", {"ref": ["b"]}]}, "*", {"ref": ["c"]}]}, "(a + b) * c"), + ({"xpr": [{"ref": ["a"]}, "is", "null"]}, "a IS NULL"), + ({"xpr": ["not", {"ref": ["a"]}]}, "NOT a"), + ({"xpr": [{"ref": ["a"]}, "in", {"list": [{"val": 1}, {"val": 2}]}]}, "a IN (1, 2)"), + ( + {"xpr": ["case", "when", {"xpr": [{"ref": ["a"]}, ">", {"val": 0}]}, "then", {"val": "pos"}, "else", {"val": "neg"}, "end"]}, + "CASE WHEN (a > 0) THEN 'pos' ELSE 'neg' END", + ), + ({"func": "upper", "args": [{"ref": ["a"]}]}, "UPPER(a)"), + ({"func": "count", "args": ["*"]}, "COUNT(*)"), + ({"func": "count", "args": [{"xpr": ["distinct", {"ref": ["a"]}]}]}, "COUNT(DISTINCT a)"), + ({"ref": ["a"], "cast": {"type": "cds.Integer"}}, "CAST(a AS INT)"), + ({"ref": ["assoc", "name"]}, "Target.name"), + ], +) +def test_cxn_to_sql(cxn, sql): + assert cxn_to_sql(cxn, lambda path: f"Target.{path[1]}") == sql + + +@pytest.mark.parametrize( + "sql", + [ + "a", + "a * 2", + "(a + b) * c", + "a IS NULL", + "NOT a", + "a IN (1, 2)", + "a BETWEEN 1 AND 5", + "a LIKE 'x%'", + "CASE WHEN a > 0 THEN 'pos' ELSE 'neg' END", + "UPPER(a)", + "COALESCE(a, b, 0)", + "COUNT(DISTINCT a)", + "CAST(a AS INT)", + "a || 'x'", + ], +) +def test_sql_survives_a_round_trip_through_cxn(sql): + cxn = sql_to_cxn(sql, _bare) + again = cxn_to_sql(cxn) + assert sql_to_cxn(again, _bare) == cxn + + +@pytest.mark.parametrize( + "sql", + [ + "SUM(a) OVER (PARTITION BY b)", + "(SELECT 1)", + "CAST(a AS GEOGRAPHY)", + "a IS NOT DISTINCT FROM b", + ], +) +def test_untranslatable_sql_is_refused_not_guessed(sql): + with pytest.raises(UnsupportedExpression): + sql_to_cxn(sql, _bare) + + +@pytest.mark.parametrize( + "cxn", + [ + {"ref": ["$session", "user"]}, + {"ref": [{"id": "a", "where": []}]}, + {"func": "f", "args": {"x": {"ref": ["a"]}}}, + {"SELECT": {}}, + {"ref": ["a", "b", "c"]}, + ], +) +def test_untranslatable_cxn_is_refused_not_guessed(cxn): + with pytest.raises(UnsupportedExpression): + cxn_to_sql(cxn) + + +@pytest.mark.parametrize("name, rendered", [("from", '"from"'), ("plus", "plus"), ("my col", '"my col"'), ('a"b', '"a""b"')]) +def test_identifiers_that_are_not_plain_words_are_quoted(name, rendered): + assert cxn_to_sql({"ref": [name]}) == rendered + + +def test_quoted_identifier_round_trips_through_sql(): + sql = cxn_to_sql({"xpr": [{"ref": ["select"]}, "+", {"val": 1}]}) + assert sql == '"select" + 1' + assert sql_to_cxn(sql, _bare) == {"xpr": [{"ref": ["select"]}, "+", {"val": 1}]} + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) +def test_non_finite_literals_are_refused(value): + with pytest.raises(UnsupportedExpression): + cxn_to_sql({"val": value}) + + +def test_parse_error_text_carries_no_ansi_escapes(): + with pytest.raises(UnsupportedExpression) as info: + sql_to_cxn("this is not sql !!!", _bare) + assert "\x1b" not in str(info.value) diff --git a/converters/sap-csn/tests/test_ossie_to_csn.py b/converters/sap-csn/tests/test_ossie_to_csn.py new file mode 100644 index 00000000..161dfdc4 --- /dev/null +++ b/converters/sap-csn/tests/test_ossie_to_csn.py @@ -0,0 +1,393 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest +from ossie import OssieDocument + +from ossie_sap_csn import OssieToCsnConverter +from ossie_sap_csn.converter_issues import ConverterError, ConverterIssueType + +from .helpers import load_tpcds + + +def raw_document(**overrides) -> dict: + def field(name, datatype="Integer", sql=None, **extra): + return { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql or name}]}, + "datatype": datatype, + **extra, + } + + raw = { + "version": "0.2.0.dev0", + "name": "m", + "datasets": [ + {"name": "cust", "source": "db.cust", "primary_key": ["id"], "fields": [field("id"), field("name", "String")]}, + { + "name": "orders", + "source": "db.orders", + "primary_key": ["id"], + "fields": [field("id"), field("cust_id"), field("qty"), field("price", "Decimal")], + }, + {"name": "other", "source": "db.other", "primary_key": ["id"], "fields": [field("id")]}, + ], + "relationships": [ + {"name": "orders_cust", "from": "orders", "to": "cust", "from_columns": ["cust_id"], "to_columns": ["id"]} + ], + "metrics": [], + } + raw.update(overrides) + return raw + + +def document(**overrides) -> OssieDocument: + return OssieDocument.model_validate(raw_document(**overrides)) + + +def metric(name, sql, **extra): + return {"name": name, "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": sql}]}, **extra} + + +def convert(doc): + result = OssieToCsnConverter().convert(doc) + return result.output, result.issues + + +@pytest.fixture(scope="module") +def tpcds(): + return convert(OssieDocument.model_validate(load_tpcds())) + + +def test_csn_shape_and_entities(): + csn, issues = convert(document()) + assert csn["$version"] == "2.0" + assert set(csn["definitions"]) == {"cust", "orders", "other"} + assert csn["definitions"]["orders"]["kind"] == "entity" + assert issues == [] + + +def test_elements_types_and_keys(): + csn, _ = convert(document()) + elements = csn["definitions"]["orders"]["elements"] + assert elements["id"] == {"type": "cds.Integer", "key": True} + assert elements["price"]["type"] == "cds.Decimal" + assert "key" not in elements["qty"] + + +def test_relationship_becomes_an_association_with_an_on_condition(): + csn, _ = convert(document()) + assoc = csn["definitions"]["orders"]["elements"]["orders_cust"] + assert assoc["type"] == "cds.Association" + assert assoc["target"] == "cust" + assert assoc["on"] == [{"ref": ["orders_cust", "id"]}, "=", {"ref": ["cust_id"]}] + + +def test_composite_relationship_joins_every_column_pair(): + doc = document( + relationships=[ + { + "name": "r", + "from": "orders", + "to": "cust", + "from_columns": ["cust_id", "qty"], + "to_columns": ["id", "name"], + } + ] + ) + csn, _ = convert(doc) + assert csn["definitions"]["orders"]["elements"]["r"]["on"] == [ + {"ref": ["r", "id"]}, "=", {"ref": ["cust_id"]}, + "and", + {"ref": ["r", "name"]}, "=", {"ref": ["qty"]}, + ] + + +def test_calculated_field_becomes_a_value_expression(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][1]["fields"].append( + { + "name": "total", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "qty * price"}]}, + "datatype": "Decimal", + } + ) + csn, _ = convert(OssieDocument.model_validate(raw)) + assert csn["definitions"]["orders"]["elements"]["total"]["value"] == { + "xpr": [{"ref": ["qty"]}, "*", {"ref": ["price"]}] + } + + +def test_untranslatable_field_expression_is_reported_and_kept_in_an_annotation(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][1]["fields"].append( + { + "name": "rank", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "RANK() OVER (ORDER BY qty)"}]}, + "datatype": "Integer", + } + ) + csn, issues = convert(OssieDocument.model_validate(raw)) + element = csn["definitions"]["orders"]["elements"]["rank"] + assert "value" not in element + assert element["@Ossie.sql"] == {"dialect": "ANSI_SQL", "expression": "RANK() OVER (ORDER BY qty)"} + assert [i.issue_type for i in issues] == [ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE] + + +def test_field_without_ansi_sql_is_reported(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][1]["fields"].append( + {"name": "x", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "qty::INT"}]}, "datatype": "Integer"} + ) + csn, issues = convert(OssieDocument.model_validate(raw)) + assert csn["definitions"]["orders"]["elements"]["x"]["@Ossie.sql"]["dialect"] == "SNOWFLAKE" + assert [i.issue_type for i in issues] == [ConverterIssueType.DIALECT_UNAVAILABLE] + + +def test_metric_over_a_column_of_the_same_name_annotates_the_element(): + csn, issues = convert(document(metrics=[metric("qty", "SUM(orders.qty)")])) + element = csn["definitions"]["orders"]["elements"]["qty"] + assert element["@Aggregation.default"] == {"#": "SUM"} + assert csn["definitions"]["orders"]["@Analytics.dataCategory"] == {"#": "CUBE"} + assert issues == [] + + +def test_metric_over_an_expression_becomes_a_derived_measure(): + csn, issues = convert(document(metrics=[metric("revenue", "SUM(orders.qty * orders.price)", datatype="Decimal")])) + element = csn["definitions"]["orders"]["elements"]["revenue"] + assert element["value"] == {"xpr": [{"ref": ["qty"]}, "*", {"ref": ["price"]}]} + assert element["@Aggregation.default"] == {"#": "SUM"} + assert element["type"] == "cds.Decimal" + assert issues == [] + + +def test_derived_measure_may_reach_other_datasets_through_associations(): + csn, _ = convert(document(metrics=[metric("n", "COUNT(DISTINCT cust.name)", datatype="Integer"), + metric("w", "SUM(orders.qty * cust.id)")])) + # `n` names only `cust`: it is homed on cust, not on orders. + assert csn["definitions"]["cust"]["elements"]["n"]["@Aggregation.default"] == {"#": "COUNT_DISTINCT"} + w = csn["definitions"]["orders"]["elements"]["w"] + assert w["value"] == {"xpr": [{"ref": ["qty"]}, "*", {"ref": ["orders_cust", "id"]}]} + + +@pytest.mark.parametrize( + "sql", + [ + "COUNT(*)", + "SUM(orders.qty) / COUNT(DISTINCT cust.id)", # two datasets and two aggregates + "SUM(orders.qty * other.id)", # no association reaches `other` + "SUM(qty)", # unqualified + "SUM(orders.qty) OVER (PARTITION BY orders.id)", + ], +) +def test_metric_csn_cannot_express_travels_in_meta_and_is_reported(sql): + csn, issues = convert(document(metrics=[metric("m1", sql)])) + assert [m["name"] for m in csn["meta"]["ossie"]["metrics"]] == ["m1"] + assert [i.issue_type for i in issues] == [ConverterIssueType.METRIC_NOT_MODELED] + assert "m1" not in csn["definitions"]["orders"]["elements"] + + +def test_measure_name_colliding_with_an_element_goes_to_meta(): + csn, issues = convert(document(metrics=[metric("qty", "SUM(orders.price)")])) + assert [m["name"] for m in csn["meta"]["ossie"]["metrics"]] == ["qty"] + assert "@Aggregation.default" not in csn["definitions"]["orders"]["elements"]["qty"] + assert [i.issue_type for i in issues] == [ConverterIssueType.METRIC_NOT_MODELED] + + +def test_unique_keys_use_cap_unique_constraints(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][0]["unique_keys"] = [["name"]] + csn, _ = convert(OssieDocument.model_validate(raw)) + assert csn["definitions"]["cust"]["@assert.unique.uk1"] == [{"=": "name"}] + + +def test_ai_context_and_other_vendors_extensions_ride_in_annotations(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][0]["ai_context"] = {"synonyms": ["clients"]} + raw["datasets"][0]["custom_extensions"] = [{"vendor_name": "DBT", "data": "{\"a\": 1}"}] + csn, _ = convert(OssieDocument.model_validate(raw)) + entity = csn["definitions"]["cust"] + assert entity["@Ossie.aiContext"] == {"synonyms": ["clients"]} + assert entity["@Ossie.customExtensions"] == [{"vendor_name": "DBT", "data": "{\"a\": 1}"}] + + +def test_field_without_a_datatype_assumes_a_string_and_says_so(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + del raw["datasets"][0]["fields"][1]["datatype"] + csn, issues = convert(OssieDocument.model_validate(raw)) + assert csn["definitions"]["cust"]["elements"]["name"]["type"] == "cds.String" + assert [i.issue_type for i in issues] == [ConverterIssueType.ASSUMED_DATATYPE] + + +def test_relationship_to_a_missing_dataset_is_dropped_and_reported(): + doc = document( + relationships=[{"name": "r", "from": "orders", "to": "ghost", "from_columns": ["cust_id"], "to_columns": ["id"]}] + ) + csn, issues = convert(doc) + assert [i.issue_type for i in issues] == [ConverterIssueType.RELATIONSHIP_DROPPED] + assert "r" not in csn["definitions"]["orders"]["elements"] + + +def test_relationship_name_colliding_with_a_column_is_made_distinct(): + doc = document( + relationships=[{"name": "cust_id", "from": "orders", "to": "cust", "from_columns": ["cust_id"], "to_columns": ["id"]}] + ) + csn, _ = convert(doc) + elements = csn["definitions"]["orders"]["elements"] + assert elements["cust_id"]["type"] == "cds.Integer" + assert elements["cust_id_assoc"]["type"] == "cds.Association" + + +def test_corrupt_vendor_extension_is_an_error(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][0]["custom_extensions"] = [{"vendor_name": "SAP_CSN", "data": "{nope"}] + with pytest.raises(ConverterError): + convert(OssieDocument.model_validate(raw)) + + +def test_tpcds_example_converts(tpcds): + csn, issues = tpcds + assert {i.element_name for i in issues if i.issue_type is ConverterIssueType.PRIMARY_KEY_COLUMN_UNRESOLVED} == {"store_sales"} + assert set(csn["definitions"]) == {"store_sales", "date_dim", "customer", "item", "store"} + sales = csn["definitions"]["store_sales"]["elements"] + assert sales["ss_item_sk"]["key"] + assert csn["definitions"]["store_sales"]["@Ossie.primaryKey"] == ["ss_item_sk", "ss_ticket_number"] # no such field in the example + assert sales["store_sales_to_date"]["target"] == "date_dim" + + +def test_tpcds_metrics_are_placed_or_reported(tpcds): + csn, issues = tpcds + meta_metrics = {m["name"] for m in csn["meta"]["ossie"]["metrics"]} + reported = {i.element_name for i in issues if i.issue_type is ConverterIssueType.METRIC_NOT_MODELED} + assert meta_metrics == reported + assert "total_sales" not in meta_metrics # SUM over one dataset: a measure element + element = csn["definitions"]["store_sales"]["elements"]["total_sales"] + assert element["@Aggregation.default"] == {"#": "SUM"} + assert "cumulative_sales" in meta_metrics # a window function + + +def test_key_naming_a_missing_column_is_reported_and_carried_whole(): + doc = document() + raw = json.loads(doc.to_ossie_json()) + raw["datasets"][1]["primary_key"] = ["id", "ghost"] + raw["datasets"][1]["unique_keys"] = [["id", "ghost"]] + csn, issues = convert(OssieDocument.model_validate(raw)) + orders = csn["definitions"]["orders"] + assert orders["@Ossie.primaryKey"] == ["id", "ghost"] + assert orders["@Ossie.uniqueKeys"] == [["id", "ghost"]] + assert orders["elements"]["id"]["key"] is True + assert not any(k.startswith("@assert.unique") for k in orders) + assert {i.issue_type for i in issues} == { + ConverterIssueType.PRIMARY_KEY_COLUMN_UNRESOLVED, + ConverterIssueType.UNIQUE_KEY_COLUMN_UNRESOLVED, + } + + +def test_data_category_is_added_only_to_entities_authored_in_ossie(): + csn, _ = convert(document(metrics=[metric("qty", "SUM(orders.qty)")])) + assert csn["definitions"]["orders"]["@Analytics.dataCategory"] == {"#": "CUBE"} + assert "@Analytics.dataCategory" not in csn["definitions"]["cust"] + + +def test_two_associations_to_one_target_make_a_path_through_it_ambiguous(): + raw = raw_document( + relationships=[ + {"name": "bill", "from": "orders", "to": "cust", "from_columns": ["cust_id"], "to_columns": ["id"]}, + {"name": "ship", "from": "orders", "to": "cust", "from_columns": ["qty"], "to_columns": ["id"]}, + ], + ) + raw["datasets"][1]["fields"].append( + { + "name": "next_id", + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "cust.id + 1"}]}, + "datatype": "Integer", + } + ) + csn, issues = convert(OssieDocument.model_validate(raw)) + assert "value" not in csn["definitions"]["orders"]["elements"]["next_id"] + assert [i.issue_type for i in issues] == [ConverterIssueType.EXPRESSION_NOT_TRANSLATABLE] + assert "ambiguous" in issues[0].detail + + +def test_metric_through_an_ambiguous_association_goes_to_meta(): + doc = document( + relationships=[ + {"name": "bill", "from": "orders", "to": "cust", "from_columns": ["cust_id"], "to_columns": ["id"]}, + {"name": "ship", "from": "orders", "to": "cust", "from_columns": ["qty"], "to_columns": ["id"]}, + ], + metrics=[metric("total", "SUM(orders.qty * cust.id)")], + ) + csn, issues = convert(doc) + assert [m["name"] for m in csn["meta"]["ossie"]["metrics"]] == ["total"] + assert [i.issue_type for i in issues] == [ConverterIssueType.METRIC_NOT_MODELED] + + +def test_unreadable_expression_message_carries_no_ansi_escapes(): + raw = raw_document() + raw["datasets"][0]["fields"][1]["expression"]["dialects"][0]["expression"] = "this is not sql !!!" + _, issues = convert(OssieDocument.model_validate(raw)) + assert issues and all("\x1b" not in i.detail for i in issues) + + +def test_measure_named_like_an_association_goes_to_meta(): + doc = document(metrics=[metric("orders_cust", "SUM(orders.cust_id)")]) + csn, issues = convert(doc) + assert csn["definitions"]["orders"]["elements"]["orders_cust"]["type"] == "cds.Association" + assert [m["name"] for m in csn["meta"]["ossie"]["metrics"]] == ["orders_cust"] + assert [i.issue_type for i in issues] == [ConverterIssueType.METRIC_NOT_MODELED] + + +def test_key_managed_association_keeps_key_true(): + raw = raw_document() + orders = raw["datasets"][1] + orders["primary_key"] = ["id", "orders_cust_id"] + orders["fields"][1].update( + name="orders_cust_id", + expression={"dialects": [{"dialect": "ANSI_SQL", "expression": "orders_cust_id"}]}, + custom_extensions=[ + {"vendor_name": "SAP_CSN", "data": json.dumps({"generated_foreign_key": True, "association": "orders_cust"})} + ], + ) + raw["relationships"][0]["from_columns"] = ["orders_cust_id"] + csn, _ = convert(OssieDocument.model_validate(raw)) + elements = csn["definitions"]["orders"]["elements"] + assert "orders_cust_id" not in elements + assert elements["orders_cust"]["key"] is True + assert elements["orders_cust"]["keys"] == [{"ref": ["id"]}] + + +def test_source_is_exported_as_the_persistence_name(): + csn, _ = convert(document()) + assert csn["definitions"]["cust"]["@cds.persistence.name"] == "db.cust" + + +def test_edited_source_replaces_a_stale_persistence_name(): + raw = raw_document() + raw["datasets"][0]["custom_extensions"] = [ + {"vendor_name": "SAP_CSN", "data": json.dumps({"annotations": {"@cds.persistence.name": "old.name"}})} + ] + csn, _ = convert(OssieDocument.model_validate(raw)) + assert csn["definitions"]["cust"]["@cds.persistence.name"] == "db.cust" diff --git a/converters/sap-csn/tests/test_roundtrip.py b/converters/sap-csn/tests/test_roundtrip.py new file mode 100644 index 00000000..151b8102 --- /dev/null +++ b/converters/sap-csn/tests/test_roundtrip.py @@ -0,0 +1,114 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json + +import pytest +import yaml +from ossie import OssieDataType, OssieDocument + +from ossie_sap_csn import CsnToOssieConverter, OssieToCsnConverter + +from .helpers import by_name, load_fixture, load_tpcds, validate_against_schema + + +def strip_additions(csn: dict) -> dict: + """Drop what export always adds: its creator stamp and the meta.ossie envelope.""" + csn = json.loads(json.dumps(csn)) + csn["meta"].pop("ossie", None) + if csn["meta"] == {"creator": "apache-ossie-sap-csn"}: + del csn["meta"] + return csn + + +def test_csn_survives_csn_ossie_csn(): + original = load_fixture("bookshop.csn.json") + ossie = CsnToOssieConverter().convert(original, name="bookshop").output + back = OssieToCsnConverter().convert(ossie).output + got, want = strip_additions(back), json.loads(json.dumps(original)) + # A managed association without `keys` comes back with the keys CDS would infer. + want["definitions"]["sap.capire.bookshop.Orders"]["elements"]["customers"]["keys"] = [{"ref": ["ID"]}] + assert got == want + + +def test_csn_survives_through_yaml(): + original = load_fixture("bookshop.csn.json") + ossie = CsnToOssieConverter().convert(original, name="bookshop").output + reloaded = OssieDocument.model_validate(yaml.safe_load(ossie.to_ossie_yaml())) + assert reloaded == ossie + assert OssieToCsnConverter().convert(reloaded).output == OssieToCsnConverter().convert(ossie).output + + +def test_second_csn_to_ossie_pass_is_stable(): + original = load_fixture("bookshop.csn.json") + first = CsnToOssieConverter().convert(original, name="bookshop").output + csn = OssieToCsnConverter().convert(first).output + second = CsnToOssieConverter().convert(csn, name="bookshop").output + assert second == first + + +def test_tpcds_survives_ossie_csn_ossie(): + original = OssieDocument.model_validate(load_tpcds()) + csn = OssieToCsnConverter().convert(original).output + back = CsnToOssieConverter().convert(csn).output + validate_against_schema(back) + + assert back.name == original.name + assert back.description == original.description + assert back.ai_context == original.ai_context + assert back.custom_extensions is not None and original.custom_extensions is not None + assert [e for e in back.custom_extensions if e.vendor_name != "SAP_CSN"] == original.custom_extensions + + assert set(by_name(back.datasets)) == set(by_name(original.datasets)) + for name, want in by_name(original.datasets).items(): + got = by_name(back.datasets)[name] + assert got.primary_key == want.primary_key + assert got.unique_keys == want.unique_keys + assert got.description == want.description + assert got.ai_context == want.ai_context + assert [(f.name, f.description, f.ai_context) for f in got.fields] == [ + (f.name, f.description, f.ai_context) for f in want.fields + ] + # CSN requires a type, so a field declared without one comes back as a String + # (the export reports ASSUMED_DATATYPE); every declared datatype is kept. + for got_field, want_field in zip(got.fields, want.fields): + assert got_field.datatype == (want_field.datatype or OssieDataType.STRING) + + for name, want in by_name(original.relationships).items(): + got = by_name(back.relationships)[name] + assert (got.from_dataset, got.to, got.from_columns, got.to_columns) == ( + want.from_dataset, want.to, want.from_columns, want.to_columns + ) + + +def sql(metric): + return metric.expression.dialects[0].expression.replace(" ", "") + + +def test_tpcds_metrics_all_survive_whatever_their_home(): + original = OssieDocument.model_validate(load_tpcds()) + back = CsnToOssieConverter().convert(OssieToCsnConverter().convert(original).output).output + got, want = by_name(back.metrics), by_name(original.metrics) + assert set(got) == set(want) + for name in want: + assert sql(got[name]) == sql(want[name]), name + assert got[name].description == want[name].description + + +@pytest.mark.parametrize("name", ["bookshop.csn.json"]) +def test_converted_fixture_validates_against_the_schema(name): + validate_against_schema(CsnToOssieConverter().convert(load_fixture(name), name="m").output)